commit f877c37fc6d4bf204960db5588ad6dc71fc4b1a5 Author: wehub-resource-sync Date: Mon Jul 13 12:37:51 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.github/CONTRIBUTOR_AGREEMENT.md b/.github/CONTRIBUTOR_AGREEMENT.md new file mode 100644 index 0000000..da9f244 --- /dev/null +++ b/.github/CONTRIBUTOR_AGREEMENT.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | | +| GitHub username | | +| Website (optional) | | diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..c9f30d1 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: [https://explosion.ai/merch, https://explosion.ai/tailored-solutions] diff --git a/.github/ISSUE_TEMPLATE/01_bugs.md b/.github/ISSUE_TEMPLATE/01_bugs.md new file mode 100644 index 0000000..f0d0ba9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/01_bugs.md @@ -0,0 +1,17 @@ +--- +name: "\U0001F6A8 Submit a Bug Report" +about: Use this template if you came across a bug or unexpected behaviour differing from the docs. + +--- + + + +## How to reproduce the behaviour + + +## Your Environment + +* Operating System: +* Python Version Used: +* spaCy Version Used: +* Environment Information: diff --git a/.github/ISSUE_TEMPLATE/02_docs.md b/.github/ISSUE_TEMPLATE/02_docs.md new file mode 100644 index 0000000..0df41ab --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02_docs.md @@ -0,0 +1,10 @@ +--- +name: "\U0001F4DA Submit a Documentation Report" +about: Did you spot a mistake in the docs, is anything unclear or do you have a + suggestion? + +--- + + +## Which page or section is this issue related to? + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..31f89f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false +contact_links: + - name: 🗯 Discussions Forum + url: https://github.com/explosion/spaCy/discussions + about: Install issues, usage questions, general discussion and anything else that isn't a bug report. + - name: 📖 spaCy FAQ & Troubleshooting + url: https://github.com/explosion/spaCy/discussions/8226 + about: Before you post, check out the FAQ for answers to common community questions! + - name: 💫 spaCy Usage Guides & API reference + url: https://spacy.io/usage + about: Everything you need to know about spaCy and how to use it. + - name: 🛠 Submit a Pull Request + url: https://github.com/explosion/spaCy/pulls + about: Did you spot a mistake and know how to fix it? Feel free to submit a PR straight away! diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..b48b2c5 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,19 @@ + + +## Description + + +### Types of change + + +## Checklist + +- [ ] I confirm that I have the right to submit this contribution under the project's MIT license. +- [ ] I ran the tests, and all new and existing tests passed. +- [ ] My changes don't require a change to the documentation, or if they do, I've added all required information. diff --git a/.github/contributors/0x2b3bfa0.md b/.github/contributors/0x2b3bfa0.md new file mode 100644 index 0000000..017aae5 --- /dev/null +++ b/.github/contributors/0x2b3bfa0.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Helio Machado | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-02-03 | +| GitHub username | 0x2b3bfa0 | +| Website (optional) | | diff --git a/.github/contributors/5hirish.md b/.github/contributors/5hirish.md new file mode 100644 index 0000000..c060423 --- /dev/null +++ b/.github/contributors/5hirish.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Shirish Kadam | +| Company name (if applicable) | SlicePay | +| Title or role (if applicable) | Android Developer | +| Date | 2017-11-13 | +| GitHub username | 5hirish | +| Website (optional) | https://shirishkadam.com | diff --git a/.github/contributors/ALSchwalm.md b/.github/contributors/ALSchwalm.md new file mode 100644 index 0000000..546ad73 --- /dev/null +++ b/.github/contributors/ALSchwalm.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Adam Schwalm | +| Company name (if applicable) | Star Lab | +| Title or role (if applicable) | Software Engineer | +| Date | 2018-11-28 | +| GitHub username | ALSchwalm | +| Website (optional) | https://alschwalm.com | diff --git a/.github/contributors/AMArostegui.md b/.github/contributors/AMArostegui.md new file mode 100644 index 0000000..0778a00 --- /dev/null +++ b/.github/contributors/AMArostegui.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Antonio Miras | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11/01/2020 | +| GitHub username | AMArostegui | +| Website (optional) | | diff --git a/.github/contributors/AlJohri.md b/.github/contributors/AlJohri.md new file mode 100644 index 0000000..4b2797a --- /dev/null +++ b/.github/contributors/AlJohri.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Al Johri | +| Company name (if applicable) | N/A | +| Title or role (if applicable) | N/A | +| Date | December 27th, 2019 | +| GitHub username | AlJohri | +| Website (optional) | http://aljohri.com/ | diff --git a/.github/contributors/Arvindcheenu.md b/.github/contributors/Arvindcheenu.md new file mode 100644 index 0000000..707a982 --- /dev/null +++ b/.github/contributors/Arvindcheenu.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Arvind Srinivasan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-06-13 | +| GitHub username | arvindcheenu | +| Website (optional) | | diff --git a/.github/contributors/AyushExel.md b/.github/contributors/AyushExel.md new file mode 100644 index 0000000..281fd0c --- /dev/null +++ b/.github/contributors/AyushExel.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ayush Chaurasia | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-03-12 | +| GitHub username | AyushExel | +| Website (optional) | | diff --git a/.github/contributors/Azagh3l.md b/.github/contributors/Azagh3l.md new file mode 100644 index 0000000..b0bf670 --- /dev/null +++ b/.github/contributors/Azagh3l.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ranty Baptiste | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11/06/2019 | +| GitHub username | Azagh3l | +| Website (optional) | | diff --git a/.github/contributors/Baciccin.md b/.github/contributors/Baciccin.md new file mode 100644 index 0000000..c7a940c --- /dev/null +++ b/.github/contributors/Baciccin.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Giovanni Battista Parodi | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-19 | +| GitHub username | Baciccin | +| Website (optional) | | diff --git a/.github/contributors/Bharat123rox.md b/.github/contributors/Bharat123rox.md new file mode 100644 index 0000000..5e2ad91 --- /dev/null +++ b/.github/contributors/Bharat123rox.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Bharat Raghunathan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 19 - 03 - 2019 | +| GitHub username | Bharat123rox | +| Website (optional) | | diff --git a/.github/contributors/BigstickCarpet.md b/.github/contributors/BigstickCarpet.md new file mode 100644 index 0000000..07b3564 --- /dev/null +++ b/.github/contributors/BigstickCarpet.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | James Messinger | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | May 23, 2018 | +| GitHub username | BigstickCarpet | +| Website (optional) | | diff --git a/.github/contributors/BramVanroy.md b/.github/contributors/BramVanroy.md new file mode 100644 index 0000000..cd71dbe --- /dev/null +++ b/.github/contributors/BramVanroy.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------| +| Name | Bram Vanroy | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | October 19, 2018 | +| GitHub username | BramVanroy | +| Website (optional) | https://bramvanroy.be | diff --git a/.github/contributors/BreakBB.md b/.github/contributors/BreakBB.md new file mode 100644 index 0000000..6cab317 --- /dev/null +++ b/.github/contributors/BreakBB.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Björn Böing | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 15.04.2019 | +| GitHub username | BreakBB | +| Website (optional) | | diff --git a/.github/contributors/Bri-Will.md b/.github/contributors/Bri-Will.md new file mode 100644 index 0000000..66bcd1e --- /dev/null +++ b/.github/contributors/Bri-Will.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschrnkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an x on one of the applicable statement below. Please do NOT +mark both statements: + + * [ X ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Brian Williams | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12/11/2017 | +| GitHub username | Bri-Will | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/Brixjohn.md b/.github/contributors/Brixjohn.md new file mode 100644 index 0000000..056103f --- /dev/null +++ b/.github/contributors/Brixjohn.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [X] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Brixter John Lumabi | +| Company name (if applicable) | Stratpoint | +| Title or role (if applicable) | Software Developer | +| Date | 18 December 2018 | +| GitHub username | Brixjohn | +| Website (optional) | | diff --git a/.github/contributors/Cinnamy.md b/.github/contributors/Cinnamy.md new file mode 100644 index 0000000..4f46bc3 --- /dev/null +++ b/.github/contributors/Cinnamy.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Marina Lysyuk | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13.10.2018 | +| GitHub username | Cinnamy | +| Website (optional) | | diff --git a/.github/contributors/DeNeutoy.md b/.github/contributors/DeNeutoy.md new file mode 100644 index 0000000..4cb3fbf --- /dev/null +++ b/.github/contributors/DeNeutoy.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Mark Neumann | +| Company name (if applicable) |Allen Institute for AI | +| Title or role (if applicable) |Research Engineer | +| Date | 13/01/2019 | +| GitHub username |@Deneutoy | +| Website (optional) |markneumann.xyz | diff --git a/.github/contributors/DimaBryuhanov.md b/.github/contributors/DimaBryuhanov.md new file mode 100644 index 0000000..78e5056 --- /dev/null +++ b/.github/contributors/DimaBryuhanov.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Dmitry Briukhanov | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 7/24/2018 | +| GitHub username | DimaBryuhanov | +| Website (optional) | | diff --git a/.github/contributors/Dobita21.md b/.github/contributors/Dobita21.md new file mode 100644 index 0000000..e206bbe --- /dev/null +++ b/.github/contributors/Dobita21.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nattapol | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 18.04.2019 | +| GitHub username | Dobita21 | +| Website (optional) | | diff --git a/.github/contributors/DoomCoder.md b/.github/contributors/DoomCoder.md new file mode 100644 index 0000000..0b9938b --- /dev/null +++ b/.github/contributors/DoomCoder.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Piotr Książek | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22.11.2018 | +| GitHub username | DoomCoder | +| Website (optional) | | diff --git a/.github/contributors/DuyguA.md b/.github/contributors/DuyguA.md new file mode 100644 index 0000000..817d25a --- /dev/null +++ b/.github/contributors/DuyguA.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Duygu Altinok | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13 November 2017 | +| GitHub username | DuyguA | +| Website (optional) | | diff --git a/.github/contributors/EARL_GREYT.md b/.github/contributors/EARL_GREYT.md new file mode 100644 index 0000000..3ee7d4f --- /dev/null +++ b/.github/contributors/EARL_GREYT.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | David Weßling | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 27.09.19 | +| GitHub username | EarlGreyT | +| Website (optional) | | diff --git a/.github/contributors/Eleni170.md b/.github/contributors/Eleni170.md new file mode 100644 index 0000000..723b63a --- /dev/null +++ b/.github/contributors/Eleni170.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Eleni Partalidou | +| Company name (if applicable) | DataScouting | +| Title or role (if applicable) | Software Engineer | +| Date | 06.07.2018 | +| GitHub username | Eleni170 | +| Website (optional) | | diff --git a/.github/contributors/EmilStenstrom.md b/.github/contributors/EmilStenstrom.md new file mode 100644 index 0000000..4736394 --- /dev/null +++ b/.github/contributors/EmilStenstrom.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------- | +| Name | Emil Stenström | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 2018-07-28 | +| GitHub username | EmilStenstrom | +| Website (optional) | https://friendlybit.com | diff --git a/.github/contributors/F0rge1cE.md b/.github/contributors/F0rge1cE.md new file mode 100644 index 0000000..f9f9871 --- /dev/null +++ b/.github/contributors/F0rge1cE.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Icarus Xu | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 05/06/2019 | +| GitHub username | F0rge1cE | +| Website (optional) | | diff --git a/.github/contributors/FallakAsad.md b/.github/contributors/FallakAsad.md new file mode 100644 index 0000000..0a0b00f --- /dev/null +++ b/.github/contributors/FallakAsad.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | | +| GitHub username | | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/GiorgioPorgio.md b/.github/contributors/GiorgioPorgio.md new file mode 100644 index 0000000..ffa1f69 --- /dev/null +++ b/.github/contributors/GiorgioPorgio.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | George Ketsopoulos | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 23 October 2019 | +| GitHub username | GiorgioPorgio | +| Website (optional) | | diff --git a/.github/contributors/Gizzio.md b/.github/contributors/Gizzio.md new file mode 100644 index 0000000..b9ca424 --- /dev/null +++ b/.github/contributors/Gizzio.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Stanisław Giziński | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 21.11.2018 | +| GitHub username | Gizzio | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/GuiGel.md b/.github/contributors/GuiGel.md new file mode 100644 index 0000000..43fb0f7 --- /dev/null +++ b/.github/contributors/GuiGel.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Guillaume Gelabert | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-11-15 | +| GitHub username | GuiGel | +| Website (optional) | | diff --git a/.github/contributors/Hazoom.md b/.github/contributors/Hazoom.md new file mode 100644 index 0000000..762cb5b --- /dev/null +++ b/.github/contributors/Hazoom.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Moshe Hazoom | +| Company name (if applicable) | Amenity Analytics | +| Title or role (if applicable) | NLP Engineer | +| Date | 2019-09-15 | +| GitHub username | Hazoom | +| Website (optional) | | diff --git a/.github/contributors/HiromuHota.md b/.github/contributors/HiromuHota.md new file mode 100644 index 0000000..24dfb1d --- /dev/null +++ b/.github/contributors/HiromuHota.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Hiromu Hota | +| Company name (if applicable) | Hitachi America, Ltd.| +| Title or role (if applicable) | Researcher | +| Date | 2019-03-25 | +| GitHub username | HiromuHota | +| Website (optional) | | diff --git a/.github/contributors/ICLRandD.md b/.github/contributors/ICLRandD.md new file mode 100644 index 0000000..8e8a49d --- /dev/null +++ b/.github/contributors/ICLRandD.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | --------------------------- | +| Name | Daniel Hoadley | +| Company name (if applicable) | ICLR&D | +| Title or role (if applicable) | Head of Research | +| Date | 9 Aug 2019 | +| GitHub username | ICLRandD | +| Website (optional) | https://research.iclr.co.uk | diff --git a/.github/contributors/IsaacHaze.md b/.github/contributors/IsaacHaze.md new file mode 100644 index 0000000..6e1ebd9 --- /dev/null +++ b/.github/contributors/IsaacHaze.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Isaac Sijaranamual | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 2017-12-10 | +| GitHub username | IsaacHaze | +| Website (optional) | - | diff --git a/.github/contributors/JKhakpour.md b/.github/contributors/JKhakpour.md new file mode 100644 index 0000000..622fb87 --- /dev/null +++ b/.github/contributors/JKhakpour.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ja'far Khakpour | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-09-24 | +| GitHub username | JKhakpour | +| Website (optional) | | diff --git a/.github/contributors/Jan-711.md b/.github/contributors/Jan-711.md new file mode 100644 index 0000000..6029764 --- /dev/null +++ b/.github/contributors/Jan-711.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jan Jessewitsch | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 16.02.2020 | +| GitHub username | Jan-711 | +| Website (optional) | | diff --git a/.github/contributors/JannisTriesToCode.md b/.github/contributors/JannisTriesToCode.md new file mode 100644 index 0000000..d834794 --- /dev/null +++ b/.github/contributors/JannisTriesToCode.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------------- | +| Name | Jannis Rauschke | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22.05.2020 | +| GitHub username | JannisTriesToCode | +| Website (optional) | https://twitter.com/JRauschke | diff --git a/.github/contributors/Jette16.md b/.github/contributors/Jette16.md new file mode 100644 index 0000000..c064f1d --- /dev/null +++ b/.github/contributors/Jette16.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Henriette Behr | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 23.09.2021 | +| GitHub username | Jette16 | +| Website (optional) | | diff --git a/.github/contributors/KKsharma99.md b/.github/contributors/KKsharma99.md new file mode 100644 index 0000000..4687702 --- /dev/null +++ b/.github/contributors/KKsharma99.md @@ -0,0 +1,108 @@ + + +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kunal Sharma | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10/19/2020 | +| GitHub username | KKsharma99 | +| Website (optional) | | diff --git a/.github/contributors/KennethEnevoldsen.md b/.github/contributors/KennethEnevoldsen.md new file mode 100644 index 0000000..0bbb28d --- /dev/null +++ b/.github/contributors/KennethEnevoldsen.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------- | +| Name | Kenneth Enevoldsen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-07-13 | +| GitHub username | KennethEnevoldsen | +| Website (optional) | www.kennethenevoldsen.com | diff --git a/.github/contributors/Kimahriman.md b/.github/contributors/Kimahriman.md new file mode 100644 index 0000000..19744e7 --- /dev/null +++ b/.github/contributors/Kimahriman.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Adam Binford | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 1/29/2018 | +| GitHub username | Kimahriman | +| Website (optional) | | diff --git a/.github/contributors/LRAbbade.md b/.github/contributors/LRAbbade.md new file mode 100644 index 0000000..bbc2a79 --- /dev/null +++ b/.github/contributors/LRAbbade.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Lucas Riêra Abbade | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-05-08 | +| GitHub username | LRAbbade | +| Website (optional) | | diff --git a/.github/contributors/Loghijiaha.md b/.github/contributors/Loghijiaha.md new file mode 100644 index 0000000..f3769e6 --- /dev/null +++ b/.github/contributors/Loghijiaha.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Loghi Perinpanayagam | +| Company name (if applicable) | | +| Title or role (if applicable) | Student | +| Date | 13 Jan, 2019 | +| GitHub username | loghijiaha | +| Website (optional) | | diff --git a/.github/contributors/Lucaterre.md b/.github/contributors/Lucaterre.md new file mode 100644 index 0000000..5da763b --- /dev/null +++ b/.github/contributors/Lucaterre.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- |---------------| +| Name | Lucas Terriel | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2022-06-20 | +| GitHub username | Lucaterre | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/MartinoMensio.md b/.github/contributors/MartinoMensio.md new file mode 100644 index 0000000..27e4536 --- /dev/null +++ b/.github/contributors/MartinoMensio.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Martino Mensio | +| Company name (if applicable) | The Open University | +| Title or role (if applicable) | PhD Student | +| Date | 17 November 2017 | +| GitHub username | MartinoMensio | +| Website (optional) | https://martinomensio.github.io/ | diff --git a/.github/contributors/MateuszOlko.md b/.github/contributors/MateuszOlko.md new file mode 100644 index 0000000..04467c7 --- /dev/null +++ b/.github/contributors/MateuszOlko.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mateusz Olko | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22.11.2018 | +| GitHub username | MateuszOlko | +| Website (optional) | | diff --git a/.github/contributors/MathiasDesch.md b/.github/contributors/MathiasDesch.md new file mode 100644 index 0000000..7ffec60 --- /dev/null +++ b/.github/contributors/MathiasDesch.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mathias Deschamps | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 9 November 2017 | +| GitHub username | MathiasDesch | +| Website (optional) | | diff --git a/.github/contributors/MiniLau.md b/.github/contributors/MiniLau.md new file mode 100644 index 0000000..14d6fe3 --- /dev/null +++ b/.github/contributors/MiniLau.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Desausoi Laurent | +| Company name (if applicable) | / | +| Title or role (if applicable) | / | +| Date | 22 November 2019 | +| GitHub username | MiniLau | +| Website (optional) | / | diff --git a/.github/contributors/MisterKeefe.md b/.github/contributors/MisterKeefe.md new file mode 100644 index 0000000..5216fc1 --- /dev/null +++ b/.github/contributors/MisterKeefe.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Tom Keefe | +| Company name (if applicable) | / | +| Title or role (if applicable) | / | +| Date | 18 February 2020 | +| GitHub username | MisterKeefe | +| Website (optional) | / | diff --git a/.github/contributors/Mlawrence95.md b/.github/contributors/Mlawrence95.md new file mode 100644 index 0000000..505d6c1 --- /dev/null +++ b/.github/contributors/Mlawrence95.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ x ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mike Lawrence | +| Company name (if applicable) | NA | +| Title or role (if applicable) | NA | +| Date | April 17, 2020 | +| GitHub username | Mlawrence95 | +| Website (optional) | | diff --git a/.github/contributors/NSchrading.md b/.github/contributors/NSchrading.md new file mode 100644 index 0000000..304a03a --- /dev/null +++ b/.github/contributors/NSchrading.md @@ -0,0 +1,98 @@ +# Syllogism contributor agreement + +This Syllogism Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +Syllogism Co. The term **"you"** shall mean the person or entity identified +below. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. Syllogism Co. may publicly + disclose your participation in the project, including the fact that you have + signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | J Nicolas Schrading | +| Company's name (if applicable) | | +| Title or Role (if applicable) | | +| Date | 2015-08-24 | +| GitHub username | NSchrading | +| Website (optional) | nicschrading.com | diff --git a/.github/contributors/NirantK.md b/.github/contributors/NirantK.md new file mode 100644 index 0000000..50f13dc --- /dev/null +++ b/.github/contributors/NirantK.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nirant Kasliwal | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | | +| GitHub username | NirantK | +| Website (optional) | https://nirantk.com | diff --git a/.github/contributors/Nuccy90.md b/.github/contributors/Nuccy90.md new file mode 100644 index 0000000..2d1adb8 --- /dev/null +++ b/.github/contributors/Nuccy90.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Elena Fano | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-09-21 | +| GitHub username | Nuccy90 | +| Website (optional) | | diff --git a/.github/contributors/Olamyy.md b/.github/contributors/Olamyy.md new file mode 100644 index 0000000..7111448 --- /dev/null +++ b/.github/contributors/Olamyy.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ x ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Olamilekan Wahab | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 8/11/2019 | +| GitHub username | Olamyy | +| Website (optional) | | diff --git a/.github/contributors/Pantalaymon.md b/.github/contributors/Pantalaymon.md new file mode 100644 index 0000000..f017f29 --- /dev/null +++ b/.github/contributors/Pantalaymon.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Valentin-Gabriel Soumah| +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-11-23 | +| GitHub username | Pantalaymon | +| Website (optional) | | diff --git a/.github/contributors/Pavle992.md b/.github/contributors/Pavle992.md new file mode 100644 index 0000000..74ba69f --- /dev/null +++ b/.github/contributors/Pavle992.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pavle Vidanović | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | August 4, 2019 | +| GitHub username | Pavle992 | +| Website (optional) | - | diff --git a/.github/contributors/PeterGilles.md b/.github/contributors/PeterGilles.md new file mode 100644 index 0000000..17fe3e2 --- /dev/null +++ b/.github/contributors/PeterGilles.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Peter Gilles | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10.10. | +| GitHub username | Peter Gilles | +| Website (optional) | | diff --git a/.github/contributors/PluieElectrique.md b/.github/contributors/PluieElectrique.md new file mode 100644 index 0000000..97e0165 --- /dev/null +++ b/.github/contributors/PluieElectrique.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pluie | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-06-18 | +| GitHub username | PluieElectrique | +| Website (optional) | | diff --git a/.github/contributors/Poluglottos.md b/.github/contributors/Poluglottos.md new file mode 100644 index 0000000..629fd82 --- /dev/null +++ b/.github/contributors/Poluglottos.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ryan Ford | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Mar 13 2019 | +| GitHub username | Poluglottos | +| Website (optional) | | diff --git a/.github/contributors/PolyglotOpenstreetmap.md b/.github/contributors/PolyglotOpenstreetmap.md new file mode 100644 index 0000000..575560f --- /dev/null +++ b/.github/contributors/PolyglotOpenstreetmap.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jo | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-01-26 | +| GitHub username | PolyglotOpenstreetmap| +| Website (optional) | | diff --git a/.github/contributors/R1j1t.md b/.github/contributors/R1j1t.md new file mode 100644 index 0000000..a92f1e0 --- /dev/null +++ b/.github/contributors/R1j1t.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Rajat | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 24 May 2020 | +| GitHub username | R1j1t | +| Website (optional) | | diff --git a/.github/contributors/RvanNieuwpoort.md b/.github/contributors/RvanNieuwpoort.md new file mode 100755 index 0000000..922ea5f --- /dev/null +++ b/.github/contributors/RvanNieuwpoort.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------- | +| Name | Rob van Nieuwpoort | +| Signing on behalf of | Dafne van Kuppevelt, Janneke van der Zwaan, Willem van Hage | +| Company name (if applicable) | Netherlands eScience center | +| Title or role (if applicable) | Director of technology | +| Date | 14-12-2016 | +| GitHub username | RvanNieuwpoort | +| Website (optional) | https://www.esciencecenter.nl/ | diff --git a/.github/contributors/SamEdwardes.md b/.github/contributors/SamEdwardes.md new file mode 100644 index 0000000..4e6453a --- /dev/null +++ b/.github/contributors/SamEdwardes.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sam Edwardes | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-04-02 | +| GitHub username | SamEdwardes | +| Website (optional) | samedwardes.com | diff --git a/.github/contributors/SamuelLKane.md b/.github/contributors/SamuelLKane.md new file mode 100644 index 0000000..9c74dcd --- /dev/null +++ b/.github/contributors/SamuelLKane.md @@ -0,0 +1,104 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Samuel Kane | +| Date | 3/20/19 | +| GitHub username | SamuelLKane | +| Website (optional) | samuel-kane.com | \ No newline at end of file diff --git a/.github/contributors/Schibsted.png b/.github/contributors/Schibsted.png new file mode 100644 index 0000000..2183418 Binary files /dev/null and b/.github/contributors/Schibsted.png differ diff --git a/.github/contributors/Stannislav.md b/.github/contributors/Stannislav.md new file mode 100644 index 0000000..899d6b0 --- /dev/null +++ b/.github/contributors/Stannislav.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Stanislav Schmidt | +| Company name (if applicable) | Blue Brain Project | +| Title or role (if applicable) | ML Engineer | +| Date | 2020-10-02 | +| GitHub username | Stannislav | +| Website (optional) | | diff --git a/.github/contributors/Tiljander.md b/.github/contributors/Tiljander.md new file mode 100644 index 0000000..89e70ef --- /dev/null +++ b/.github/contributors/Tiljander.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Henrik Tiljander | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 24/3/2020 | +| GitHub username | Tiljander | +| Website (optional) | | diff --git a/.github/contributors/YohannesDatasci.md b/.github/contributors/YohannesDatasci.md new file mode 100644 index 0000000..129c455 --- /dev/null +++ b/.github/contributors/YohannesDatasci.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yohannes | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-02 | +| GitHub username | YohannesDatasci | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/ZeeD.md b/.github/contributors/ZeeD.md new file mode 100644 index 0000000..460f91e --- /dev/null +++ b/.github/contributors/ZeeD.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Vito De Tullio | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-06-01 | +| GitHub username | ZeeD | +| Website (optional) | | diff --git a/.github/contributors/aajanki.md b/.github/contributors/aajanki.md new file mode 100644 index 0000000..de7dc7f --- /dev/null +++ b/.github/contributors/aajanki.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Antti Ajanki | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-11-30 | +| GitHub username | aajanki | +| Website (optional) | | diff --git a/.github/contributors/aaronkub.md b/.github/contributors/aaronkub.md new file mode 100644 index 0000000..c2a7f49 --- /dev/null +++ b/.github/contributors/aaronkub.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aaron Kub | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-05-09 | +| GitHub username | aaronkub | +| Website (optional) | | diff --git a/.github/contributors/aashishg.md b/.github/contributors/aashishg.md new file mode 100644 index 0000000..b58cf0c --- /dev/null +++ b/.github/contributors/aashishg.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aashish Gangwani | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 7/08/2018 | +| GitHub username | aashishg | +| Website (optional) | | diff --git a/.github/contributors/abchapman93.md b/.github/contributors/abchapman93.md new file mode 100644 index 0000000..5af0cb8 --- /dev/null +++ b/.github/contributors/abchapman93.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alec Chapman | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 7/17/2020 | +| GitHub username | abchapman93 | +| Website (optional) | | diff --git a/.github/contributors/abhi18av.md b/.github/contributors/abhi18av.md new file mode 100644 index 0000000..71a6671 --- /dev/null +++ b/.github/contributors/abhi18av.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Abhinav Sharma | +| Company name (if applicable) | Fourtek I.T. Solutions Pvt. Ltd. | +| Title or role (if applicable) | Machine Learning Engineer | +| Date | 3 November 2017 | +| GitHub username | abhi18av | +| Website (optional) | https://abhi18av.github.io/ | diff --git a/.github/contributors/adrianeboyd.md b/.github/contributors/adrianeboyd.md new file mode 100644 index 0000000..2c5ccb4 --- /dev/null +++ b/.github/contributors/adrianeboyd.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Adriane Boyd | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 28 January 2019 | +| GitHub username | adrianeboyd | +| Website (optional) | | diff --git a/.github/contributors/adrienball.md b/.github/contributors/adrienball.md new file mode 100644 index 0000000..29d957b --- /dev/null +++ b/.github/contributors/adrienball.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------------- | +| Name | Adrien Ball | +| Company name (if applicable) | | +| Title or role (if applicable) | Machine Learning Engineer | +| Date | 2019-03-07 | +| GitHub username | adrienball | +| Website (optional) | https://medium.com/@adrien_ball | \ No newline at end of file diff --git a/.github/contributors/ajrader.md b/.github/contributors/ajrader.md new file mode 100644 index 0000000..646d5b5 --- /dev/null +++ b/.github/contributors/ajrader.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Andrew J Rader | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | August 14, 2019 | +| GitHub username | ajrader | +| Website (optional) | | diff --git a/.github/contributors/akki2825.md b/.github/contributors/akki2825.md new file mode 100644 index 0000000..bc4cda0 --- /dev/null +++ b/.github/contributors/akki2825.md @@ -0,0 +1,87 @@ +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Akhilesh K R | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-02-12 | +| GitHub username | akki2825 | +| Website (optional) | | diff --git a/.github/contributors/akornilo.md b/.github/contributors/akornilo.md new file mode 100644 index 0000000..a38846e --- /dev/null +++ b/.github/contributors/akornilo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Anastassia Kornilova | +| Company name (if applicable) | / | +| Title or role (if applicable) | / | +| Date | 01.08.2019 | +| GitHub username | akornilo | +| Website (optional) | akornilo.com | diff --git a/.github/contributors/alexcombessie.md b/.github/contributors/alexcombessie.md new file mode 100644 index 0000000..7c5b22a --- /dev/null +++ b/.github/contributors/alexcombessie.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alex COMBESSIE | +| Company name (if applicable) | Dataiku | +| Title or role (if applicable) | R&D Engineer | +| Date | 2020-10-27 | +| GitHub username | alexcombessie | +| Website (optional) | | diff --git a/.github/contributors/alexvy86.md b/.github/contributors/alexvy86.md new file mode 100644 index 0000000..c8e9ce7 --- /dev/null +++ b/.github/contributors/alexvy86.md @@ -0,0 +1,87 @@ +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alejandro Villarreal | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-05-01 | +| GitHub username | alexvy86 | +| Website (optional) | | diff --git a/.github/contributors/aliiae.md b/.github/contributors/aliiae.md new file mode 100644 index 0000000..7e9cd48 --- /dev/null +++ b/.github/contributors/aliiae.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aliia Erofeeva | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13 June 2018 | +| GitHub username | aliiae | +| Website (optional) | | diff --git a/.github/contributors/alldefector.md b/.github/contributors/alldefector.md new file mode 100644 index 0000000..a32a6de --- /dev/null +++ b/.github/contributors/alldefector.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Feng Niu | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Feb 21, 2018 | +| GitHub username | alldefector | +| Website (optional) | | diff --git a/.github/contributors/alvaroabascar.md b/.github/contributors/alvaroabascar.md new file mode 100644 index 0000000..f035fd6 --- /dev/null +++ b/.github/contributors/alvaroabascar.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Álvaro Abella | +| Company name (if applicable) | IOMED | +| Title or role (if applicable) | CSO | +| Date | 21/12/2018 | +| GitHub username | alvaroabascar | +| Website (optional) | | diff --git a/.github/contributors/alvations.md b/.github/contributors/alvations.md new file mode 100644 index 0000000..2b9ba4a --- /dev/null +++ b/.github/contributors/alvations.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Liling | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 04 Jan 2019 | +| GitHub username | alvations | +| Website (optional) | | diff --git a/.github/contributors/ameyuuno.md b/.github/contributors/ameyuuno.md new file mode 100644 index 0000000..3db9d5a --- /dev/null +++ b/.github/contributors/ameyuuno.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alexey Kim | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-07-09 | +| GitHub username | ameyuuno | +| Website (optional) | https://ameyuuno.io | diff --git a/.github/contributors/amitness.md b/.github/contributors/amitness.md new file mode 100644 index 0000000..dd27e74 --- /dev/null +++ b/.github/contributors/amitness.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Amit Chaudhary | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | April 29, 2019 | +| GitHub username | amitness | +| Website (optional) | https://amitness.com | diff --git a/.github/contributors/amperinet.md b/.github/contributors/amperinet.md new file mode 100644 index 0000000..7ef458f --- /dev/null +++ b/.github/contributors/amperinet.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------- | +| Name | Amandine Périnet | +| Company name (if applicable) | 365Talents | +| Title or role (if applicable) | Data Science Researcher | +| Date | 28/01/2019 | +| GitHub username | amperinet | +| Website (optional) | | diff --git a/.github/contributors/aniruddha-adhikary.md b/.github/contributors/aniruddha-adhikary.md new file mode 100644 index 0000000..be8532d --- /dev/null +++ b/.github/contributors/aniruddha-adhikary.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aniruddha Adhikary | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-09-05 | +| GitHub username | aniruddha-adhikary | +| Website (optional) | https://adhikary.net | diff --git a/.github/contributors/ansgar-t.md b/.github/contributors/ansgar-t.md new file mode 100644 index 0000000..a45dea3 --- /dev/null +++ b/.github/contributors/ansgar-t.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ansgar Tümmers | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-05-26 | +| GitHub username | ansgar-t | +| Website (optional) | | diff --git a/.github/contributors/aongko.md b/.github/contributors/aongko.md new file mode 100644 index 0000000..148ac91 --- /dev/null +++ b/.github/contributors/aongko.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Andrew Ongko | +| Company name (if applicable) | Kurio | +| Title or role (if applicable) | Senior Data Science | +| Date | Sep 10, 2018 | +| GitHub username | aongko | +| Website (optional) | | diff --git a/.github/contributors/aristorinjuang.md b/.github/contributors/aristorinjuang.md new file mode 100644 index 0000000..17cb692 --- /dev/null +++ b/.github/contributors/aristorinjuang.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------- | +| Name | Aristo Rinjuang | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | May 22, 2018 | +| GitHub username | aristorinjuang | +| Website (optional) | https://aristorinjuang.com | diff --git a/.github/contributors/armsp.md b/.github/contributors/armsp.md new file mode 100644 index 0000000..45607d6 --- /dev/null +++ b/.github/contributors/armsp.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shantam Raj | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10/4/2021 | +| GitHub username | armsp | +| Website (optional) |https://shantamraj.com| diff --git a/.github/contributors/aryaprabhudesai.md b/.github/contributors/aryaprabhudesai.md new file mode 100644 index 0000000..f7858c1 --- /dev/null +++ b/.github/contributors/aryaprabhudesai.md @@ -0,0 +1,54 @@ +spaCy contributor agreement +This spaCy Contributor Agreement ("SCA") is based on the Oracle Contributor Agreement. The SCA applies to any contribution that you make to any product or project managed by us (the "project"), and sets out the intellectual property rights you grant to us in the contributed materials. The term "us" shall mean ExplosionAI UG (haftungsbeschränkt). The term "you" shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested below and include the filled-in version with your first pull request, under the folder .github/contributors/. The name of the file should be your GitHub username, with the extension .md. For example, the user example_user would create the file .github/contributors/example_user.md. + +Read this agreement carefully before signing. These terms and conditions constitute a binding legal agreement. + +Contributor Agreement +The term "contribution" or "contributed materials" means any source code, object code, patch, tool, sample, graphic, specification, manual, documentation, or any other material posted or submitted by you to the project. + +With respect to any worldwide copyrights, or copyright applications and registrations, in your contribution: + +you hereby assign to us joint ownership, and to the extent that such assignment is or becomes invalid, ineffective or unenforceable, you hereby grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, royalty-free, unrestricted license to exercise all rights under those copyrights. This includes, at our option, the right to sublicense these same rights to third parties through multiple levels of sublicensees or other licensing arrangements; + +you agree that each of us can do all things in relation to your contribution as if each of us were the sole owners, and if one of us makes a derivative work of your contribution, the one who makes the derivative work (or has it made will be the sole owner of that derivative work; + +you agree that you will not assert any moral rights in your contribution against us, our licensees or transferees; + +you agree that we may register a copyright in your contribution and exercise all ownership rights associated with it; and + +you agree that neither of us has any duty to consult with, obtain the consent of, pay or render an accounting to the other for any use or distribution of your contribution. + +With respect to any patents you own, or that you can license without payment to any third party, you hereby grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, royalty-free license to: + +make, have made, use, sell, offer to sell, import, and otherwise transfer your contribution in whole or in part, alone or in combination with or included in any product, work or materials arising out of the project to which your contribution was submitted, and + +at our option, to sublicense these same rights to third parties through multiple levels of sublicensees or other licensing arrangements. + +Except as set out above, you keep all right, title, and interest in your contribution. The rights that you grant to us under these terms are effective on the date you first submitted a contribution to us, even if your submission took place before the date you sign these terms. + +You covenant, represent, warrant and agree that: + +Each contribution that you submit is and shall be an original work of authorship and you can legally grant the rights set out in this SCA; + +to the best of your knowledge, each contribution will not violate any third party's copyrights, trademarks, patents, or other intellectual property rights; and + +each contribution shall be in compliance with U.S. export control laws and other applicable export and import laws. You agree to notify us if you become aware of any circumstance which would make any of the foregoing representations inaccurate in any respect. We may publicly disclose your participation in the project, including the fact that you have signed the SCA. + +This SCA is governed by the laws of the State of California and applicable U.S. Federal law. Any choice of law rules will not apply. + +Please place an “x” on one of the applicable statement below. Please do NOT mark both statements: + + [X] I am signing on behalf of myself as an individual and no other person or entity, including my employer, has or will have rights with respect to my contributions. + + I am signing on behalf of my employer or a legal entity and I have the actual authority to contractually bind that entity. + +Contributor Details +Field Entry +Name Arya Prabhudesai +Company name (if applicable) - +Title or role (if applicable) - +Date 2018-08-17 +GitHub username aryaprabhudesai +Website (optional) - diff --git a/.github/contributors/askhogan.md b/.github/contributors/askhogan.md new file mode 100644 index 0000000..9fbf9df --- /dev/null +++ b/.github/contributors/askhogan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Patrick Hogan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 7/7/2019 | +| GitHub username | askhogan@gmail.com | +| Website (optional) | | diff --git a/.github/contributors/avadhpatel.md b/.github/contributors/avadhpatel.md new file mode 100644 index 0000000..6609c7d --- /dev/null +++ b/.github/contributors/avadhpatel.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Avadh Patel | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 17.01.2018 | +| GitHub username | avadhpatel | +| Website (optional) | | diff --git a/.github/contributors/avi197.md b/.github/contributors/avi197.md new file mode 100644 index 0000000..903d7db --- /dev/null +++ b/.github/contributors/avi197.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Son Pham | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 09/10/2021 | +| GitHub username | Avi197 | +| Website (optional) | | diff --git a/.github/contributors/avramandrei.md b/.github/contributors/avramandrei.md new file mode 100644 index 0000000..6d4a696 --- /dev/null +++ b/.github/contributors/avramandrei.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Andrei-Marius Avram | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 20/08/2019 | +| GitHub username | avramandrei | +| Website (optional) | | diff --git a/.github/contributors/azarezade.md b/.github/contributors/azarezade.md new file mode 100644 index 0000000..4d33fab --- /dev/null +++ b/.github/contributors/azarezade.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ali Zarezade | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-01-23 | +| GitHub username | azarezade | +| Website (optional) | azarezade.github.io | diff --git a/.github/contributors/b1uec0in.md b/.github/contributors/b1uec0in.md new file mode 100644 index 0000000..2e2cd08 --- /dev/null +++ b/.github/contributors/b1uec0in.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Bae, Yong-Ju | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-07-25 | +| GitHub username | b1uec0in | +| Website (optional) | | diff --git a/.github/contributors/bbieniek.md b/.github/contributors/bbieniek.md new file mode 100644 index 0000000..4050946 --- /dev/null +++ b/.github/contributors/bbieniek.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Baltazar Bieniek | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021.08.19 | +| GitHub username | bbieniek | +| Website (optional) | https://baltazar.bieniek.org.pl/ | \ No newline at end of file diff --git a/.github/contributors/bdewilde.md b/.github/contributors/bdewilde.md new file mode 100644 index 0000000..f65a625 --- /dev/null +++ b/.github/contributors/bdewilde.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------------- | +| Name | Burton DeWilde | +| Company name (if applicable) | - | +| Title or role (if applicable) | data scientist | +| Date | 20 November 2017 | +| GitHub username | bdewilde | +| Website (optional) | https://bdewilde.github.io/ | diff --git a/.github/contributors/beatesi.md b/.github/contributors/beatesi.md new file mode 100644 index 0000000..2210965 --- /dev/null +++ b/.github/contributors/beatesi.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Beate Sildnes | +| Company name (if applicable) | NAV | +| Title or role (if applicable) | Data Scientist | +| Date | 04.12.2018 | +| GitHub username | beatesi | +| Website (optional) | | diff --git a/.github/contributors/bellabie.md b/.github/contributors/bellabie.md new file mode 100644 index 0000000..6543965 --- /dev/null +++ b/.github/contributors/bellabie.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | B Cavello | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-05-06 | +| GitHub username | bellabie | +| Website (optional) | bcavello.com | diff --git a/.github/contributors/bintay.md b/.github/contributors/bintay.md new file mode 100644 index 0000000..c64f2db --- /dev/null +++ b/.github/contributors/bintay.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ben Taylor | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | October 2, 2019 | +| GitHub username | bintay | +| Website (optional) | bentaylor.xyz | diff --git a/.github/contributors/bittlingmayer.md b/.github/contributors/bittlingmayer.md new file mode 100644 index 0000000..69ec98a --- /dev/null +++ b/.github/contributors/bittlingmayer.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Adam Bittlingmayer | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12 Aug 2020 | +| GitHub username | bittlingmayer | +| Website (optional) | | + diff --git a/.github/contributors/bjascob.md b/.github/contributors/bjascob.md new file mode 100644 index 0000000..4870c49 --- /dev/null +++ b/.github/contributors/bjascob.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Brad Jascob | +| Company name (if applicable) | n/a | +| Title or role (if applicable) | Software Engineer | +| Date | 04/25/2019 | +| GitHub username | bjascob | +| Website (optional) | n/a | diff --git a/.github/contributors/bodak.md b/.github/contributors/bodak.md new file mode 100644 index 0000000..f87224f --- /dev/null +++ b/.github/contributors/bodak.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kristian Boda | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 18.05.2021 | +| GitHub username | bodak | +| Website (optional) | | diff --git a/.github/contributors/boena.md b/.github/contributors/boena.md new file mode 100644 index 0000000..3178182 --- /dev/null +++ b/.github/contributors/boena.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Björn Lennartsson | +| Company name (if applicable) | Uptrail AB | +| Title or role (if applicable) | CTO | +| Date | 2019-01-15 | +| GitHub username | boena | +| Website (optional) | www.uptrail.com | diff --git a/.github/contributors/borijang.md b/.github/contributors/borijang.md new file mode 100644 index 0000000..0f2763f --- /dev/null +++ b/.github/contributors/borijang.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Borijan Georgievski | +| Company name (if applicable) | Netcetera | +| Title or role (if applicable) | Deta Scientist | +| Date | 2020.10.09 | +| GitHub username | borijang | +| Website (optional) | | diff --git a/.github/contributors/bratao.md b/.github/contributors/bratao.md new file mode 100644 index 0000000..9bc187c --- /dev/null +++ b/.github/contributors/bratao.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Bruno Souza Cabral | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 24/12/2020 | +| GitHub username | bratao | +| Website (optional) | | diff --git a/.github/contributors/broaddeep.md b/.github/contributors/broaddeep.md new file mode 100644 index 0000000..d6c4b3c --- /dev/null +++ b/.github/contributors/broaddeep.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Dongjun Park | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-03-06 | +| GitHub username | broaddeep | +| Website (optional) | | diff --git a/.github/contributors/bryant1410.md b/.github/contributors/bryant1410.md new file mode 100644 index 0000000..83f3a7a --- /dev/null +++ b/.github/contributors/bryant1410.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Santiago Castro | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-04-09 | +| GitHub username | bryant1410 | +| Website (optional) | | diff --git a/.github/contributors/bsweileh.md b/.github/contributors/bsweileh.md new file mode 100644 index 0000000..13f78a4 --- /dev/null +++ b/.github/contributors/bsweileh.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Belal | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | March 13, 2021 | +| GitHub username | bsweileh | +| Website (optional) | | diff --git a/.github/contributors/btrungchi.md b/.github/contributors/btrungchi.md new file mode 100644 index 0000000..1c27566 --- /dev/null +++ b/.github/contributors/btrungchi.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Bui Trung Chi | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-06-30 | +| GitHub username | btrungchi | +| Website (optional) | | diff --git a/.github/contributors/calumcalder.md b/.github/contributors/calumcalder.md new file mode 100644 index 0000000..f2c4442 --- /dev/null +++ b/.github/contributors/calumcalder.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Calum Calder | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22 March 2018 | +| GitHub username | calumcalder | +| Website (optional) | | diff --git a/.github/contributors/cbilgili.md b/.github/contributors/cbilgili.md new file mode 100644 index 0000000..934e3fa --- /dev/null +++ b/.github/contributors/cbilgili.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Canbey Bilgili | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 1 December 2017 | +| GitHub username | cbilgili | +| Website (optional) | | diff --git a/.github/contributors/cclauss.md b/.github/contributors/cclauss.md new file mode 100644 index 0000000..62ebbab --- /dev/null +++ b/.github/contributors/cclauss.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Chris Clauss | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 20 November 2017 | +| GitHub username | cclauss | +| Website (optional) | | diff --git a/.github/contributors/cedar101.md b/.github/contributors/cedar101.md new file mode 100644 index 0000000..4d04eba --- /dev/null +++ b/.github/contributors/cedar101.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Kim, Baeg-il | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-07-03 | +| GitHub username | cedar101 | +| Website (optional) | | diff --git a/.github/contributors/celikomer.md b/.github/contributors/celikomer.md new file mode 100644 index 0000000..793d169 --- /dev/null +++ b/.github/contributors/celikomer.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Omer Celik | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 04/11/2019 | +| GitHub username | celikomer | +| Website (optional) | www.ocelik.com | diff --git a/.github/contributors/ceteri.md b/.github/contributors/ceteri.md new file mode 100644 index 0000000..f29bfd3 --- /dev/null +++ b/.github/contributors/ceteri.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------- | +| Name | Paco Nathan | +| Company name (if applicable) | Derwen, Inc. | +| Title or role (if applicable) | Managing Partner | +| Date | 2020-01-25 | +| GitHub username | ceteri | +| Website (optional) | https://derwen.ai/paco | diff --git a/.github/contributors/charlax.md b/.github/contributors/charlax.md new file mode 100644 index 0000000..3cd799a --- /dev/null +++ b/.github/contributors/charlax.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Charles-Axel Dein | +| Company name (if applicable) | Skrib | +| Title or role (if applicable) | CEO | +| Date | 27/09/2018 | +| GitHub username | charlax | +| Website (optional) | www.dein.fr | diff --git a/.github/contributors/chezou.md b/.github/contributors/chezou.md new file mode 100644 index 0000000..e47a1e3 --- /dev/null +++ b/.github/contributors/chezou.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aki Ariga | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 07/12/2018 | +| GitHub username | chezou | +| Website (optional) | chezo.uno | diff --git a/.github/contributors/chopeen.md b/.github/contributors/chopeen.md new file mode 100644 index 0000000..d293c98 --- /dev/null +++ b/.github/contributors/chopeen.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Marek Grzenkowicz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020.04.10 | +| GitHub username | chopeen | +| Website (optional) | | diff --git a/.github/contributors/chrisdubois.md b/.github/contributors/chrisdubois.md new file mode 100644 index 0000000..beac1c5 --- /dev/null +++ b/.github/contributors/chrisdubois.md @@ -0,0 +1,98 @@ +# Syllogism contributor agreement + +This Syllogism Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +Syllogism Co. The term **"you"** shall mean the person or entity identified +below. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. Syllogism Co. may publicly + disclose your participation in the project, including the fact that you have + signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Chris DuBois | +| Company's name (if applicable) | | +| Title or Role (if applicable) | | +| Date | 2015.10.07 | +| GitHub username | chrisdubois | +| Website (optional) | | diff --git a/.github/contributors/cicorias.md b/.github/contributors/cicorias.md new file mode 100644 index 0000000..3dac47f --- /dev/null +++ b/.github/contributors/cicorias.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shawn Cicoria | +| Company name (if applicable) | Microsoft | +| Title or role (if applicable) | Principal Software Engineer | +| Date | November 20, 2018 | +| GitHub username | cicorias | +| Website (optional) | www.cicoria.com | \ No newline at end of file diff --git a/.github/contributors/clarus.md b/.github/contributors/clarus.md new file mode 100644 index 0000000..108d852 --- /dev/null +++ b/.github/contributors/clarus.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Guillaume Claret | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | June 19, 2019 | +| GitHub username | clarus | +| Website (optional) | guillaume.claret.me | diff --git a/.github/contributors/clippered.md b/.github/contributors/clippered.md new file mode 100644 index 0000000..2cd93d2 --- /dev/null +++ b/.github/contributors/clippered.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kenneth Cruz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-12-07 | +| GitHub username | clippered | +| Website (optional) | | diff --git a/.github/contributors/connorbrinton.md b/.github/contributors/connorbrinton.md new file mode 100644 index 0000000..25d03b4 --- /dev/null +++ b/.github/contributors/connorbrinton.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Connor Brinton | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | July 20th, 2021 | +| GitHub username | connorbrinton | +| Website (optional) | | diff --git a/.github/contributors/coryhurst.md b/.github/contributors/coryhurst.md new file mode 100644 index 0000000..39ba15b --- /dev/null +++ b/.github/contributors/coryhurst.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -----------------------------| +| Name | Cory Hurst | +| Company name (if applicable) | Samtec Smart Platform Group | +| Title or role (if applicable) | SoftwareDeveloper | +| Date | 2017-11-13 | +| GitHub username | cjhurst | +| Website (optional) | https://blog.spg.ai/ | diff --git a/.github/contributors/cristianasp.md b/.github/contributors/cristianasp.md new file mode 100644 index 0000000..a829098 --- /dev/null +++ b/.github/contributors/cristianasp.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Cristiana S Parada | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-11-04 | +| GitHub username | cristianasp | +| Website (optional) | | + diff --git a/.github/contributors/d99kris.md b/.github/contributors/d99kris.md new file mode 100644 index 0000000..8261b57 --- /dev/null +++ b/.github/contributors/d99kris.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kristofer Berggren | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 2017-12-17 | +| GitHub username | d99kris | +| Website (optional) | - | diff --git a/.github/contributors/danielhers.md b/.github/contributors/danielhers.md new file mode 100644 index 0000000..1a43dde --- /dev/null +++ b/.github/contributors/danielhers.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Daniel Hershcovich | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 8 November 2017 | +| GitHub username | danielhers | +| Website (optional) | www.cs.huji.ac.il/~danielh | diff --git a/.github/contributors/danielkingai2.md b/.github/contributors/danielkingai2.md new file mode 100644 index 0000000..dd2f600 --- /dev/null +++ b/.github/contributors/danielkingai2.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Daniel King | +| Company name (if applicable) | Allen Institute for Artificial Intelligence | +| Title or role (if applicable) | Predoctoral Young Investigator | +| Date | 03/06/2019 | +| GitHub username | danielkingai2 | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/danielruf.md b/.github/contributors/danielruf.md new file mode 100644 index 0000000..dd35ca7 --- /dev/null +++ b/.github/contributors/danielruf.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | --------------------- | +| Name | Daniel Ruf | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 07.06.2018 | +| GitHub username | danielruf | +| Website (optional) | https://daniel-ruf.de | diff --git a/.github/contributors/danielvasic.md b/.github/contributors/danielvasic.md new file mode 100644 index 0000000..03606c8 --- /dev/null +++ b/.github/contributors/danielvasic.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Daniel Vasić | +| Company name (if applicable) | University of Mostar | +| Title or role (if applicable) | Teaching asistant | +| Date | 13/10/2020 | +| GitHub username | danielvasic | +| Website (optional) | | diff --git a/.github/contributors/dardoria.md b/.github/contributors/dardoria.md new file mode 100644 index 0000000..0c7202f --- /dev/null +++ b/.github/contributors/dardoria.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Boian Tzonev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 18.02.2021 | +| GitHub username | dardoria | +| Website (optional) | | diff --git a/.github/contributors/darindf.md b/.github/contributors/darindf.md new file mode 100644 index 0000000..1e0cc29 --- /dev/null +++ b/.github/contributors/darindf.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Darin DeForest | +| Company name (if applicable) | Ipro Tech | +| Title or role (if applicable) | Senior Software Engineer | +| Date | 2018-09-26 | +| GitHub username | darindf | +| Website (optional) | | diff --git a/.github/contributors/delzac.md b/.github/contributors/delzac.md new file mode 100644 index 0000000..0fcfe6f --- /dev/null +++ b/.github/contributors/delzac.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Matthew Chin | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-09-22 | +| GitHub username | delzac | +| Website (optional) | | diff --git a/.github/contributors/demfier.md b/.github/contributors/demfier.md new file mode 100644 index 0000000..1a730fc --- /dev/null +++ b/.github/contributors/demfier.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gaurav Sahu | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-10-18 | +| GitHub username | demfier | +| Website (optional) | | diff --git a/.github/contributors/demongolem.md b/.github/contributors/demongolem.md new file mode 100644 index 0000000..29cb088 --- /dev/null +++ b/.github/contributors/demongolem.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gregory Werner | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 5/29/2019 | +| GitHub username | demongolem | +| Website (optional) | | diff --git a/.github/contributors/dhpollack.md b/.github/contributors/dhpollack.md new file mode 100644 index 0000000..444d97d --- /dev/null +++ b/.github/contributors/dhpollack.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | David Pollack | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Mar 5. 2020 | +| GitHub username | dhpollack | +| Website (optional) | | diff --git a/.github/contributors/dhruvrnaik.md b/.github/contributors/dhruvrnaik.md new file mode 100644 index 0000000..9639b6c --- /dev/null +++ b/.github/contributors/dhruvrnaik.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Dhruv Naik | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 26-01-2021 | +| GitHub username | dhruvrnaik | +| Website (optional) | | diff --git a/.github/contributors/doug-descombaz.md b/.github/contributors/doug-descombaz.md new file mode 100644 index 0000000..210bb72 --- /dev/null +++ b/.github/contributors/doug-descombaz.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Doug DesCombaz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-03-15 | +| GitHub username | doug-descombaz | +| Website (optional) | https://medium.com/@doug.descombaz | diff --git a/.github/contributors/drndos.md b/.github/contributors/drndos.md new file mode 100644 index 0000000..897959a --- /dev/null +++ b/.github/contributors/drndos.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Filip Bednárik | +| Company name (if applicable) | Ardevop, s. r. o. | +| Title or role (if applicable) | IT Consultant | +| Date | 2020-01-26 | +| GitHub username | drndos | +| Website (optional) | https://ardevop.sk | diff --git a/.github/contributors/dvsrepo.md b/.github/contributors/dvsrepo.md new file mode 100644 index 0000000..c9edcf1 --- /dev/null +++ b/.github/contributors/dvsrepo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Daniel Vila-Suero | +| Company name (if applicable) | recogn.ai | +| Title or role (if applicable) | | +| Date | 07-04-2017 | +| GitHub username | dvsrepo | +| Website (optional) | recogn.ai | diff --git a/.github/contributors/elbaulp.md b/.github/contributors/elbaulp.md new file mode 100644 index 0000000..1e9604c --- /dev/null +++ b/.github/contributors/elbaulp.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alejandro Alcalde | +| Company name (if applicable) | | +| Title or role (if applicable) | Data Scientist | +| Date | 07/17/2019 | +| GitHub username | elbaulp | +| Website (optional) | [elbauldelprogramador.com](https://elbauldelprogramador.com/en/) | diff --git a/.github/contributors/elben10 b/.github/contributors/elben10 new file mode 100644 index 0000000..1eb4656 --- /dev/null +++ b/.github/contributors/elben10 @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jakob Jul Elben | +| Company name (if applicable) | N/A | +| Title or role (if applicable) | N/A | +| Date | April 16th, 2020 | +| GitHub username | elben10 | +| Website (optional) | N/A | diff --git a/.github/contributors/emulbreh.md b/.github/contributors/emulbreh.md new file mode 100644 index 0000000..60388d2 --- /dev/null +++ b/.github/contributors/emulbreh.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Johannes Dollinger | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-02-13 | +| GitHub username | emulbreh | +| Website (optional) | | diff --git a/.github/contributors/enerrio.md b/.github/contributors/enerrio.md new file mode 100644 index 0000000..85ed022 --- /dev/null +++ b/.github/contributors/enerrio.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aaron Marquez | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2/15/2018 | +| GitHub username | enerrio | +| Website (optional) | | diff --git a/.github/contributors/er-raoniz.md b/.github/contributors/er-raoniz.md new file mode 100644 index 0000000..9521028 --- /dev/null +++ b/.github/contributors/er-raoniz.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Rahul Soni | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 30th September, 2019 | +| GitHub username | er-raoniz | +| Website (optional) | | diff --git a/.github/contributors/erip.md b/.github/contributors/erip.md new file mode 100644 index 0000000..56df073 --- /dev/null +++ b/.github/contributors/erip.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Elijah Rippeth | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-11-16 | +| GitHub username | erip | +| Website (optional) | | diff --git a/.github/contributors/estr4ng7d.md b/.github/contributors/estr4ng7d.md new file mode 100644 index 0000000..35c095c --- /dev/null +++ b/.github/contributors/estr4ng7d.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Amey Baviskar | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 21-May-2019 | +| GitHub username | estr4ng7d | +| Website (optional) | | diff --git a/.github/contributors/ezorita.md b/.github/contributors/ezorita.md new file mode 100644 index 0000000..e5f3f52 --- /dev/null +++ b/.github/contributors/ezorita.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Eduard Zorita | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 06/17/2021 | +| GitHub username | ezorita | +| Website (optional) | | diff --git a/.github/contributors/fgaim.md b/.github/contributors/fgaim.md new file mode 100644 index 0000000..1c3b409 --- /dev/null +++ b/.github/contributors/fgaim.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Fitsum Gaim | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-08-07 | +| GitHub username | fgaim | +| Website (optional) | | diff --git a/.github/contributors/filipecaixeta.md b/.github/contributors/filipecaixeta.md new file mode 100644 index 0000000..526583c --- /dev/null +++ b/.github/contributors/filipecaixeta.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Filipe Caixeta | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 09.12.2018 | +| GitHub username | filipecaixeta | +| Website (optional) | filipecaixeta.com.br | diff --git a/.github/contributors/fizban99.md b/.github/contributors/fizban99.md new file mode 100644 index 0000000..7970fd7 --- /dev/null +++ b/.github/contributors/fizban99.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | A.I.M. | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 16.04.2019 | +| GitHub username | fizban99 | +| Website (optional) | | diff --git a/.github/contributors/florijanstamenkovic.md b/.github/contributors/florijanstamenkovic.md new file mode 100644 index 0000000..65da875 --- /dev/null +++ b/.github/contributors/florijanstamenkovic.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Florijan Stamenkovic | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-10-05 | +| GitHub username | florijanstamenkovic | +| Website (optional) | | diff --git a/.github/contributors/fonfonx.md b/.github/contributors/fonfonx.md new file mode 100644 index 0000000..7fb01ca --- /dev/null +++ b/.github/contributors/fonfonx.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Xavier Fontaine | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2022-04-13 | +| GitHub username | fonfonx | +| Website (optional) | | diff --git a/.github/contributors/forest1988.md b/.github/contributors/forest1988.md new file mode 100644 index 0000000..486764b --- /dev/null +++ b/.github/contributors/forest1988.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yusuke Mori | +| Company name (if applicable) | | +| Title or role (if applicable) | Ph.D. student | +| Date | 2020/11/22 | +| GitHub username | forest1988 | +| Website (optional) | https://forest1988.github.io | diff --git a/.github/contributors/foufaster.md b/.github/contributors/foufaster.md new file mode 100644 index 0000000..824b99d --- /dev/null +++ b/.github/contributors/foufaster.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Anès Foufa | +| Company name (if applicable) | | +| Title or role (if applicable) |NLP developer | +| Date |21/01/2019 | +| GitHub username |foufaster | +| Website (optional) | | diff --git a/.github/contributors/frascuchon.md b/.github/contributors/frascuchon.md new file mode 100644 index 0000000..ed6be60 --- /dev/null +++ b/.github/contributors/frascuchon.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Francisco Aranda | +| Company name (if applicable) | recognai | +| Title or role (if applicable) | | +| Date | | +| GitHub username | frascuchon | +| Website (optional) | https://recogn.ai | diff --git a/.github/contributors/free-variation.md b/.github/contributors/free-variation.md new file mode 100644 index 0000000..81dc455 --- /dev/null +++ b/.github/contributors/free-variation.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | John Stewart | +| Company name (if applicable) | Amplify | +| Title or role (if applicable) | SVP Research | +| Date | 14/09/2018 | +| GitHub username | free-variation | +| Website (optional) | | diff --git a/.github/contributors/fsonntag.md b/.github/contributors/fsonntag.md new file mode 100644 index 0000000..0d84015 --- /dev/null +++ b/.github/contributors/fsonntag.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Felix Sonntag | +| Company name (if applicable) | - | +| Title or role (if applicable) | Student | +| Date | 2017-11-19 | +| GitHub username | fsonntag | +| Website (optional) | http://github.com/fsonntag/ | diff --git a/.github/contributors/fucking-signup.md b/.github/contributors/fucking-signup.md new file mode 100644 index 0000000..b34e23e --- /dev/null +++ b/.github/contributors/fucking-signup.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kit | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 2018/01/08 | +| GitHub username | fucking-signup | +| Website (optional) | - | diff --git a/.github/contributors/gandersen101.md b/.github/contributors/gandersen101.md new file mode 100644 index 0000000..cae4ad0 --- /dev/null +++ b/.github/contributors/gandersen101.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Grant Andersen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 07.06.2020 | +| GitHub username | gandersen101 | +| Website (optional) | | diff --git a/.github/contributors/gavrieltal.md b/.github/contributors/gavrieltal.md new file mode 100644 index 0000000..16ed381 --- /dev/null +++ b/.github/contributors/gavrieltal.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gavriel Loria | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Nov 29, 2018 | +| GitHub username | gavrieltal | +| Website (optional) | | diff --git a/.github/contributors/giannisdaras.md b/.github/contributors/giannisdaras.md new file mode 100644 index 0000000..2f403f4 --- /dev/null +++ b/.github/contributors/giannisdaras.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------------- | +| Name | Ioannis Daras | +| Company name (if applicable) | | +| Title or role (if applicable) | Computer Engineering Student | +| Date | 2018-07-17 | +| GitHub username | giannisdaras | +| Website (optional) | https://giannisdaras.github.io | diff --git a/.github/contributors/graue70.md b/.github/contributors/graue70.md new file mode 100644 index 0000000..7f9aa03 --- /dev/null +++ b/.github/contributors/graue70.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Thomas | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-08-11 | +| GitHub username | graue70 | +| Website (optional) | | diff --git a/.github/contributors/graus.md b/.github/contributors/graus.md new file mode 100644 index 0000000..3848dfe --- /dev/null +++ b/.github/contributors/graus.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | David Graus | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 28.03.2019 | +| GitHub username | graus | +| Website (optional) | graus.nu | + diff --git a/.github/contributors/greenriverrus.md b/.github/contributors/greenriverrus.md new file mode 100644 index 0000000..a7d6ba2 --- /dev/null +++ b/.github/contributors/greenriverrus.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | --------------------| +| Name | Vadim Mazaev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 26 November 2017 | +| GitHub username | GreenRiverRUS | +| Website (optional) | | diff --git a/.github/contributors/grivaz.md b/.github/contributors/grivaz.md new file mode 100644 index 0000000..0ebdbcc --- /dev/null +++ b/.github/contributors/grivaz.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |C. Grivaz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date |08.22.2018 | +| GitHub username |grivaz | +| Website (optional) | | diff --git a/.github/contributors/gtoffoli.md b/.github/contributors/gtoffoli.md new file mode 100644 index 0000000..5d5d712 --- /dev/null +++ b/.github/contributors/gtoffoli.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Giovanni Toffoli | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-05-12 | +| GitHub username | gtoffoli | +| Website (optional) | | diff --git a/.github/contributors/guerda.md b/.github/contributors/guerda.md new file mode 100644 index 0000000..6ac418e --- /dev/null +++ b/.github/contributors/guerda.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Philip Gillißen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-24 | +| GitHub username | guerda | +| Website (optional) | | diff --git a/.github/contributors/gustavengstrom.md b/.github/contributors/gustavengstrom.md new file mode 100644 index 0000000..d3e0da5 --- /dev/null +++ b/.github/contributors/gustavengstrom.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gustav Engström | +| Company name (if applicable) | Davcon | +| Title or role (if applicable) | Data scientis | +| Date | 2019-10-10 | +| GitHub username | gustavengstrom | +| Website (optional) | | diff --git a/.github/contributors/henry860916.md b/.github/contributors/henry860916.md new file mode 100644 index 0000000..b01f81e --- /dev/null +++ b/.github/contributors/henry860916.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Henry Zhang | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-04-30 | +| GitHub username | henry860916 | +| Website (optional) | | diff --git a/.github/contributors/hertelm.md b/.github/contributors/hertelm.md new file mode 100644 index 0000000..ba4250b --- /dev/null +++ b/.github/contributors/hertelm.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Matthias Hertel | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | June 29, 2020 | +| GitHub username | hertelm | +| Website (optional) | | diff --git a/.github/contributors/himkt.md b/.github/contributors/himkt.md new file mode 100644 index 0000000..e21aafd --- /dev/null +++ b/.github/contributors/himkt.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ x ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ x ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------- | +| Name | himkt | +| Company name (if applicable) | University of Tsukuba | +| Title or role (if applicable) | | +| Date | 2018-06-07 | +| GitHub username | himkt | +| Website (optional) | https://himkt.github.io | diff --git a/.github/contributors/hiroshi-matsuda-rit.md b/.github/contributors/hiroshi-matsuda-rit.md new file mode 100644 index 0000000..bf19125 --- /dev/null +++ b/.github/contributors/hiroshi-matsuda-rit.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Hiroshi Matsuda | +| Company name (if applicable) | Megagon Labs, Tokyo | +| Title or role (if applicable) | Research Scientist | +| Date | June 6, 2020 | +| GitHub username | hiroshi-matsuda-rit | +| Website (optional) | | diff --git a/.github/contributors/hlasse.md b/.github/contributors/hlasse.md new file mode 100644 index 0000000..b64b3c6 --- /dev/null +++ b/.github/contributors/hlasse.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------- | +| Name | Lasse Hansen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-08-11 | +| GitHub username | HLasse | +| Website (optional) | www.lassehansen.me | diff --git a/.github/contributors/holubvl3.md b/.github/contributors/holubvl3.md new file mode 100644 index 0000000..f2047b1 --- /dev/null +++ b/.github/contributors/holubvl3.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Vladimir Holubec | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 30.07.2020 | +| GitHub username | holubvl3 | +| Website (optional) | | diff --git a/.github/contributors/honnibal.md b/.github/contributors/honnibal.md new file mode 100644 index 0000000..3a700b7 --- /dev/null +++ b/.github/contributors/honnibal.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Matthew Honnibal | +| Company name (if applicable) | Explosion AI | +| Title or role (if applicable) | Founder | +| Date | 2017-10-18 | +| GitHub username | honnibal | +| Website (optional) | https://explosion.ai | diff --git a/.github/contributors/howl-anderson.md b/.github/contributors/howl-anderson.md new file mode 100644 index 0000000..902d354 --- /dev/null +++ b/.github/contributors/howl-anderson.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Xiaoquan Kong | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-03-23 | +| GitHub username | howl-anderson | +| Website (optional) | | diff --git a/.github/contributors/hugovk.md b/.github/contributors/hugovk.md new file mode 100644 index 0000000..1b96f68 --- /dev/null +++ b/.github/contributors/hugovk.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Hugo van Kemenade | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 26 November 2017 | +| GitHub username | hugovk | +| Website (optional) | | diff --git a/.github/contributors/iann0036.md b/.github/contributors/iann0036.md new file mode 100644 index 0000000..969c9ae --- /dev/null +++ b/.github/contributors/iann0036.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ian Mckay | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22/03/2018 | +| GitHub username | iann0036 | +| Website (optional) | | diff --git a/.github/contributors/idealley.md b/.github/contributors/idealley.md new file mode 100644 index 0000000..9aa7d4a --- /dev/null +++ b/.github/contributors/idealley.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pouyt Samuel | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 26.05.2018 | +| GitHub username | Idealley | +| Website (optional) | | diff --git a/.github/contributors/idoshr.md b/.github/contributors/idoshr.md new file mode 100644 index 0000000..26e9015 --- /dev/null +++ b/.github/contributors/idoshr.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ido Shraga | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 20-09-2020 | +| GitHub username | idoshr | +| Website (optional) | | diff --git a/.github/contributors/iechevarria.md b/.github/contributors/iechevarria.md new file mode 100644 index 0000000..f0c05ef --- /dev/null +++ b/.github/contributors/iechevarria.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | --------------------- | +| Name | Ivan Echevarria | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-12-24 | +| GitHub username | iechevarria | +| Website (optional) | https://echevarria.io | diff --git a/.github/contributors/ilivans.md b/.github/contributors/ilivans.md new file mode 100644 index 0000000..d471fde --- /dev/null +++ b/.github/contributors/ilivans.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Ilia Ivanov | +| Company name (if applicable) | Chattermill | +| Title or role (if applicable) | DL Engineer | +| Date | 2020-05-14 | +| GitHub username | ilivans | +| Website (optional) | | diff --git a/.github/contributors/ines.md b/.github/contributors/ines.md new file mode 100644 index 0000000..5cd57b0 --- /dev/null +++ b/.github/contributors/ines.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ines Montani | +| Company name (if applicable) | Explosion AI | +| Title or role (if applicable) | Founder | +| Date | 2017/10/18 | +| GitHub username | ines | +| Website (optional) | https://explosion.ai | diff --git a/.github/contributors/intrafindBreno.md b/.github/contributors/intrafindBreno.md new file mode 100644 index 0000000..204d20c --- /dev/null +++ b/.github/contributors/intrafindBreno.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Breno Faria | +| Company name (if applicable) | IntraFind | +| Title or role (if applicable) | Product Lead | +| Date | 03.06.2019 | +| GitHub username | intrafindBreno | +| Website (optional) | | diff --git a/.github/contributors/isaric.md b/.github/contributors/isaric.md new file mode 100644 index 0000000..698eb1d --- /dev/null +++ b/.github/contributors/isaric.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ivan Šarić | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 18.08.2019. | +| GitHub username | isaric | +| Website (optional) | | diff --git a/.github/contributors/iurshina.md b/.github/contributors/iurshina.md new file mode 100644 index 0000000..2268130 --- /dev/null +++ b/.github/contributors/iurshina.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Anastasiia Iurshina | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 28.12.2019 | +| GitHub username | iurshina | +| Website (optional) | | diff --git a/.github/contributors/ivigamberdiev.md b/.github/contributors/ivigamberdiev.md new file mode 100644 index 0000000..9c5496e --- /dev/null +++ b/.github/contributors/ivigamberdiev.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Igor Igamberdiev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | April 2, 2019 | +| GitHub username | ivigamberdiev | +| Website (optional) | | diff --git a/.github/contributors/ivyleavedtoadflax.md b/.github/contributors/ivyleavedtoadflax.md new file mode 100644 index 0000000..f8bb22e --- /dev/null +++ b/.github/contributors/ivyleavedtoadflax.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Matthew Upson | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date |2018-04-24 | +| GitHub username |ivyleavedtoadflax | +| Website (optional) |www.machinegurning.com| diff --git a/.github/contributors/jabortell.md b/.github/contributors/jabortell.md new file mode 100644 index 0000000..4f87891 --- /dev/null +++ b/.github/contributors/jabortell.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jacob Bortell | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-11-20 | +| GitHub username | jabortell | +| Website (optional) | | diff --git a/.github/contributors/jacopofar.md b/.github/contributors/jacopofar.md new file mode 100644 index 0000000..555d4d8 --- /dev/null +++ b/.github/contributors/jacopofar.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jacopo Farina | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-10-12 | +| GitHub username | jacopofar | +| Website (optional) | jacopofarina.eu | diff --git a/.github/contributors/jacse.md b/.github/contributors/jacse.md new file mode 100644 index 0000000..7face10 --- /dev/null +++ b/.github/contributors/jacse.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jacob Lauritzen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-30 | +| GitHub username | jacse | +| Website (optional) | | diff --git a/.github/contributors/janimo.md b/.github/contributors/janimo.md new file mode 100644 index 0000000..dfa4079 --- /dev/null +++ b/.github/contributors/janimo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jani Monoses | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 5/10/2018 | +| GitHub username | janimo | +| Website (optional) | | diff --git a/.github/contributors/jankrepl.md b/.github/contributors/jankrepl.md new file mode 100644 index 0000000..eda5a29 --- /dev/null +++ b/.github/contributors/jankrepl.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jan Krepl | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-03-09 | +| GitHub username | jankrepl | +| Website (optional) | | diff --git a/.github/contributors/jarib.md b/.github/contributors/jarib.md new file mode 100644 index 0000000..e6b19c4 --- /dev/null +++ b/.github/contributors/jarib.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jari Bakken | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-12-21 | +| GitHub username | jarib | +| Website (optional) | | diff --git a/.github/contributors/jaydeepborkar.md b/.github/contributors/jaydeepborkar.md new file mode 100644 index 0000000..32199d5 --- /dev/null +++ b/.github/contributors/jaydeepborkar.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jaydeep Borkar | +| Company name (if applicable) | Pune University, India | +| Title or role (if applicable) | CS Undergrad | +| Date | 9/26/2019 | +| GitHub username | jaydeepborkar | +| Website (optional) | http://jaydeepborkar.github.io | diff --git a/.github/contributors/jbesomi.md b/.github/contributors/jbesomi.md new file mode 100644 index 0000000..ac43a3b --- /dev/null +++ b/.github/contributors/jbesomi.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jonathan B. | +| Company name (if applicable) | besomi.ai | +| Title or role (if applicable) | - | +| Date | 07.07.2020 | +| GitHub username | jbesomi | +| Website (optional) | besomi.ai | diff --git a/.github/contributors/jeannefukumaru.md b/.github/contributors/jeannefukumaru.md new file mode 100644 index 0000000..bde2c78 --- /dev/null +++ b/.github/contributors/jeannefukumaru.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | jeanneC | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 4th April 2019 | +| GitHub username | jeannefukumaru | +| Website (optional) | | diff --git a/.github/contributors/jenojp.md b/.github/contributors/jenojp.md new file mode 100644 index 0000000..3ba5ed2 --- /dev/null +++ b/.github/contributors/jenojp.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jeno Pizarro | +| Company name (if applicable) | IBM | +| Title or role (if applicable) | Data Scientist and Software Developer | +| Date | 3 August 2019 | +| GitHub username | jenojp | +| Website (optional) | https://www.linkedin.com/in/jenojp/ | diff --git a/.github/contributors/jerbob92.md b/.github/contributors/jerbob92.md new file mode 100644 index 0000000..bb0430d --- /dev/null +++ b/.github/contributors/jerbob92.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jeroen Bobbeldijk | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22-10-2017 | +| GitHub username | jerbob92 | +| Website (optional) | | diff --git a/.github/contributors/jganseman.md b/.github/contributors/jganseman.md new file mode 100644 index 0000000..dc25bee --- /dev/null +++ b/.github/contributors/jganseman.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Joachim Ganseman | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 26/01/2021 | +| GitHub username | jganseman | +| Website (optional) | www.ganseman.be | diff --git a/.github/contributors/jgutix.md b/.github/contributors/jgutix.md new file mode 100644 index 0000000..4bda948 --- /dev/null +++ b/.github/contributors/jgutix.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Juan Gutiérrez | +| Company name (if applicable) | Ojtli | +| Title or role (if applicable) | | +| Date | 2020-08-28 | +| GitHub username | jgutix | +| Website (optional) | ojtli.app | diff --git a/.github/contributors/jimregan.md b/.github/contributors/jimregan.md new file mode 100644 index 0000000..dd8fe3d --- /dev/null +++ b/.github/contributors/jimregan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jim O'Regan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-06-24 | +| GitHub username | jimregan | +| Website (optional) | | diff --git a/.github/contributors/jklaise.md b/.github/contributors/jklaise.md new file mode 100644 index 0000000..66d77ee --- /dev/null +++ b/.github/contributors/jklaise.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Janis Klaise | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date |26/04/2021 | +| GitHub username |jklaise | +| Website (optional) |janisklaise.com | diff --git a/.github/contributors/jmargeta.md b/.github/contributors/jmargeta.md new file mode 100644 index 0000000..ab13424 --- /dev/null +++ b/.github/contributors/jmargeta.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jan Margeta | +| Company name (if applicable) | KardioMe | +| Title or role (if applicable) | Founder | +| Date | 2020-10-16 | +| GitHub username | jmargeta | +| Website (optional) | kardio.me | diff --git a/.github/contributors/jmyerston.md b/.github/contributors/jmyerston.md new file mode 100644 index 0000000..be5db54 --- /dev/null +++ b/.github/contributors/jmyerston.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | ----------------------------------- | +| Name | Jacobo Myerston | +| Company name (if applicable) | University of California, San Diego | +| Title or role (if applicable) | Academic | +| Date | 07/05/2021 | +| GitHub username | jmyerston | +| Website (optional) | diogenet.ucsd.edu | diff --git a/.github/contributors/johnhaley81.md b/.github/contributors/johnhaley81.md new file mode 100644 index 0000000..277b312 --- /dev/null +++ b/.github/contributors/johnhaley81.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | John Haley | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 19/10/2017 | +| GitHub username | johnhaley81 | +| Website (optional) | | diff --git a/.github/contributors/jonesmartins.md b/.github/contributors/jonesmartins.md new file mode 100644 index 0000000..5663f61 --- /dev/null +++ b/.github/contributors/jonesmartins.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jones Martins | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-06-10 | +| GitHub username | jonesmartins | +| Website (optional) | | diff --git a/.github/contributors/juliamakogon.md b/.github/contributors/juliamakogon.md new file mode 100644 index 0000000..90133b5 --- /dev/null +++ b/.github/contributors/juliamakogon.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Julia Makogon | +| Company name (if applicable) | Semantrum | +| Title or role (if applicable) | | +| Date | 07.02.2019 | +| GitHub username | juliamakogon | +| Website (optional) | | diff --git a/.github/contributors/julien-talkair.md b/.github/contributors/julien-talkair.md new file mode 100644 index 0000000..f8a1933 --- /dev/null +++ b/.github/contributors/julien-talkair.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Julien Rossi | +| Company name (if applicable) | TalkAir BV | +| Title or role (if applicable) | CTO, Partner | +| Date | June 28 2021 | +| GitHub username | julien-talkair | +| Website (optional) | | diff --git a/.github/contributors/juliensalinas.md b/.github/contributors/juliensalinas.md new file mode 100644 index 0000000..0062426 --- /dev/null +++ b/.github/contributors/juliensalinas.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | ------------------- | +| Name | Julien Salinas | +| Company name (if applicable) | NLP Cloud | +| Title or role (if applicable) | Founder and CTO | +| Date | Mayb 14th 2021 | +| GitHub username | juliensalinas | +| Website (optional) | https://nlpcloud.io | diff --git a/.github/contributors/jumasheff.md b/.github/contributors/jumasheff.md new file mode 100644 index 0000000..1ce6d23 --- /dev/null +++ b/.github/contributors/jumasheff.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Murat Jumashev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 25.01.2021 | +| GitHub username | jumasheff | +| Website (optional) | | diff --git a/.github/contributors/justindujardin.md b/.github/contributors/justindujardin.md new file mode 100644 index 0000000..35d403a --- /dev/null +++ b/.github/contributors/justindujardin.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Justin DuJardin | +| Company name (if applicable) | DuJardin Consulting, LLC | +| Title or role (if applicable) | | +| Date | 2018-03-23 | +| GitHub username | justindujardin | +| Website (optional) | https://justindujardin.com | diff --git a/.github/contributors/kabirkhan.md b/.github/contributors/kabirkhan.md new file mode 100644 index 0000000..4dfb3df --- /dev/null +++ b/.github/contributors/kabirkhan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Kabir Khan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-04-08 | +| GitHub username | kabirkhan | +| Website (optional) | | diff --git a/.github/contributors/katarkor.md b/.github/contributors/katarkor.md new file mode 100644 index 0000000..a8b6293 --- /dev/null +++ b/.github/contributors/katarkor.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Katarzyna Korsak | +| Company name (if applicable) | Nav | +| Title or role (if applicable) | | +| Date | 18.07.2018 | +| GitHub username | katarkor | +| Website (optional) | | diff --git a/.github/contributors/katrinleinweber.md b/.github/contributors/katrinleinweber.md new file mode 100644 index 0000000..ca0c908 --- /dev/null +++ b/.github/contributors/katrinleinweber.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Katrin Leinweber | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-03-30 | +| GitHub username | katrinleinweber | +| Website (optional) | | diff --git a/.github/contributors/kbulygin.md b/.github/contributors/kbulygin.md new file mode 100644 index 0000000..4679868 --- /dev/null +++ b/.github/contributors/kbulygin.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kirill Bulygin | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-12-18 | +| GitHub username | kbulygin | +| Website (optional) | | diff --git a/.github/contributors/keshan.md b/.github/contributors/keshan.md new file mode 100644 index 0000000..96b5d89 --- /dev/null +++ b/.github/contributors/keshan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Keshan Sodimana | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Sep 21, 2018 | +| GitHub username | keshan | +| Website (optional) | | diff --git a/.github/contributors/keshav.md b/.github/contributors/keshav.md new file mode 100644 index 0000000..4f9eeac --- /dev/null +++ b/.github/contributors/keshav.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Keshav Garg | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Jan 22, 2021 | +| GitHub username | KeshavG-lb | +| Website (optional) | | diff --git a/.github/contributors/kevinlu1248.md b/.github/contributors/kevinlu1248.md new file mode 100644 index 0000000..fc974ec --- /dev/null +++ b/.github/contributors/kevinlu1248.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kevin Lu| +| Company name (if applicable) | | +| Title or role (if applicable) | Student| +| Date | | +| GitHub username | kevinlu1248| +| Website (optional) | | diff --git a/.github/contributors/khellan.md b/.github/contributors/khellan.md new file mode 100644 index 0000000..3e7f78e --- /dev/null +++ b/.github/contributors/khellan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Knut O. Hellan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 02.07.2019 | +| GitHub username | khellan | +| Website (optional) | knuthellan.com | diff --git a/.github/contributors/kimfalk.md b/.github/contributors/kimfalk.md new file mode 100644 index 0000000..52a3e01 --- /dev/null +++ b/.github/contributors/kimfalk.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kim Falk | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 19 December 2017 | +| GitHub username | kimfalk | +| Website (optional) | | diff --git a/.github/contributors/knoxdw.md b/.github/contributors/knoxdw.md new file mode 100644 index 0000000..261e563 --- /dev/null +++ b/.github/contributors/knoxdw.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Douglas Knox | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-04-27 | +| GitHub username | knoxdw | +| Website (optional) | | diff --git a/.github/contributors/koaning.md b/.github/contributors/koaning.md new file mode 100644 index 0000000..ddb28ca --- /dev/null +++ b/.github/contributors/koaning.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Vincent D. Warmerdam | +| Company name (if applicable) | | +| Title or role (if applicable) | Data Person | +| Date | 2020-03-01 | +| GitHub username | koaning | +| Website (optional) | https://koaning.io | diff --git a/.github/contributors/kognate.md b/.github/contributors/kognate.md new file mode 100644 index 0000000..9ff2c75 --- /dev/null +++ b/.github/contributors/kognate.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Joshua B. Smith | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | July 7, 2019 | +| GitHub username | kognate | +| Website (optional) | | diff --git a/.github/contributors/kororo.md b/.github/contributors/kororo.md new file mode 100644 index 0000000..1c17559 --- /dev/null +++ b/.github/contributors/kororo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Robertus Johansyah | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018/07/19 | +| GitHub username | kororo | +| Website (optional) | https://kororo.co | diff --git a/.github/contributors/kowaalczyk.md b/.github/contributors/kowaalczyk.md new file mode 100644 index 0000000..c367c91 --- /dev/null +++ b/.github/contributors/kowaalczyk.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Krzysztof Kowalczyk | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date |22.11.2018 | +| GitHub username |kowaalczyk | +| Website (optional) |kowaalczyk.pl | diff --git a/.github/contributors/kwhumphreys.md b/.github/contributors/kwhumphreys.md new file mode 100644 index 0000000..0df94f4 --- /dev/null +++ b/.github/contributors/kwhumphreys.md @@ -0,0 +1,107 @@ + +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------- | +| Name | Kevin Humphreys | +| Company name (if applicable) | Textio Inc. | +| Title or role (if applicable) | | +| Date | 01-03-2018 | +| GitHub username | kwhumphreys | +| Website (optional) | | diff --git a/.github/contributors/laszabine.md b/.github/contributors/laszabine.md new file mode 100644 index 0000000..c1a4a3a --- /dev/null +++ b/.github/contributors/laszabine.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sabine Laszakovits | +| Company name (if applicable) | Austrian Academy of Sciences | +| Title or role (if applicable) | Data analyst | +| Date | 2020-04-16 | +| GitHub username | laszabine | +| Website (optional) | https://sabine.laszakovits.net | diff --git a/.github/contributors/lauraBaakman.md b/.github/contributors/lauraBaakman.md new file mode 100644 index 0000000..0cc0134 --- /dev/null +++ b/.github/contributors/lauraBaakman.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Laura Baakman | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | February 7, 2019 | +| GitHub username | lauraBaakman | +| Website (optional) | | diff --git a/.github/contributors/ldorigo.md b/.github/contributors/ldorigo.md new file mode 100644 index 0000000..c37e8bf --- /dev/null +++ b/.github/contributors/ldorigo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Luca Dorigo | +| Company name (if applicable) | / | +| Title or role (if applicable) | / | +| Date | 08.05.2019 | +| GitHub username | ldorigo | +| Website (optional) | / | diff --git a/.github/contributors/leicmi.md b/.github/contributors/leicmi.md new file mode 100644 index 0000000..6a65a48 --- /dev/null +++ b/.github/contributors/leicmi.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Michael Leichtfried | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 30.03.2020 | +| GitHub username | leicmi | +| Website (optional) | | diff --git a/.github/contributors/leomrocha.md b/.github/contributors/leomrocha.md new file mode 100644 index 0000000..4956541 --- /dev/null +++ b/.github/contributors/leomrocha.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Leonardo M. Rocha | +| Company name (if applicable) | | +| Title or role (if applicable) | Eng. | +| Date | 31/05/2020 | +| GitHub username | leomrocha | +| Website (optional) | | diff --git a/.github/contributors/leyendecker.md b/.github/contributors/leyendecker.md new file mode 100644 index 0000000..74e6cdd --- /dev/null +++ b/.github/contributors/leyendecker.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------------- | +| Name | Gustavo Zadrozny Leyendecker | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | July 29, 2020 | +| GitHub username | leyendecker | +| Website (optional) | | diff --git a/.github/contributors/lfiedler.md b/.github/contributors/lfiedler.md new file mode 100644 index 0000000..61f8ffe --- /dev/null +++ b/.github/contributors/lfiedler.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Leander Fiedler | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 06 April 2020 | +| GitHub username | lfiedler | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/ligser.md b/.github/contributors/ligser.md new file mode 100644 index 0000000..defda44 --- /dev/null +++ b/.github/contributors/ligser.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Roman Domrachev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11 November 2017 | +| GitHub username | ligser | +| Website (optional) | | diff --git a/.github/contributors/lizhe2004.md b/.github/contributors/lizhe2004.md new file mode 100644 index 0000000..6011506 --- /dev/null +++ b/.github/contributors/lizhe2004.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Zhe li | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-07-24 | +| GitHub username | lizhe2004 | +| Website (optional) | http://www.huahuaxia.net| diff --git a/.github/contributors/lorenanda.md b/.github/contributors/lorenanda.md new file mode 100644 index 0000000..90c4c54 --- /dev/null +++ b/.github/contributors/lorenanda.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Lorena Ciutacu | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-12-23 | +| GitHub username | lorenanda | +| Website (optional) | lorenaciutacu.com/ | diff --git a/.github/contributors/louisguitton.md b/.github/contributors/louisguitton.md new file mode 100644 index 0000000..8c5f30d --- /dev/null +++ b/.github/contributors/louisguitton.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Louis Guitton | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-25 | +| GitHub username | louisguitton | +| Website (optional) | https://guitton.co/ | diff --git a/.github/contributors/luvogels.md b/.github/contributors/luvogels.md new file mode 100644 index 0000000..c967c1c --- /dev/null +++ b/.github/contributors/luvogels.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Leif Uwe Vogelsang | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 4/27/2017 | +| GitHub username | luvogels | +| Website (optional) | | diff --git a/.github/contributors/mabraham.md b/.github/contributors/mabraham.md new file mode 100644 index 0000000..c91c950 --- /dev/null +++ b/.github/contributors/mabraham.md @@ -0,0 +1,89 @@ + + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | | +| GitHub username | | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/magnusburton.md b/.github/contributors/magnusburton.md new file mode 100644 index 0000000..9c9e296 --- /dev/null +++ b/.github/contributors/magnusburton.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------- | +| Name | Magnus Burton | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 17-12-2016 | +| GitHub username | magnusburton | +| Website (optional) | | diff --git a/.github/contributors/mahnerak.md b/.github/contributors/mahnerak.md new file mode 100644 index 0000000..cc77396 --- /dev/null +++ b/.github/contributors/mahnerak.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Karen Hambardzumyan | +| Company name (if applicable) | YerevaNN | +| Title or role (if applicable) | Researcher | +| Date | 2020-06-19 | +| GitHub username | mahnerak | +| Website (optional) | https://mahnerak.com/| diff --git a/.github/contributors/mariosasko.md b/.github/contributors/mariosasko.md new file mode 100644 index 0000000..1f5acc9 --- /dev/null +++ b/.github/contributors/mariosasko.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mario Šaško | +| Company name (if applicable) | TakeLab FER | +| Title or role (if applicable) | R&D Intern | +| Date | 2021-07-12 | +| GitHub username | mariosasko | +| Website (optional) | | diff --git a/.github/contributors/markulrich.md b/.github/contributors/markulrich.md new file mode 100644 index 0000000..b31bba0 --- /dev/null +++ b/.github/contributors/markulrich.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mark Ulrich | +| Company name (if applicable) | | +| Title or role (if applicable) | Machine Learning Engineer | +| Date | 22 November 2017 | +| GitHub username | markulrich | +| Website (optional) | | diff --git a/.github/contributors/mauryaland.md b/.github/contributors/mauryaland.md new file mode 100644 index 0000000..e178d4e --- /dev/null +++ b/.github/contributors/mauryaland.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Amaury Fouret | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 05/08/2018 | +| GitHub username | mauryaland | +| Website (optional) | | diff --git a/.github/contributors/mbkupfer.md b/.github/contributors/mbkupfer.md new file mode 100644 index 0000000..776368c --- /dev/null +++ b/.github/contributors/mbkupfer.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Maxim Kupfer | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | Sep 6, 2018 | +| GitHub username | mbkupfer | +| Website (optional) | | diff --git a/.github/contributors/mdaudali.md b/.github/contributors/mdaudali.md new file mode 100644 index 0000000..39ca26c --- /dev/null +++ b/.github/contributors/mdaudali.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mohammed Daudali | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 30 July 2019 | +| GitHub username | mdaudali | +| Website (optional) | | diff --git a/.github/contributors/mdcclv.md b/.github/contributors/mdcclv.md new file mode 100644 index 0000000..14ebfae --- /dev/null +++ b/.github/contributors/mdcclv.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------- | +| Name | Orion Montoya | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 04-10-2017 | +| GitHub username | mdcclv | +| Website (optional) | http://www.mdcclv.com/ | diff --git a/.github/contributors/mdda.md b/.github/contributors/mdda.md new file mode 100644 index 0000000..6345f63 --- /dev/null +++ b/.github/contributors/mdda.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Martin Andrews | +| Company name (if applicable) | - | +| Title or role (if applicable) | - | +| Date | 2017-12-18 | +| GitHub username | mdda | +| Website (optional) | - | diff --git a/.github/contributors/meghanabhange.md b/.github/contributors/meghanabhange.md new file mode 100644 index 0000000..2aaa57d --- /dev/null +++ b/.github/contributors/meghanabhange.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Meghana Bhange | +| Company name (if applicable) | Verloop.io | +| Title or role (if applicable) | ML Engineer | +| Date | 2020-04-21 | +| GitHub username | meghanbhange | +| Website (optional) | https://meghana.blog | + diff --git a/.github/contributors/melanuria.pdf b/.github/contributors/melanuria.pdf new file mode 100644 index 0000000..39da1bb Binary files /dev/null and b/.github/contributors/melanuria.pdf differ diff --git a/.github/contributors/merrcury.md b/.github/contributors/merrcury.md new file mode 100644 index 0000000..056a790 --- /dev/null +++ b/.github/contributors/merrcury.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Himanshu Garg | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-10 | +| GitHub username | merrcury | +| Website (optional) | | diff --git a/.github/contributors/michael-k.md b/.github/contributors/michael-k.md new file mode 100644 index 0000000..4ecc5be --- /dev/null +++ b/.github/contributors/michael-k.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Michael Käufl | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-23 | +| GitHub username | michael-k | +| Website (optional) | | diff --git a/.github/contributors/mihaigliga21.md b/.github/contributors/mihaigliga21.md new file mode 100644 index 0000000..c643a3a --- /dev/null +++ b/.github/contributors/mihaigliga21.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------- | +| Name | Mihai Gliga | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | September 9, 2019 | +| GitHub username | mihaigliga21 | +| Website (optional) | | diff --git a/.github/contributors/mikeizbicki.md b/.github/contributors/mikeizbicki.md new file mode 100644 index 0000000..6e9d8c0 --- /dev/null +++ b/.github/contributors/mikeizbicki.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mike Izbicki | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 02 Jun 2020 | +| GitHub username | mikeizbicki | +| Website (optional) | https://izbicki.me | diff --git a/.github/contributors/mikelibg.md b/.github/contributors/mikelibg.md new file mode 100644 index 0000000..81f9e2f --- /dev/null +++ b/.github/contributors/mikelibg.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Michael Liberman | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-11-08 | +| GitHub username | mikelibg | +| Website (optional) | | diff --git a/.github/contributors/mirfan899.md b/.github/contributors/mirfan899.md new file mode 100644 index 0000000..46f59eb --- /dev/null +++ b/.github/contributors/mirfan899.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Muhammad Irfan | +| Company name (if applicable) | | +| Title or role (if applicable) | AI & ML Developer | +| Date | 2018-09-06 | +| GitHub username | mirfan899 | +| Website (optional) | | diff --git a/.github/contributors/miroli.md b/.github/contributors/miroli.md new file mode 100644 index 0000000..6565c54 --- /dev/null +++ b/.github/contributors/miroli.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Robin Linderborg | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-04-23 | +| GitHub username | miroli | +| Website (optional) | | diff --git a/.github/contributors/mmaybeno.md b/.github/contributors/mmaybeno.md new file mode 100644 index 0000000..603cd5b --- /dev/null +++ b/.github/contributors/mmaybeno.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Matt Maybeno | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-11-19 | +| GitHub username | mmaybeno | +| Website (optional) | | diff --git a/.github/contributors/mn3mos.md b/.github/contributors/mn3mos.md new file mode 100644 index 0000000..e1eae0a --- /dev/null +++ b/.github/contributors/mn3mos.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Gaëtan PRUVOST | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13/04/2018 | +| GitHub username | mn3mos | +| Website (optional) | | diff --git a/.github/contributors/mollerhoj.md b/.github/contributors/mollerhoj.md new file mode 100644 index 0000000..f9b961f --- /dev/null +++ b/.github/contributors/mollerhoj.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jens Dahl Mollerhoj | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 4/04/2018 | +| GitHub username | mollerhoj | +| Website (optional) | | diff --git a/.github/contributors/moreymat.md b/.github/contributors/moreymat.md new file mode 100644 index 0000000..b02f780 --- /dev/null +++ b/.github/contributors/moreymat.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mathieu Morey | +| Company name (if applicable) | Datactivist | +| Title or role (if applicable) | Researcher | +| Date | 2019-01-07 | +| GitHub username | moreymat | +| Website (optional) | | diff --git a/.github/contributors/mpszumowski.md b/.github/contributors/mpszumowski.md new file mode 100644 index 0000000..226c400 --- /dev/null +++ b/.github/contributors/mpszumowski.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Maciej Szumowski | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 30.05.2018 | +| GitHub username | mpszumowski | +| Website (optional) | | diff --git a/.github/contributors/mpuig.md b/.github/contributors/mpuig.md new file mode 100644 index 0000000..fb9ebdc --- /dev/null +++ b/.github/contributors/mpuig.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Marc Puig | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-11-17 | +| GitHub username | mpuig | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/mr-bjerre.md b/.github/contributors/mr-bjerre.md new file mode 100644 index 0000000..ad3695b --- /dev/null +++ b/.github/contributors/mr-bjerre.md @@ -0,0 +1,87 @@ +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nicolai Bjerre Pedersen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-12-06 | +| GitHub username | mr_bjerre | +| Website (optional) | | diff --git a/.github/contributors/msklvsk.md b/.github/contributors/msklvsk.md new file mode 100644 index 0000000..f561649 --- /dev/null +++ b/.github/contributors/msklvsk.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------- | +| Name | Bohdan Moskalevskyi | +| Company name (if applicable) | Institute for Ukrainian, NGO | +| Title or role (if applicable) | | +| Date | 2018-06-08 | +| GitHub username | msklvsk | +| Website (optional) | https://mova.institute | diff --git a/.github/contributors/munozbravo.md b/.github/contributors/munozbravo.md new file mode 100644 index 0000000..b36133e --- /dev/null +++ b/.github/contributors/munozbravo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Germán Muñoz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-06-01 | +| GitHub username | munozbravo | +| Website (optional) | | diff --git a/.github/contributors/myavrum.md b/.github/contributors/myavrum.md new file mode 100644 index 0000000..dc8f1bb --- /dev/null +++ b/.github/contributors/myavrum.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Marat M. Yavrumyan | +| Company name (if applicable) | YSU, UD_Armenian Project | +| Title or role (if applicable) | Dr., Principal Investigator | +| Date | 2020-06-19 | +| GitHub username | myavrum | +| Website (optional) | http://armtreebank.yerevann.com/ | diff --git a/.github/contributors/narayanacharya6.md b/.github/contributors/narayanacharya6.md new file mode 100644 index 0000000..e4bf770 --- /dev/null +++ b/.github/contributors/narayanacharya6.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Narayan Acharya | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 29 APR 2021 | +| GitHub username | narayanacharya6 | +| Website (optional) | narayanacharya.com | \ No newline at end of file diff --git a/.github/contributors/neelkamath.md b/.github/contributors/neelkamath.md new file mode 100644 index 0000000..76502e7 --- /dev/null +++ b/.github/contributors/neelkamath.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------- | +| Name | Neel Kamath | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | October 30, 2019 | +| GitHub username | neelkamath | +| Website (optional) | https://neelkamath.com | diff --git a/.github/contributors/nikhilsaldanha.md b/.github/contributors/nikhilsaldanha.md new file mode 100644 index 0000000..f8d37d7 --- /dev/null +++ b/.github/contributors/nikhilsaldanha.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nikhil Saldanha | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-17 | +| GitHub username | nikhilsaldanha | +| Website (optional) | | diff --git a/.github/contributors/nipunsadvilkar.md b/.github/contributors/nipunsadvilkar.md new file mode 100644 index 0000000..aa0ff46 --- /dev/null +++ b/.github/contributors/nipunsadvilkar.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nipun Sadvilkar | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 31st May, 2019 | +| GitHub username | nipunsadvilkar| +| Website (optional) |https://nipunsadvilkar.github.io/| diff --git a/.github/contributors/njsmith.md b/.github/contributors/njsmith.md new file mode 100644 index 0000000..0054f30 --- /dev/null +++ b/.github/contributors/njsmith.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nathaniel J. Smith | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-08-26 | +| GitHub username | njsmith | +| Website (optional) | https://vorpus.org | diff --git a/.github/contributors/nlptown.md b/.github/contributors/nlptown.md new file mode 100644 index 0000000..8618f25 --- /dev/null +++ b/.github/contributors/nlptown.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yves Peirsman | +| Company name (if applicable) | NLP Town (Island Constraints BVBA) | +| Title or role (if applicable) | Co-founder | +| Date | 14.03.2019 | +| GitHub username | nlptown | +| Website (optional) | http://www.nlp.town | diff --git a/.github/contributors/nourshalabi.md b/.github/contributors/nourshalabi.md new file mode 100644 index 0000000..b8eff6c --- /dev/null +++ b/.github/contributors/nourshalabi.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Nour Shalabi | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-06-07 | +| GitHub username | NourShalabi | +| Website (optional) | | diff --git a/.github/contributors/nsorros.md b/.github/contributors/nsorros.md new file mode 100644 index 0000000..a449c52 --- /dev/null +++ b/.github/contributors/nsorros.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Nick Sorros | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2/8/2021 | +| GitHub username | nsorros | +| Website (optional) | | diff --git a/.github/contributors/ohenrik.md b/.github/contributors/ohenrik.md new file mode 100644 index 0000000..4e04b99 --- /dev/null +++ b/.github/contributors/ohenrik.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ole Henrik Skogstrøm | +| Company name (if applicable) | Amplify AS | +| Title or role (if applicable) | CEO / Developer | +| Date | 25.01.2018 | +| GitHub username | ohenrik | +| Website (optional) | amplify.no | diff --git a/.github/contributors/onlyanegg.md b/.github/contributors/onlyanegg.md new file mode 100644 index 0000000..eb1ac01 --- /dev/null +++ b/.github/contributors/onlyanegg.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | ---------------- | +| Name | Tyler Couto | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | January 29, 2020 | +| GitHub username | onlyanegg | +| Website (optional) | | diff --git a/.github/contributors/ophelielacroix.md b/.github/contributors/ophelielacroix.md new file mode 100644 index 0000000..2abdfde --- /dev/null +++ b/.github/contributors/ophelielacroix.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|-------------------------------|-----------------| +| Name | Ophélie Lacroix | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | | +| GitHub username | ophelielacroix | +| Website (optional) | | diff --git a/.github/contributors/oroszgy.md b/.github/contributors/oroszgy.md new file mode 100644 index 0000000..8e69b40 --- /dev/null +++ b/.github/contributors/oroszgy.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | György Orosz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2016-12-26 | +| GitHub username | oroszgy | +| Website (optional) | gyorgy.orosz.link | diff --git a/.github/contributors/osori.md b/.github/contributors/osori.md new file mode 100644 index 0000000..93b5c7d --- /dev/null +++ b/.github/contributors/osori.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ilkyu Ju | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-05-17 | +| GitHub username | osori | +| Website (optional) | | diff --git a/.github/contributors/ottosulin.md b/.github/contributors/ottosulin.md new file mode 100644 index 0000000..83975b7 --- /dev/null +++ b/.github/contributors/ottosulin.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschr�nkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an �x� on one of the applicable statement below. Please do NOT +mark both statements: + + * [ X ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Otto Sulin | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 23/03/2018 | +| GitHub username | ottosulin | +| Website (optional) | | diff --git a/.github/contributors/oxinabox.md b/.github/contributors/oxinabox.md new file mode 100644 index 0000000..8e58c4e --- /dev/null +++ b/.github/contributors/oxinabox.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Lyndon White | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 9/2/2018 | +| GitHub username | oxinabox | +| Website (optional) | white.ucc.asn.au | diff --git a/.github/contributors/ozcankasal.md b/.github/contributors/ozcankasal.md new file mode 100644 index 0000000..998e701 --- /dev/null +++ b/.github/contributors/ozcankasal.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ozcan Kasal | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | December 21, 2018 | +| GitHub username | ozcankasal | +| Website (optional) | | diff --git a/.github/contributors/paoloq.md b/.github/contributors/paoloq.md new file mode 100644 index 0000000..0fac70c --- /dev/null +++ b/.github/contributors/paoloq.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Paolo Arduin | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 9 April 2020 | +| GitHub username | paoloq | +| Website (optional) | | diff --git a/.github/contributors/pberba.md b/.github/contributors/pberba.md new file mode 100644 index 0000000..34feb3e --- /dev/null +++ b/.github/contributors/pberba.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [X] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pepe Berba | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-10-18 | +| GitHub username | pberba | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/pbnsilva.md b/.github/contributors/pbnsilva.md new file mode 100644 index 0000000..95bc4eb --- /dev/null +++ b/.github/contributors/pbnsilva.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pedro Silva | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-01-11 | +| GitHub username | pbnsilva | +| Website (optional) | | diff --git a/.github/contributors/peter-exos.md b/.github/contributors/peter-exos.md new file mode 100644 index 0000000..e0ef134 --- /dev/null +++ b/.github/contributors/peter-exos.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Peter Baumann | +| Company name (if applicable) | Exos Financial | +| Title or role (if applicable) | data scientist | +| Date | Feb 1st, 2021 | +| GitHub username | peter-exos | +| Website (optional) | | diff --git a/.github/contributors/phiedulxp.md b/.github/contributors/phiedulxp.md new file mode 100644 index 0000000..fa2666e --- /dev/null +++ b/.github/contributors/phiedulxp.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Xiepeng Li | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 20190810 | +| GitHub username | phiedulxp | +| Website (optional) | | diff --git a/.github/contributors/philipvollet.md b/.github/contributors/philipvollet.md new file mode 100644 index 0000000..0bf58a7 --- /dev/null +++ b/.github/contributors/philipvollet.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Philip Vollet | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22.09.2021 | +| GitHub username | philipvollet | +| Website (optional) | | diff --git a/.github/contributors/phojnacki.md b/.github/contributors/phojnacki.md new file mode 100644 index 0000000..048f240 --- /dev/null +++ b/.github/contributors/phojnacki.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ X ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------------------- | +| Name | Przemysław Hojnacki | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12/09/2018 | +| GitHub username | phojnacki | +| Website (optional) | https://about.me/przemyslaw.hojnacki | \ No newline at end of file diff --git a/.github/contributors/pickfire.md b/.github/contributors/pickfire.md new file mode 100644 index 0000000..74f57b7 --- /dev/null +++ b/.github/contributors/pickfire.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ivan Tham Jun Hoe | +| Company name (if applicable) | Semut | +| Title or role (if applicable) | Data Analyst | +| Date | Apr 11, 2019 | +| GitHub username | pickfire | +| Website (optional) | https://pickfire.tk | diff --git a/.github/contributors/pinealan.md b/.github/contributors/pinealan.md new file mode 100644 index 0000000..699b405 --- /dev/null +++ b/.github/contributors/pinealan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alan Chan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-15 | +| GitHub username | pinealan | +| Website (optional) | http://pinealan.xyz | diff --git a/.github/contributors/pktippa.md b/.github/contributors/pktippa.md new file mode 100644 index 0000000..740944a --- /dev/null +++ b/.github/contributors/pktippa.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pradeep Kumar Tippa | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 07-02-2018 | +| GitHub username | pktippa | +| Website (optional) | | diff --git a/.github/contributors/plison.md b/.github/contributors/plison.md new file mode 100644 index 0000000..e98b096 --- /dev/null +++ b/.github/contributors/plison.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pierre Lison | +| Company name (if applicable) | Norsk Regnesentral | +| Title or role (if applicable) | Senior Researcher | +| Date | 22.04.2021 | +| GitHub username | plison | +| Website (optional) | www.nr.no/~plison | diff --git a/.github/contributors/pmbaumgartner.md b/.github/contributors/pmbaumgartner.md new file mode 100644 index 0000000..f548cb8 --- /dev/null +++ b/.github/contributors/pmbaumgartner.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Peter Baumgartner | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | July 14, 2019 | +| GitHub username | pmbaumgartner | +| Website (optional) | pmbaumgartner.github.io | diff --git a/.github/contributors/polm.md b/.github/contributors/polm.md new file mode 100644 index 0000000..a2aa0cb --- /dev/null +++ b/.github/contributors/polm.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Paul McCann | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-10-14 | +| GitHub username | polm | +| Website (optional) | http://dampfkraft.com| diff --git a/.github/contributors/prilopes.md b/.github/contributors/prilopes.md new file mode 100644 index 0000000..ad111d4 --- /dev/null +++ b/.github/contributors/prilopes.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Priscilla Lopes | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-11-06 | +| GitHub username | prilopes | +| Website (optional) | | diff --git a/.github/contributors/punitvara.md b/.github/contributors/punitvara.md new file mode 100644 index 0000000..dde8104 --- /dev/null +++ b/.github/contributors/punitvara.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Punit Vara | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-26 | +| GitHub username | punitvara | +| Website (optional) | https://punitvara.com | + diff --git a/.github/contributors/pzelasko.md b/.github/contributors/pzelasko.md new file mode 100644 index 0000000..72ec411 --- /dev/null +++ b/.github/contributors/pzelasko.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Piotr Żelasko | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 04-09-2018 | +| GitHub username | pzelasko | +| Website (optional) | | diff --git a/.github/contributors/questoph.md b/.github/contributors/questoph.md new file mode 100644 index 0000000..24559c0 --- /dev/null +++ b/.github/contributors/questoph.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Christoph Purschke | +| Company name (if applicable) | University of Luxembourg | +| Title or role (if applicable) | | +| Date | 14/11/2019 | +| GitHub username | questoph | +| Website (optional) | https://purschke.info | diff --git a/.github/contributors/rafguns.md b/.github/contributors/rafguns.md new file mode 100644 index 0000000..61398e4 --- /dev/null +++ b/.github/contributors/rafguns.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Raf Guns | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-12-09 | +| GitHub username | rafguns | +| Website (optional) | | diff --git a/.github/contributors/rahul1990gupta.md b/.github/contributors/rahul1990gupta.md new file mode 100644 index 0000000..eab41b3 --- /dev/null +++ b/.github/contributors/rahul1990gupta.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Rahul Gupta | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 28 July 2020 | +| GitHub username | rahul1990gupta | +| Website (optional) | | diff --git a/.github/contributors/ramananbalakrishnan.md b/.github/contributors/ramananbalakrishnan.md new file mode 100644 index 0000000..804c41f --- /dev/null +++ b/.github/contributors/ramananbalakrishnan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ramanan Balakrishnan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-10-19 | +| GitHub username | ramananbalakrishnan | +| Website (optional) | | diff --git a/.github/contributors/rameshhpathak.md b/.github/contributors/rameshhpathak.md new file mode 100644 index 0000000..30a5433 --- /dev/null +++ b/.github/contributors/rameshhpathak.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ramesh Pathak | +| Company name (if applicable) | Diyo AI | +| Title or role (if applicable) | AI Engineer | +| Date | June 21, 2020 | +| GitHub username | rameshhpathak | +| Website (optional) |rameshhpathak.github.io| | diff --git a/.github/contributors/rasyidf.md b/.github/contributors/rasyidf.md new file mode 100644 index 0000000..4a70547 --- /dev/null +++ b/.github/contributors/rasyidf.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Muhammad Fahmi Rasyid | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-09-23 | +| GitHub username | rasyidf | +| Website (optional) | http://rasyidf.github.io | diff --git a/.github/contributors/reneoctavio.md b/.github/contributors/reneoctavio.md new file mode 100644 index 0000000..c0a4d1a --- /dev/null +++ b/.github/contributors/reneoctavio.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | -------------------- | +| Name | Rene Octavio Q. Dias | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-02-03 | +| GitHub username | reneoctavio | +| Website (optional) | | diff --git a/.github/contributors/retnuh.md b/.github/contributors/retnuh.md new file mode 100644 index 0000000..a7e996b --- /dev/null +++ b/.github/contributors/retnuh.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | ------------ | +| Name | Hunter Kelly | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-01-10 | +| GitHub username | retnuh | +| Website (optional) | | diff --git a/.github/contributors/revuel.md b/.github/contributors/revuel.md new file mode 100644 index 0000000..51cc5fa --- /dev/null +++ b/.github/contributors/revuel.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Miguel Revuelta | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-11-17 | +| GitHub username | revuel | +| Website (optional) | | diff --git a/.github/contributors/richardliaw.md b/.github/contributors/richardliaw.md new file mode 100644 index 0000000..2af4ce8 --- /dev/null +++ b/.github/contributors/richardliaw.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Richard Liaw | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 06/22/2020 | +| GitHub username | richardliaw | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/richardpaulhudson.md b/.github/contributors/richardpaulhudson.md new file mode 100644 index 0000000..3d68b98 --- /dev/null +++ b/.github/contributors/richardpaulhudson.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Richard Paul Hudson | +| Company name (if applicable) | msg systems ag | +| Title or role (if applicable) | Principal IT Consultant| +| Date | 06. May 2019 | +| GitHub username | richardpaulhudson | +| Website (optional) | | diff --git a/.github/contributors/robertsipek.md b/.github/contributors/robertsipek.md new file mode 100644 index 0000000..888d798 --- /dev/null +++ b/.github/contributors/robertsipek.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Robert Šípek | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 22.10.2020 | +| GitHub username | @robertsipek | +| Website (optional) | | diff --git a/.github/contributors/rokasramas.md b/.github/contributors/rokasramas.md new file mode 100644 index 0000000..746149a --- /dev/null +++ b/.github/contributors/rokasramas.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ----------------------- | +| Name | Rokas Ramanauskas | +| Company name (if applicable) | TokenMill | +| Title or role (if applicable) | Software Engineer | +| Date | 2019-07-02 | +| GitHub username | rokasramas | +| Website (optional) | http://www.tokenmill.lt | diff --git a/.github/contributors/roshni-b.md b/.github/contributors/roshni-b.md new file mode 100644 index 0000000..1b967ae --- /dev/null +++ b/.github/contributors/roshni-b.md @@ -0,0 +1,107 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Roshni Biswas | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 02-17-2019 | +| GitHub username | roshni-b | +| Website (optional) | | + diff --git a/.github/contributors/ryanzhe.md b/.github/contributors/ryanzhe.md new file mode 100644 index 0000000..afb250a --- /dev/null +++ b/.github/contributors/ryanzhe.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ziming He | +| Company name (if applicable) | Georgia Tech | +| Title or role (if applicable) | Student | +| Date | 2019-07-24 | +| GitHub username | RyanZHe | +| Website (optional) | www.papermachine.me | diff --git a/.github/contributors/sabiqueqb.md b/.github/contributors/sabiqueqb.md new file mode 100644 index 0000000..da0f2f2 --- /dev/null +++ b/.github/contributors/sabiqueqb.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sabique Ahammed Lava | +| Company name (if applicable) | QBurst | +| Title or role (if applicable) | Senior Engineer | +| Date | 24 Apr 2020 | +| GitHub username | sabiqueqb | +| Website (optional) | | diff --git a/.github/contributors/sainathadapa.md b/.github/contributors/sainathadapa.md new file mode 100644 index 0000000..0b40775 --- /dev/null +++ b/.github/contributors/sainathadapa.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sainath Adapa | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-09-06 | +| GitHub username | sainathadapa | +| Website (optional) | | diff --git a/.github/contributors/sammous.md b/.github/contributors/sammous.md new file mode 100644 index 0000000..b419738 --- /dev/null +++ b/.github/contributors/sammous.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sami Moustachir | +| Company name (if applicable) | | +| Title or role (if applicable) | Data Scientist | +| Date | 2018-08-02 | +| GitHub username | sammous | +| Website (optional) | https://samimoustachir.com | diff --git a/.github/contributors/savkov.md b/.github/contributors/savkov.md new file mode 100644 index 0000000..ca38582 --- /dev/null +++ b/.github/contributors/savkov.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Aleksandar Savkov | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11.01.2018 | +| GitHub username | savkov | +| Website (optional) | sasho.io | diff --git a/.github/contributors/seanBE.md b/.github/contributors/seanBE.md new file mode 100644 index 0000000..5e4b4de --- /dev/null +++ b/.github/contributors/seanBE.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------- | +| Name | Sean Löfgren | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-09-17 | +| GitHub username | seanBE | +| Website (optional) | http://seanbe.github.io | diff --git a/.github/contributors/sebastienharinck.md b/.github/contributors/sebastienharinck.md new file mode 100644 index 0000000..e0fddeb --- /dev/null +++ b/.github/contributors/sebastienharinck.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------------------- | +| Name | Sébastien Harinck | +| Company name (if applicable) | Odaxiom | +| Title or role (if applicable) | ML Engineer | +| Date | 2020-04-15 | +| GitHub username | sebastienharinck | +| Website (optional) | [https://odaxiom.com](https://odaxiom.com) | \ No newline at end of file diff --git a/.github/contributors/sevdimali.md b/.github/contributors/sevdimali.md new file mode 100644 index 0000000..6b96abd --- /dev/null +++ b/.github/contributors/sevdimali.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sevdimali | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10/4/2021 | +| GitHub username | sevdimali | +| Website (optional) | https://sevdimali.me | diff --git a/.github/contributors/shigapov.md b/.github/contributors/shigapov.md new file mode 100644 index 0000000..3c24c79 --- /dev/null +++ b/.github/contributors/shigapov.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Renat Shigapov | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021-09-09 | +| GitHub username | shigapov | +| Website (optional) | | diff --git a/.github/contributors/shuvanon.md b/.github/contributors/shuvanon.md new file mode 100644 index 0000000..82d02d8 --- /dev/null +++ b/.github/contributors/shuvanon.md @@ -0,0 +1,108 @@ + + +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shuvanon Razik | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 3/12/2017 | +| GitHub username | shuvanon | +| Website (optional) | | diff --git a/.github/contributors/skrcode.md b/.github/contributors/skrcode.md new file mode 100644 index 0000000..40a4a08 --- /dev/null +++ b/.github/contributors/skrcode.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Suraj Rajan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 31/Mar/2018 | +| GitHub username | skrcode | +| Website (optional) | | diff --git a/.github/contributors/sloev.md b/.github/contributors/sloev.md new file mode 100644 index 0000000..d151d46 --- /dev/null +++ b/.github/contributors/sloev.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Johannes Valbjørn | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-13 | +| GitHub username | sloev | +| Website (optional) | https://sloev.github.io | diff --git a/.github/contributors/snsten.md b/.github/contributors/snsten.md new file mode 100644 index 0000000..0d7c288 --- /dev/null +++ b/.github/contributors/snsten.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shashank Shekhar | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-08-23 | +| GitHub username | snsten | +| Website (optional) | | diff --git a/.github/contributors/socool.md b/.github/contributors/socool.md new file mode 100644 index 0000000..a8c1437 --- /dev/null +++ b/.github/contributors/socool.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Kamolsit Mongkolsrisawat | +| Company name (if applicable) | Mojito | +| Title or role (if applicable) | | +| Date | 02-4-2019 | +| GitHub username | socool | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/solarmist.md b/.github/contributors/solarmist.md new file mode 100644 index 0000000..6bfb216 --- /dev/null +++ b/.github/contributors/solarmist.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------- | +| Name | Joshua Olson | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-07-22 | +| GitHub username | solarmist | +| Website (optional) | http://blog.solarmist.net | diff --git a/.github/contributors/sorenlind.md b/.github/contributors/sorenlind.md new file mode 100644 index 0000000..73e4263 --- /dev/null +++ b/.github/contributors/sorenlind.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Søren Lind Kristiansen | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 24 November 2017 | +| GitHub username | sorenlind | +| Website (optional) | | diff --git a/.github/contributors/suchow.md b/.github/contributors/suchow.md new file mode 100644 index 0000000..b1d5ed0 --- /dev/null +++ b/.github/contributors/suchow.md @@ -0,0 +1,98 @@ +# Syllogism contributor agreement + +This Syllogism Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +Syllogism Co. The term **"you"** shall mean the person or entity identified +below. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. Syllogism Co. may publicly + disclose your participation in the project, including the fact that you have + signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jordan Suchow | +| Company's name (if applicable) | | +| Title or Role (if applicable) | | +| Date | 2015-04-19 | +| GitHub username | suchow | +| Website (optional) | http://suchow.io | diff --git a/.github/contributors/svlandeg.md b/.github/contributors/svlandeg.md new file mode 100644 index 0000000..f53ab6a --- /dev/null +++ b/.github/contributors/svlandeg.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschrnkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an x on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sofie Van Landeghem | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 29 Nov 2018 | +| GitHub username | svlandeg | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/swfarnsworth.md b/.github/contributors/swfarnsworth.md new file mode 100644 index 0000000..c289e66 --- /dev/null +++ b/.github/contributors/swfarnsworth.md @@ -0,0 +1,88 @@ +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Steele Farnsworth | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13 August, 2021 | +| GitHub username | swfarnsworth | +| Website (optional) | | + diff --git a/.github/contributors/syrull.md b/.github/contributors/syrull.md new file mode 100644 index 0000000..82cdade --- /dev/null +++ b/.github/contributors/syrull.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Dimitar Ganev | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2021/8/2 | +| GitHub username | syrull | +| Website (optional) | | diff --git a/.github/contributors/tamuhey.md b/.github/contributors/tamuhey.md new file mode 100644 index 0000000..6a63e3b --- /dev/null +++ b/.github/contributors/tamuhey.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yohei Tamura | +| Company name (if applicable) | PKSHA | +| Title or role (if applicable) | | +| Date | 2019/9/12 | +| GitHub username | tamuhey | +| Website (optional) | | diff --git a/.github/contributors/therealronnie.md b/.github/contributors/therealronnie.md new file mode 100644 index 0000000..b1095b0 --- /dev/null +++ b/.github/contributors/therealronnie.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ronnie Gonzalez | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 17.04.2018 | +| GitHub username | therealronnie | +| Website (optional) | | diff --git a/.github/contributors/theudas.md b/.github/contributors/theudas.md new file mode 100644 index 0000000..3d8a2bd --- /dev/null +++ b/.github/contributors/theudas.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Philipp Sodmann | +| Company name (if applicable) | Empolis | +| Title or role (if applicable) | | +| Date | 2017-05-06 | +| GitHub username | theudas | +| Website (optional) | | diff --git a/.github/contributors/thomasbird.md b/.github/contributors/thomasbird.md new file mode 100644 index 0000000..44b8cb7 --- /dev/null +++ b/.github/contributors/thomasbird.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------- | +| Name | Thomas Bird | +| Company name (if applicable) | Leap Beyond Group | +| Title or role (if applicable) | Data Scientist | +| Date | 15/12/2020 | +| GitHub username | thomasbird | +| Website (optional) | https://leapbeyond.ai | diff --git a/.github/contributors/thomashacker.md b/.github/contributors/thomashacker.md new file mode 100644 index 0000000..d88727d --- /dev/null +++ b/.github/contributors/thomashacker.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Edward Schmuhl | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 09.07.2021 | +| GitHub username | thomashacker | +| Website (optional) | | diff --git a/.github/contributors/thomasopsomer.md b/.github/contributors/thomasopsomer.md new file mode 100644 index 0000000..240c1d0 --- /dev/null +++ b/.github/contributors/thomasopsomer.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | thomas opsomer | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 28-01-2018 | +| GitHub username | thomasopsomer | +| Website (optional) | | diff --git a/.github/contributors/thomasthiebaud.md b/.github/contributors/thomasthiebaud.md new file mode 100644 index 0000000..bdbf0ec --- /dev/null +++ b/.github/contributors/thomasthiebaud.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | --------------- | +| Name | Thomas Thiebaud | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-07 | +| GitHub username | thomasthiebaud | +| Website (optional) | | diff --git a/.github/contributors/thoppe.md b/.github/contributors/thoppe.md new file mode 100644 index 0000000..9271a26 --- /dev/null +++ b/.github/contributors/thoppe.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Travis Hoppe | +| Company name (if applicable) | | +| Title or role (if applicable) | Data Scientist | +| Date | 07 May 2020 | +| GitHub username | thoppe | +| Website (optional) | http://thoppe.github.io/ | diff --git a/.github/contributors/tiangolo.md b/.github/contributors/tiangolo.md new file mode 100644 index 0000000..5fd253f --- /dev/null +++ b/.github/contributors/tiangolo.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Sebastián Ramírez | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-07-01 | +| GitHub username | tiangolo | +| Website (optional) | | diff --git a/.github/contributors/tilusnet.md b/.github/contributors/tilusnet.md new file mode 100644 index 0000000..1618bac --- /dev/null +++ b/.github/contributors/tilusnet.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Attila Szász | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12 Aug 2020 | +| GitHub username | tilusnet | +| Website (optional) | | diff --git a/.github/contributors/tjkemp.md b/.github/contributors/tjkemp.md new file mode 100644 index 0000000..669da94 --- /dev/null +++ b/.github/contributors/tjkemp.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Tero Kemppi | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 14-07-2018 | +| GitHub username | tjkemp | +| Website (optional) | | diff --git a/.github/contributors/tmetzl.md b/.github/contributors/tmetzl.md new file mode 100644 index 0000000..e3c8529 --- /dev/null +++ b/.github/contributors/tmetzl.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Tim Metzler | +| Company name (if applicable) | University of Applied Sciences Bonn-Rhein-Sieg | +| Title or role (if applicable) | | +| Date | 03/10/2019 | +| GitHub username | tmetzl | +| Website (optional) | | diff --git a/.github/contributors/tokestermw.md b/.github/contributors/tokestermw.md new file mode 100644 index 0000000..2d69d77 --- /dev/null +++ b/.github/contributors/tokestermw.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Motoki Wu | +| Company name (if applicable) | WriteLab | +| Title or role (if applicable) | NLP / Deep Learning Engineer | +| Date | 17 November 2017 | +| GitHub username | tokestermw | +| Website (optional) | https://twitter.com/plusepsilon | diff --git a/.github/contributors/tommilligan.md b/.github/contributors/tommilligan.md new file mode 100644 index 0000000..475df5a --- /dev/null +++ b/.github/contributors/tommilligan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, + object code, patch, tool, sample, graphic, specification, manual, + documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and + registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment + to any third party, you hereby grant to us a perpetual, irrevocable, + non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your + contribution. The rights that you grant to us under these terms are effective + on the date you first submitted a contribution to us, even if your submission + took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + - Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + - to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + - each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable + U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT + mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +| ----------------------------- | ------------ | +| Name | Tom Milligan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-03-24 | +| GitHub username | tommilligan | +| Website (optional) | | diff --git a/.github/contributors/trungtv.md b/.github/contributors/trungtv.md new file mode 100644 index 0000000..906ad28 --- /dev/null +++ b/.github/contributors/trungtv.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Viet-Trung Tran | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-03-28 | +| GitHub username | trungtv | +| Website (optional) | https://datalab.vn | diff --git a/.github/contributors/tupui.md b/.github/contributors/tupui.md new file mode 100644 index 0000000..5f53a72 --- /dev/null +++ b/.github/contributors/tupui.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Pamphile Roy | +| Company name (if applicable) | N/A | +| Title or role (if applicable) | N/A | +| Date | January 29th, 2021 | +| GitHub username | tupui | +| Website (optional) | N/A | diff --git a/.github/contributors/tyburam.md b/.github/contributors/tyburam.md new file mode 100644 index 0000000..379e30b --- /dev/null +++ b/.github/contributors/tyburam.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Mateusz Tybura | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 08.09.2018 | +| GitHub username | tyburam | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/tzano.md b/.github/contributors/tzano.md new file mode 100644 index 0000000..2c20c59 --- /dev/null +++ b/.github/contributors/tzano.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Tahar Zanouda | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 09-05-2018 | +| GitHub username | tzano | +| Website (optional) | | diff --git a/.github/contributors/ujwal-narayan.md b/.github/contributors/ujwal-narayan.md new file mode 100644 index 0000000..622bb5d --- /dev/null +++ b/.github/contributors/ujwal-narayan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ujwal Narayan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 17/05/2019 | +| GitHub username | ujwal-narayan | +| Website (optional) | | diff --git a/.github/contributors/umarbutler.md b/.github/contributors/umarbutler.md new file mode 100644 index 0000000..8df8251 --- /dev/null +++ b/.github/contributors/umarbutler.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Umar Butler | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-04-09 | +| GitHub username | umarbutler | +| Website (optional) | https://umarbutler.com | diff --git a/.github/contributors/ursachec.md b/.github/contributors/ursachec.md new file mode 100644 index 0000000..45a85f1 --- /dev/null +++ b/.github/contributors/ursachec.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------- | +| Name | Claudiu-Vlad Ursache | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-02-04 | +| GitHub username | ursachec | +| Website (optional) | https://www.cvursache.com | diff --git a/.github/contributors/uwol.md b/.github/contributors/uwol.md new file mode 100644 index 0000000..ddc82d2 --- /dev/null +++ b/.github/contributors/uwol.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ulrich Wolffgang | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-11-05 | +| GitHub username | uwol | +| Website (optional) | https://uwol.github.io/ | diff --git a/.github/contributors/veer-bains.md b/.github/contributors/veer-bains.md new file mode 100644 index 0000000..2b67dcb --- /dev/null +++ b/.github/contributors/veer-bains.md @@ -0,0 +1,54 @@ +spaCy contributor agreement +This spaCy Contributor Agreement ("SCA") is based on the Oracle Contributor Agreement. The SCA applies to any contribution that you make to any product or project managed by us (the "project"), and sets out the intellectual property rights you grant to us in the contributed materials. The term "us" shall mean ExplosionAI UG (haftungsbeschränkt). The term "you" shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested below and include the filled-in version with your first pull request, under the folder .github/contributors/. The name of the file should be your GitHub username, with the extension .md. For example, the user example_user would create the file .github/contributors/example_user.md. + +Read this agreement carefully before signing. These terms and conditions constitute a binding legal agreement. + +Contributor Agreement +The term "contribution" or "contributed materials" means any source code, object code, patch, tool, sample, graphic, specification, manual, documentation, or any other material posted or submitted by you to the project. + +With respect to any worldwide copyrights, or copyright applications and registrations, in your contribution: + +you hereby assign to us joint ownership, and to the extent that such assignment is or becomes invalid, ineffective or unenforceable, you hereby grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, royalty-free, unrestricted license to exercise all rights under those copyrights. This includes, at our option, the right to sublicense these same rights to third parties through multiple levels of sublicensees or other licensing arrangements; + +you agree that each of us can do all things in relation to your contribution as if each of us were the sole owners, and if one of us makes a derivative work of your contribution, the one who makes the derivative work (or has it made will be the sole owner of that derivative work; + +you agree that you will not assert any moral rights in your contribution against us, our licensees or transferees; + +you agree that we may register a copyright in your contribution and exercise all ownership rights associated with it; and + +you agree that neither of us has any duty to consult with, obtain the consent of, pay or render an accounting to the other for any use or distribution of your contribution. + +With respect to any patents you own, or that you can license without payment to any third party, you hereby grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, royalty-free license to: + +make, have made, use, sell, offer to sell, import, and otherwise transfer your contribution in whole or in part, alone or in combination with or included in any product, work or materials arising out of the project to which your contribution was submitted, and + +at our option, to sublicense these same rights to third parties through multiple levels of sublicensees or other licensing arrangements. + +Except as set out above, you keep all right, title, and interest in your contribution. The rights that you grant to us under these terms are effective on the date you first submitted a contribution to us, even if your submission took place before the date you sign these terms. + +You covenant, represent, warrant and agree that: + +Each contribution that you submit is and shall be an original work of authorship and you can legally grant the rights set out in this SCA; + +to the best of your knowledge, each contribution will not violate any third party's copyrights, trademarks, patents, or other intellectual property rights; and + +each contribution shall be in compliance with U.S. export control laws and other applicable export and import laws. You agree to notify us if you become aware of any circumstance which would make any of the foregoing representations inaccurate in any respect. We may publicly disclose your participation in the project, including the fact that you have signed the SCA. + +This SCA is governed by the laws of the State of California and applicable U.S. Federal law. Any choice of law rules will not apply. + +Please place an “x” on one of the applicable statement below. Please do NOT mark both statements: + + I am signing on behalf of myself as an individual and no other person or entity, including my employer, has or will have rights with respect to my contributions. + + I am signing on behalf of my employer or a legal entity and I have the actual authority to contractually bind that entity. + +Contributor Details +Field Entry +Name Veer Bains +Company name (if applicable) +Title or role (if applicable) +Date 2010-08-05 +GitHub username veer-bains +Website (optional) diff --git a/.github/contributors/vha14.md b/.github/contributors/vha14.md new file mode 100644 index 0000000..0d7c9b5 --- /dev/null +++ b/.github/contributors/vha14.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Vu Ha | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10-23-2020 | +| GitHub username | vha14 | +| Website (optional) | | diff --git a/.github/contributors/vikaskyadav.md b/.github/contributors/vikaskyadav.md new file mode 100644 index 0000000..871fc11 --- /dev/null +++ b/.github/contributors/vikaskyadav.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | vikas yadav | +| Company name (if applicable) | | +| Title or role (if applicable) | Data Scientist | +| Date | 1 August 2018 | +| GitHub username | vikaskyadav | +| Website (optional) | www.vikaskyadav.tk | diff --git a/.github/contributors/vishnumenon.md b/.github/contributors/vishnumenon.md new file mode 100644 index 0000000..26c4613 --- /dev/null +++ b/.github/contributors/vishnumenon.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Vishnu Menon | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12 May 2018 | +| GitHub username | vishnumenon | +| Website (optional) | | diff --git a/.github/contributors/vishnupriyavr.md b/.github/contributors/vishnupriyavr.md new file mode 100644 index 0000000..73657a7 --- /dev/null +++ b/.github/contributors/vishnupriyavr.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Vishnu Priya VR | +| Company name (if applicable) | Uniphore | +| Title or role (if applicable) | NLP/AI Engineer | +| Date | 2020-05-03 | +| GitHub username | vishnupriyavr | +| Website (optional) | | diff --git a/.github/contributors/vondersam.md b/.github/contributors/vondersam.md new file mode 100644 index 0000000..8add703 --- /dev/null +++ b/.github/contributors/vondersam.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------| +| Name | Samuel Rodríguez Medina | +| Company name (if applicable) | | +| Title or role (if applicable) | Computational linguist | +| Date | 28 April 2020 | +| GitHub username | vondersam | +| Website (optional) | | diff --git a/.github/contributors/vsolovyov.md b/.github/contributors/vsolovyov.md new file mode 100644 index 0000000..1ef72f8 --- /dev/null +++ b/.github/contributors/vsolovyov.md @@ -0,0 +1,98 @@ +# Syllogism contributor agreement + +This Syllogism Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +Syllogism Co. The term **"you"** shall mean the person or entity identified +below. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. Syllogism Co. may publicly + disclose your participation in the project, including the fact that you have + signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Vsevolod Solovyov | +| Company's name (if applicable) | | +| Title or Role (if applicable) | | +| Date | 2015-08-24 | +| GitHub username | vsolovyov | +| Website (optional) | | diff --git a/.github/contributors/w4nderlust.md b/.github/contributors/w4nderlust.md new file mode 100644 index 0000000..1e18845 --- /dev/null +++ b/.github/contributors/w4nderlust.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ---------------------- | +| Name | Piero Molino | +| Company name (if applicable) | Uber | +| Title or role (if applicable) | Sr. Research Scientist | +| Date | 06 - 04 - 2019 | +| GitHub username | w4nderlust | +| Website (optional) | http://w4nderlu.st | diff --git a/.github/contributors/wallinm1.md b/.github/contributors/wallinm1.md new file mode 100644 index 0000000..9c6d0bb --- /dev/null +++ b/.github/contributors/wallinm1.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------------------- | +| Name | Michael Wallin | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-02-04 | +| GitHub username | wallinm1 | +| Website (optional) | | diff --git a/.github/contributors/walterhenry.md b/.github/contributors/walterhenry.md new file mode 100644 index 0000000..76026e1 --- /dev/null +++ b/.github/contributors/walterhenry.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Walter Henry | +| Company name (if applicable) | ExplosionAI GmbH | +| Title or role (if applicable) | Executive Assistant | +| Date | September 14, 2020 | +| GitHub username | walterhenry | +| Website (optional) | | diff --git a/.github/contributors/wannaphongcom.md b/.github/contributors/wannaphongcom.md new file mode 100644 index 0000000..67aae70 --- /dev/null +++ b/.github/contributors/wannaphongcom.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Wannaphong Phatthiyaphaibun | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 25-3-2019 | +| GitHub username | wannaphongcom | +| Website (optional) | | diff --git a/.github/contributors/werew.md b/.github/contributors/werew.md new file mode 100644 index 0000000..04a79ed --- /dev/null +++ b/.github/contributors/werew.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Luigi Coniglio | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 10/01/2021 | +| GitHub username | werew | +| Website (optional) | | diff --git a/.github/contributors/willismonroe.md b/.github/contributors/willismonroe.md new file mode 100644 index 0000000..3a6f105 --- /dev/null +++ b/.github/contributors/willismonroe.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [x] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Willis Monroe | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-3-5 | +| GitHub username | willismonroe | +| Website (optional) | | diff --git a/.github/contributors/willprice.md b/.github/contributors/willprice.md new file mode 100644 index 0000000..119f428 --- /dev/null +++ b/.github/contributors/willprice.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | --------------------- | +| Name | Will Price | +| Company name (if applicable) | N/A | +| Title or role (if applicable) | N/A | +| Date | 26/12/2018 | +| GitHub username | willprice | +| Website (optional) | https://willprice.org | diff --git a/.github/contributors/wojtuch.md b/.github/contributors/wojtuch.md new file mode 100644 index 0000000..ef44421 --- /dev/null +++ b/.github/contributors/wojtuch.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Wojciech Lukasiewicz | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13.08.2018 | +| GitHub username | wojtuch | +| Website (optional) | | diff --git a/.github/contributors/wxv.md b/.github/contributors/wxv.md new file mode 100644 index 0000000..bef854d --- /dev/null +++ b/.github/contributors/wxv.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Jason Xu | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2018-11-29 | +| GitHub username | wxv | +| Website (optional) | | diff --git a/.github/contributors/x-ji.md b/.github/contributors/x-ji.md new file mode 100644 index 0000000..14dd964 --- /dev/null +++ b/.github/contributors/x-ji.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|-------------------------------|--------------| +| Name | Xiang Ji | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 07 July 2018 | +| GitHub username | x-ji | +| Website (optional) | | diff --git a/.github/contributors/xadrianzetx.md b/.github/contributors/xadrianzetx.md new file mode 100644 index 0000000..65603e9 --- /dev/null +++ b/.github/contributors/xadrianzetx.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name |Adrian Zuber | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date |20-06-2021 | +| GitHub username |xadrianzetx | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/xssChauhan.md b/.github/contributors/xssChauhan.md new file mode 100644 index 0000000..cad50c8 --- /dev/null +++ b/.github/contributors/xssChauhan.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Shikhar Chauhan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 12/11/2019 | +| GitHub username | xssChauhan | +| Website (optional) | | diff --git a/.github/contributors/yanaiela.md b/.github/contributors/yanaiela.md new file mode 100644 index 0000000..ee76318 --- /dev/null +++ b/.github/contributors/yanaiela.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [ ] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yanai Elazar | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 14/8/2019 | +| GitHub username | yanaiela | +| Website (optional) | https://yanaiela.github.io | \ No newline at end of file diff --git a/.github/contributors/yaph.md b/.github/contributors/yaph.md new file mode 100644 index 0000000..d3697bc --- /dev/null +++ b/.github/contributors/yaph.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Ramiro Gómez | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-04-29 | +| GitHub username | yaph | +| Website (optional) | http://ramiro.org/ | diff --git a/.github/contributors/yashpatadia.md b/.github/contributors/yashpatadia.md new file mode 100644 index 0000000..2dcf921 --- /dev/null +++ b/.github/contributors/yashpatadia.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yash Patadia | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 11/07/2019 | +| GitHub username | yash1994 | +| Website (optional) | | \ No newline at end of file diff --git a/.github/contributors/yohasebe.md b/.github/contributors/yohasebe.md new file mode 100644 index 0000000..c6f6167 --- /dev/null +++ b/.github/contributors/yohasebe.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Yoichiro Hasebe | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | July 4th, 2021 | +| GitHub username | yohasebe | +| Website (optional) | https://yohasebe.com | diff --git a/.github/contributors/yosiasz.md b/.github/contributors/yosiasz.md new file mode 100644 index 0000000..244cc30 --- /dev/null +++ b/.github/contributors/yosiasz.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Josiah Solomon | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2020-12-15 | +| GitHub username | yosiasz | +| Website (optional) | | diff --git a/.github/contributors/yuukos.md b/.github/contributors/yuukos.md new file mode 100644 index 0000000..aecafee --- /dev/null +++ b/.github/contributors/yuukos.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Alexey Kim | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 13-12-2017 | +| GitHub username | yuukos | +| Website (optional) | | diff --git a/.github/contributors/zaibacu.md b/.github/contributors/zaibacu.md new file mode 100644 index 0000000..365b898 --- /dev/null +++ b/.github/contributors/zaibacu.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Šarūnas Navickas | +| Company name (if applicable) | TokenMill | +| Title or role (if applicable) | Data Engineer | +| Date | 2020-09-24 | +| GitHub username | zaibacu | +| Website (optional) | | diff --git a/.github/contributors/zhuorulin.md b/.github/contributors/zhuorulin.md new file mode 100644 index 0000000..8fef757 --- /dev/null +++ b/.github/contributors/zhuorulin.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | ------------------------ | +| Name | Zhuoru Lin | +| Company name (if applicable) | Bombora Inc. | +| Title or role (if applicable) | Data Scientist | +| Date | 2017-11-13 | +| GitHub username | ZhuoruLin | +| Website (optional) | | diff --git a/.github/contributors/zqhZY.md b/.github/contributors/zqhZY.md new file mode 100644 index 0000000..6fe4038 --- /dev/null +++ b/.github/contributors/zqhZY.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI UG (haftungsbeschränkt)](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | QingHeng Zhang | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2017-12-28 | +| GitHub username | zqhZY | +| Website (optional) | https://someth.duapp.com/ | diff --git a/.github/contributors/zqianem.md b/.github/contributors/zqianem.md new file mode 100644 index 0000000..13f6ab2 --- /dev/null +++ b/.github/contributors/zqianem.md @@ -0,0 +1,106 @@ +# spaCy contributor agreement + +This spaCy Contributor Agreement (**"SCA"**) is based on the +[Oracle Contributor Agreement](http://www.oracle.com/technetwork/oca-405177.pdf). +The SCA applies to any contribution that you make to any product or project +managed by us (the **"project"**), and sets out the intellectual property rights +you grant to us in the contributed materials. The term **"us"** shall mean +[ExplosionAI GmbH](https://explosion.ai/legal). The term +**"you"** shall mean the person or entity identified below. + +If you agree to be bound by these terms, fill in the information requested +below and include the filled-in version with your first pull request, under the +folder [`.github/contributors/`](/.github/contributors/). The name of the file +should be your GitHub username, with the extension `.md`. For example, the user +example_user would create the file `.github/contributors/example_user.md`. + +Read this agreement carefully before signing. These terms and conditions +constitute a binding legal agreement. + +## Contributor Agreement + +1. The term "contribution" or "contributed materials" means any source code, +object code, patch, tool, sample, graphic, specification, manual, +documentation, or any other material posted or submitted by you to the project. + +2. With respect to any worldwide copyrights, or copyright applications and +registrations, in your contribution: + + * you hereby assign to us joint ownership, and to the extent that such + assignment is or becomes invalid, ineffective or unenforceable, you hereby + grant to us a perpetual, irrevocable, non-exclusive, worldwide, no-charge, + royalty-free, unrestricted license to exercise all rights under those + copyrights. This includes, at our option, the right to sublicense these same + rights to third parties through multiple levels of sublicensees or other + licensing arrangements; + + * you agree that each of us can do all things in relation to your + contribution as if each of us were the sole owners, and if one of us makes + a derivative work of your contribution, the one who makes the derivative + work (or has it made will be the sole owner of that derivative work; + + * you agree that you will not assert any moral rights in your contribution + against us, our licensees or transferees; + + * you agree that we may register a copyright in your contribution and + exercise all ownership rights associated with it; and + + * you agree that neither of us has any duty to consult with, obtain the + consent of, pay or render an accounting to the other for any use or + distribution of your contribution. + +3. With respect to any patents you own, or that you can license without payment +to any third party, you hereby grant to us a perpetual, irrevocable, +non-exclusive, worldwide, no-charge, royalty-free license to: + + * make, have made, use, sell, offer to sell, import, and otherwise transfer + your contribution in whole or in part, alone or in combination with or + included in any product, work or materials arising out of the project to + which your contribution was submitted, and + + * at our option, to sublicense these same rights to third parties through + multiple levels of sublicensees or other licensing arrangements. + +4. Except as set out above, you keep all right, title, and interest in your +contribution. The rights that you grant to us under these terms are effective +on the date you first submitted a contribution to us, even if your submission +took place before the date you sign these terms. + +5. You covenant, represent, warrant and agree that: + + * Each contribution that you submit is and shall be an original work of + authorship and you can legally grant the rights set out in this SCA; + + * to the best of your knowledge, each contribution will not violate any + third party's copyrights, trademarks, patents, or other intellectual + property rights; and + + * each contribution shall be in compliance with U.S. export control laws and + other applicable export and import laws. You agree to notify us if you + become aware of any circumstance which would make any of the foregoing + representations inaccurate in any respect. We may publicly disclose your + participation in the project, including the fact that you have signed the SCA. + +6. This SCA is governed by the laws of the State of California and applicable +U.S. Federal law. Any choice of law rules will not apply. + +7. Please place an “x” on one of the applicable statement below. Please do NOT +mark both statements: + + * [x] I am signing on behalf of myself as an individual and no other person + or entity, including my employer, has or will have rights with respect to my + contributions. + + * [ ] I am signing on behalf of my employer or a legal entity and I have the + actual authority to contractually bind that entity. + +## Contributor Details + +| Field | Entry | +|------------------------------- | -------------------- | +| Name | Em Zhan | +| Company name (if applicable) | | +| Title or role (if applicable) | | +| Date | 2019-09-25 | +| GitHub username | zqianem | +| Website (optional) | | diff --git a/.github/spacy_universe_alert.py b/.github/spacy_universe_alert.py new file mode 100644 index 0000000..cea876e --- /dev/null +++ b/.github/spacy_universe_alert.py @@ -0,0 +1,67 @@ +import os +import sys +import json +from datetime import datetime + +from slack_sdk.web.client import WebClient + +CHANNEL = "#alerts-universe" +SLACK_TOKEN = os.environ.get("SLACK_BOT_TOKEN", "ENV VAR not available!") +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + +client = WebClient(SLACK_TOKEN) +github_context = json.loads(sys.stdin.read()) + +event = github_context['event'] +pr_title = event['pull_request']["title"] +pr_link = event['pull_request']["patch_url"].replace(".patch", "") +pr_author_url = event['sender']["html_url"] +pr_author_name = pr_author_url.rsplit('/')[-1] +pr_created_at_dt = datetime.strptime( + event['pull_request']["created_at"], + DATETIME_FORMAT +) +pr_created_at = pr_created_at_dt.strftime("%c") +pr_updated_at_dt = datetime.strptime( + event['pull_request']["updated_at"], + DATETIME_FORMAT +) +pr_updated_at = pr_updated_at_dt.strftime("%c") + +blocks = [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "📣 New spaCy Universe Project Alert ✨" + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": f"*Pull Request:*\n<{pr_link}|{pr_title}>" + }, + { + "type": "mrkdwn", + "text": f"*Author:*\n<{pr_author_url}|{pr_author_name}>" + }, + { + "type": "mrkdwn", + "text": f"*Created at:*\n {pr_created_at}" + }, + { + "type": "mrkdwn", + "text": f"*Last Updated:*\n {pr_updated_at}" + } + ] + } + ] + + +client.chat_postMessage( + channel=CHANNEL, + text="spaCy universe project PR alert", + blocks=blocks +) diff --git a/.github/validate_universe_json.py b/.github/validate_universe_json.py new file mode 100644 index 0000000..b96b7b3 --- /dev/null +++ b/.github/validate_universe_json.py @@ -0,0 +1,19 @@ +import json +import re +import sys +from pathlib import Path + + +def validate_json(document): + universe_file = Path(document) + with universe_file.open() as f: + universe_data = json.load(f) + for entry in universe_data["resources"]: + if "github" in entry: + assert not re.match( + r"^(http:)|^(https:)", entry["github"] + ), "Github field should be user/repo, not a url" + + +if __name__ == "__main__": + validate_json(str(sys.argv[1])) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml new file mode 100644 index 0000000..0e102cd --- /dev/null +++ b/.github/workflows/cibuildwheel.yml @@ -0,0 +1,107 @@ +name: Build + +on: + push: + tags: + # ytf did they invent their own syntax that's almost regex? + # ** matches 'zero or more of any character' + - 'release-v[0-9]+.[0-9]+.[0-9]+**' + - 'prerelease-v[0-9]+.[0-9]+.[0-9]+**' + +permissions: {} + +jobs: + build_wheels: + uses: explosion/gha-cibuildwheel/.github/workflows/cibuildwheel.yml@2c98f757f13d112cf73fcf4b627249f1fffb5aae # main + permissions: + contents: write + actions: read + with: + wheel-name-pattern: "spacy-*.whl" + pure-python: false + secrets: + gh-token: ${{ secrets.GITHUB_TOKEN }} + + smoke_test: + name: Smoke test + needs: build_wheels + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1 + with: + tag: ${{ github.ref_name }} + fileName: "spacy-*cp312*manylinux*x86_64*.whl" + out-file-path: "dist" + + - name: Install from wheel + run: | + WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1) + pip install "$WHEEL" + + - name: Test import + run: python -c "import spacy; print('spacy==' + spacy.__version__)" + + - name: Download and load model + run: | + python -m spacy download en_core_web_sm + python -c " + import spacy + nlp = spacy.load('en_core_web_sm') + doc = nlp('Apple is looking at buying U.K. startup for \$1 billion') + assert len(doc.ents) > 0, 'No entities found' + print('Model load OK:', nlp.meta['name'], '@', nlp.meta['version']) + print('Entities:', [(ent.text, ent.label_) for ent in doc.ents]) + " + + upgrade_test: + name: Upgrade test + needs: build_wheels + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Install previous spaCy version + run: | + pip install "spacy>=3.8.0,<${{ github.ref_name }}" || pip install "spacy<4" + python -m spacy download en_core_web_sm + python -c " + import spacy + nlp = spacy.load('en_core_web_sm') + print('Pre-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version']) + " + + - uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1 + with: + tag: ${{ github.ref_name }} + fileName: "spacy-*cp312*manylinux*x86_64*.whl" + out-file-path: "dist" + + - name: Upgrade to new version + run: | + WHEEL=$(ls dist/spacy-*cp312*manylinux*x86_64*.whl | head -1) + pip install "$WHEEL" + + - name: Test model still loads after upgrade + run: | + python -c " + import spacy + nlp = spacy.load('en_core_web_sm') + doc = nlp('Apple is looking at buying U.K. startup for \$1 billion') + assert len(doc.ents) > 0, 'No entities found after upgrade' + print('Post-upgrade:', spacy.__version__, nlp.meta['name'], '@', nlp.meta['version']) + print('Entities:', [(ent.text, ent.label_) for ent in doc.ents]) + " diff --git a/.github/workflows/explosionbot.yml b/.github/workflows/explosionbot.yml new file mode 100644 index 0000000..979385c --- /dev/null +++ b/.github/workflows/explosionbot.yml @@ -0,0 +1,32 @@ +name: Explosion Bot + +on: + issue_comment: + types: + - created + - edited + +permissions: {} + +jobs: + explosion-bot: + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - name: Install and run explosion-bot + run: | + git config --global url."https://x-access-token:${EXPLOSIONBOT_TOKEN}@github.com/".insteadOf "https://github.com/" + pip install git+https://github.com/explosion/explosion-bot + python -m explosionbot + env: + EXPLOSIONBOT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }} + INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }} + INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }} + ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu" + ALLOWED_TEAMS: "spaCy" diff --git a/.github/workflows/gputests.yml.disabled b/.github/workflows/gputests.yml.disabled new file mode 100644 index 0000000..66e0707 --- /dev/null +++ b/.github/workflows/gputests.yml.disabled @@ -0,0 +1,22 @@ +name: Weekly GPU tests + +on: + schedule: + - cron: '0 1 * * MON' + +jobs: + weekly-gputests: + strategy: + fail-fast: false + matrix: + branch: [master, v4] + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Trigger buildkite build + uses: buildkite/trigger-pipeline-action@v1.2.0 + env: + PIPELINE: explosion-ai/spacy-slow-gpu-tests + BRANCH: ${{ matrix.branch }} + MESSAGE: ":github: Weekly GPU + slow tests - triggered from a GitHub Action" + BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_SECRET }} diff --git a/.github/workflows/issue-manager.yml b/.github/workflows/issue-manager.yml new file mode 100644 index 0000000..2647074 --- /dev/null +++ b/.github/workflows/issue-manager.yml @@ -0,0 +1,40 @@ +name: Issue Manager + +on: + schedule: + - cron: "0 0 * * *" + issue_comment: + types: + - created + - edited + issues: + types: + - labeled + +permissions: {} + +jobs: + issue-manager: + permissions: + issues: write + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - uses: tiangolo/issue-manager@4d1b7e05935a404dc8337d30bd23be46be8bb8e5 # 0.4.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + config: > + { + "resolved": { + "delay": "P7D", + "message": "This issue has been automatically closed because it was answered and there was no follow-up discussion.", + "remove_label_on_comment": true, + "remove_label_on_close": true + }, + "more-info-needed": { + "delay": "P7D", + "message": "This issue has been automatically closed because there has been no response to a request for more information from the original author. With only the information that is currently in the issue, there's not enough information to take action. If you're the original author, feel free to reopen the issue if you have or find the answers needed to investigate further.", + "remove_label_on_comment": true, + "remove_label_on_close": true + } + } diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000..8fcf302 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,26 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' # check every day + workflow_dispatch: + +permissions: + issues: write + +concurrency: + group: lock + +jobs: + action: + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5 + with: + process-only: 'issues' + issue-inactive-days: '30' + issue-comment: > + This thread has been automatically locked since there + has not been any recent activity after it was closed. + Please open a new issue for related bugs. diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..ac54255 --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,31 @@ +# The cibuildwheel action triggers on creation of a release, this +# triggers on publication. +# The expected workflow is to create a draft release and let the wheels +# upload, and then hit 'publish', which uploads to PyPi. + +on: + release: + types: + - published + +permissions: {} + +jobs: + upload_pypi: + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/spacy + permissions: + id-token: write + contents: read + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: robinraju/release-downloader@daf26c55d821e836577a15f77d86ddc078948b05 # v1 + with: + tag: ${{ github.event.release.tag_name }} + fileName: '*' + out-file-path: 'dist' + - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 diff --git a/.github/workflows/slowtests.yml.disabled b/.github/workflows/slowtests.yml.disabled new file mode 100644 index 0000000..17d8989 --- /dev/null +++ b/.github/workflows/slowtests.yml.disabled @@ -0,0 +1,38 @@ +name: Daily slow tests + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + daily-slowtests: + strategy: + fail-fast: false + matrix: + branch: [master, v4] + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ matrix.branch }} + - name: Get commits from past 24 hours + id: check_commits + run: | + today=$(date '+%Y-%m-%d %H:%M:%S') + yesterday=$(date -d "yesterday" '+%Y-%m-%d %H:%M:%S') + if git log --after="$yesterday" --before="$today" | grep commit ; then + echo run_tests=true >> $GITHUB_OUTPUT + else + echo run_tests=false >> $GITHUB_OUTPUT + fi + + - name: Trigger buildkite build + if: steps.check_commits.outputs.run_tests == 'true' + uses: buildkite/trigger-pipeline-action@v1.2.0 + env: + PIPELINE: explosion-ai/spacy-slow-tests + BRANCH: ${{ matrix.branch }} + MESSAGE: ":github: Daily slow tests - triggered from a GitHub Action" + BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_SECRET }} diff --git a/.github/workflows/spacy_universe_alert.yml b/.github/workflows/spacy_universe_alert.yml new file mode 100644 index 0000000..6798e27 --- /dev/null +++ b/.github/workflows/spacy_universe_alert.yml @@ -0,0 +1,28 @@ +name: spaCy universe project alert + +on: + pull_request_target: + paths: + - "website/meta/universe.json" + +permissions: {} + +jobs: + build: + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.10' + - name: Install Bernadette app dependency and send an alert + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + GITHUB_CONTEXT: ${{ toJson(github) }} + CHANNEL: "#alerts-universe" + run: | + pip install slack-sdk==3.17.2 aiohttp==3.8.1 + echo "$CHANNEL" + echo "$GITHUB_CONTEXT" | python .github/spacy_universe_alert.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..fab5a85 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,167 @@ +name: tests + +on: + push: + tags-ignore: + - '**' + branches-ignore: + - "spacy.io" + - "nightly.spacy.io" + - "v2.spacy.io" + paths-ignore: + - "*.md" + - "*.mdx" + - "website/**" + pull_request: + types: [opened, synchronize, reopened, edited] + paths-ignore: + - "*.md" + - "*.mdx" + - "website/**" + +permissions: + contents: read + +jobs: + validate: + name: Validate + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Configure Python version + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.10" + + - name: ruff format + run: | + python -m pip install ruff -c requirements.txt + python -m ruff format spacy --check + - name: ruff isort + run: | + python -m ruff check spacy --select I + + tests: + name: Test + needs: Validate + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python_version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Configure Python version + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ matrix.python_version }} + + - name: Install dependencies + run: | + python -m pip install -U build pip setuptools + python -m pip install -U -r requirements.txt + + - name: Build sdist + run: | + python -m build --sdist + + - name: Run mypy + run: | + python -m mypy spacy + if: matrix.python_version != '3.7' + + - name: Delete source directory and .egg-info + run: | + rm -rf spacy *.egg-info + shell: bash + + - name: Uninstall all packages + run: | + python -m pip freeze + python -m pip freeze --exclude pywin32 > installed.txt + python -m pip uninstall -y -r installed.txt + + - name: Install from sdist + run: | + SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) + SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST + shell: bash + + - name: Test import + run: python -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic' -c "import spacy" + + - name: "Test download CLI" + run: | + python -m spacy download ca_core_news_sm + python -m spacy download ca_core_news_md + python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" + if: matrix.python_version == '3.9' + + - name: "Test download_url in info CLI" + run: | + python -W error -m spacy info ca_core_news_sm | grep -q download_url + if: matrix.python_version == '3.9' + + - name: "Test no warnings on load (#11713)" + run: | + python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" + if: matrix.python_version == '3.9' + + - name: "Test convert CLI" + run: | + python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json . + if: matrix.python_version == '3.9' + + - name: "Test debug config CLI" + run: | + python -m spacy init config -p ner -l ca ner.cfg + python -m spacy debug config ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy + if: matrix.python_version == '3.9' + + - name: "Test debug data CLI" + run: | + # will have errors due to sparse data, check for summary in output + python -m spacy debug data ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy | grep -q Summary + if: matrix.python_version == '3.9' + + - name: "Test train CLI" + run: | + python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1 + if: matrix.python_version == '3.9' + + - name: "Test assemble CLI" + run: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" + python -m spacy assemble ner_source_sm.cfg output_dir + env: + PYTHONWARNINGS: "error,ignore::DeprecationWarning" + if: matrix.python_version == '3.9' + + - name: "Test assemble CLI vectors warning" + run: | + python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" + python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 + if: matrix.python_version == '3.9' + + - name: "Install test requirements" + run: | + python -m pip install -U -r requirements.txt + + - name: "Run CPU tests" + run: | + python -m pytest --pyargs spacy -W error -W 'ignore:Core Pydantic V1:UserWarning:pydantic' + if: "!(startsWith(matrix.os, 'macos') && matrix.python_version == '3.11')" + + - name: "Run CPU tests with thinc-apple-ops" + run: | + python -m pip install 'spacy[apple]' + python -m pytest --pyargs spacy + if: startsWith(matrix.os, 'macos') && matrix.python_version == '3.11' diff --git a/.github/workflows/universe_validation.yml b/.github/workflows/universe_validation.yml new file mode 100644 index 0000000..de1de07 --- /dev/null +++ b/.github/workflows/universe_validation.yml @@ -0,0 +1,35 @@ +name: universe validation + +on: + push: + branches-ignore: + - "spacy.io" + - "nightly.spacy.io" + - "v2.spacy.io" + paths: + - "website/meta/universe.json" + pull_request: + types: [opened, synchronize, reopened, edited] + paths: + - "website/meta/universe.json" + +permissions: + contents: read + +jobs: + validate: + name: Validate + if: github.repository_owner == 'explosion' + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Configure Python version + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.7" + + - name: Validate website/meta/universe.json + run: | + python .github/validate_universe_json.py website/meta/universe.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af75a4d --- /dev/null +++ b/.gitignore @@ -0,0 +1,117 @@ +# spaCy +spacy/data/ +corpora/ +/models/ +keys/ +*.json.gz + +# Tests +spacy/tests/package/setup.cfg +spacy/tests/package/pyproject.toml +spacy/tests/package/requirements.txt + +# Cython / C extensions +cythonize.json +spacy/*.html +*.cpp +*.c +*.so + +# Vim / VSCode / editors +*.swp +*.sw* +Profile.prof +.vscode +.sass-cache + +# Python +.Python +.python-version +__pycache__/ +.pytest_cache +*.py[cod] +.env/ +.env* +.~env/ +.venv +env3.6/ +venv/ +env3.*/ +.dev +.denv +.pypyenv +.pytest_cache/ +.mypy_cache/ +.hypothesis/ + +# Distribution / packaging +env/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheelhouse/ +*.egg-info/ +pip-wheel-metadata/ +Pipfile.lock +.installed.cfg +*.egg +.eggs +MANIFEST +spacy/git_info.py + +# Temporary files +*.~* +tmp/ + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Windows +*.bat +Thumbs.db +Desktop.ini + +# Mac OS X +*.DS_Store + +# Komodo project files +*.komodoproject + +# Other +*.tgz + +# Pycharm project files +*.idea + +# IPython +.ipynb_checkpoints/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7d57c3a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.0 + hooks: + - id: ruff + args: ['--fix'] + - id: ruff-format diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..88c05b2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,16 @@ +cff-version: 1.2.0 +preferred-citation: + type: article + message: "If you use spaCy, please cite it as below." + authors: + - family-names: "Honnibal" + given-names: "Matthew" + - family-names: "Montani" + given-names: "Ines" + - family-names: "Van Landeghem" + given-names: "Sofie" + - family-names: "Boyd" + given-names: "Adriane" + title: "spaCy: Industrial-strength Natural Language Processing in Python" + doi: "10.5281/zenodo.1212303" + year: 2020 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b0a20b6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,465 @@ + + +# Contribute to spaCy + +Thanks for your interest in contributing to spaCy 🎉 This page will give you a quick +overview of how things are organized and most importantly, how to get involved. + +## Table of contents + +1. [Issues and bug reports](#issues-and-bug-reports) +2. [Contributing to the code base](#contributing-to-the-code-base) +3. [Code conventions](#code-conventions) +4. [Adding tests](#adding-tests) +5. [Updating the website](#updating-the-website) +6. [Publishing extensions and plugins](#publishing-spacy-extensions-and-plugins) +7. [Code of conduct](#code-of-conduct) + +## Issues and bug reports + +First, [do a quick search](https://github.com/issues?q=+is%3Aissue+user%3Aexplosion) +to see if the issue has already been reported. If so, it's often better to just +leave a comment on an existing issue, rather than creating a new one. Old issues +also often include helpful tips and solutions to common problems. You should +also check the [troubleshooting guide](https://spacy.io/usage/#troubleshooting) +to see if your problem is already listed there. + +If you're looking for help with your code, consider posting a question on the +[GitHub Discussions board](https://github.com/explosion/spaCy/discussions) or +[Stack Overflow](http://stackoverflow.com/questions/tagged/spacy). Please +understand that we won't be able to provide individual support via email. We +also believe that help is much more valuable if it's **shared publicly**, +so that more people can benefit from it. + +### Submitting issues + +When opening an issue, use a **descriptive title** and include your +**environment** (operating system, Python version, spaCy version). Our +[issue templates](https://github.com/explosion/spaCy/issues/new/choose) help you +remember the most important details to include. If you've discovered a bug, you +can also submit a [regression test](#fixing-bugs) straight away. When you're +opening an issue to report the bug, simply refer to your pull request in the +issue body. A few more tips: + +- **Describing your issue:** Try to provide as many details as possible. What + exactly goes wrong? _How_ is it failing? Is there an error? + "XY doesn't work" usually isn't that helpful for tracking down problems. Always + remember to include the code you ran and if possible, extract only the relevant + parts and don't just dump your entire script. This will make it easier for us to + reproduce the error. + +- **Getting info about your spaCy installation and environment:** You can use the command line interface to print details and + even format them as Markdown to copy-paste into GitHub issues: + `python -m spacy info --markdown`. + +- **Checking the model compatibility:** If you're having problems with a + [statistical model](https://spacy.io/models), it may be because the + model is incompatible with your spaCy installation. In spaCy v2.0+, you can check + this on the command line by running `python -m spacy validate`. + +- **Sharing a model's output, like dependencies and entities:** spaCy + comes with [built-in visualizers](https://spacy.io/usage/visualizers) that + you can run from within your script or a Jupyter notebook. For some issues, it's + helpful to **include a screenshot** of the visualization. You can simply drag and + drop the image into GitHub's editor and it will be uploaded and included. + +- **Sharing long blocks of code or logs:** If you need to include long code, + logs or tracebacks, you can wrap them in `
` and `
`. This + [collapses the content](https://developer.mozilla.org/en/docs/Web/HTML/Element/details) + so it only becomes visible on click, making the issue easier to read and follow. + +### Issue labels + +[See this page](https://github.com/explosion/spaCy/labels) for an overview of +the system we use to tag our issues and pull requests. + +## Contributing to the code base + +You don't have to be an NLP expert or Python pro to contribute, and we're happy +to help you get started. If you're new to spaCy, a good place to start is the +[spaCy 101 guide](https://spacy.io/usage/spacy-101) and the +[`help wanted (easy)`](https://github.com/explosion/spaCy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted+%28easy%29%22) +label, which we use to tag bugs and feature requests that are easy and +self-contained. If you've decided to take on one of these problems and you're +making good progress, don't forget to add a quick comment to the issue. You can +also use the issue to ask questions, or share your work in progress. + +### What belongs in spaCy? + +Every library has a different inclusion philosophy — a policy of what should be +shipped in the core library, and what could be provided in other packages. Our +philosophy is to prefer a smaller core library. We generally ask the following +questions: + +- **What would this feature look like if implemented in a separate package?** + Some features would be very difficult to implement externally – for example, + changes to spaCy's built-in methods. In contrast, a library of word + alignment functions could easily live as a separate package that depended on + spaCy — there's little difference between writing `import word_aligner` and + `import spacy.word_aligner`. spaCy makes it easy to implement + [custom pipeline components](https://spacy.io/usage/processing-pipelines#custom-components), + and add your own attributes, properties and methods to the `Doc`, `Token` and + `Span`. If you're looking to implement a new spaCy feature, starting with a + custom component package is usually the best strategy. You won't have to worry + about spaCy's internals and you can test your module in an isolated + environment. And if it works well, we can always integrate it into the core + library later. + +- **Would the feature be easier to implement if it relied on "heavy" dependencies spaCy doesn't currently require?** + Python has a very rich ecosystem. Libraries like PyTorch, TensorFlow, scikit-learn, SciPy or Gensim + do lots of useful things — but we don't want to have them as default + dependencies. If the feature requires functionality in one of these libraries, + it's probably better to break it out into a different package. + +- **Is the feature orthogonal to the current spaCy functionality, or overlapping?** + spaCy strongly prefers to avoid having 6 different ways of doing the same thing. + As better techniques are developed, we prefer to drop support for "the old way". + However, it's rare that one approach _entirely_ dominates another. It's very + common that there's still a use-case for the "obsolete" approach. For instance, + [WordNet](https://wordnet.princeton.edu/) is still very useful — but word + vectors are better for most use-cases, and the two approaches to lexical + semantics do a lot of the same things. spaCy therefore only supports word + vectors, and support for WordNet is currently left for other packages. + +- **Do you need the feature to get basic things done?** We do want spaCy to be + at least somewhat self-contained. If we keep needing some feature in our + recipes, that does provide some argument for bringing it "in house". + +### Getting started + +To make changes to spaCy's code base, you need to fork then clone the GitHub repository +and build spaCy from source. You'll need to make sure that you have a +development environment consisting of a Python distribution including header +files, a compiler, [pip](https://pip.pypa.io/en/latest/installing/), +[virtualenv](https://virtualenv.pypa.io/en/stable/) and +[git](https://git-scm.com) installed. The compiler is usually the trickiest part. + +If you've made changes to `.pyx` files, you need to **recompile spaCy** before you +can test your changes by re-running `python setup.py build_ext --inplace`. +Changes to `.py` files will be effective immediately. + +📖 **For more details and instructions, see the documentation on [compiling spaCy from source](https://spacy.io/usage/#source) and the [quickstart widget](https://spacy.io/usage/#section-quickstart) to get the right commands for your platform and Python version.** + +### Fixing bugs + +When fixing a bug, first create an +[issue](https://github.com/explosion/spaCy/issues) if one does not already +exist. The description text can be very short – we don't want to make this too +bureaucratic. + +Next, add a test to the relevant file in the +[`spacy/tests`](spacy/tests)folder. Then add a [pytest +mark](https://docs.pytest.org/en/6.2.x/example/markers.html#working-with-custom-markers), +`@pytest.mark.issue(NUMBER)`, to reference the issue number. + +```python +# Assume you're fixing Issue #1234 +@pytest.mark.issue(1234) +def test_issue1234(): + ... +``` + +Test for the bug you're fixing, and make sure the test fails. Next, add and +commit your test file. Finally, fix the bug, make sure your test passes and +reference the issue number in your pull request description. + +📖 **For more information on how to add tests, check out the [tests README](spacy/tests/README.md).** + +## Code conventions + +Code should loosely follow [pep8](https://www.python.org/dev/peps/pep-0008/). +spaCy uses [`black`](https://github.com/ambv/black) for code +formatting and [`flake8`](http://flake8.pycqa.org/en/latest/) for linting its +Python modules. If you've built spaCy from source, you'll already have both +tools installed. + +As a general rule of thumb, we use f-strings for any formatting of strings. +One exception are calls to Python's `logging` functionality. +To avoid unnecessary string conversions in these cases, we use string formatting +templates with `%s` and `%d` etc. + +**⚠️ Note that formatting and linting is currently only possible for Python +modules in `.py` files, not Cython modules in `.pyx` and `.pxd` files.** + +### Pre-Commit Hooks + +After cloning the repo, after installing the packages from `requirements.txt`, enter the repo folder and run `pre-commit install`. +Each time a `git commit` is initiated, `black` and `flake8` will run automatically on the modified files only. + +In case of error, or when `black` modified a file, the modified file needs to be `git add` once again and a new +`git commit` has to be issued. + +### Code formatting + +[`black`](https://github.com/ambv/black) is an opinionated Python code +formatter, optimized to produce readable code and small diffs. You can run +`black` from the command-line, or via your code editor. For example, if you're +using [Visual Studio Code](https://code.visualstudio.com/), you can add the +following to your `settings.json` to use `black` for formatting and auto-format +your files on save: + +```json +{ + "python.formatting.provider": "black", + "[python]": { + "editor.formatOnSave": true + } +} +``` + +[See here](https://github.com/ambv/black#editor-integration) for the full +list of available editor integrations. + +#### Disabling formatting + +There are a few cases where auto-formatting doesn't improve readability – for +example, in some of the language data files or in the tests that construct `Doc` objects from lists of words and other labels. +Wrapping a block in `# fmt: off` and `# fmt: on` lets you disable formatting +for that particular code. Here's an example: + +```python +# fmt: off +text = "I look forward to using Thingamajig. I've been told it will make my life easier..." +heads = [1, 1, 1, 1, 3, 4, 1, 6, 11, 11, 11, 11, 14, 14, 11, 16, 17, 14, 11] +deps = ["nsubj", "ROOT", "advmod", "prep", "pcomp", "dobj", "punct", "", + "nsubjpass", "aux", "auxpass", "ROOT", "nsubj", "aux", "ccomp", + "poss", "nsubj", "ccomp", "punct"] +# fmt: on +``` + +### Code linting + +[`flake8`](http://flake8.pycqa.org/en/latest/) is a tool for enforcing code +style. It scans one or more files and outputs errors and warnings. This feedback +can help you stick to general standards and conventions, and can be very useful +for spotting potential mistakes and inconsistencies in your code. The most +important things to watch out for are syntax errors and undefined names, but you +also want to keep an eye on unused declared variables or repeated +(i.e. overwritten) dictionary keys. If your code was formatted with `black` +(see above), you shouldn't see any formatting-related warnings. + +The `flake8` section in [`setup.cfg`](setup.cfg) defines the configuration we use for this +codebase. For example, we're not super strict about the line length, and we're +excluding very large files like lemmatization and tokenizer exception tables. + +Ideally, running the following command from within the repo directory should +not return any errors or warnings: + +```bash +flake8 spacy +``` + +#### Disabling linting + +Sometimes, you explicitly want to write code that's not compatible with our +rules. For example, a module's `__init__.py` might import a function so other +modules can import it from there, but `flake8` will complain about an unused +import. And although it's generally discouraged, there might be cases where it +makes sense to use a bare `except`. + +To ignore a given line, you can add a comment like `# noqa: F401`, specifying +the code of the error or warning we want to ignore. It's also possible to +ignore several comma-separated codes at once, e.g. `# noqa: E731,E123`. Here +are some examples: + +```python +# The imported class isn't used in this file, but imported here, so it can be +# imported *from* here by another module. +from .submodule import SomeClass # noqa: F401 + +try: + do_something() +except: # noqa: E722 + # This bare except is justified, for some specific reason + do_something_else() +``` + +### Python conventions + +All Python code must be written **compatible with Python 3.6+**. More detailed +code conventions can be found in the [developer docs](https://github.com/explosion/spaCy/blob/master/extra/DEVELOPER_DOCS/Code%20Conventions.md). + +#### I/O and handling paths + +Code that interacts with the file-system should accept objects that follow the +`pathlib.Path` API, without assuming that the object inherits from `pathlib.Path`. +If the function is user-facing and takes a path as an argument, it should check +whether the path is provided as a string. Strings should be converted to +`pathlib.Path` objects. Serialization and deserialization functions should always +accept **file-like objects**, as it makes the library IO-agnostic. Working on +buffers makes the code more general, easier to test, and compatible with Python +3's asynchronous IO. + +#### Composition vs. inheritance + +Although spaCy uses a lot of classes, **inheritance is viewed with some suspicion** +— it's seen as a mechanism of last resort. You should discuss plans to extend +the class hierarchy before implementing. + +#### Naming conventions + +We have a number of conventions around variable naming that are still being +documented, and aren't 100% strict. A general policy is that instances of the +class `Doc` should by default be called `doc`, `Token` → `token`, `Lexeme` → `lex`, +`Vocab` → `vocab` and `Language` → `nlp`. You should avoid naming variables that are +of other types these names. For instance, don't name a text string `doc` — you +should usually call this `text`. Two general code style preferences further help +with naming. First, **lean away from introducing temporary variables**, as these +clutter your namespace. This is one reason why comprehension expressions are +often preferred. Second, **keep your functions shortish**, so they can work in a +smaller scope. Of course, this is a question of trade-offs. + +### Cython conventions + +spaCy's core data structures are implemented as [Cython](http://cython.org/) `cdef` +classes. Memory is managed through the `cymem.cymem.Pool` class, which allows +you to allocate memory which will be freed when the `Pool` object is garbage +collected. This means you usually don't have to worry about freeing memory. You +just have to decide which Python object owns the memory, and make it own the +`Pool`. When that object goes out of scope, the memory will be freed. You do +have to take care that no pointers outlive the object that owns them — but this +is generally quite easy. + +All Cython modules should have the `# cython: infer_types=True` compiler +directive at the top of the file. This makes the code much cleaner, as it avoids +the need for many type declarations. If possible, you should prefer to declare +your functions `nogil`, even if you don't especially care about multi-threading. +The reason is that `nogil` functions help the Cython compiler reason about your +code quite a lot — you're telling the compiler that no Python dynamics are +possible. This lets many errors be raised, and ensures your function will run +at C speed. + +Cython gives you many choices of sequences: you could have a Python list, a +numpy array, a memory view, a C++ vector, or a pointer. Pointers are preferred, +because they are fastest, have the most explicit semantics, and let the compiler +check your code more strictly. C++ vectors are also great — but you should only +use them internally in functions. It's less friendly to accept a vector as an +argument, because that asks the user to do much more work. + +Here's how to get a pointer from a numpy array, memory view or vector: + +```cython +cdef void get_pointers(np.ndarray[int, mode='c'] numpy_array, vector[int] cpp_vector, int[::1] memory_view) nogil: + pointer1 = numpy_array.data + pointer2 = cpp_vector.data() + pointer3 = &memory_view[0] +``` + +Both C arrays and C++ vectors reassure the compiler that no Python operations +are possible on your variable. This is a big advantage: it lets the Cython +compiler raise many more errors for you. + +When getting a pointer from a numpy array or memoryview, take care that the data +is actually stored in C-contiguous order — otherwise you'll get a pointer to +nonsense. The type-declarations in the code above should generate runtime errors +if buffers with incorrect memory layouts are passed in. + +To iterate over the array, the following style is preferred: + +```cython +cdef int c_total(const int* int_array, int length) nogil: + total = 0 + for item in int_array[:length]: + total += item + return total +``` + +If this is confusing, consider that the compiler couldn't deal with +`for item in int_array:` — there's no length attached to a raw pointer, so how +could we figure out where to stop? The length is provided in the slice notation +as a solution to this. Note that we don't have to declare the type of `item` in +the code above — the compiler can easily infer it. This gives us tidy code that +looks quite like Python, but is exactly as fast as C — because we've made sure +the compilation to C is trivial. + +Your functions cannot be declared `nogil` if they need to create Python objects +or call Python functions. This is perfectly okay — you shouldn't torture your +code just to get `nogil` functions. However, if your function isn't `nogil`, you +should compile your module with `cython -a --cplus my_module.pyx` and open the +resulting `my_module.html` file in a browser. This will let you see how Cython +is compiling your code. Calls into the Python run-time will be in bright yellow. +This lets you easily see whether Cython is able to correctly type your code, or +whether there are unexpected problems. + +Finally, if you're new to Cython, you should expect to find the first steps a +bit frustrating. It's a very large language, since it's essentially a superset +of Python and C++, with additional complexity and syntax from numpy. The +[documentation](http://docs.cython.org/en/latest/) isn't great, and there are +many "traps for new players". Working in Cython is very rewarding once you're +over the initial learning curve. As with C and C++, the first way you write +something in Cython will often be the performance-optimal approach. In contrast, +Python optimization generally requires a lot of experimentation. Is it faster to +have an `if item in my_dict` check, or to use `.get()`? What about `try`/`except`? +Does this numpy operation create a copy? There's no way to guess the answers to +these questions, and you'll usually be dissatisfied with your results — so +there's no way to know when to stop this process. In the worst case, you'll make +a mess that invites the next reader to try their luck too. This is like one of +those [volcanic gas-traps](http://www.wemjournal.org/article/S1080-6032%2809%2970088-2/abstract), +where the rescuers keep passing out from low oxygen, causing another rescuer to +follow — only to succumb themselves. In short, just say no to optimizing your +Python. If it's not fast enough the first time, just switch to Cython. + +### Resources to get you started + +- [PEP 8 Style Guide for Python Code](https://www.python.org/dev/peps/pep-0008/) (python.org) +- [Official Cython documentation](http://docs.cython.org/en/latest/) (cython.org) +- [Writing C in Cython](https://explosion.ai/blog/writing-c-in-cython) (explosion.ai) +- [Multi-threading spaCy’s parser and named entity recognizer](https://explosion.ai/blog/multithreading-with-cython) (explosion.ai) + +## Adding tests + +spaCy uses the [pytest](http://doc.pytest.org/) framework for testing. For more +info on this, see the [pytest documentation](http://docs.pytest.org/en/latest/contents.html). +Tests for spaCy modules and classes live in their own directories of the same +name. For example, tests for the `Tokenizer` can be found in +[`/spacy/tests/tokenizer`](spacy/tests/tokenizer). To be interpreted and run, +all test files and test functions need to be prefixed with `test_`. + +When adding tests, make sure to use descriptive names, keep the code short and +concise and only test for one behavior at a time. Try to `parametrize` test +cases wherever possible, use our pre-defined fixtures for spaCy components and +avoid unnecessary imports. Extensive tests that take a long time should be marked with `@pytest.mark.slow`. + +📖 **For more guidelines and information on how to add tests, check out the [tests README](spacy/tests/README.md).** + +## Updating the website + +For instructions on how to build and run the [website](https://spacy.io) locally see **[Setup and installation](https://github.com/explosion/spaCy/blob/master/website/README.md#setup-and-installation-setup)** in the _website_ directory's README. + +The docs can always use another example or more detail, and they should always +be up to date and not misleading. To quickly find the correct file to edit, +simply click on the "Suggest edits" button at the bottom of a page. + +📖 **For more info and troubleshooting guides, check out the [website README](website).** + +## Publishing spaCy extensions and plugins + +We're very excited about all the new possibilities for **community extensions** +and plugins in spaCy v3.0, and we can't wait to see what you build with it! + +- An extension or plugin should add substantial functionality, be + **well-documented** and **open-source**. It should be available for users to download + and install as a Python package – for example via [PyPi](http://pypi.python.org). + +- Extensions that write to `Doc`, `Token` or `Span` attributes should be wrapped + as [pipeline components](https://spacy.io/usage/processing-pipelines#custom-components) + that users can **add to their processing pipeline** using `nlp.add_pipe()`. + +- When publishing your extension on GitHub, **tag it** with the topics + [`spacy`](https://github.com/topics/spacy?o=desc&s=stars) and + [`spacy-extensions`](https://github.com/topics/spacy-extension?o=desc&s=stars) + to make it easier to find. Those are also the topics we're linking to from the + spaCy website. If you're sharing your project on X, feel free to tag + [@spacy_io](https://x.com/spacy_io) so we can check it out. + +- Once your extension is published, you can open a + [PR](https://github.com/explosion/spaCy/pulls) to suggest it for the + [Universe](https://spacy.io/universe) page. + +📖 **For more tips and best practices, see the [checklist for developing spaCy extensions](https://spacy.io/usage/processing-pipelines#extensions).** + +## Code of conduct + +spaCy adheres to the +[Contributor Covenant Code of Conduct](http://contributor-covenant.org/version/1/4/). +By participating, you are expected to uphold this code. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6cb7810 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (C) 2016-2024 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1caf758 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +recursive-include spacy *.pyi *.pyx *.pxd *.txt *.cfg *.jinja *.toml *.hh +include LICENSE +include README.md +include pyproject.toml +include spacy/py.typed +recursive-include spacy/cli *.yml +recursive-include spacy/tests *.json +recursive-include licenses * +recursive-exclude spacy *.cpp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c8f68be --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +SHELL := /bin/bash + +ifndef SPACY_EXTRAS +override SPACY_EXTRAS = spacy-lookups-data==1.0.3 +endif + +ifndef PYVER +override PYVER = 3.8 +endif + +VENV := ./env$(PYVER) + +version := $(shell "bin/get-version.sh") +package := $(shell "bin/get-package.sh") + +ifndef SPACY_BIN +override SPACY_BIN = $(package)-$(version).pex +endif + +ifndef WHEELHOUSE +override WHEELHOUSE = "./wheelhouse" +endif + + +dist/$(SPACY_BIN) : $(WHEELHOUSE)/spacy-$(PYVER)-$(version).stamp + $(VENV)/bin/pex \ + -f $(WHEELHOUSE) \ + --no-index \ + --disable-cache \ + -o $@ \ + $(package)==$(version) \ + $(SPACY_EXTRAS) + chmod a+rx $@ + cp $@ dist/spacy.pex + +dist/pytest.pex : $(WHEELHOUSE)/pytest-*.whl + $(VENV)/bin/pex -f $(WHEELHOUSE) --no-index --disable-cache -m pytest -o $@ pytest pytest-timeout mock + chmod a+rx $@ + +$(WHEELHOUSE)/spacy-$(PYVER)-$(version).stamp : $(VENV)/bin/pex setup.py spacy/*.py* spacy/*/*.py* + $(VENV)/bin/pip wheel . -w $(WHEELHOUSE) + $(VENV)/bin/pip wheel $(SPACY_EXTRAS) -w $(WHEELHOUSE) + + touch $@ + +$(WHEELHOUSE)/pytest-%.whl : $(VENV)/bin/pex + $(VENV)/bin/pip wheel pytest pytest-timeout mock -w $(WHEELHOUSE) + +$(VENV)/bin/pex : + python$(PYVER) -m venv $(VENV) + $(VENV)/bin/pip install -U pip setuptools pex wheel + $(VENV)/bin/pip install numpy + +.PHONY : clean test + +test : dist/spacy-$(version).pex dist/pytest.pex + ( . $(VENV)/bin/activate ; \ + PEX_PATH=dist/spacy-$(version).pex ./dist/pytest.pex --pyargs spacy -x ; ) + +clean : setup.py + rm -rf dist/* + rm -rf $(WHEELHOUSE)/* + rm -rf $(VENV) + python setup.py clean --all diff --git a/README.md b/README.md new file mode 100644 index 0000000..cab0d80 --- /dev/null +++ b/README.md @@ -0,0 +1,292 @@ + + +# spaCy: Industrial-strength NLP + +spaCy is a library for **advanced Natural Language Processing** in Python and +Cython. It's built on the very latest research, and was designed from day one to +be used in real products. + +spaCy comes with [pretrained pipelines](https://spacy.io/models) and currently +supports tokenization and training for **70+ languages**. It features +state-of-the-art speed and **neural network models** for tagging, parsing, +**named entity recognition**, **text classification** and more, multi-task +learning with pretrained **transformers** like BERT, as well as a +production-ready [**training system**](https://spacy.io/usage/training) and easy +model packaging, deployment and workflow management. spaCy is commercial +open-source software, released under the +[MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE). + +💫 **Version 3.8 out now!** +[Check out the release notes here.](https://github.com/explosion/spaCy/releases) + +[![tests](https://github.com/explosion/spaCy/actions/workflows/tests.yml/badge.svg)](https://github.com/explosion/spaCy/actions/workflows/tests.yml) +[![Current Release Version](https://img.shields.io/github/release/explosion/spacy.svg?style=flat-square&logo=github)](https://github.com/explosion/spaCy/releases) +[![pypi Version](https://img.shields.io/pypi/v/spacy.svg?style=flat-square&logo=pypi&logoColor=white)](https://pypi.org/project/spacy/) +[![conda Version](https://img.shields.io/conda/vn/conda-forge/spacy.svg?style=flat-square&logo=conda-forge&logoColor=white)](https://anaconda.org/conda-forge/spacy) +[![Python wheels](https://img.shields.io/badge/wheels-%E2%9C%93-4c1.svg?longCache=true&style=flat-square&logo=python&logoColor=white)](https://github.com/explosion/wheelwright/releases) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/ambv/black) +
+[![PyPi downloads](https://static.pepy.tech/personalized-badge/spacy?period=total&units=international_system&left_color=grey&right_color=orange&left_text=pip%20downloads)](https://pypi.org/project/spacy/) +[![Conda downloads](https://img.shields.io/conda/dn/conda-forge/spacy?label=conda%20downloads)](https://anaconda.org/conda-forge/spacy) + +## 📖 Documentation + +| Documentation | | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ⭐️ **[spaCy 101]** | New to spaCy? Here's everything you need to know! | +| 📚 **[Usage Guides]** | How to use spaCy and its features. | +| 🚀 **[New in v3.0]** | New features, backwards incompatibilities and migration guide. | +| 🪐 **[Project Templates]** | End-to-end workflows you can clone, modify and run. | +| 🎛 **[API Reference]** | The detailed reference for spaCy's API. | +| ⏩ **[GPU Processing]** | Use spaCy with CUDA-compatible GPU processing. | +| 📦 **[Models]** | Download trained pipelines for spaCy. | +| 🦙 **[Large Language Models]** | Integrate LLMs into spaCy pipelines. | +| 🌌 **[Universe]** | Plugins, extensions, demos and books from the spaCy ecosystem. | +| ⚙️ **[spaCy VS Code Extension]** | Additional tooling and features for working with spaCy's config files. | +| 👩‍🏫 **[Online Course]** | Learn spaCy in this free and interactive online course. | +| 📰 **[Blog]** | Read about current spaCy and Prodigy development, releases, talks and more from Explosion. | +| 📺 **[Videos]** | Our YouTube channel with video tutorials, talks and more. | +| 🔴 **[Live Stream]** | Join Matt as he works on spaCy and chat about NLP. | +| 🛠 **[Changelog]** | Changes and version history. | +| 💝 **[Contribute]** | How to contribute to the spaCy project and code base. | +| 👕 **[Swag]** | Support us and our work with unique, custom-designed swag! | +| Tailored Solutions | Custom NLP consulting, implementation and strategic advice by spaCy’s core development team. Streamlined, production-ready, predictable and maintainable. Send us an email or take our 5-minute questionnaire, and well'be in touch! **[Learn more →](https://explosion.ai/tailored-solutions)** | + +[spacy 101]: https://spacy.io/usage/spacy-101 +[new in v3.0]: https://spacy.io/usage/v3 +[usage guides]: https://spacy.io/usage/ +[api reference]: https://spacy.io/api/ +[gpu processing]: https://spacy.io/usage#gpu +[models]: https://spacy.io/models +[large language models]: https://spacy.io/usage/large-language-models +[universe]: https://spacy.io/universe +[spacy vs code extension]: https://github.com/explosion/spacy-vscode +[videos]: https://www.youtube.com/c/ExplosionAI +[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c +[online course]: https://course.spacy.io +[blog]: https://explosion.ai +[project templates]: https://github.com/explosion/projects +[changelog]: https://spacy.io/usage#changelog +[contribute]: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md +[swag]: https://explosion.ai/merch + +## 💬 Where to ask questions + +The spaCy project is maintained by the [spaCy team](https://explosion.ai/about). +Please understand that we won't be able to provide individual support via email. +We also believe that help is much more valuable if it's shared publicly, so that +more people can benefit from it. + +| Type | Platforms | +| ------------------------------- | --------------------------------------- | +| 🚨 **Bug Reports** | [GitHub Issue Tracker] | +| 🎁 **Feature Requests & Ideas** | [GitHub Discussions] · [Live Stream] | +| 👩‍💻 **Usage Questions** | [GitHub Discussions] · [Stack Overflow] | +| 🗯 **General Discussion** | [GitHub Discussions] · [Live Stream] | + +[github issue tracker]: https://github.com/explosion/spaCy/issues +[github discussions]: https://github.com/explosion/spaCy/discussions +[stack overflow]: https://stackoverflow.com/questions/tagged/spacy +[live stream]: https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c + +## Features + +- Support for **70+ languages** +- **Trained pipelines** for different languages and tasks +- Multi-task learning with pretrained **transformers** like BERT +- Support for pretrained **word vectors** and embeddings +- State-of-the-art speed +- Production-ready **training system** +- Linguistically-motivated **tokenization** +- Components for named **entity recognition**, part-of-speech-tagging, + dependency parsing, sentence segmentation, **text classification**, + lemmatization, morphological analysis, entity linking and more +- Easily extensible with **custom components** and attributes +- Support for custom models in **PyTorch**, **TensorFlow** and other frameworks +- Built in **visualizers** for syntax and NER +- Easy **model packaging**, deployment and workflow management +- Robust, rigorously evaluated accuracy + +📖 **For more details, see the +[facts, figures and benchmarks](https://spacy.io/usage/facts-figures).** + +## ⏳ Install spaCy + +For detailed installation instructions, see the +[documentation](https://spacy.io/usage). + +- **Operating system**: macOS / OS X · Linux · Windows (Cygwin, MinGW, Visual + Studio) +- **Python version**: Python >=3.7, <3.13 (only 64 bit) +- **Package managers**: [pip] · [conda] (via `conda-forge`) + +[pip]: https://pypi.org/project/spacy/ +[conda]: https://anaconda.org/conda-forge/spacy + +### pip + +Using pip, spaCy releases are available as source packages and binary wheels. +Before you install spaCy and its dependencies, make sure that your `pip`, +`setuptools` and `wheel` are up to date. + +```bash +pip install -U pip setuptools wheel +pip install spacy +``` + +To install additional data tables for lemmatization and normalization you can +run `pip install spacy[lookups]` or install +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) +separately. The lookups package is needed to create blank models with +lemmatization data, and to lemmatize in languages that don't yet come with +pretrained models and aren't powered by third-party libraries. + +When using pip it is generally recommended to install packages in a virtual +environment to avoid modifying system state: + +```bash +python -m venv .env +source .env/bin/activate +pip install -U pip setuptools wheel +pip install spacy +``` + +### conda + +You can also install spaCy from `conda` via the `conda-forge` channel. For the +feedstock including the build recipe and configuration, check out +[this repository](https://github.com/conda-forge/spacy-feedstock). + +```bash +conda install -c conda-forge spacy +``` + +### Updating spaCy + +Some updates to spaCy may require downloading new statistical models. If you're +running spaCy v2.0 or higher, you can use the `validate` command to check if +your installed models are compatible and if not, print details on how to update +them: + +```bash +pip install -U spacy +python -m spacy validate +``` + +If you've trained your own models, keep in mind that your training and runtime +inputs must match. After updating spaCy, we recommend **retraining your models** +with the new version. + +📖 **For details on upgrading from spaCy 2.x to spaCy 3.x, see the +[migration guide](https://spacy.io/usage/v3#migrating).** + +## 📦 Download model packages + +Trained pipelines for spaCy can be installed as **Python packages**. This means +that they're a component of your application, just like any other module. Models +can be installed using spaCy's [`download`](https://spacy.io/api/cli#download) +command, or manually by pointing pip to a path or URL. + +| Documentation | | +| -------------------------- | ---------------------------------------------------------------- | +| **[Available Pipelines]** | Detailed pipeline descriptions, accuracy figures and benchmarks. | +| **[Models Documentation]** | Detailed usage and installation instructions. | +| **[Training]** | How to train your own pipelines on your data. | + +[available pipelines]: https://spacy.io/models +[models documentation]: https://spacy.io/usage/models +[training]: https://spacy.io/usage/training + +```bash +# Download best-matching version of specific model for your spaCy installation +python -m spacy download en_core_web_sm + +# pip install .tar.gz archive or .whl from path or URL +pip install /Users/you/en_core_web_sm-3.0.0.tar.gz +pip install /Users/you/en_core_web_sm-3.0.0-py3-none-any.whl +pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz +``` + +### Loading and using models + +To load a model, use [`spacy.load()`](https://spacy.io/api/top-level#spacy.load) +with the model name or a path to the model data directory. + +```python +import spacy +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence.") +``` + +You can also `import` a model directly via its full name and then call its +`load()` method with no arguments. + +```python +import spacy +import en_core_web_sm + +nlp = en_core_web_sm.load() +doc = nlp("This is a sentence.") +``` + +📖 **For more info and examples, check out the +[models documentation](https://spacy.io/docs/usage/models).** + +## ⚒ Compile from source + +The other way to install spaCy is to clone its +[GitHub repository](https://github.com/explosion/spaCy) and build it from +source. That is the common way if you want to make changes to the code base. +You'll need to make sure that you have a development environment consisting of a +Python distribution including header files, a compiler, +[pip](https://pip.pypa.io/en/latest/installing/), +[virtualenv](https://virtualenv.pypa.io/en/latest/) and +[git](https://git-scm.com) installed. The compiler part is the trickiest. How to +do that depends on your system. + +| Platform | | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Ubuntu** | Install system-level dependencies via `apt-get`: `sudo apt-get install build-essential python-dev git` . | +| **Mac** | Install a recent version of [XCode](https://developer.apple.com/xcode/), including the so-called "Command Line Tools". macOS and OS X ship with Python and git preinstalled. | +| **Windows** | Install a version of the [Visual C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) or [Visual Studio Express](https://visualstudio.microsoft.com/vs/express/) that matches the version that was used to compile your Python interpreter. | + +For more details and instructions, see the documentation on +[compiling spaCy from source](https://spacy.io/usage#source) and the +[quickstart widget](https://spacy.io/usage#section-quickstart) to get the right +commands for your platform and Python version. + +```bash +git clone https://github.com/explosion/spaCy +cd spaCy + +python -m venv .env +source .env/bin/activate + +# make sure you are using the latest pip +python -m pip install -U pip setuptools wheel + +pip install -r requirements.txt +pip install --no-build-isolation --editable . +``` + +To install with extras: + +```bash +pip install --no-build-isolation --editable .[lookups,cuda102] +``` + +## 🚦 Run tests + +spaCy comes with an [extensive test suite](spacy/tests). In order to run the +tests, you'll usually want to clone the repository and build spaCy from source. +This will also install the required development dependencies and test utilities +defined in the [`requirements.txt`](requirements.txt). + +Alternatively, you can run `pytest` on the tests from within the installed +`spacy` package. Don't forget to also install the test utilities via spaCy's +[`requirements.txt`](requirements.txt): + +```bash +pip install -r requirements.txt +python -m pytest --pyargs spacy +``` diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..8e3783e --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`explosion/spaCy` +- 原始仓库：https://github.com/explosion/spaCy +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/bin/get-package.sh b/bin/get-package.sh new file mode 100755 index 0000000..d60b930 --- /dev/null +++ b/bin/get-package.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +version=$(grep "__title__ = " spacy/about.py) +version=${version/__title__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo $version diff --git a/bin/get-version.sh b/bin/get-version.sh new file mode 100755 index 0000000..5a12ddd --- /dev/null +++ b/bin/get-version.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +version=$(grep "__version__ = " spacy/about.py) +version=${version/__version__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo $version diff --git a/bin/push-tag.sh b/bin/push-tag.sh new file mode 100755 index 0000000..50b50c9 --- /dev/null +++ b/bin/push-tag.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +# Insist repository is clean +git diff-index --quiet HEAD + +git checkout $1 +git pull origin $1 +git push origin $1 + +version=$(grep "__version__ = " spacy/about.py) +version=${version/__version__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} +git tag "v$version" +git push origin "v$version" diff --git a/bin/release.sh b/bin/release.sh new file mode 100755 index 0000000..03ac80d --- /dev/null +++ b/bin/release.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +# Insist repository is clean +git diff-index --quiet HEAD + +version=$(grep "__version__ = " spacy/about.py) +version=${version/__version__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo "Pushing release-v"$version + +git tag -d release-v$version || true +git push origin :release-v$version || true +git tag release-v$version +git push origin release-v$version diff --git a/build-constraints.txt b/build-constraints.txt new file mode 100644 index 0000000..94ebdc0 --- /dev/null +++ b/build-constraints.txt @@ -0,0 +1,2 @@ +# build version constraints for use with wheelwright +numpy>=2.0.0,<3.0.0 diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..23ff59a --- /dev/null +++ b/examples/README.md @@ -0,0 +1,130 @@ + + +# spaCy examples + +For spaCy v3 we've converted many of the [v2 example +scripts](https://github.com/explosion/spaCy/tree/v2.3.x/examples/) into +end-to-end [spacy projects](https://spacy.io/usage/projects) workflows. The +workflows include all the steps to go from data to packaged spaCy models. + +## 🪐 Pipeline component demos + +The simplest demos for training a single pipeline component are in the +[`pipelines`](https://github.com/explosion/projects/blob/v3/pipelines) category +including: + +- [`pipelines/ner_demo`](https://github.com/explosion/projects/blob/v3/pipelines/ner_demo): + Train a named entity recognizer +- [`pipelines/textcat_demo`](https://github.com/explosion/projects/blob/v3/pipelines/textcat_demo): + Train a text classifier +- [`pipelines/parser_intent_demo`](https://github.com/explosion/projects/blob/v3/pipelines/parser_intent_demo): + Train a dependency parser for custom semantics + +## 🪐 Tutorials + +The [`tutorials`](https://github.com/explosion/projects/blob/v3/tutorials) +category includes examples that work through specific NLP use cases end-to-end: + +- [`tutorials/textcat_goemotions`](https://github.com/explosion/projects/blob/v3/tutorials/textcat_goemotions): + Train a text classifier to categorize emotions in Reddit posts +- [`tutorials/nel_emerson`](https://github.com/explosion/projects/blob/v3/tutorials/nel_emerson): + Use an entity linker to disambiguate mentions of the same name + +Check out the [projects documentation](https://spacy.io/usage/projects) and +browse through the [available +projects](https://github.com/explosion/projects/)! + +## 🚀 Get started with a demo project + +The +[`pipelines/ner_demo`](https://github.com/explosion/projects/blob/v3/pipelines/ner_demo) +project converts the spaCy v2 +[`train_ner.py`](https://github.com/explosion/spaCy/blob/v2.3.x/examples/training/train_ner.py) +demo script into a spaCy v3 project. + +1. Clone the project: + + ```bash + python -m spacy project clone pipelines/ner_demo + ``` + +2. Install requirements and download any data assets: + + ```bash + cd ner_demo + python -m pip install -r requirements.txt + python -m spacy project assets + ``` + +3. Run the default workflow to convert, train and evaluate: + + ```bash + python -m spacy project run all + ``` + + Sample output: + + ```none + ℹ Running workflow 'all' + + ================================== convert ================================== + Running command: /home/user/venv/bin/python scripts/convert.py en assets/train.json corpus/train.spacy + Running command: /home/user/venv/bin/python scripts/convert.py en assets/dev.json corpus/dev.spacy + + =============================== create-config =============================== + Running command: /home/user/venv/bin/python -m spacy init config --lang en --pipeline ner configs/config.cfg --force + ℹ Generated config template specific for your use case + - Language: en + - Pipeline: ner + - Optimize for: efficiency + - Hardware: CPU + - Transformer: None + ✔ Auto-filled config with all values + ✔ Saved config + configs/config.cfg + You can now add your data and train your pipeline: + python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy + + =================================== train =================================== + Running command: /home/user/venv/bin/python -m spacy train configs/config.cfg --output training/ --paths.train corpus/train.spacy --paths.dev corpus/dev.spacy --training.eval_frequency 10 --training.max_steps 100 --gpu-id -1 + ℹ Using CPU + + =========================== Initializing pipeline =========================== + [2021-03-11 19:34:59,101] [INFO] Set up nlp object from config + [2021-03-11 19:34:59,109] [INFO] Pipeline: ['tok2vec', 'ner'] + [2021-03-11 19:34:59,113] [INFO] Created vocabulary + [2021-03-11 19:34:59,113] [INFO] Finished initializing nlp object + [2021-03-11 19:34:59,265] [INFO] Initialized pipeline components: ['tok2vec', 'ner'] + ✔ Initialized pipeline + + ============================= Training pipeline ============================= + ℹ Pipeline: ['tok2vec', 'ner'] + ℹ Initial learn rate: 0.001 + E # LOSS TOK2VEC LOSS NER ENTS_F ENTS_P ENTS_R SCORE + --- ------ ------------ -------- ------ ------ ------ ------ + 0 0 0.00 7.90 0.00 0.00 0.00 0.00 + 10 10 0.11 71.07 0.00 0.00 0.00 0.00 + 20 20 0.65 22.44 50.00 50.00 50.00 0.50 + 30 30 0.22 6.38 80.00 66.67 100.00 0.80 + 40 40 0.00 0.00 80.00 66.67 100.00 0.80 + 50 50 0.00 0.00 80.00 66.67 100.00 0.80 + 60 60 0.00 0.00 100.00 100.00 100.00 1.00 + 70 70 0.00 0.00 100.00 100.00 100.00 1.00 + 80 80 0.00 0.00 100.00 100.00 100.00 1.00 + 90 90 0.00 0.00 100.00 100.00 100.00 1.00 + 100 100 0.00 0.00 100.00 100.00 100.00 1.00 + ✔ Saved pipeline to output directory + training/model-last + ``` + +4. Package the model: + + ```bash + python -m spacy project run package + ``` + +5. Visualize the model's output with [Streamlit](https://streamlit.io): + + ```bash + python -m spacy project run visualize-model + ``` diff --git a/examples/training/README.md b/examples/training/README.md new file mode 100644 index 0000000..34689ce --- /dev/null +++ b/examples/training/README.md @@ -0,0 +1,5 @@ + + +# spaCy examples + +See [examples/README.md](../README.md) diff --git a/extra/DEVELOPER_DOCS/Code Conventions.md b/extra/DEVELOPER_DOCS/Code Conventions.md new file mode 100644 index 0000000..7294ac3 --- /dev/null +++ b/extra/DEVELOPER_DOCS/Code Conventions.md @@ -0,0 +1,580 @@ +# Code Conventions + +For a general overview of code conventions for contributors, see the [section in the contributing guide](https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md#code-conventions). + +1. [Code compatibility](#code-compatibility) +2. [Auto-formatting](#auto-formatting) +3. [Linting](#linting) +4. [Documenting code](#documenting-code) +5. [Type hints](#type-hints) +6. [Structuring logic](#structuring-logic) +7. [Naming](#naming) +8. [Error handling](#error-handling) +9. [Writing tests](#writing-tests) + +## Code compatibility + +spaCy supports **Python 3.6** and above, so all code should be written compatible with 3.6. This means that there are certain new syntax features that we won't be able to use until we drop support for older Python versions. Some newer features provide backports that we can conditionally install for older versions, although we only want to do this if it's absolutely necessary. If we need to use conditional imports based on the Python version or other custom compatibility-specific helpers, those should live in `compat.py`. + +## Auto-formatting + +spaCy uses `black` for auto-formatting (which is also available as a pre-commit hook). It's recommended to configure your editor to perform this automatically, either triggered manually or whenever you save a file. We also have a GitHub action that regularly formats the code base and submits a PR if changes are available. Note that auto-formatting is currently only available for `.py` (Python) files, not for `.pyx` (Cython). + +As a rule of thumb, if the auto-formatting produces output that looks messy, it can often indicate that there's a better way to structure the code to make it more concise. + +```diff +- range_suggester = registry.misc.get("spacy.ngram_range_suggester.v1")( +- min_size=1, max_size=3 +- ) ++ suggester_factory = registry.misc.get("spacy.ngram_range_suggester.v1") ++ range_suggester = suggester_factory(min_size=1, max_size=3) +``` + +In some specific cases, e.g. in the tests, it can make sense to disable auto-formatting for a specific block. You can do this by wrapping the code in `# fmt: off` and `# fmt: on`: + +```diff ++ # fmt: off +text = "I look forward to using Thingamajig. I've been told it will make my life easier..." +deps = ["nsubj", "ROOT", "advmod", "prep", "pcomp", "dobj", "punct", "", + "nsubjpass", "aux", "auxpass", "ROOT", "nsubj", "aux", "ccomp", + "poss", "nsubj", "ccomp", "punct"] ++ # fmt: on +``` + +## Linting + +[`flake8`](http://flake8.pycqa.org/en/latest/) is a tool for enforcing code style. It scans one or more files and outputs errors and warnings. This feedback can help you stick to general standards and conventions, and can be very useful for spotting potential mistakes and inconsistencies in your code. Code you write should be compatible with our flake8 rules and not cause any warnings. + +```bash +flake8 spacy +``` + +The most common problems surfaced by linting are: + +- **Trailing or missing whitespace.** This is related to formatting and should be fixed automatically by running `black`. +- **Unused imports.** Those should be removed if the imports aren't actually used. If they're required, e.g. to expose them so they can be imported from the given module, you can add a comment and `# noqa: F401` exception (see details below). +- **Unused variables.** This can often indicate bugs, e.g. a variable that's declared and not correctly passed on or returned. To prevent ambiguity here, your code shouldn't contain unused variables. If you're unpacking a list of tuples and end up with variables you don't need, you can call them `_` to indicate that they're unused. +- **Redefinition of function.** This can also indicate bugs, e.g. a copy-pasted function that you forgot to rename and that now replaces the original function. +- **Repeated dictionary keys.** This either indicates a bug or unnecessary duplication. +- **Comparison with `True`, `False`, `None`**. This is mostly a stylistic thing: when checking whether a value is `True`, `False` or `None`, you should be using `is` instead of `==`. For example, `if value is None`. + +### Ignoring linter rules for special cases + +To ignore a given line, you can add a comment like `# noqa: F401`, specifying the code of the error or warning we want to ignore. It's also possible to ignore several comma-separated codes at once, e.g. `# noqa: E731,E123`. In general, you should always **specify the code(s)** you want to ignore – otherwise, you may end up missing actual problems. + +```python +# The imported class isn't used in this file, but imported here, so it can be +# imported *from* here by another module. +from .submodule import SomeClass # noqa: F401 + +try: + do_something() +except: # noqa: E722 + # This bare except is justified, for some specific reason + do_something_else() +``` + +## Documenting code + +All functions and methods you write should be documented with a docstring inline. The docstring can contain a simple summary, and an overview of the arguments and their (simplified) types. Modern editors will show this information to users when they call the function or method in their code. + +If it's part of the public API and there's a documentation section available, we usually add the link as `DOCS:` at the end. This allows us to keep the docstrings simple and concise, while also providing additional information and examples if necessary. + +```python +def has_pipe(self, name: str) -> bool: + """Check if a component name is present in the pipeline. Equivalent to + `name in nlp.pipe_names`. + + name (str): Name of the component. + RETURNS (bool): Whether a component of the name exists in the pipeline. + + DOCS: https://spacy.io/api/language#has_pipe + """ + ... +``` + +We specifically chose this approach of maintaining the docstrings and API reference separately, instead of auto-generating the API docs from the docstrings like other packages do. We want to be able to provide extensive explanations and examples in the documentation and use our own custom markup for it that would otherwise clog up the docstrings. We also want to be able to update the documentation independently of the code base. It's slightly more work, but it's absolutely worth it in terms of user and developer experience. + +### Inline code comments + +We don't expect you to add inline comments for everything you're doing – this should be obvious from reading the code. If it's not, the first thing to check is whether your code can be improved to make it more explicit. That said, if your code includes complex logic or aspects that may be unintuitive at first glance (or even included a subtle bug that you ended up fixing), you should leave a quick comment that provides more context. + +```diff +token_index = indices[value] ++ # Index describes Token.i of last token but Span indices are inclusive +span = doc[prev_token_index:token_index + 1] +``` + +```diff ++ # To create the components we need to use the final interpolated config ++ # so all values are available (if component configs use variables). ++ # Later we replace the component config with the raw config again. +interpolated = filled.interpolate() if not filled.is_interpolated else filled +``` + +Don't be shy about including comments for tricky parts that _you_ found hard to implement or get right – those may come in handy for the next person working on this code, or even future you! + +If your change implements a fix to a specific issue, it can often be helpful to include the issue number in the comment, especially if it's a relatively straightforward adjustment: + +```diff ++ # Ensure object is a Span, not a Doc (#1234) +if isinstance(obj, Doc): + obj = obj[obj.start:obj.end] +``` + +### Including TODOs + +It's fine to include code comments that indicate future TODOs, using the `TODO:` prefix. Modern editors typically format this in a different color, so it's easy to spot. TODOs don't necessarily have to be things that are absolutely critical to fix fight now – those should already be addressed in your pull request once it's ready for review. But they can include notes about potential future improvements. + +```diff ++ # TODO: this is currently pretty slow +dir_checksum = hashlib.md5() +for sub_file in sorted(fp for fp in path.rglob("*") if fp.is_file()): + dir_checksum.update(sub_file.read_bytes()) +``` + +If any of the TODOs you've added are important and should be fixed soon, you should add a task for this on Explosion's internal Ora board or an issue on the public issue tracker to make sure we don't forget to address it. + +## Type hints + +We use Python type hints across the `.py` files wherever possible. This makes it easy to understand what a function expects and returns, and modern editors will be able to show this information to you when you call an annotated function. Type hints are not currently used in the `.pyx` (Cython) code, except for definitions of registered functions and component factories, where they're used for config validation. Ideally when developing, run `mypy spacy` on the code base to inspect any issues. + +If possible, you should always use the more descriptive type hints like `List[str]` or even `List[Any]` instead of only `list`. We also annotate arguments and return types of `Callable` – although, you can simplify this if the type otherwise gets too verbose (e.g. functions that return factories to create callbacks). Remember that `Callable` takes two values: a **list** of the argument type(s) in order, and the return values. + +```diff +- def func(some_arg: dict) -> None: ++ def func(some_arg: Dict[str, Any]) -> None: + ... +``` + +```python +def create_callback(some_arg: bool) -> Callable[[str, int], List[str]]: + def callback(arg1: str, arg2: int) -> List[str]: + ... + + return callback +``` + +For typing variables, we prefer the explicit format. + +```diff +- var = value # type: Type ++ var: Type = value +``` + +For model architectures, Thinc also provides a collection of [custom types](https://thinc.ai/docs/api-types), including more specific types for arrays and model inputs/outputs. Even outside of static type checking, using these types will make the code a lot easier to read and follow, since it's always clear what array types are expected (and what might go wrong if the output is different from the expected type). + +```python +def build_tagger_model( + tok2vec: Model[List[Doc], List[Floats2d]], nO: Optional[int] = None +) -> Model[List[Doc], List[Floats2d]]: + ... +``` + +If you need to use a type hint that refers to something later declared in the same module, or the class that a method belongs to, you can use a string value instead: + +```python +class SomeClass: + def from_bytes(self, data: bytes) -> "SomeClass": + ... +``` + +In some cases, you won't be able to import a class from a different module to use it as a type hint because it'd cause circular imports. For instance, `spacy/util.py` includes various helper functions that return an instance of `Language`, but we couldn't import it, because `spacy/language.py` imports `util` itself. In this case, we can provide `"Language"` as a string and make the import conditional on `typing.TYPE_CHECKING` so it only runs when the code is evaluated by a type checker: + +```python +from typing TYPE_CHECKING + +if TYPE_CHECKING: + from .language import Language + +def load_model(name: str) -> "Language": + ... +``` + +Note that we typically put the `from typing` import statements on the first line(s) of the Python module. + +## Structuring logic + +### Positional and keyword arguments + +We generally try to avoid writing functions and methods with too many arguments, and use keyword-only arguments wherever possible. Python lets you define arguments as keyword-only by separating them with a `, *`. If you're writing functions with additional arguments that customize the behavior, you typically want to make those arguments keyword-only, so their names have to be provided explicitly. + +```diff +- def do_something(name: str, validate: bool = False): ++ def do_something(name: str, *, validate: bool = False): + ... + +- do_something("some_name", True) ++ do_something("some_name", validate=True) +``` + +This makes the function calls easier to read, because it's immediately clear what the additional values mean. It also makes it easier to extend arguments or change their order later on, because you don't end up with any function calls that depend on a specific positional order. + +### Avoid mutable default arguments + +A common Python gotcha are [mutable default arguments](https://docs.python-guide.org/writing/gotchas/#mutable-default-arguments): if your argument defines a mutable default value like `[]` or `{}` and then goes and mutates it, the default value is created _once_ when the function is created and the same object is then mutated every time the function is called. This can be pretty unintuitive when you first encounter it. We therefore avoid writing logic that does this. + +If your arguments need to default to an empty list or dict, you can use the `SimpleFrozenList` and `SimpleFrozenDict` helpers provided by spaCy. They are simple frozen implementations that raise an error if they're being mutated to prevent bugs and logic that accidentally mutates default arguments. + +```diff +- def to_bytes(self, *, exclude: List[str] = []): ++ def to_bytes(self, *, exclude: List[str] = SimpleFrozenList()): + ... +``` + +```diff +def do_something(values: List[str] = SimpleFrozenList()): + if some_condition: +- values.append("foo") # raises an error ++ values = [*values, "foo"] + return values +``` + +### Don't use `try`/`except` for control flow + +We strongly discourage using `try`/`except` blocks for anything that's not third-party error handling or error handling that we otherwise have little control over. There's typically always a way to anticipate the _actual_ problem and **check for it explicitly**, which makes the code easier to follow and understand, and prevents bugs: + +```diff +- try: +- token = doc[i] +- except IndexError: +- token = doc[-1] + ++ if i < len(doc): ++ token = doc[i] ++ else: ++ token = doc[-1] +``` + +Even if you end up having to check for multiple conditions explicitly, this is still preferred over a catch-all `try`/`except`. It can be very helpful to think about the exact scenarios you need to cover, and what could go wrong at each step, which often leads to better code and fewer bugs. `try/except` blocks can also easily mask _other_ bugs and problems that raise the same errors you're catching, which is obviously bad. + +If you have to use `try`/`except`, make sure to only include what's **absolutely necessary** in the `try` block and define the exception(s) explicitly. Otherwise, you may end up masking very different exceptions caused by other bugs. + +```diff +- try: +- value1 = get_some_value() +- value2 = get_some_other_value() +- score = external_library.compute_some_score(value1, value2) +- except: +- score = 0.0 + ++ value1 = get_some_value() ++ value2 = get_some_other_value() ++ try: ++ score = external_library.compute_some_score(value1, value2) ++ except ValueError: ++ score = 0.0 +``` + +### Avoid lambda functions + +`lambda` functions can be useful for defining simple anonymous functions in a single line, but they also introduce problems: for instance, they require [additional logic](https://stackoverflow.com/questions/25348532/can-python-pickle-lambda-functions) in order to be pickled and are pretty ugly to type-annotate. So we typically avoid them in the code base and only use them in the serialization handlers and within tests for simplicity. Instead of `lambda`s, check if your code can be refactored to not need them, or use helper functions instead. + +```diff +- split_string: Callable[[str], List[str]] = lambda value: [v.strip() for v in value.split(",")] + ++ def split_string(value: str) -> List[str]: ++ return [v.strip() for v in value.split(",")] +``` + +### Numeric comparisons + +For numeric comparisons, as a general rule we always use `<` and `>=` and avoid the usage of `<=` and `>`. This is to ensure we consistently +apply inclusive lower bounds and exclusive upper bounds, helping to prevent off-by-one errors. + +One exception to this rule is the ternary case. With a chain like + +```python +if value >= 0 and value < max: + ... +``` + +it's fine to rewrite this to the shorter form + +```python +if 0 <= value < max: + ... +``` + +even though this requires the usage of the `<=` operator. + +### Iteration and comprehensions + +We generally avoid using built-in functions like `filter` or `map` in favor of list or generator comprehensions. + +```diff +- filtered = filter(lambda x: x in ["foo", "bar"], values) ++ filtered = (x for x in values if x in ["foo", "bar"]) +- filtered = list(filter(lambda x: x in ["foo", "bar"], values)) ++ filtered = [x for x in values if x in ["foo", "bar"]] + +- result = map(lambda x: { x: x in ["foo", "bar"]}, values) ++ result = ({x: x in ["foo", "bar"]} for x in values) +- result = list(map(lambda x: { x: x in ["foo", "bar"]}, values)) ++ result = [{x: x in ["foo", "bar"]} for x in values] +``` + +If your logic is more complex, it's often better to write a loop instead, even if it adds more lines of code in total. The result will be much easier to follow and understand. + +```diff +- result = [{"key": key, "scores": {f"{i}": score for i, score in enumerate(scores)}} for key, scores in values] + ++ result = [] ++ for key, scores in values: ++ scores_dict = {f"{i}": score for i, score in enumerate(scores)} ++ result.append({"key": key, "scores": scores_dict}) +``` + +### Composition vs. inheritance + +Although spaCy uses a lot of classes, **inheritance is viewed with some suspicion** — it's seen as a mechanism of last resort. You should discuss plans to extend the class hierarchy before implementing. Unless you're implementing a new data structure or pipeline component, you typically shouldn't have to use classes at all. + +### Don't use `print` + +The core library never `print`s anything. While we encourage using `print` statements for simple debugging (it's the most straightforward way of looking at what's happening), make sure to clean them up once you're ready to submit your pull request. If you want to output warnings or debugging information for users, use the respective dedicated mechanisms for this instead (see sections on warnings and logging for details). + +The only exceptions are the CLI functions, which pretty-print messages for the user, and methods that are explicitly intended for printing things, e.g. `Language.analyze_pipes` with `pretty=True` enabled. For this, we use our lightweight helper library [`wasabi`](https://github.com/ines/wasabi). + +## Naming + +Naming is hard and often a topic of long internal discussions. We don't expect you to come up with the perfect names for everything you write – finding the right names is often an iterative and collaborative process. That said, we do try to follow some basic conventions. + +Consistent with general Python conventions, we use `CamelCase` for class names including dataclasses, `snake_case` for methods, functions and variables, and `UPPER_SNAKE_CASE` for constants, typically defined at the top of a module. We also avoid using variable names that shadow the names of built-in functions, e.g. `input`, `help` or `list`. + +### Naming variables + +Variable names should always make it clear _what exactly_ the variable is and what it's used for. Instances of common classes should use the same consistent names. For example, you should avoid naming a text string (or anything else that's not a `Doc` object) `doc`. The most common class-to-variable mappings are: + +| Class | Variable | Example | +| ---------- | --------------------- | ------------------------------------------- | +| `Language` | `nlp` | `nlp = spacy.blank("en")` | +| `Doc` | `doc` | `doc = nlp("Some text")` | +| `Span` | `span`, `ent`, `sent` | `span = doc[1:4]`, `ent = doc.ents[0]` | +| `Token` | `token` | `token = doc[0]` | +| `Lexeme` | `lexeme`, `lex` | `lex = nlp.vocab["foo"]` | +| `Vocab` | `vocab` | `vocab = Vocab()` | +| `Example` | `example`, `eg` | `example = Example.from_dict(doc, gold)` | +| `Config` | `config`, `cfg` | `config = Config().from_disk("config.cfg")` | + +We try to avoid introducing too many temporary variables, as these clutter your namespace. It's okay to re-assign to an existing variable, but only if the value has the same type. + +```diff +ents = get_a_list_of_entities() +ents = [ent for ent in doc.ents if ent.label_ == "PERSON"] +- ents = {(ent.start, ent.end): ent.label_ for ent in ents} ++ ent_mappings = {(ent.start, ent.end): ent.label_ for ent in ents} +``` + +### Naming methods and functions + +Try choosing short and descriptive names wherever possible and imperative verbs for methods that do something, e.g. `disable_pipes`, `add_patterns` or `get_vector`. Private methods and functions that are not intended to be part of the user-facing API should be prefixed with an underscore `_`. It's often helpful to look at the existing classes for inspiration. + +Objects that can be serialized, e.g. data structures and pipeline components, should implement the same consistent methods for serialization. Those usually include at least `to_disk`, `from_disk`, `to_bytes` and `from_bytes`. Some objects can also implement more specific methods like `{to/from}_dict` or `{to/from}_str`. + +## Error handling + +We always encourage writing helpful and detailed custom error messages for everything we can anticipate going wrong, and including as much detail as possible. spaCy provides a directory of error messages in `errors.py` with unique codes for each message. This allows us to keep the code base more concise and avoids long and nested blocks of texts throughout the code that disrupt the reading flow. The codes make it easy to find references to the same error in different places, and also helps identify problems reported by users (since we can just search for the error code). + +Errors can be referenced via their code, e.g. `Errors.E123`. Messages can also include placeholders for values, that can be populated by formatting the string with `.format()`. + +```python +class Errors: + E123 = "Something went wrong" + E456 = "Unexpected value: {value}" +``` + +```diff +if something_went_wrong: +- raise ValueError("Something went wrong!") ++ raise ValueError(Errors.E123) + +if not isinstance(value, int): +- raise ValueError(f"Unexpected value: {value}") ++ raise ValueError(Errors.E456.format(value=value)) +``` + +As a general rule of thumb, all error messages raised within the **core library** should be added to `Errors`. The only place where we write errors and messages as strings is `spacy.cli`, since these functions typically pretty-print and generate a lot of output that'd otherwise be very difficult to separate from the actual logic. + +### Re-raising exceptions + +If we anticipate possible errors in third-party code that we don't control, or our own code in a very different context, we typically try to provide custom and more specific error messages if possible. If we need to re-raise an exception within a `try`/`except` block, we can re-raise a custom exception. + +[Re-raising `from`](https://docs.python.org/3/tutorial/errors.html#exception-chaining) the original caught exception lets us chain the exceptions, so the user sees both the original error, as well as the custom message with a note "The above exception was the direct cause of the following exception". + +```diff +try: + run_third_party_code_that_might_fail() +except ValueError as e: ++ raise ValueError(Errors.E123) from e +``` + +In some cases, it makes sense to suppress the original exception, e.g. if we know what it is and know that it's not particularly helpful. In that case, we can raise `from None`. This prevents clogging up the user's terminal with multiple and irrelevant chained exceptions. + +```diff +try: + run_our_own_code_that_might_fail_confusingly() +except ValueError: ++ raise ValueError(Errors.E123) from None +``` + +### Avoid using naked `assert` + +During development, it can sometimes be helpful to add `assert` statements throughout your code to make sure that the values you're working with are what you expect. However, as you clean up your code, those should either be removed or replaced by more explicit error handling: + +```diff +- assert score >= 0.0 ++ if score < 0.0: ++ raise ValueError(Errors.789.format(score=score)) +``` + +Otherwise, the user will get to see a naked `AssertionError` with no further explanation, which is very unhelpful. Instead of adding an error message to `assert`, it's always better to `raise` more explicit errors for specific conditions. If you're checking for something that _has to be right_ and would otherwise be a bug in spaCy, you can express this in the error message: + +```python +E161 = ("Found an internal inconsistency when predicting entity links. " + "This is likely a bug in spaCy, so feel free to open an issue: " + "https://github.com/explosion/spaCy/issues") +``` + +### Warnings + +Instead of raising an error, some parts of the code base can raise warnings to notify the user of a potential problem. This is done using Python's `warnings.warn` and the messages defined in `Warnings` in the `errors.py`. Whether or not warnings are shown can be controlled by the user, including custom filters for disabling specific warnings using a regular expression matching our internal codes, e.g. `W123`. + +```diff +- print("Warning: No examples provided for validation") ++ warnings.warn(Warnings.W123) +``` + +When adding warnings, make sure you're not calling `warnings.warn` repeatedly, e.g. in a loop, which will clog up the terminal output. Instead, you can collect the potential problems first and then raise a single warning. If the problem is critical, consider raising an error instead. + +```diff ++ n_empty = 0 +for spans in lots_of_annotations: + if len(spans) == 0: +- warnings.warn(Warnings.456) ++ n_empty += 1 ++ warnings.warn(Warnings.456.format(count=n_empty)) +``` + +### Logging + +Log statements can be added via spaCy's `logger`, which uses Python's native `logging` module under the hood. We generally only use logging for debugging information that **the user may choose to see** in debugging mode or that's **relevant during training** but not at runtime. + +```diff ++ logger.info("Set up nlp object from config") +config = nlp.config.interpolate() +``` + +`spacy train` and similar CLI commands will enable all log statements of level `INFO` by default (which is not the case at runtime). This allows outputting specific information within certain parts of the core library during training, without having it shown at runtime. `DEBUG`-level logs are only shown if the user enables `--verbose` logging during training. They can be used to provide more specific and potentially more verbose details, especially in areas that can indicate bugs or problems, or to surface more details about what spaCy does under the hood. You should only use logging statements if absolutely necessary and important. + +## Writing tests + +spaCy uses the [`pytest`](http://doc.pytest.org/) framework for testing. Tests for spaCy modules and classes live in their own directories of the same name and all test files should be prefixed with `test_`. Tests included in the core library only cover the code and do not depend on any trained pipelines. When implementing a new feature or fixing a bug, it's usually good to start by writing some tests that describe what _should_ happen. As you write your code, you can then keep running the relevant tests until all of them pass. + +### Test suite structure + +When adding tests, make sure to use descriptive names and only test for one behavior at a time. Tests should be grouped into modules dedicated to the same type of functionality and some test modules are organized as directories of test files related to the same larger area of the library, e.g. `matcher` or `tokenizer`. + +Regression tests are tests that refer to bugs reported in specific issues. They should live in the relevant module of the test suite, named according to the issue number (e.g., `test_issue1234.py`), and [marked](https://docs.pytest.org/en/6.2.x/example/markers.html#working-with-custom-markers) appropriately (e.g. `@pytest.mark.issue(1234)`). This system allows us to relate tests for specific bugs back to the original reported issue, which is especially useful if we introduce a regression and a previously passing regression tests suddenly fails again. When fixing a bug, it's often useful to create a regression test for it first. + +The test suite also provides [fixtures](https://github.com/explosion/spaCy/blob/master/spacy/tests/conftest.py) for different language tokenizers that can be used as function arguments of the same name and will be passed in automatically. Those should only be used for tests related to those specific languages. We also have [test utility functions](https://github.com/explosion/spaCy/blob/master/spacy/tests/util.py) for common operations, like creating a temporary file. + +### Testing Cython Code + +If you're developing Cython code (`.pyx` files), those extensions will need to be built before the test runner can test that code - otherwise it's going to run the tests with stale code from the last time the extension was built. You can build the extensions locally with `python setup.py build_ext -i`. + +### Constructing objects and state + +Test functions usually follow the same simple structure: they set up some state, perform the operation you want to test and `assert` conditions that you expect to be true, usually before and after the operation. + +Tests should focus on exactly what they're testing and avoid dependencies on other unrelated library functionality wherever possible. If all your test needs is a `Doc` object with certain annotations set, you should always construct it manually: + +```python +def test_doc_creation_with_pos(): + doc = Doc(Vocab(), words=["hello", "world"], pos=["NOUN", "VERB"]) + assert doc[0].pos_ == "NOUN" + assert doc[1].pos_ == "VERB" +``` + +### Parametrizing tests + +If you need to run the same test function over different input examples, you usually want to parametrize the test cases instead of using a loop within your test. This lets you keep a better separation between test cases and test logic, and it'll result in more useful output because `pytest` will be able to tell you which exact test case failed. + +The `@pytest.mark.parametrize` decorator takes two arguments: a string defining one or more comma-separated arguments that should be passed to the test function and a list of corresponding test cases (or a list of tuples to provide multiple arguments). + +```python +@pytest.mark.parametrize("words", [["hello", "world"], ["this", "is", "a", "test"]]) +def test_doc_length(words): + doc = Doc(Vocab(), words=words) + assert len(doc) == len(words) +``` + +```python +@pytest.mark.parametrize("text,expected_len", [("hello world", 2), ("I can't!", 4)]) +def test_token_length(en_tokenizer, text, expected_len): # en_tokenizer is a fixture + doc = en_tokenizer(text) + assert len(doc) == expected_len +``` + +You can also stack `@pytest.mark.parametrize` decorators, although this is not recommended unless it's absolutely needed or required for the test. When stacking decorators, keep in mind that this will run the test with all possible combinations of the respective parametrized values, which is often not what you want and can slow down the test suite. + +### Handling failing tests + +`xfail` means that a test **should pass but currently fails**, i.e. is expected to fail. You can mark a test as currently xfailing by adding the `@pytest.mark.xfail` decorator. This should only be used for tests that don't yet work, not for logic that cause errors we raise on purpose (see the section on testing errors for this). It's often very helpful to implement tests for edge cases that we don't yet cover and mark them as `xfail`. You can also provide a `reason` keyword argument to the decorator with an explanation of why the test currently fails. + +```diff ++ @pytest.mark.xfail(reason="Issue #225 - not yet implemented") +def test_en_tokenizer_splits_em_dash_infix(en_tokenizer): + doc = en_tokenizer("Will this road take me to Puddleton?\u2014No.") + assert doc[8].text == "\u2014" +``` + +When you run the test suite, you may come across tests that are reported as `xpass`. This means that they're marked as `xfail` but didn't actually fail. This is worth looking into: sometimes, it can mean that we have since fixed a bug that caused the test to previously fail, so we can remove the decorator. In other cases, especially when it comes to machine learning model implementations, it can also indicate that the **test is flaky**: it sometimes passes and sometimes fails. This can be caused by a bug, or by constraints being too narrowly defined. If a test shows different behavior depending on whether its run in isolation or not, this can indicate that it reacts to global state set in a previous test, which is unideal and should be avoided. + +### Writing slow tests + +If a test is useful but potentially quite slow, you can mark it with the `@pytest.mark.slow` decorator. This is a special marker we introduced and tests decorated with it only run if you run the test suite with `--slow`, but not as part of the main CI process. Before introducing a slow test, double-check that there isn't another and more efficient way to test for the behavior. You should also consider adding a simpler test with maybe only a subset of the test cases that can always run, so we at least have some coverage. + +### Skipping tests + +The `@pytest.mark.skip` decorator lets you skip tests entirely. You only want to do this for failing tests that may be slow to run or cause memory errors or segfaults, which would otherwise terminate the entire process and wouldn't be caught by `xfail`. We also sometimes use the `skip` decorator for old and outdated regression tests that we want to keep around but that don't apply anymore. When using the `skip` decorator, make sure to provide the `reason` keyword argument with a quick explanation of why you chose to skip this test. + +### Testing errors and warnings + +`pytest` lets you check whether a given error is raised by using the `pytest.raises` contextmanager. This is very useful when implementing custom error handling, so make sure you're not only testing for the correct behavior but also for errors resulting from incorrect inputs. If you're testing errors, you should always check for `pytest.raises` explicitly and not use `xfail`. + +```python +words = ["a", "b", "c", "d", "e"] +ents = ["Q-PERSON", "I-PERSON", "O", "I-PERSON", "I-GPE"] +with pytest.raises(ValueError): + Doc(Vocab(), words=words, ents=ents) +``` + +You can also use the `pytest.warns` contextmanager to check that a given warning type is raised. The first argument is the warning type or `None` (which will capture a list of warnings that you can `assert` is empty). + +```python +def test_phrase_matcher_validation(en_vocab): + doc1 = Doc(en_vocab, words=["Test"], deps=["ROOT"]) + doc2 = Doc(en_vocab, words=["Test"]) + matcher = PhraseMatcher(en_vocab, validate=True) + with pytest.warns(UserWarning): + # Warn about unnecessarily parsed document + matcher.add("TEST1", [doc1]) + with pytest.warns(None) as record: + matcher.add("TEST2", [docs]) + assert not record.list +``` + +Keep in mind that your tests will fail if you're using the `pytest.warns` contextmanager with a given warning and the warning is _not_ shown. So you should only use it to check that spaCy handles and outputs warnings correctly. If your test outputs a warning that's expected but not relevant to what you're testing, you can use the `@pytest.mark.filterwarnings` decorator and ignore specific warnings starting with a given code: + +```python +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_matcher_empty(en_vocab): + matcher = Matcher(en_vocab) + matcher(Doc(en_vocab, words=["test"])) +``` + +### Testing trained pipelines + +Our regular test suite does not depend on any of the trained pipelines, since their outputs can vary and aren't generally required to test the library functionality. We test pipelines separately using the tests included in the [`spacy-models`](https://github.com/explosion/spacy-models) repository, which run whenever we train a new suite of models. The tests here mostly focus on making sure that the packages can be loaded and that the predictions seam reasonable, and they include checks for common bugs we encountered previously. If your test does not primarily focus on verifying a model's predictions, it should be part of the core library tests and construct the required objects manually, instead of being added to the models tests. + +Keep in mind that specific predictions may change, and we can't test for all incorrect predictions reported by users. Different models make different mistakes, so even a model that's significantly more accurate overall may end up making wrong predictions that it previously didn't. However, some surprising incorrect predictions may indicate deeper bugs that we definitely want to investigate. diff --git a/extra/DEVELOPER_DOCS/ExplosionBot.md b/extra/DEVELOPER_DOCS/ExplosionBot.md new file mode 100644 index 0000000..606fe93 --- /dev/null +++ b/extra/DEVELOPER_DOCS/ExplosionBot.md @@ -0,0 +1,56 @@ +# Explosion-bot + +Explosion-bot is a robot that can be invoked to help with running particular test commands. + +## Permissions + +Only maintainers have permissions to summon explosion-bot. Each of the open source repos that use explosion-bot has its own team(s) of maintainers, and only github users who are members of those teams can successfully run bot commands. + +## Running robot commands + +To summon the robot, write a github comment on the issue/PR you wish to test. The comment must be in the following format: + +``` +@explosion-bot please test_gpu +``` + +Some things to note: + +- The `@explosion-bot please` must be the beginning of the command - you cannot add anything in front of this or else the robot won't know how to parse it. Adding anything at the end aside from the test name will also confuse the robot, so keep it simple! +- The command name (such as `test_gpu`) must be one of the tests that the bot knows how to run. The available commands are documented in the bot's [workflow config](https://github.com/explosion/spaCy/blob/master/.github/workflows/explosionbot.yml#L26) and must match exactly one of the commands listed there. +- The robot can't do multiple things at once, so if you want it to run multiple tests, you'll have to summon it with one comment per test. + +### Examples + +- Execute spaCy slow GPU tests with a custom thinc branch from a spaCy PR: + + ``` + @explosion-bot please test_slow_gpu --thinc-branch + ``` + + `branch_name` can either be a named branch, e.g: `develop`, or an unmerged PR, e.g: `refs/pull//head`. + +- Execute spaCy Transformers GPU tests from a spaCy PR: + + ``` + @explosion-bot please test_gpu --run-on spacy-transformers --run-on-branch master --spacy-branch current_pr + ``` + + This will launch the GPU pipeline for the `spacy-transformers` repo on its `master` branch, using the current spaCy PR's branch to build spaCy. The name of the repository passed to `--run-on` is case-sensitive, e.g: use `spaCy` instead of `spacy`. + +- General info about supported commands. + + ``` + @explosion-bot please info + ``` + +- Help text for a specific command + ``` + @explosion-bot please --help + ``` + +## Troubleshooting + +If the robot isn't responding to commands as expected, you can check its logs in the [Github Action](https://github.com/explosion/spaCy/actions/workflows/explosionbot.yml). + +For each command sent to the bot, there should be a run of the `explosion-bot` workflow. In the `Install and run explosion-bot` step, towards the ends of the logs you should see info about the configuration that the bot was run with, as well as any errors that the bot encountered. diff --git a/extra/DEVELOPER_DOCS/Language.md b/extra/DEVELOPER_DOCS/Language.md new file mode 100644 index 0000000..f4fc850 --- /dev/null +++ b/extra/DEVELOPER_DOCS/Language.md @@ -0,0 +1,150 @@ +# Language + +> Reference: `spacy/language.py` + +1. [Constructing the `nlp` object from a config](#1-constructing-the-nlp-object-from-a-config) + - [A. Overview of `Language.from_config`](#1a-overview) + - [B. Component factories](#1b-how-pipeline-component-factories-work-in-the-config) + - [C. Sourcing a component](#1c-sourcing-a-pipeline-component) + - [D. Tracking components as they're modified](#1d-tracking-components-as-theyre-modified) + - [E. spaCy's config utility function](#1e-spacys-config-utility-functions) +2. [Initialization](#initialization) + - [A. Initialization for training](#2a-initialization-for-training): `init_nlp` + - [B. Initializing the `nlp` object](#2b-initializing-the-nlp-object): `Language.initialize` + - [C. Initializing the vocab](#2c-initializing-the-vocab): `init_vocab` + +## 1. Constructing the `nlp` object from a config + +### 1A. Overview + +Most of the functions referenced in the config are regular functions with arbitrary arguments registered via the function registry. However, the pipeline components are a bit special: they don't only receive arguments passed in via the config file, but also the current `nlp` object and the string `name` of the individual component instance (so a user can have multiple components created with the same factory, e.g. `ner_one` and `ner_two`). This name can then be used by the components to add to the losses and scores. This special requirement means that pipeline components can't just be resolved via the config the "normal" way: we need to retrieve the component functions manually and pass them their arguments, plus the `nlp` and `name`. + +The `Language.from_config` classmethod takes care of constructing the `nlp` object from a config. It's the single place where this happens and what `spacy.load` delegates to under the hood. Its main responsibilities are: + +- **Load and validate the config**, and optionally **auto-fill** all missing values that we either have defaults for in the config template or that registered function arguments define defaults for. This helps ensure backwards-compatibility, because we're able to add a new argument `foo: str = "bar"` to an existing function, without breaking configs that don't specity it. +- **Execute relevant callbacks** for pipeline creation, e.g. optional functions called before and after creation of the `nlp` object and pipeline. +- **Initialize language subclass and create tokenizer**. The `from_config` classmethod will always be called on a language subclass, e.g. `English`, not on `Language` directly. Initializing the subclass takes a callback to create the tokenizer. +- **Set up the pipeline components**. Components can either refer to a component factory or a `source`, i.e. an existing pipeline that's loaded and that the component is then copied from. We also need to ensure that we update the information about which components are disabled. +- **Manage listeners.** If sourced components "listen" to other components (`tok2vec`, `transformer`), we need to ensure that the references are valid. If the config specifies that listeners should be replaced by copies (e.g. to give the `ner` component its own `tok2vec` model instead of listening to the shared `tok2vec` component in the pipeline), we also need to take care of that. + +Note that we only resolve and load **selected sections** in `Language.from_config`, i.e. only the parts that are relevant at runtime, which is `[nlp]` and `[components]`. We don't want to be resolving anything related to training or initialization, since this would mean loading and constructing unnecessary functions, including functions that require information that isn't necessarily available at runtime, like `paths.train`. + +### 1B. How pipeline component factories work in the config + +As opposed to regular registered functions that refer to a registry and function name (e.g. `"@misc": "foo.v1"`), pipeline components follow a different format and refer to their component `factory` name. This corresponds to the name defined via the `@Language.component` or `@Language.factory` decorator. We need this decorator to define additional meta information for the components, like their default config and score weights. + +```ini +[components.my_component] +factory = "foo" +some_arg = "bar" +other_arg = ${paths.some_path} +``` + +This means that we need to create and resolve the `config["components"]` separately from the rest of the config. There are some important considerations and things we need to manage explicitly to avoid unexpected behavior: + +#### Variable interpolation + +When a config is resolved, references to variables are replaced, so that the functions receive the correct value instead of just the variable name. To interpolate a config, we need it in its entirety: we couldn't just interpolate a subsection that refers to variables defined in a different subsection. So we first interpolate the entire config. + +However, the `nlp.config` should include the original config with variables intact – otherwise, loading a pipeline and saving it to disk will destroy all logic implemented via variables and hard-code the values all over the place. This means that when we create the components, we need to keep two versions of the config: the interpolated config with the "real" values and the `raw_config` including the variable references. + +#### Factory registry + +Component factories are special and use the `@Language.factory` or `@Language.component` decorator to register themselves and their meta. When the decorator runs, it performs some basic validation, stores the meta information for the factory on the `Language` class (default config, scores etc.) and then adds the factory function to `registry.factories`. The `component` decorator can be used for registering simple functions that just take a `Doc` object and return it so in that case, we create the factory for the user automatically. + +There's one important detail to note about how factories are registered via entry points: A package that wants to expose spaCy components still needs to register them via the `@Language` decorators so we have the component meta information and can perform required checks. All we care about here is that the decorated function is **loaded and imported**. When it is, the `@Language` decorator takes care of everything, including actually registering the component factory. + +Normally, adding to the registry via an entry point will just add the function to the registry under the given name. But for `spacy_factories`, we don't actually want that: all we care about is that the function decorated with `@Language` is imported so the decorator runs. So we only exploit Python's entry point system to automatically import the function, and the `spacy_factories` entry point group actually adds to a **separate registry**, `registry._factories`, under the hood. Its only purpose is that the functions are imported. The decorator then runs, creates the factory if needed and adds it to the `registry.factories` registry. + +#### Language-specific factories + +spaCy supports registering factories on the `Language` base class, as well as language-specific subclasses like `English` or `German`. This allows providing different factories depending on the language, e.g. a different default lemmatizer. The `Language.get_factory_name` classmethod constructs the factory name as `{lang}.{name}` if a language is available (i.e. if it's a subclass) and falls back to `{name}` otherwise. So `@German.factory("foo")` will add a factory `de.foo` under the hood. If you add `nlp.add_pipe("foo")`, we first check if there's a factory for `{nlp.lang}.foo` and if not, we fall back to checking for a factory `foo`. + +#### Creating a pipeline component from a factory + +`Language.add_pipe` takes care of adding a pipeline component, given its factory name, its config. If no source pipeline to copy the component from is provided, it delegates to `Language.create_pipe`, which sets up the actual component function. + +- Validate the config and make sure that the factory was registered via the decorator and that we have meta for it. +- Update the component config with any defaults specified by the component's `default_config`, if available. This is done by merging the values we receive into the defaults. It ensures that you can still add a component without having to specify its _entire_ config including more complex settings like `model`. If no `model` is defined, we use the default. +- Check if we have a language-specific factory for the given `nlp.lang` and if not, fall back to the global factory. +- Construct the component config, consisting of whatever arguments were provided, plus the current `nlp` object and `name`, which are default expected arguments of all factories. We also add a reference to the `@factories` registry, so we can resolve the config via the registry, like any other config. With the added `nlp` and `name`, it should now include all expected arguments of the given function. +- Fill the config to make sure all unspecified defaults from the function arguments are added and update the `raw_config` (uninterpolated with variables intact) with that information, so the component config we store in `nlp.config` is up to date. We do this by adding the `raw_config` _into_ the filled config – otherwise, the references to variables would be overwritten. +- Resolve the config and create all functions it refers to (e.g. `model`). This gives us the actual component function that we can insert into the pipeline. + +### 1C. Sourcing a pipeline component + +```ini +[components.ner] +source = "en_core_web_sm" +``` + +spaCy also allows ["sourcing" a component](https://spacy.io/usage/processing-pipelines#sourced-components), which will copy it over from an existing pipeline. In this case, `Language.add_pipe` will delegate to `Language.create_pipe_from_source`. In order to copy a component effectively and validate it, the source pipeline first needs to be loaded. This is done in `Language.from_config`, so a source pipeline only has to be loaded once if multiple components source from it. Sourcing a component will perform the following checks and modifications: + +- For each sourced pipeline component loaded in `Language.from_config`, a hash of the vectors data from the source pipeline is stored in the pipeline meta so we're able to check whether the vectors match and warn if not (since different vectors that are used as features in components can lead to degraded performance). Because the vectors are not loaded at the point when components are sourced, the check is postponed to `init_vocab` as part of `Language.initialize`. +- If the sourced pipeline component is loaded through `Language.add_pipe(source=)`, the vectors are already loaded and can be compared directly. The check compares the shape and keys first and finally falls back to comparing the actual byte representation of the vectors (which is slower). +- Ensure that the component is available in the pipeline. +- Interpolate the entire config of the source pipeline so all variables are replaced and the component's config that's copied over doesn't include references to variables that are not available in the destination config. +- Add the source `vocab.strings` to the destination's `vocab.strings` so we don't end up with unavailable strings in the final pipeline (which would also include labels used by the sourced component). + +Note that there may be other incompatibilities that we're currently not checking for and that could cause a sourced component to not work in the destination pipeline. We're interested in adding more checks here but there'll always be a small number of edge cases we'll never be able to catch, including a sourced component depending on other pipeline state that's not available in the destination pipeline. + +### 1D. Tracking components as they're modified + +The `Language` class implements methods for removing, replacing or renaming pipeline components. Whenever we make these changes, we need to update the information stored on the `Language` object to ensure that it matches the current state of the pipeline. If a user just writes to `nlp.config` manually, we obviously can't ensure that the config matches the reality – but since we offer modification via the pipe methods, it's expected that spaCy keeps the config in sync under the hood. Otherwise, saving a modified pipeline to disk and loading it back wouldn't work. The internal attributes we need to keep in sync here are: + +| Attribute | Type | Description | +| ------------------------ | ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Language._components` | `List[Tuple[str, Callable]]` | All pipeline components as `(name, func)` tuples. This is used as the source of truth for `Language.pipeline`, `Language.pipe_names` and `Language.components`. | +| `Language._pipe_meta` | `Dict[str, FactoryMeta]` | The meta information of a component's factory, keyed by component name. This can include multiple components referring to the same factory meta. | +| `Language._pipe_configs` | `Dict[str, Config]` | The component's config, keyed by component name. | +| `Language._disabled` | `Set[str]` | Names of components that are currently disabled. | +| `Language._config` | `Config` | The underlying config. This is only internals and will be used as the basis for constructing the config in the `Language.config` property. | + +In addition to the actual component settings in `[components]`, the config also allows specifying component-specific arguments via the `[initialize.components]` block, which are passed to the component's `initialize` method during initialization if it's available. So we also need to keep this in sync in the underlying config. + +### 1E. spaCy's config utility functions + +When working with configs in spaCy, make sure to use the utility functions provided by spaCy if available, instead of calling the respective `Config` methods. The utilities take care of providing spaCy-specific error messages and ensure a consistent order of config sections by setting the `section_order` argument. This ensures that exported configs always have the same consistent format. + +- `util.load_config`: load a config from a file +- `util.load_config_from_str`: load a confirm from a string representation +- `util.copy_config`: deepcopy a config + +## 2. Initialization + +Initialization is a separate step of the [config lifecycle](https://spacy.io/usage/training#config-lifecycle) that's not performed at runtime. It's implemented via the `training.initialize.init_nlp` helper and calls into `Language.initialize` method, which sets up the pipeline and component models before training. The `initialize` method takes a callback that returns a sample of examples, which is used to initialize the component models, add all required labels and perform shape inference if applicable. + +Components can also define custom initialization setting via the `[initialize.components]` block, e.g. if they require external data like lookup tables to be loaded in. All config settings defined here will be passed to the component's `initialize` method, if it implements one. Components are expected to handle their own serialization after they're initialized so that any data or settings they require are saved with the pipeline and will be available from disk when the pipeline is loaded back at runtime. + +### 2A. Initialization for training + +The `init_nlp` function is called before training and returns an initialized `nlp` object that can be updated with the examples. It only needs the config and does the following: + +- Load and validate the config. In order to validate certain settings like the `seed`, we also interpolate the config to get the final value (because in theory, a user could provide this via a variable). +- Set up the GPU allocation, if required. +- Create the `nlp` object from the raw, uninterpolated config, which delegates to `Language.from_config`. Since this method may modify and auto-fill the config and pipeline component settings, we then use the interpolated version of `nlp.config` going forward, to ensure that what we're training with is up to date. +- Resolve the `[training]` block of the config and perform validation, e.g. to check that the corpora are available. +- Determine the components that should be frozen (not updated during training) or resumed (sourced components from a different pipeline that should be updated from the examples and not reset and re-initialized). To resume training, we can call the `nlp.resume_training` method. +- Initialize the `nlp` object via `nlp.initialize` and pass it a `get_examples` callback that returns the training corpus (used for shape inference, setting up labels etc.). If the training corpus is streamed, we only provide a small sample of the data, which can potentially be infinite. `nlp.initialize` will delegate to the components as well and pass the data sample forward. +- Check the listeners and warn about components dependencies, e.g. if a frozen component listens to a component that is retrained, or vice versa (which can degrade results). + +### 2B. Initializing the `nlp` object + +The `Language.initialize` method does the following: + +- **Resolve the config** defined in the `[initialize]` block separately (since everything else is already available in the loaded `nlp` object), based on the fully interpolated config. +- **Execute callbacks**, i.e. `before_init` and `after_init`, if they're defined. +- **Initialize the vocab**, including vocab data, lookup tables and vectors. +- **Initialize the tokenizer** if it implements an `initialize` method. This is not the case for the default tokenizers, but it allows custom tokenizers to depend on external data resources that are loaded in on initialization. +- **Initialize all pipeline components** if they implement an `initialize` method and pass them the `get_examples` callback, the current `nlp` object as well as well additional initialization config settings provided in the component-specific block. +- **Initialize pretraining** if a `[pretraining]` block is available in the config. This allows loading pretrained tok2vec weights in `spacy pretrain`. +- **Register listeners** if token-to-vector embedding layers of a component model "listen" to a previous component (`tok2vec`, `transformer`) in the pipeline. +- **Create an optimizer** on the `Language` class, either by adding the optimizer passed as `sgd` to `initialize`, or by creating the optimizer defined in the config's training settings. + +### 2C. Initializing the vocab + +Vocab initialization is handled in the `training.initialize.init_vocab` helper. It takes the relevant loaded functions and values from the config and takes care of the following: + +- Add lookup tables defined in the config initialization, e.g. custom lemmatization tables. Those will be added to `nlp.vocab.lookups` from where they can be accessed by components. +- Add JSONL-formatted [vocabulary data](https://spacy.io/api/data-formats#vocab-jsonl) to pre-populate the lexical attributes. +- Load vectors into the pipeline. Vectors are defined as a name or path to a saved `nlp` object containing the vectors, e.g. `en_vectors_web_lg`. It's loaded and the vectors are ported over, while ensuring that all source strings are available in the destination strings. We also warn if there's a mismatch between sourced vectors, since this can lead to problems. diff --git a/extra/DEVELOPER_DOCS/Listeners.md b/extra/DEVELOPER_DOCS/Listeners.md new file mode 100644 index 0000000..72c0368 --- /dev/null +++ b/extra/DEVELOPER_DOCS/Listeners.md @@ -0,0 +1,235 @@ +# Listeners + +- [1. Overview](#1-overview) +- [2. Initialization](#2-initialization) + - [2A. Linking listeners to the embedding component](#2a-linking-listeners-to-the-embedding-component) + - [2B. Shape inference](#2b-shape-inference) +- [3. Internal communication](#3-internal-communication) + - [3A. During prediction](#3a-during-prediction) + - [3B. During training](#3b-during-training) + - [Training with multiple listeners](#training-with-multiple-listeners) + - [3C. Frozen components](#3c-frozen-components) + - [The Tok2Vec or Transformer is frozen](#the-tok2vec-or-transformer-is-frozen) + - [The upstream component is frozen](#the-upstream-component-is-frozen) +- [4. Replacing listener with standalone](#4-replacing-listener-with-standalone) + +## 1. Overview + +Trainable spaCy components typically use some sort of `tok2vec` layer as part of the `model` definition. +This `tok2vec` layer produces embeddings and is either a standard `Tok2Vec` layer, or a Transformer-based one. +Both versions can be used either inline/standalone, which means that they are defined and used +by only one specific component (e.g. NER), or +[shared](https://spacy.io/usage/embeddings-transformers#embedding-layers), +in which case the embedding functionality becomes a separate component that can +feed embeddings to multiple components downstream, using a listener-pattern. + +| Type | Usage | Model Architecture | +| ------------- | ---------- | -------------------------------------------------------------------------------------------------- | +| `Tok2Vec` | standalone | [`spacy.Tok2Vec`](https://spacy.io/api/architectures#Tok2Vec) | +| `Tok2Vec` | listener | [`spacy.Tok2VecListener`](https://spacy.io/api/architectures#Tok2VecListener) | +| `Transformer` | standalone | [`spacy-transformers.Tok2VecTransformer`](https://spacy.io/api/architectures#Tok2VecTransformer) | +| `Transformer` | listener | [`spacy-transformers.TransformerListener`](https://spacy.io/api/architectures#TransformerListener) | + +Here we discuss the listener pattern and its implementation in code in more detail. + +## 2. Initialization + +### 2A. Linking listeners to the embedding component + +To allow sharing a `tok2vec` layer, a separate `tok2vec` component needs to be defined in the config: + +``` +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v2" +``` + +A listener can then be set up by making sure the correct `upstream` name is defined, referring to the +name of the `tok2vec` component (which equals the factory name by default), or `*` as a wildcard: + +``` +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +upstream = "tok2vec" +``` + +When an [`nlp`](https://github.com/explosion/spaCy/blob/master/extra/DEVELOPER_DOCS/Language.md) object is +initialized or deserialized, it will make sure to link each `tok2vec` component to its listeners. This is +implemented in the method `nlp._link_components()` which loops over each +component in the pipeline and calls `find_listeners()` on a component if it's defined. +The [`tok2vec` component](https://github.com/explosion/spaCy/blob/master/spacy/pipeline/tok2vec.py)'s implementation +of this `find_listener()` method will specifically identify sublayers of a model definition that are of type +`Tok2VecListener` with a matching upstream name and will then add that listener to the internal `self.listener_map`. + +If it's a Transformer-based pipeline, a +[`transformer` component](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py) +has a similar implementation but its `find_listener()` function will specifically look for `TransformerListener` +sublayers of downstream components. + +### 2B. Shape inference + +Typically, the output dimension `nO` of a listener's model equals the `nO` (or `width`) of the upstream embedding layer. +For a standard `Tok2Vec`-based component, this is typically known up-front and defined as such in the config: + +``` +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +``` + +A `transformer` component however only knows its `nO` dimension after the HuggingFace transformer +is set with the function `model.attrs["set_transformer"]`, +[implemented](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/transformer_model.py) +by `set_pytorch_transformer`. +This is why, upon linking of the transformer listeners, the `transformer` component also makes sure to set +the listener's output dimension correctly. + +This shape inference mechanism also needs to happen with resumed/frozen components, which means that for some CLI +commands (`assemble` and `train`), we need to call `nlp._link_components` even before initializing the `nlp` +object. To cover all use-cases and avoid negative side effects, the code base ensures that performing the +linking twice is not harmful. + +## 3. Internal communication + +The internal communication between a listener and its downstream components is organized by sending and +receiving information across the components - either directly or implicitly. +The details are different depending on whether the pipeline is currently training, or predicting. +Either way, the `tok2vec` or `transformer` component always needs to run before the listener. + +### 3A. During prediction + +When the `Tok2Vec` pipeline component is called, its `predict()` method is executed to produce the results, +which are then stored by `set_annotations()` in the `doc.tensor` field of the document(s). +Similarly, the `Transformer` component stores the produced embeddings +in `doc._.trf_data`. Next, the `forward` pass of a +[`Tok2VecListener`](https://github.com/explosion/spaCy/blob/master/spacy/pipeline/tok2vec.py) +or a +[`TransformerListener`](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/listener.py) +accesses these fields on the `Doc` directly. Both listener implementations have a fallback mechanism for when these +properties were not set on the `Doc`: in that case an all-zero tensor is produced and returned. +We need this fallback mechanism to enable shape inference methods in Thinc, but the code +is slightly risky and at times might hide another bug - so it's a good spot to be aware of. + +### 3B. During training + +During training, the `update()` methods of the `Tok2Vec` & `Transformer` components don't necessarily set the +annotations on the `Doc` (though since 3.1 they can if they are part of the `annotating_components` list in the config). +Instead, we rely on a caching mechanism between the original embedding component and its listener. +Specifically, the produced embeddings are sent to the listeners by calling `listener.receive()` and uniquely +identifying the batch of documents with a `batch_id`. This `receive()` call also sends the appropriate `backprop` +call to ensure that gradients from the downstream component flow back to the trainable `Tok2Vec` or `Transformer` +network. + +We rely on the `nlp` object properly batching the data and sending each batch through the pipeline in sequence, +which means that only one such batch needs to be kept in memory for each listener. +When the downstream component runs and the listener should produce embeddings, it accesses the batch in memory, +runs the backpropagation, and returns the results and the gradients. + +There are two ways in which this mechanism can fail, both are detected by `verify_inputs()`: + +- `E953` if a different batch is in memory than the requested one - signaling some kind of out-of-sync state of the + training pipeline. +- `E954` if no batch is in memory at all - signaling that the pipeline is probably not set up correctly. + +#### Training with multiple listeners + +One `Tok2Vec` or `Transformer` component may be listened to by several downstream components, e.g. +a tagger and a parser could be sharing the same embeddings. In this case, we need to be careful about how we do +the backpropagation. When the `Tok2Vec` or `Transformer` sends out data to the listener with `receive()`, they will +send an `accumulate_gradient` function call to all listeners, except the last one. This function will keep track +of the gradients received so far. Only the final listener in the pipeline will get an actual `backprop` call that +will initiate the backpropagation of the `tok2vec` or `transformer` model with the accumulated gradients. + +### 3C. Frozen components + +The listener pattern can get particularly tricky in combination with frozen components. To detect components +with listeners that are not frozen consistently, `init_nlp()` (which is called by `spacy train`) goes through +the listeners and their upstream components and warns in two scenarios. + +#### The Tok2Vec or Transformer is frozen + +If the `Tok2Vec` or `Transformer` was already trained, +e.g. by [pretraining](https://spacy.io/usage/embeddings-transformers#pretraining), +it could be a valid use-case to freeze the embedding architecture and only train downstream components such +as a tagger or a parser. This used to be impossible before 3.1, but has become supported since then by putting the +embedding component in the [`annotating_components`](https://spacy.io/usage/training#annotating-components) +list of the config. This works like any other "annotating component" because it relies on the `Doc` attributes. + +However, if the `Tok2Vec` or `Transformer` is frozen, and not present in `annotating_components`, and a related +listener isn't frozen, then a `W086` warning is shown and further training of the pipeline will likely end with `E954`. + +#### The upstream component is frozen + +If an upstream component is frozen but the underlying `Tok2Vec` or `Transformer` isn't, the performance of +the upstream component will be degraded after training. In this case, a `W087` warning is shown, explaining +how to use the `replace_listeners` functionality to prevent this problem. + +## 4. Replacing listener with standalone + +The [`replace_listeners`](https://spacy.io/api/language#replace_listeners) functionality changes the architecture +of a downstream component from using a listener pattern to a standalone `tok2vec` or `transformer` layer, +effectively making the downstream component independent of any other components in the pipeline. +It is implemented by `nlp.replace_listeners()` and typically executed by `nlp.from_config()`. +First, it fetches the original `Model` of the original component that creates the embeddings: + +``` +tok2vec = self.get_pipe(tok2vec_name) +tok2vec_model = tok2vec.model +``` + +Which is either a [`Tok2Vec` model](https://github.com/explosion/spaCy/blob/master/spacy/ml/models/tok2vec.py) or a +[`TransformerModel`](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/transformer_model.py). + +In the case of the `tok2vec`, this model can be copied as-is into the configuration and architecture of the +downstream component. However, for the `transformer`, this doesn't work. +The reason is that the `TransformerListener` architecture chains the listener with +[`trfs2arrays`](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/trfs2arrays.py): + +``` +model = chain( + TransformerListener(upstream_name=upstream) + trfs2arrays(pooling, grad_factor), +) +``` + +but the standalone `Tok2VecTransformer` has an additional `split_trf_batch` chained inbetween the model +and `trfs2arrays`: + +``` +model = chain( + TransformerModel(name, get_spans, tokenizer_config), + split_trf_batch(), + trfs2arrays(pooling, grad_factor), +) +``` + +So you can't just take the model from the listener, and drop that into the component internally. You need to +adjust the model and the config. To facilitate this, `nlp.replace_listeners()` will check whether additional +[functions](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/_util.py) are +[defined](https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/layers/transformer_model.py) +in `model.attrs`, and if so, it will essentially call these to make the appropriate changes: + +``` +replace_func = tok2vec_model.attrs["replace_listener_cfg"] +new_config = replace_func(tok2vec_cfg["model"], pipe_cfg["model"]["tok2vec"]) +... +new_model = tok2vec_model.attrs["replace_listener"](new_model) +``` + +The new config and model are then properly stored on the `nlp` object. +Note that this functionality (running the replacement for a transformer listener) was broken prior to +`spacy-transformers` 1.0.5. + +In spaCy 3.7, `Language.replace_listeners` was updated to pass the following additional arguments to the `replace_listener` callback: +the listener to be replaced and the `tok2vec`/`transformer` pipe from which the new model was copied. To maintain backwards-compatiblity, +the method only passes these extra arguments for callbacks that support them: + +``` +def replace_listener_pre_37(copied_tok2vec_model): + ... + +def replace_listener_post_37(copied_tok2vec_model, replaced_listener, tok2vec_pipe): + ... +``` diff --git a/extra/DEVELOPER_DOCS/README.md b/extra/DEVELOPER_DOCS/README.md new file mode 100644 index 0000000..8ff505d --- /dev/null +++ b/extra/DEVELOPER_DOCS/README.md @@ -0,0 +1,7 @@ + + +# Developer Documentation + +This directory includes additional documentation and explanations of spaCy's internals. It's mostly intended for the spaCy core development team and contributors interested in the more complex parts of the library. The documents generally focus on more abstract implementation details and how specific methods and algorithms work, and they assume knowledge of what's already available in the [usage documentation](https://spacy.io/usage) and [API reference](https://spacy.io/api). + +If you're looking to contribute to spaCy, make sure to check out the documentation and [contributing guide](https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md) first. diff --git a/extra/DEVELOPER_DOCS/Satellite Packages.md b/extra/DEVELOPER_DOCS/Satellite Packages.md new file mode 100644 index 0000000..02b06a9 --- /dev/null +++ b/extra/DEVELOPER_DOCS/Satellite Packages.md @@ -0,0 +1,82 @@ +# spaCy Satellite Packages + +This is a list of all the active repos relevant to spaCy besides the main one, with short descriptions, history, and current status. Archived repos will not be covered. + +## Always Included in spaCy + +These packages are always pulled in when you install spaCy. Most of them are direct dependencies, but some are transitive dependencies through other packages. + +- [spacy-legacy](https://github.com/explosion/spacy-legacy): When an architecture in spaCy changes enough to get a new version, the old version is frozen and moved to spacy-legacy. This allows us to keep the core library slim while also preserving backwards compatability. +- [thinc](https://github.com/explosion/thinc): Thinc is the machine learning library that powers trainable components in spaCy. It wraps backends like Numpy, PyTorch, and Tensorflow to provide a functional interface for specifying architectures. +- [catalogue](https://github.com/explosion/catalogue): Small library for adding function registries, like those used for model architectures in spaCy. +- [confection](https://github.com/explosion/confection): This library contains the functionality for config parsing that was formerly contained directly in Thinc. +- [spacy-loggers](https://github.com/explosion/spacy-loggers): Contains loggers beyond the default logger available in spaCy's core code base. This includes loggers integrated with third-party services, which may differ in release cadence from spaCy itself. +- [wasabi](https://github.com/explosion/wasabi): A command line formatting library, used for terminal output in spaCy. +- [srsly](https://github.com/explosion/srsly): A wrapper that vendors several serialization libraries for spaCy. Includes parsers for JSON, JSONL, MessagePack, (extended) Pickle, and YAML. +- [preshed](https://github.com/explosion/preshed): A Cython library for low-level data structures like hash maps, used for memory efficient data storage. +- [cython-blis](https://github.com/explosion/cython-blis): Fast matrix multiplication using BLIS without depending on system libraries. Required by Thinc, rather than spaCy directly. +- [murmurhash](https://github.com/explosion/murmurhash): A wrapper library for a C++ murmurhash implementation, used for string IDs in spaCy and preshed. +- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython. + +## Optional Extensions for spaCy + +These are repos that can be used by spaCy but aren't part of a default installation. Many of these are wrappers to integrate various kinds of third-party libraries. + +- [spacy-transformers](https://github.com/explosion/spacy-transformers): A wrapper for the [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) library, this handles the extensive conversion necessary to coordinate spaCy's powerful `Doc` representation, training pipeline, and the Transformer embeddings. When released, this was known as `spacy-pytorch-transformers`, but it changed to the current name when HuggingFace update the name of their library as well. +- [spacy-huggingface-hub](https://github.com/explosion/spacy-huggingface-hub): This package has a CLI script for uploading a packaged spaCy pipeline (created with `spacy package`) to the [Hugging Face Hub](https://huggingface.co/models). +- [spacy-alignments](https://github.com/explosion/spacy-alignments): A wrapper for the tokenizations library (mentioned below) with a modified build system to simplify cross-platform wheel creation. Used in spacy-transformers for aligning spaCy and HuggingFace tokenizations. +- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them. +- [spacy-lookups-data](https://github.com/explosion/spacy-lookups-data): A repository of linguistic data, such as lemmas, that takes up a lot of disk space. Originally created to reduce the size of the spaCy core library. This is mainly useful if you want the data included but aren't using a pretrained pipeline; for the affected languages, the relevant data is included in pretrained pipelines directly. +- [coreferee](https://github.com/explosion/coreferee): Coreference resolution for English, French, German and Polish, optimised for limited training data and easily extensible for further languages. Used as a spaCy pipeline component. +- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy. +- [spacy-streamlit](https://github.com/explosion/spacy-streamlit): A wrapper for the Streamlit dashboard building library to help with integrating [displaCy](https://spacy.io/api/top-level/#displacy). +- [spacymoji](https://github.com/explosion/spacymoji): A library to add extra support for emoji to spaCy, such as including character names. +- [thinc-apple-ops](https://github.com/explosion/thinc-apple-ops): A special backend for OSX that uses Apple's native libraries for improved performance. +- [os-signpost](https://github.com/explosion/os-signpost): A Python package that allows you to use the `OSSignposter` API in OSX for performance analysis. +- [spacy-ray](https://github.com/explosion/spacy-ray): A wrapper to integrate spaCy with Ray, a distributed training framework. Currently a work in progress. + +## Prodigy + +[Prodigy](https://prodi.gy) is Explosion's easy to use and highly customizable tool for annotating data. Prodigy itself requires a license, but the repos below contain documentation, examples, and editor or notebook integrations. + +- [prodigy-recipes](https://github.com/explosion/prodigy-recipes): Sample recipes for Prodigy, along with notebooks and other examples of usage. +- [vscode-prodigy](https://github.com/explosion/vscode-prodigy): A VS Code extension that lets you run Prodigy inside VS Code. +- [jupyterlab-prodigy](https://github.com/explosion/jupyterlab-prodigy): An extension for JupyterLab that lets you run Prodigy inside JupyterLab. + +## Independent Tools or Projects + +These are tools that may be related to or use spaCy, but are functional independent projects in their own right as well. + +- [floret](https://github.com/explosion/floret): A modification of fastText to use Bloom Embeddings. Can be used to add vectors with subword features to spaCy, and also works independently in the same manner as fastText. +- [sense2vec](https://github.com/explosion/sense2vec): A library to make embeddings of noun phrases or words coupled with their part of speech. This library uses spaCy. +- [spacy-vectors-builder](https://github.com/explosion/spacy-vectors-builder): This is a spaCy project that builds vectors using floret and a lot of input text. It handles downloading the input data as well as the actual building of vectors. +- [holmes-extractor](https://github.com/explosion/holmes-extractor): Information extraction from English and German texts based on predicate logic. Uses spaCy. +- [healthsea](https://github.com/explosion/healthsea): Healthsea is a project to extract information from comments about health supplements. Structurally, it's a self-contained, large spaCy project. +- [spacy-pkuseg](https://github.com/explosion/spacy-pkuseg): A fork of the pkuseg Chinese tokenizer. Used for Chinese support in spaCy, but also works independently. +- [ml-datasets](https://github.com/explosion/ml-datasets): This repo includes loaders for several standard machine learning datasets, like MNIST or WikiNER, and has historically been used in spaCy example code and documentation. + +## Documentation and Informational Repos + +These repos are used to support the spaCy docs or otherwise present information about spaCy or other Explosion projects. + +- [projects](https://github.com/explosion/projects): The projects repo is used to show detailed examples of spaCy usage. Individual projects can be checked out using the spaCy command line tool, rather than checking out the projects repo directly. +- [spacy-course](https://github.com/explosion/spacy-course): Home to the interactive spaCy course for learning about how to use the library and some basic NLP principles. +- [spacy-io-binder](https://github.com/explosion/spacy-io-binder): Home to the notebooks used for interactive examples in the documentation. + +## Organizational / Meta + +These repos are used for organizing data around spaCy, but are not something an end user would need to install as part of using the library. + +- [spacy-models](https://github.com/explosion/spacy-models): This repo contains metadata (but not training data) for all the spaCy models. This includes information about where their training data came from, version compatability, and performance information. It also includes tests for the model packages, and the built models are hosted as releases of this repo. +- [wheelwright](https://github.com/explosion/wheelwright): A tool for automating our PyPI builds and releases. +- [ec2buildwheel](https://github.com/explosion/ec2buildwheel): A small project that allows you to build Python packages in the manner of cibuildwheel, but on any EC2 image. Used by wheelwright. + +## Other + +Repos that don't fit in any of the above categories. + +- [blis](https://github.com/explosion/blis): A fork of the official BLIS library. The main branch is not updated, but work continues in various branches. This is used for cython-blis. +- [tokenizations](https://github.com/explosion/tokenizations): A library originally by Yohei Tamura to align strings with tolerance to some variations in features like case and diacritics, used for aligning tokens and wordpieces. Adopted and maintained by Explosion, but usually spacy-alignments is used instead. +- [conll-2012](https://github.com/explosion/conll-2012): A repo to hold some slightly cleaned up versions of the official scripts for the CoNLL 2012 shared task involving coreference resolution. Used in the coref project. +- [fastapi-explosion-extras](https://github.com/explosion/fastapi-explosion-extras): Some small tweaks to FastAPI used at Explosion. + diff --git a/extra/DEVELOPER_DOCS/StringStore-Vocab.md b/extra/DEVELOPER_DOCS/StringStore-Vocab.md new file mode 100644 index 0000000..866ba2a --- /dev/null +++ b/extra/DEVELOPER_DOCS/StringStore-Vocab.md @@ -0,0 +1,216 @@ +# StringStore & Vocab + +> Reference: `spacy/strings.pyx` +> Reference: `spacy/vocab.pyx` + +## Overview + +spaCy represents mosts strings internally using a `uint64` in Cython which +corresponds to a hash. The magic required to make this largely transparent is +handled by the `StringStore`, and is integrated into the pipelines using the +`Vocab`, which also connects it to some other information. + +These are mostly internal details that average library users should never have +to think about. On the other hand, when developing a component it's normal to +interact with the Vocab for lexeme data or word vectors, and it's not unusual +to add labels to the `StringStore`. + +## StringStore + +### Overview + +The `StringStore` is a `cdef class` that looks a bit like a two-way dictionary, +though it is not a subclass of anything in particular. + +The main functionality of the `StringStore` is that `__getitem__` converts +hashes into strings or strings into hashes. + +The full details of the conversion are complicated. Normally you shouldn't have +to worry about them, but the first applicable case here is used to get the +return value: + +1. 0 and the empty string are special cased to each other +2. internal symbols use a lookup table (`SYMBOLS_BY_STR`) +3. normal strings or bytes are hashed +4. internal symbol IDs in `SYMBOLS_BY_INT` are handled +5. anything not yet handled is used as a hash to lookup a string + +For the symbol enums, see [`symbols.pxd`](https://github.com/explosion/spaCy/blob/master/spacy/symbols.pxd). + +Almost all strings in spaCy are stored in the `StringStore`. This naturally +includes tokens, but also includes things like labels (not just NER/POS/dep, +but also categories etc.), lemmas, lowercase forms, word shapes, and so on. One +of the main results of this is that tokens can be represented by a compact C +struct ([`LexemeC`](https://spacy.io/api/cython-structs#lexemec)/[`TokenC`](https://github.com/explosion/spaCy/issues/4854)) that mostly consists of string hashes. This also means that converting +input for the models is straightforward, and there's not a token mapping step +like in many machine learning frameworks. Additionally, because the token IDs +in spaCy are based on hashes, they are consistent across environments or +models. + +One pattern you'll see a lot in spaCy APIs is that `something.value` returns an +`int` and `something.value_` returns a string. That's implemented using the +`StringStore`. Typically the `int` is stored in a C struct and the string is +generated via a property that calls into the `StringStore` with the `int`. + +Besides `__getitem__`, the `StringStore` has functions to return specifically a +string or specifically a hash, regardless of whether the input was a string or +hash to begin with, though these are only used occasionally. + +### Implementation Details: Hashes and Allocations + +Hashes are 64-bit and are computed using [murmurhash][] on UTF-8 bytes. There is no +mechanism for detecting and avoiding collisions. To date there has never been a +reproducible collision or user report about any related issues. + +[murmurhash]: https://github.com/explosion/murmurhash + +The empty string is not hashed, it's just converted to/from 0. + +A small number of strings use indices into a lookup table (so low integers) +rather than hashes. This is mostly Universal Dependencies labels or other +strings considered "core" in spaCy. This was critical in v1, which hadn't +introduced hashing yet. Since v2 it's important for items in `spacy.attrs`, +especially lexeme flags, but is otherwise only maintained for backwards +compatibility. + +You can call `strings["mystring"]` with a string the `StringStore` has never seen +before and it will return a hash. But in order to do the reverse operation, you +need to call `strings.add("mystring")` first. Without a call to `add` the +string will not be interned. + +Example: + +``` +from spacy.strings import StringStore + +ss = StringStore() +hashval = ss["spacy"] # 10639093010105930009 +try: + # this won't work + ss[hashval] +except KeyError: + print(f"key {hashval} unknown in the StringStore.") + +ss.add("spacy") +assert ss[hashval] == "spacy" # it works now + +# There is no `.keys` property, but you can iterate over keys +# The empty string will never be in the list of keys +for key in ss: + print(key) +``` + +In normal use nothing is ever removed from the `StringStore`. In theory this +means that if you do something like iterate through all hex values of a certain +length you can have explosive memory usage. In practice this has never been an +issue. (Note that this is also different from using `sys.intern` to intern +Python strings, which does not guarantee they won't be garbage collected later.) + +Strings are stored in the `StringStore` in a peculiar way: each string uses a +union that is either an eight-byte `char[]` or a `char*`. Short strings are +stored directly in the `char[]`, while longer strings are stored in allocated +memory and prefixed with their length. This is a strategy to reduce indirection +and memory fragmentation. See `decode_Utf8Str` and `_allocate` in +`strings.pyx` for the implementation. + +### When to Use the StringStore? + +While you can ignore the `StringStore` in many cases, there are situations where +you should make use of it to avoid errors. + +Any time you introduce a string that may be set on a `Doc` field that has a hash, +you should add the string to the `StringStore`. This mainly happens when adding +labels in components, but there are some other cases: + +- syntax iterators, mainly `get_noun_chunks` +- external data used in components, like the `KnowledgeBase` in the `entity_linker` +- labels used in tests + +## Vocab + +The `Vocab` is a core component of a `Language` pipeline. Its main function is +to manage `Lexeme`s, which are structs that contain information about a token +that depends only on its surface form, without context. `Lexeme`s store much of +the data associated with `Token`s. As a side effect of this the `Vocab` also +manages the `StringStore` for a pipeline and a grab-bag of other data. + +These are things stored in the vocab: + +- `Lexeme`s +- `StringStore` +- `Morphology`: manages info used in `MorphAnalysis` objects +- `vectors`: basically a dict for word vectors +- `lookups`: language specific data like lemmas +- `writing_system`: language specific metadata +- `get_noun_chunks`: a syntax iterator +- lex attribute getters: functions like `is_punct`, set in language defaults +- `cfg`: **not** the pipeline config, this is mostly unused +- `_unused_object`: Formerly an unused object, kept around until v4 for compatability + +Some of these, like the Morphology and Vectors, are complex enough that they +need their own explanations. Here we'll just look at Vocab-specific items. + +### Lexemes + +A `Lexeme` is a type that mainly wraps a `LexemeC`, a struct consisting of ints +that identify various context-free token attributes. Lexemes are the core data +of the `Vocab`, and can be accessed using `__getitem__` on the `Vocab`. The memory +for storing `LexemeC` objects is managed by a pool that belongs to the `Vocab`. + +Note that `__getitem__` on the `Vocab` works much like the `StringStore`, in +that it accepts a hash or id, with one important difference: if you do a lookup +using a string, that value is added to the `StringStore` automatically. + +The attributes stored in a `LexemeC` are: + +- orth (the raw text) +- lower +- norm +- shape +- prefix +- suffix + +Most of these are straightforward. All of them can be customized, and (except +`orth`) probably should be since the defaults are based on English, but in +practice this is rarely done at present. + +### Lookups + +This is basically a dict of dicts, implemented using a `Table` for each +sub-dict, that stores lemmas and other language-specific lookup data. + +A `Table` is a subclass of `OrderedDict` used for string-to-string data. It uses +Bloom filters to speed up misses and has some extra serialization features. +Tables are not used outside of the lookups. + +### Lex Attribute Getters + +Lexical Attribute Getters like `is_punct` are defined on a per-language basis, +much like lookups, but take the form of functions rather than string-to-string +dicts, so they're stored separately. + +### Writing System + +This is a dict with three attributes: + +- `direction`: ltr or rtl (default ltr) +- `has_case`: bool (default `True`) +- `has_letters`: bool (default `True`, `False` only for CJK for now) + +Currently these are not used much - the main use is that `direction` is used in +visualizers, though `rtl` doesn't quite work (see +[#4854](https://github.com/explosion/spaCy/issues/4854)). In the future they +could be used when choosing hyperparameters for subwords, controlling word +shape generation, and similar tasks. + +### Other Vocab Members + +The Vocab is kind of the default place to store things from `Language.defaults` +that don't belong to the Tokenizer. The following properties are in the Vocab +just because they don't have anywhere else to go. + +- `get_noun_chunks` +- `cfg`: This is a dict that just stores `oov_prob` (hardcoded to `-20`) +- `_unused_object`: Leftover C member, should be removed in next major version + + diff --git a/extra/example_data/ner_example_data/README.md b/extra/example_data/ner_example_data/README.md new file mode 100644 index 0000000..3c6a4a8 --- /dev/null +++ b/extra/example_data/ner_example_data/README.md @@ -0,0 +1,25 @@ +## Examples of NER/IOB data that can be converted with `spacy convert` + +To convert an IOB file to `.spacy` ([`DocBin`](https://spacy.io/api/docbin)) +for spaCy v3: + +```bash +python -m spacy convert -c iob -s -n 10 -b en_core_web_sm file.iob . +``` + +See all the `spacy convert` options: https://spacy.io/api/cli#convert + +--- + +The spaCy v2 JSON training files were generated using **spaCy v2** with: + +```bash +python -m spacy convert -c iob -s -n 10 -b en file.iob +``` + +To convert an existing JSON training file to `.spacy` for spaCy v3, convert +with **spaCy v3**: + +```bash +python -m spacy convert file.json . +``` diff --git a/extra/example_data/ner_example_data/ner-sent-per-line.iob b/extra/example_data/ner_example_data/ner-sent-per-line.iob new file mode 100644 index 0000000..1ff5a1a --- /dev/null +++ b/extra/example_data/ner_example_data/ner-sent-per-line.iob @@ -0,0 +1,2 @@ +When|WRB|O Sebastian|NNP|B-PERSON Thrun|NNP|I-PERSON started|VBD|O working|VBG|O on|IN|O self|NN|O -|HYPH|O driving|VBG|O cars|NNS|O at|IN|O Google|NNP|B-ORG in|IN|O 2007|CD|B-DATE ,|,|O few|JJ|O people|NNS|O outside|RB|O of|IN|O the|DT|O company|NN|O took|VBD|O him|PRP|O seriously|RB|O .|.|O +“|''|O I|PRP|O can|MD|O tell|VB|O you|PRP|O very|RB|O senior|JJ|O CEOs|NNS|O of|IN|O major|JJ|O American|JJ|B-NORP car|NN|O companies|NNS|O would|MD|O shake|VB|O my|PRP$|O hand|NN|O and|CC|O turn|VB|O away|RB|O because|IN|O I|PRP|O was|VBD|O n’t|RB|O worth|JJ|O talking|VBG|O to|IN|O ,|,|O ”|''|O said|VBD|O Thrun|NNP|B-PERSON ,|,|O in|IN|O an|DT|O interview|NN|O with|IN|O Recode|NNP|B-ORG earlier|RBR|B-DATE this|DT|I-DATE week|NN|I-DATE .|.|O diff --git a/extra/example_data/ner_example_data/ner-sent-per-line.json b/extra/example_data/ner_example_data/ner-sent-per-line.json new file mode 100644 index 0000000..3bc3513 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-sent-per-line.json @@ -0,0 +1,349 @@ +[ + { + "id":0, + "paragraphs":[ + { + "sentences":[ + { + "tokens":[ + { + "orth":"When", + "tag":"WRB", + "ner":"O" + }, + { + "orth":"Sebastian", + "tag":"NNP", + "ner":"B-PERSON" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"L-PERSON" + }, + { + "orth":"started", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"working", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"on", + "tag":"IN", + "ner":"O" + }, + { + "orth":"self", + "tag":"NN", + "ner":"O" + }, + { + "orth":"-", + "tag":"HYPH", + "ner":"O" + }, + { + "orth":"driving", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"cars", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"at", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Google", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"2007", + "tag":"CD", + "ner":"U-DATE" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"few", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"people", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"outside", + "tag":"RB", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"the", + "tag":"DT", + "ner":"O" + }, + { + "orth":"company", + "tag":"NN", + "ner":"O" + }, + { + "orth":"took", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"him", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"seriously", + "tag":"RB", + "ner":"O" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"\u201c", + "tag":"''", + "ner":"O" + }, + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"can", + "tag":"MD", + "ner":"O" + }, + { + "orth":"tell", + "tag":"VB", + "ner":"O" + }, + { + "orth":"you", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"very", + "tag":"RB", + "ner":"O" + }, + { + "orth":"senior", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"CEOs", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"major", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"American", + "tag":"JJ", + "ner":"U-NORP" + }, + { + "orth":"car", + "tag":"NN", + "ner":"O" + }, + { + "orth":"companies", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"would", + "tag":"MD", + "ner":"O" + }, + { + "orth":"shake", + "tag":"VB", + "ner":"O" + }, + { + "orth":"my", + "tag":"PRP$", + "ner":"O" + }, + { + "orth":"hand", + "tag":"NN", + "ner":"O" + }, + { + "orth":"and", + "tag":"CC", + "ner":"O" + }, + { + "orth":"turn", + "tag":"VB", + "ner":"O" + }, + { + "orth":"away", + "tag":"RB", + "ner":"O" + }, + { + "orth":"because", + "tag":"IN", + "ner":"O" + }, + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"was", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"n\u2019t", + "tag":"RB", + "ner":"O" + }, + { + "orth":"worth", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"talking", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"to", + "tag":"IN", + "ner":"O" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"\u201d", + "tag":"''", + "ner":"O" + }, + { + "orth":"said", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"U-PERSON" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"an", + "tag":"DT", + "ner":"O" + }, + { + "orth":"interview", + "tag":"NN", + "ner":"O" + }, + { + "orth":"with", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Recode", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"earlier", + "tag":"RBR", + "ner":"B-DATE" + }, + { + "orth":"this", + "tag":"DT", + "ner":"I-DATE" + }, + { + "orth":"week", + "tag":"NN", + "ner":"L-DATE" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/ner_example_data/ner-token-per-line-conll2003.iob b/extra/example_data/ner_example_data/ner-token-per-line-conll2003.iob new file mode 100644 index 0000000..8a2c4f9 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line-conll2003.iob @@ -0,0 +1,70 @@ +-DOCSTART- -X- O O + +When WRB _ O +Sebastian NNP _ B-PERSON +Thrun NNP _ I-PERSON +started VBD _ O +working VBG _ O +on IN _ O +self NN _ O +- HYPH _ O +driving VBG _ O +cars NNS _ O +at IN _ O +Google NNP _ B-ORG +in IN _ O +2007 CD _ B-DATE +, , _ O +few JJ _ O +people NNS _ O +outside RB _ O +of IN _ O +the DT _ O +company NN _ O +took VBD _ O +him PRP _ O +seriously RB _ O +. . _ O + +“ '' _ O +I PRP _ O +can MD _ O +tell VB _ O +you PRP _ O +very RB _ O +senior JJ _ O +CEOs NNS _ O +of IN _ O +major JJ _ O +American JJ _ B-NORP +car NN _ O +companies NNS _ O +would MD _ O +shake VB _ O +my PRP$ _ O +hand NN _ O +and CC _ O +turn VB _ O +away RB _ O +because IN _ O +I PRP _ O +was VBD _ O +n’t RB _ O +worth JJ _ O +talking VBG _ O +to IN _ O +, , _ O +” '' _ O +said VBD _ O +Thrun NNP _ B-PERSON +, , _ O +in IN _ O +an DT _ O +interview NN _ O +with IN _ O +Recode NNP _ B-ORG +earlier RBR _ B-DATE +this DT _ I-DATE +week NN _ I-DATE +. . _ O + diff --git a/extra/example_data/ner_example_data/ner-token-per-line-conll2003.json b/extra/example_data/ner_example_data/ner-token-per-line-conll2003.json new file mode 100644 index 0000000..3bc3513 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line-conll2003.json @@ -0,0 +1,349 @@ +[ + { + "id":0, + "paragraphs":[ + { + "sentences":[ + { + "tokens":[ + { + "orth":"When", + "tag":"WRB", + "ner":"O" + }, + { + "orth":"Sebastian", + "tag":"NNP", + "ner":"B-PERSON" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"L-PERSON" + }, + { + "orth":"started", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"working", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"on", + "tag":"IN", + "ner":"O" + }, + { + "orth":"self", + "tag":"NN", + "ner":"O" + }, + { + "orth":"-", + "tag":"HYPH", + "ner":"O" + }, + { + "orth":"driving", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"cars", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"at", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Google", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"2007", + "tag":"CD", + "ner":"U-DATE" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"few", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"people", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"outside", + "tag":"RB", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"the", + "tag":"DT", + "ner":"O" + }, + { + "orth":"company", + "tag":"NN", + "ner":"O" + }, + { + "orth":"took", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"him", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"seriously", + "tag":"RB", + "ner":"O" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"\u201c", + "tag":"''", + "ner":"O" + }, + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"can", + "tag":"MD", + "ner":"O" + }, + { + "orth":"tell", + "tag":"VB", + "ner":"O" + }, + { + "orth":"you", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"very", + "tag":"RB", + "ner":"O" + }, + { + "orth":"senior", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"CEOs", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"major", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"American", + "tag":"JJ", + "ner":"U-NORP" + }, + { + "orth":"car", + "tag":"NN", + "ner":"O" + }, + { + "orth":"companies", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"would", + "tag":"MD", + "ner":"O" + }, + { + "orth":"shake", + "tag":"VB", + "ner":"O" + }, + { + "orth":"my", + "tag":"PRP$", + "ner":"O" + }, + { + "orth":"hand", + "tag":"NN", + "ner":"O" + }, + { + "orth":"and", + "tag":"CC", + "ner":"O" + }, + { + "orth":"turn", + "tag":"VB", + "ner":"O" + }, + { + "orth":"away", + "tag":"RB", + "ner":"O" + }, + { + "orth":"because", + "tag":"IN", + "ner":"O" + }, + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"was", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"n\u2019t", + "tag":"RB", + "ner":"O" + }, + { + "orth":"worth", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"talking", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"to", + "tag":"IN", + "ner":"O" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"\u201d", + "tag":"''", + "ner":"O" + }, + { + "orth":"said", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"U-PERSON" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"an", + "tag":"DT", + "ner":"O" + }, + { + "orth":"interview", + "tag":"NN", + "ner":"O" + }, + { + "orth":"with", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Recode", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"earlier", + "tag":"RBR", + "ner":"B-DATE" + }, + { + "orth":"this", + "tag":"DT", + "ner":"I-DATE" + }, + { + "orth":"week", + "tag":"NN", + "ner":"L-DATE" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/ner_example_data/ner-token-per-line-with-pos.iob b/extra/example_data/ner_example_data/ner-token-per-line-with-pos.iob new file mode 100644 index 0000000..0733982 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line-with-pos.iob @@ -0,0 +1,66 @@ +When WRB O +Sebastian NNP B-PERSON +Thrun NNP I-PERSON +started VBD O +working VBG O +on IN O +self NN O +- HYPH O +driving VBG O +cars NNS O +at IN O +Google NNP B-ORG +in IN O +2007 CD B-DATE +, , O +few JJ O +people NNS O +outside RB O +of IN O +the DT O +company NN O +took VBD O +him PRP O +seriously RB O +. . O +“ '' O +I PRP O +can MD O +tell VB O +you PRP O +very RB O +senior JJ O +CEOs NNS O +of IN O +major JJ O +American JJ B-NORP +car NN O +companies NNS O +would MD O +shake VB O +my PRP$ O +hand NN O +and CC O +turn VB O +away RB O +because IN O +I PRP O +was VBD O +n’t RB O +worth JJ O +talking VBG O +to IN O +, , O +” '' O +said VBD O +Thrun NNP B-PERSON +, , O +in IN O +an DT O +interview NN O +with IN O +Recode NNP B-ORG +earlier RBR B-DATE +this DT I-DATE +week NN I-DATE +. . O diff --git a/extra/example_data/ner_example_data/ner-token-per-line-with-pos.json b/extra/example_data/ner_example_data/ner-token-per-line-with-pos.json new file mode 100644 index 0000000..8645097 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line-with-pos.json @@ -0,0 +1,353 @@ +[ + { + "id":0, + "paragraphs":[ + { + "sentences":[ + { + "tokens":[ + { + "orth":"When", + "tag":"WRB", + "ner":"O" + }, + { + "orth":"Sebastian", + "tag":"NNP", + "ner":"B-PERSON" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"L-PERSON" + }, + { + "orth":"started", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"working", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"on", + "tag":"IN", + "ner":"O" + }, + { + "orth":"self", + "tag":"NN", + "ner":"O" + }, + { + "orth":"-", + "tag":"HYPH", + "ner":"O" + }, + { + "orth":"driving", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"cars", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"at", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Google", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"2007", + "tag":"CD", + "ner":"U-DATE" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"few", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"people", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"outside", + "tag":"RB", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"the", + "tag":"DT", + "ner":"O" + }, + { + "orth":"company", + "tag":"NN", + "ner":"O" + }, + { + "orth":"took", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"him", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"seriously", + "tag":"RB", + "ner":"O" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"\u201c", + "tag":"''", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"can", + "tag":"MD", + "ner":"O" + }, + { + "orth":"tell", + "tag":"VB", + "ner":"O" + }, + { + "orth":"you", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"very", + "tag":"RB", + "ner":"O" + }, + { + "orth":"senior", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"CEOs", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"of", + "tag":"IN", + "ner":"O" + }, + { + "orth":"major", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"American", + "tag":"JJ", + "ner":"U-NORP" + }, + { + "orth":"car", + "tag":"NN", + "ner":"O" + }, + { + "orth":"companies", + "tag":"NNS", + "ner":"O" + }, + { + "orth":"would", + "tag":"MD", + "ner":"O" + }, + { + "orth":"shake", + "tag":"VB", + "ner":"O" + }, + { + "orth":"my", + "tag":"PRP$", + "ner":"O" + }, + { + "orth":"hand", + "tag":"NN", + "ner":"O" + }, + { + "orth":"and", + "tag":"CC", + "ner":"O" + }, + { + "orth":"turn", + "tag":"VB", + "ner":"O" + }, + { + "orth":"away", + "tag":"RB", + "ner":"O" + }, + { + "orth":"because", + "tag":"IN", + "ner":"O" + }, + { + "orth":"I", + "tag":"PRP", + "ner":"O" + }, + { + "orth":"was", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"n\u2019t", + "tag":"RB", + "ner":"O" + }, + { + "orth":"worth", + "tag":"JJ", + "ner":"O" + }, + { + "orth":"talking", + "tag":"VBG", + "ner":"O" + }, + { + "orth":"to", + "tag":"IN", + "ner":"O" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"\u201d", + "tag":"''", + "ner":"O" + }, + { + "orth":"said", + "tag":"VBD", + "ner":"O" + }, + { + "orth":"Thrun", + "tag":"NNP", + "ner":"U-PERSON" + }, + { + "orth":",", + "tag":",", + "ner":"O" + }, + { + "orth":"in", + "tag":"IN", + "ner":"O" + }, + { + "orth":"an", + "tag":"DT", + "ner":"O" + }, + { + "orth":"interview", + "tag":"NN", + "ner":"O" + }, + { + "orth":"with", + "tag":"IN", + "ner":"O" + }, + { + "orth":"Recode", + "tag":"NNP", + "ner":"U-ORG" + }, + { + "orth":"earlier", + "tag":"RBR", + "ner":"B-DATE" + }, + { + "orth":"this", + "tag":"DT", + "ner":"I-DATE" + }, + { + "orth":"week", + "tag":"NN", + "ner":"L-DATE" + }, + { + "orth":".", + "tag":".", + "ner":"O" + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/ner_example_data/ner-token-per-line.iob b/extra/example_data/ner_example_data/ner-token-per-line.iob new file mode 100644 index 0000000..d00fb5d --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line.iob @@ -0,0 +1,66 @@ +When O +Sebastian B-PERSON +Thrun I-PERSON +started O +working O +on O +self O +- O +driving O +cars O +at O +Google B-ORG +in O +2007 B-DATE +, O +few O +people O +outside O +of O +the O +company O +took O +him O +seriously O +. O +“ O +I O +can O +tell O +you O +very O +senior O +CEOs O +of O +major O +American B-NORP +car O +companies O +would O +shake O +my O +hand O +and O +turn O +away O +because O +I O +was O +n’t O +worth O +talking O +to O +, O +” O +said O +Thrun B-PERSON +, O +in O +an O +interview O +with O +Recode B-ORG +earlier B-DATE +this I-DATE +week I-DATE +. O diff --git a/extra/example_data/ner_example_data/ner-token-per-line.json b/extra/example_data/ner_example_data/ner-token-per-line.json new file mode 100644 index 0000000..37f7776 --- /dev/null +++ b/extra/example_data/ner_example_data/ner-token-per-line.json @@ -0,0 +1,353 @@ +[ + { + "id":0, + "paragraphs":[ + { + "sentences":[ + { + "tokens":[ + { + "orth":"When", + "tag":"-", + "ner":"O" + }, + { + "orth":"Sebastian", + "tag":"-", + "ner":"B-PERSON" + }, + { + "orth":"Thrun", + "tag":"-", + "ner":"L-PERSON" + }, + { + "orth":"started", + "tag":"-", + "ner":"O" + }, + { + "orth":"working", + "tag":"-", + "ner":"O" + }, + { + "orth":"on", + "tag":"-", + "ner":"O" + }, + { + "orth":"self", + "tag":"-", + "ner":"O" + }, + { + "orth":"-", + "tag":"-", + "ner":"O" + }, + { + "orth":"driving", + "tag":"-", + "ner":"O" + }, + { + "orth":"cars", + "tag":"-", + "ner":"O" + }, + { + "orth":"at", + "tag":"-", + "ner":"O" + }, + { + "orth":"Google", + "tag":"-", + "ner":"U-ORG" + }, + { + "orth":"in", + "tag":"-", + "ner":"O" + }, + { + "orth":"2007", + "tag":"-", + "ner":"U-DATE" + }, + { + "orth":",", + "tag":"-", + "ner":"O" + }, + { + "orth":"few", + "tag":"-", + "ner":"O" + }, + { + "orth":"people", + "tag":"-", + "ner":"O" + }, + { + "orth":"outside", + "tag":"-", + "ner":"O" + }, + { + "orth":"of", + "tag":"-", + "ner":"O" + }, + { + "orth":"the", + "tag":"-", + "ner":"O" + }, + { + "orth":"company", + "tag":"-", + "ner":"O" + }, + { + "orth":"took", + "tag":"-", + "ner":"O" + }, + { + "orth":"him", + "tag":"-", + "ner":"O" + }, + { + "orth":"seriously", + "tag":"-", + "ner":"O" + }, + { + "orth":".", + "tag":"-", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"\u201c", + "tag":"-", + "ner":"O" + } + ] + }, + { + "tokens":[ + { + "orth":"I", + "tag":"-", + "ner":"O" + }, + { + "orth":"can", + "tag":"-", + "ner":"O" + }, + { + "orth":"tell", + "tag":"-", + "ner":"O" + }, + { + "orth":"you", + "tag":"-", + "ner":"O" + }, + { + "orth":"very", + "tag":"-", + "ner":"O" + }, + { + "orth":"senior", + "tag":"-", + "ner":"O" + }, + { + "orth":"CEOs", + "tag":"-", + "ner":"O" + }, + { + "orth":"of", + "tag":"-", + "ner":"O" + }, + { + "orth":"major", + "tag":"-", + "ner":"O" + }, + { + "orth":"American", + "tag":"-", + "ner":"U-NORP" + }, + { + "orth":"car", + "tag":"-", + "ner":"O" + }, + { + "orth":"companies", + "tag":"-", + "ner":"O" + }, + { + "orth":"would", + "tag":"-", + "ner":"O" + }, + { + "orth":"shake", + "tag":"-", + "ner":"O" + }, + { + "orth":"my", + "tag":"-", + "ner":"O" + }, + { + "orth":"hand", + "tag":"-", + "ner":"O" + }, + { + "orth":"and", + "tag":"-", + "ner":"O" + }, + { + "orth":"turn", + "tag":"-", + "ner":"O" + }, + { + "orth":"away", + "tag":"-", + "ner":"O" + }, + { + "orth":"because", + "tag":"-", + "ner":"O" + }, + { + "orth":"I", + "tag":"-", + "ner":"O" + }, + { + "orth":"was", + "tag":"-", + "ner":"O" + }, + { + "orth":"n\u2019t", + "tag":"-", + "ner":"O" + }, + { + "orth":"worth", + "tag":"-", + "ner":"O" + }, + { + "orth":"talking", + "tag":"-", + "ner":"O" + }, + { + "orth":"to", + "tag":"-", + "ner":"O" + }, + { + "orth":",", + "tag":"-", + "ner":"O" + }, + { + "orth":"\u201d", + "tag":"-", + "ner":"O" + }, + { + "orth":"said", + "tag":"-", + "ner":"O" + }, + { + "orth":"Thrun", + "tag":"-", + "ner":"U-PERSON" + }, + { + "orth":",", + "tag":"-", + "ner":"O" + }, + { + "orth":"in", + "tag":"-", + "ner":"O" + }, + { + "orth":"an", + "tag":"-", + "ner":"O" + }, + { + "orth":"interview", + "tag":"-", + "ner":"O" + }, + { + "orth":"with", + "tag":"-", + "ner":"O" + }, + { + "orth":"Recode", + "tag":"-", + "ner":"U-ORG" + }, + { + "orth":"earlier", + "tag":"-", + "ner":"B-DATE" + }, + { + "orth":"this", + "tag":"-", + "ner":"I-DATE" + }, + { + "orth":"week", + "tag":"-", + "ner":"L-DATE" + }, + { + "orth":".", + "tag":"-", + "ner":"O" + } + ] + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/textcat_example_data/CC0.txt b/extra/example_data/textcat_example_data/CC0.txt new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/extra/example_data/textcat_example_data/CC0.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/extra/example_data/textcat_example_data/CC_BY-SA-3.0.txt b/extra/example_data/textcat_example_data/CC_BY-SA-3.0.txt new file mode 100644 index 0000000..604209a --- /dev/null +++ b/extra/example_data/textcat_example_data/CC_BY-SA-3.0.txt @@ -0,0 +1,359 @@ +Creative Commons Legal Code + +Attribution-ShareAlike 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE +COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY +COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS +AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE +TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY +BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS +CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND +CONDITIONS. + +1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined below) for the purposes of this + License. + c. "Creative Commons Compatible License" means a license that is listed + at https://creativecommons.org/compatiblelicenses that has been + approved by Creative Commons as being essentially equivalent to this + License, including, at a minimum, because that license: (i) contains + terms that have the same purpose, meaning and effect as the License + Elements of this License; and, (ii) explicitly permits the relicensing + of adaptations of works made available under that license under this + License or a Creative Commons jurisdiction license with the same + License Elements as this License. + d. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + e. "License Elements" means the following high-level license attributes + as selected by Licensor and indicated in the title of this License: + Attribution, ShareAlike. + f. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + g. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + h. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + i. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + j. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + k. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + +2. Fair Dealing Rights. Nothing in this License is intended to reduce, +limit, or restrict any uses free from copyright or rights arising from +limitations or exceptions that are provided for in connection with the +copyright protection under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, +Licensor hereby grants You a worldwide, royalty-free, non-exclusive, +perpetual (for the duration of the applicable copyright) license to +exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + +The above rights may be exercised in all media and formats whether now +known or hereafter devised. The above rights include the right to make +such modifications as are technically necessary to exercise the rights in +other media and formats. Subject to Section 8(f), all rights not expressly +granted by Licensor are hereby reserved. + +4. Restrictions. The license granted in Section 3 above is expressly made +subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(c), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(c), as requested. + b. You may Distribute or Publicly Perform an Adaptation only under the + terms of: (i) this License; (ii) a later version of this License with + the same License Elements as this License; (iii) a Creative Commons + jurisdiction license (either this or a later license version) that + contains the same License Elements as this License (e.g., + Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible + License. If you license the Adaptation under one of the licenses + mentioned in (iv), you must comply with the terms of that license. If + you license the Adaptation under the terms of any of the licenses + mentioned in (i), (ii) or (iii) (the "Applicable License"), you must + comply with the terms of the Applicable License generally and the + following provisions: (I) You must include a copy of, or the URI for, + the Applicable License with every copy of each Adaptation You + Distribute or Publicly Perform; (II) You may not offer or impose any + terms on the Adaptation that restrict the terms of the Applicable + License or the ability of the recipient of the Adaptation to exercise + the rights granted to that recipient under the terms of the Applicable + License; (III) You must keep intact all notices that refer to the + Applicable License and to the disclaimer of warranties with every copy + of the Work as included in the Adaptation You Distribute or Publicly + Perform; (IV) when You Distribute or Publicly Perform the Adaptation, + You may not impose any effective technological measures on the + Adaptation that restrict the ability of a recipient of the Adaptation + from You to exercise the rights granted to that recipient under the + terms of the Applicable License. This Section 4(b) applies to the + Adaptation as incorporated in a Collection, but this does not require + the Collection apart from the Adaptation itself to be made subject to + the terms of the Applicable License. + c. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Ssection 3(b), in the case of an + Adaptation, a credit identifying the use of the Work in the Adaptation + (e.g., "French translation of the Work by Original Author," or + "Screenplay based on original Work by Original Author"). The credit + required by this Section 4(c) may be implemented in any reasonable + manner; provided, however, that in the case of a Adaptation or + Collection, at a minimum such credit will appear, if a credit for all + contributing authors of the Adaptation or Collection appears, then as + part of these credits and in a manner at least as prominent as the + credits for the other contributing authors. For the avoidance of + doubt, You may only use the credit required by this Section for the + purpose of attribution in the manner set out above and, by exercising + Your rights under this License, You may not implicitly or explicitly + assert or imply any connection with, sponsorship or endorsement by the + Original Author, Licensor and/or Attribution Parties, as appropriate, + of You or Your use of the Work, without the separate, express prior + written permission of the Original Author, Licensor and/or Attribution + Parties. + d. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR +OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY +KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, +INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, +FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF +LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, +WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE +LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR +ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES +ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + +8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + +Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of the License. + + Creative Commons may be contacted at https://creativecommons.org/. diff --git a/extra/example_data/textcat_example_data/CC_BY-SA-4.0.txt b/extra/example_data/textcat_example_data/CC_BY-SA-4.0.txt new file mode 100644 index 0000000..a73481c --- /dev/null +++ b/extra/example_data/textcat_example_data/CC_BY-SA-4.0.txt @@ -0,0 +1,428 @@ +Attribution-ShareAlike 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + + including for purposes of Section 3(b); and + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/extra/example_data/textcat_example_data/README.md b/extra/example_data/textcat_example_data/README.md new file mode 100644 index 0000000..1165f02 --- /dev/null +++ b/extra/example_data/textcat_example_data/README.md @@ -0,0 +1,34 @@ +## Examples of textcat training data + +spacy JSON training files were generated from JSONL with: + +``` +python textcatjsonl_to_trainjson.py -m en file.jsonl . +``` + +`cooking.json` is an example with mutually-exclusive classes with two labels: + +* `baking` +* `not_baking` + +`jigsaw-toxic-comment.json` is an example with multiple labels per instance: + +* `insult` +* `obscene` +* `severe_toxic` +* `toxic` + +### Data Sources + +* `cooking.jsonl`: https://cooking.stackexchange.com. The meta IDs link to the + original question as `https://cooking.stackexchange.com/questions/ID`, e.g., + `https://cooking.stackexchange.com/questions/2` for the first instance. +* `jigsaw-toxic-comment.jsonl`: [Jigsaw Toxic Comments Classification + Challenge](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge) + +### Data Licenses + +* `cooking.jsonl`: CC BY-SA 4.0 ([`CC_BY-SA-4.0.txt`](CC_BY-SA-4.0.txt)) +* `jigsaw-toxic-comment.jsonl`: + * text: CC BY-SA 3.0 ([`CC_BY-SA-3.0.txt`](CC_BY-SA-3.0.txt)) + * annotation: CC0 ([`CC0.txt`](CC0.txt)) diff --git a/extra/example_data/textcat_example_data/cooking.json b/extra/example_data/textcat_example_data/cooking.json new file mode 100644 index 0000000..4bad4db --- /dev/null +++ b/extra/example_data/textcat_example_data/cooking.json @@ -0,0 +1,3487 @@ +[ + { + "id":0, + "paragraphs":[ + { + "raw":"How should I cook bacon in an oven?\nI've heard of people cooking bacon in an oven by laying the strips out on a cookie sheet. When using this method, how long should I cook the bacon for, and at what temperature?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"How", + "ner":"O" + }, + { + "id":1, + "orth":"should", + "ner":"O" + }, + { + "id":2, + "orth":"I", + "ner":"O" + }, + { + "id":3, + "orth":"cook", + "ner":"O" + }, + { + "id":4, + "orth":"bacon", + "ner":"O" + }, + { + "id":5, + "orth":"in", + "ner":"O" + }, + { + "id":6, + "orth":"an", + "ner":"O" + }, + { + "id":7, + "orth":"oven", + "ner":"O" + }, + { + "id":8, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":9, + "orth":"\n", + "ner":"O" + }, + { + "id":10, + "orth":"I", + "ner":"O" + }, + { + "id":11, + "orth":"'ve", + "ner":"O" + }, + { + "id":12, + "orth":"heard", + "ner":"O" + }, + { + "id":13, + "orth":"of", + "ner":"O" + }, + { + "id":14, + "orth":"people", + "ner":"O" + }, + { + "id":15, + "orth":"cooking", + "ner":"O" + }, + { + "id":16, + "orth":"bacon", + "ner":"O" + }, + { + "id":17, + "orth":"in", + "ner":"O" + }, + { + "id":18, + "orth":"an", + "ner":"O" + }, + { + "id":19, + "orth":"oven", + "ner":"O" + }, + { + "id":20, + "orth":"by", + "ner":"O" + }, + { + "id":21, + "orth":"laying", + "ner":"O" + }, + { + "id":22, + "orth":"the", + "ner":"O" + }, + { + "id":23, + "orth":"strips", + "ner":"O" + }, + { + "id":24, + "orth":"out", + "ner":"O" + }, + { + "id":25, + "orth":"on", + "ner":"O" + }, + { + "id":26, + "orth":"a", + "ner":"O" + }, + { + "id":27, + "orth":"cookie", + "ner":"O" + }, + { + "id":28, + "orth":"sheet", + "ner":"O" + }, + { + "id":29, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":30, + "orth":"When", + "ner":"O" + }, + { + "id":31, + "orth":"using", + "ner":"O" + }, + { + "id":32, + "orth":"this", + "ner":"O" + }, + { + "id":33, + "orth":"method", + "ner":"O" + }, + { + "id":34, + "orth":",", + "ner":"O" + }, + { + "id":35, + "orth":"how", + "ner":"O" + }, + { + "id":36, + "orth":"long", + "ner":"O" + }, + { + "id":37, + "orth":"should", + "ner":"O" + }, + { + "id":38, + "orth":"I", + "ner":"O" + }, + { + "id":39, + "orth":"cook", + "ner":"O" + }, + { + "id":40, + "orth":"the", + "ner":"O" + }, + { + "id":41, + "orth":"bacon", + "ner":"O" + }, + { + "id":42, + "orth":"for", + "ner":"O" + }, + { + "id":43, + "orth":",", + "ner":"O" + }, + { + "id":44, + "orth":"and", + "ner":"O" + }, + { + "id":45, + "orth":"at", + "ner":"O" + }, + { + "id":46, + "orth":"what", + "ner":"O" + }, + { + "id":47, + "orth":"temperature", + "ner":"O" + }, + { + "id":48, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":49, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"What is the difference between white and brown eggs?\nI always use brown extra large eggs, but I can't honestly say why I do this other than habit at this point. Are there any distinct advantages or disadvantages like flavor, shelf life, etc?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"What", + "ner":"O" + }, + { + "id":1, + "orth":"is", + "ner":"O" + }, + { + "id":2, + "orth":"the", + "ner":"O" + }, + { + "id":3, + "orth":"difference", + "ner":"O" + }, + { + "id":4, + "orth":"between", + "ner":"O" + }, + { + "id":5, + "orth":"white", + "ner":"O" + }, + { + "id":6, + "orth":"and", + "ner":"O" + }, + { + "id":7, + "orth":"brown", + "ner":"O" + }, + { + "id":8, + "orth":"eggs", + "ner":"O" + }, + { + "id":9, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":10, + "orth":"\n", + "ner":"O" + }, + { + "id":11, + "orth":"I", + "ner":"O" + }, + { + "id":12, + "orth":"always", + "ner":"O" + }, + { + "id":13, + "orth":"use", + "ner":"O" + }, + { + "id":14, + "orth":"brown", + "ner":"O" + }, + { + "id":15, + "orth":"extra", + "ner":"O" + }, + { + "id":16, + "orth":"large", + "ner":"O" + }, + { + "id":17, + "orth":"eggs", + "ner":"O" + }, + { + "id":18, + "orth":",", + "ner":"O" + }, + { + "id":19, + "orth":"but", + "ner":"O" + }, + { + "id":20, + "orth":"I", + "ner":"O" + }, + { + "id":21, + "orth":"ca", + "ner":"O" + }, + { + "id":22, + "orth":"n't", + "ner":"O" + }, + { + "id":23, + "orth":"honestly", + "ner":"O" + }, + { + "id":24, + "orth":"say", + "ner":"O" + }, + { + "id":25, + "orth":"why", + "ner":"O" + }, + { + "id":26, + "orth":"I", + "ner":"O" + }, + { + "id":27, + "orth":"do", + "ner":"O" + }, + { + "id":28, + "orth":"this", + "ner":"O" + }, + { + "id":29, + "orth":"other", + "ner":"O" + }, + { + "id":30, + "orth":"than", + "ner":"O" + }, + { + "id":31, + "orth":"habit", + "ner":"O" + }, + { + "id":32, + "orth":"at", + "ner":"O" + }, + { + "id":33, + "orth":"this", + "ner":"O" + }, + { + "id":34, + "orth":"point", + "ner":"O" + }, + { + "id":35, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":36, + "orth":"Are", + "ner":"O" + }, + { + "id":37, + "orth":"there", + "ner":"O" + }, + { + "id":38, + "orth":"any", + "ner":"O" + }, + { + "id":39, + "orth":"distinct", + "ner":"O" + }, + { + "id":40, + "orth":"advantages", + "ner":"O" + }, + { + "id":41, + "orth":"or", + "ner":"O" + }, + { + "id":42, + "orth":"disadvantages", + "ner":"O" + }, + { + "id":43, + "orth":"like", + "ner":"O" + }, + { + "id":44, + "orth":"flavor", + "ner":"O" + }, + { + "id":45, + "orth":",", + "ner":"O" + }, + { + "id":46, + "orth":"shelf", + "ner":"O" + }, + { + "id":47, + "orth":"life", + "ner":"O" + }, + { + "id":48, + "orth":",", + "ner":"O" + }, + { + "id":49, + "orth":"etc", + "ner":"O" + }, + { + "id":50, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":51, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"What is the difference between baking soda and baking powder?\nAnd can I use one in place of the other in certain recipes?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"What", + "ner":"O" + }, + { + "id":1, + "orth":"is", + "ner":"O" + }, + { + "id":2, + "orth":"the", + "ner":"O" + }, + { + "id":3, + "orth":"difference", + "ner":"O" + }, + { + "id":4, + "orth":"between", + "ner":"O" + }, + { + "id":5, + "orth":"baking", + "ner":"O" + }, + { + "id":6, + "orth":"soda", + "ner":"O" + }, + { + "id":7, + "orth":"and", + "ner":"O" + }, + { + "id":8, + "orth":"baking", + "ner":"O" + }, + { + "id":9, + "orth":"powder", + "ner":"O" + }, + { + "id":10, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":11, + "orth":"\n", + "ner":"O" + }, + { + "id":12, + "orth":"And", + "ner":"O" + }, + { + "id":13, + "orth":"can", + "ner":"O" + }, + { + "id":14, + "orth":"I", + "ner":"O" + }, + { + "id":15, + "orth":"use", + "ner":"O" + }, + { + "id":16, + "orth":"one", + "ner":"O" + }, + { + "id":17, + "orth":"in", + "ner":"O" + }, + { + "id":18, + "orth":"place", + "ner":"O" + }, + { + "id":19, + "orth":"of", + "ner":"O" + }, + { + "id":20, + "orth":"the", + "ner":"O" + }, + { + "id":21, + "orth":"other", + "ner":"O" + }, + { + "id":22, + "orth":"in", + "ner":"O" + }, + { + "id":23, + "orth":"certain", + "ner":"O" + }, + { + "id":24, + "orth":"recipes", + "ner":"O" + }, + { + "id":25, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":26, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"In a tomato sauce recipe, how can I cut the acidity?\nIt seems that every time I make a tomato sauce for pasta, the sauce is a little bit too acid for my taste. I've tried using sugar or sodium bicarbonate, but I'm not satisfied with the results.\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"In", + "ner":"O" + }, + { + "id":1, + "orth":"a", + "ner":"O" + }, + { + "id":2, + "orth":"tomato", + "ner":"O" + }, + { + "id":3, + "orth":"sauce", + "ner":"O" + }, + { + "id":4, + "orth":"recipe", + "ner":"O" + }, + { + "id":5, + "orth":",", + "ner":"O" + }, + { + "id":6, + "orth":"how", + "ner":"O" + }, + { + "id":7, + "orth":"can", + "ner":"O" + }, + { + "id":8, + "orth":"I", + "ner":"O" + }, + { + "id":9, + "orth":"cut", + "ner":"O" + }, + { + "id":10, + "orth":"the", + "ner":"O" + }, + { + "id":11, + "orth":"acidity", + "ner":"O" + }, + { + "id":12, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":13, + "orth":"\n", + "ner":"O" + }, + { + "id":14, + "orth":"It", + "ner":"O" + }, + { + "id":15, + "orth":"seems", + "ner":"O" + }, + { + "id":16, + "orth":"that", + "ner":"O" + }, + { + "id":17, + "orth":"every", + "ner":"O" + }, + { + "id":18, + "orth":"time", + "ner":"O" + }, + { + "id":19, + "orth":"I", + "ner":"O" + }, + { + "id":20, + "orth":"make", + "ner":"O" + }, + { + "id":21, + "orth":"a", + "ner":"O" + }, + { + "id":22, + "orth":"tomato", + "ner":"O" + }, + { + "id":23, + "orth":"sauce", + "ner":"O" + }, + { + "id":24, + "orth":"for", + "ner":"O" + }, + { + "id":25, + "orth":"pasta", + "ner":"O" + }, + { + "id":26, + "orth":",", + "ner":"O" + }, + { + "id":27, + "orth":"the", + "ner":"O" + }, + { + "id":28, + "orth":"sauce", + "ner":"O" + }, + { + "id":29, + "orth":"is", + "ner":"O" + }, + { + "id":30, + "orth":"a", + "ner":"O" + }, + { + "id":31, + "orth":"little", + "ner":"O" + }, + { + "id":32, + "orth":"bit", + "ner":"O" + }, + { + "id":33, + "orth":"too", + "ner":"O" + }, + { + "id":34, + "orth":"acid", + "ner":"O" + }, + { + "id":35, + "orth":"for", + "ner":"O" + }, + { + "id":36, + "orth":"my", + "ner":"O" + }, + { + "id":37, + "orth":"taste", + "ner":"O" + }, + { + "id":38, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":39, + "orth":"I", + "ner":"O" + }, + { + "id":40, + "orth":"'ve", + "ner":"O" + }, + { + "id":41, + "orth":"tried", + "ner":"O" + }, + { + "id":42, + "orth":"using", + "ner":"O" + }, + { + "id":43, + "orth":"sugar", + "ner":"O" + }, + { + "id":44, + "orth":"or", + "ner":"O" + }, + { + "id":45, + "orth":"sodium", + "ner":"O" + }, + { + "id":46, + "orth":"bicarbonate", + "ner":"O" + }, + { + "id":47, + "orth":",", + "ner":"O" + }, + { + "id":48, + "orth":"but", + "ner":"O" + }, + { + "id":49, + "orth":"I", + "ner":"O" + }, + { + "id":50, + "orth":"'m", + "ner":"O" + }, + { + "id":51, + "orth":"not", + "ner":"O" + }, + { + "id":52, + "orth":"satisfied", + "ner":"O" + }, + { + "id":53, + "orth":"with", + "ner":"O" + }, + { + "id":54, + "orth":"the", + "ner":"O" + }, + { + "id":55, + "orth":"results", + "ner":"O" + }, + { + "id":56, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":57, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"What ingredients (available in specific regions) can I substitute for parsley?\nI have a recipe that calls for fresh parsley. I have substituted other fresh herbs for their dried equivalents but I don't have fresh or dried parsley. Is there something else (ex another dried herb) that I can use instead of parsley?\nI know it is used mainly for looks rather than taste but I have a pasta recipe that calls for 2 tablespoons of parsley in the sauce and then another 2 tablespoons on top when it is done. I know the parsley on top is more for looks but there must be something about the taste otherwise it would call for parsley within the sauce as well.\nI would especially like to hear about substitutes available in Southeast Asia and other parts of the world where the obvious answers (such as cilantro) are not widely available.\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"What", + "ner":"O" + }, + { + "id":1, + "orth":"ingredients", + "ner":"O" + }, + { + "id":2, + "orth":"(", + "ner":"O" + }, + { + "id":3, + "orth":"available", + "ner":"O" + }, + { + "id":4, + "orth":"in", + "ner":"O" + }, + { + "id":5, + "orth":"specific", + "ner":"O" + }, + { + "id":6, + "orth":"regions", + "ner":"O" + }, + { + "id":7, + "orth":")", + "ner":"O" + }, + { + "id":8, + "orth":"can", + "ner":"O" + }, + { + "id":9, + "orth":"I", + "ner":"O" + }, + { + "id":10, + "orth":"substitute", + "ner":"O" + }, + { + "id":11, + "orth":"for", + "ner":"O" + }, + { + "id":12, + "orth":"parsley", + "ner":"O" + }, + { + "id":13, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":14, + "orth":"\n", + "ner":"O" + }, + { + "id":15, + "orth":"I", + "ner":"O" + }, + { + "id":16, + "orth":"have", + "ner":"O" + }, + { + "id":17, + "orth":"a", + "ner":"O" + }, + { + "id":18, + "orth":"recipe", + "ner":"O" + }, + { + "id":19, + "orth":"that", + "ner":"O" + }, + { + "id":20, + "orth":"calls", + "ner":"O" + }, + { + "id":21, + "orth":"for", + "ner":"O" + }, + { + "id":22, + "orth":"fresh", + "ner":"O" + }, + { + "id":23, + "orth":"parsley", + "ner":"O" + }, + { + "id":24, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":25, + "orth":"I", + "ner":"O" + }, + { + "id":26, + "orth":"have", + "ner":"O" + }, + { + "id":27, + "orth":"substituted", + "ner":"O" + }, + { + "id":28, + "orth":"other", + "ner":"O" + }, + { + "id":29, + "orth":"fresh", + "ner":"O" + }, + { + "id":30, + "orth":"herbs", + "ner":"O" + }, + { + "id":31, + "orth":"for", + "ner":"O" + }, + { + "id":32, + "orth":"their", + "ner":"O" + }, + { + "id":33, + "orth":"dried", + "ner":"O" + }, + { + "id":34, + "orth":"equivalents", + "ner":"O" + }, + { + "id":35, + "orth":"but", + "ner":"O" + }, + { + "id":36, + "orth":"I", + "ner":"O" + }, + { + "id":37, + "orth":"do", + "ner":"O" + }, + { + "id":38, + "orth":"n't", + "ner":"O" + }, + { + "id":39, + "orth":"have", + "ner":"O" + }, + { + "id":40, + "orth":"fresh", + "ner":"O" + }, + { + "id":41, + "orth":"or", + "ner":"O" + }, + { + "id":42, + "orth":"dried", + "ner":"O" + }, + { + "id":43, + "orth":"parsley", + "ner":"O" + }, + { + "id":44, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":45, + "orth":"Is", + "ner":"O" + }, + { + "id":46, + "orth":"there", + "ner":"O" + }, + { + "id":47, + "orth":"something", + "ner":"O" + }, + { + "id":48, + "orth":"else", + "ner":"O" + }, + { + "id":49, + "orth":"(", + "ner":"O" + }, + { + "id":50, + "orth":"ex", + "ner":"O" + }, + { + "id":51, + "orth":"another", + "ner":"O" + }, + { + "id":52, + "orth":"dried", + "ner":"O" + }, + { + "id":53, + "orth":"herb", + "ner":"O" + }, + { + "id":54, + "orth":")", + "ner":"O" + }, + { + "id":55, + "orth":"that", + "ner":"O" + }, + { + "id":56, + "orth":"I", + "ner":"O" + }, + { + "id":57, + "orth":"can", + "ner":"O" + }, + { + "id":58, + "orth":"use", + "ner":"O" + }, + { + "id":59, + "orth":"instead", + "ner":"O" + }, + { + "id":60, + "orth":"of", + "ner":"O" + }, + { + "id":61, + "orth":"parsley", + "ner":"O" + }, + { + "id":62, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":63, + "orth":"\n", + "ner":"O" + }, + { + "id":64, + "orth":"I", + "ner":"O" + }, + { + "id":65, + "orth":"know", + "ner":"O" + }, + { + "id":66, + "orth":"it", + "ner":"O" + }, + { + "id":67, + "orth":"is", + "ner":"O" + }, + { + "id":68, + "orth":"used", + "ner":"O" + }, + { + "id":69, + "orth":"mainly", + "ner":"O" + }, + { + "id":70, + "orth":"for", + "ner":"O" + }, + { + "id":71, + "orth":"looks", + "ner":"O" + }, + { + "id":72, + "orth":"rather", + "ner":"O" + }, + { + "id":73, + "orth":"than", + "ner":"O" + }, + { + "id":74, + "orth":"taste", + "ner":"O" + }, + { + "id":75, + "orth":"but", + "ner":"O" + }, + { + "id":76, + "orth":"I", + "ner":"O" + }, + { + "id":77, + "orth":"have", + "ner":"O" + }, + { + "id":78, + "orth":"a", + "ner":"O" + }, + { + "id":79, + "orth":"pasta", + "ner":"O" + }, + { + "id":80, + "orth":"recipe", + "ner":"O" + }, + { + "id":81, + "orth":"that", + "ner":"O" + }, + { + "id":82, + "orth":"calls", + "ner":"O" + }, + { + "id":83, + "orth":"for", + "ner":"O" + }, + { + "id":84, + "orth":"2", + "ner":"O" + }, + { + "id":85, + "orth":"tablespoons", + "ner":"O" + }, + { + "id":86, + "orth":"of", + "ner":"O" + }, + { + "id":87, + "orth":"parsley", + "ner":"O" + }, + { + "id":88, + "orth":"in", + "ner":"O" + }, + { + "id":89, + "orth":"the", + "ner":"O" + }, + { + "id":90, + "orth":"sauce", + "ner":"O" + }, + { + "id":91, + "orth":"and", + "ner":"O" + }, + { + "id":92, + "orth":"then", + "ner":"O" + }, + { + "id":93, + "orth":"another", + "ner":"O" + }, + { + "id":94, + "orth":"2", + "ner":"O" + }, + { + "id":95, + "orth":"tablespoons", + "ner":"O" + }, + { + "id":96, + "orth":"on", + "ner":"O" + }, + { + "id":97, + "orth":"top", + "ner":"O" + }, + { + "id":98, + "orth":"when", + "ner":"O" + }, + { + "id":99, + "orth":"it", + "ner":"O" + }, + { + "id":100, + "orth":"is", + "ner":"O" + }, + { + "id":101, + "orth":"done", + "ner":"O" + }, + { + "id":102, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":103, + "orth":"I", + "ner":"O" + }, + { + "id":104, + "orth":"know", + "ner":"O" + }, + { + "id":105, + "orth":"the", + "ner":"O" + }, + { + "id":106, + "orth":"parsley", + "ner":"O" + }, + { + "id":107, + "orth":"on", + "ner":"O" + }, + { + "id":108, + "orth":"top", + "ner":"O" + }, + { + "id":109, + "orth":"is", + "ner":"O" + }, + { + "id":110, + "orth":"more", + "ner":"O" + }, + { + "id":111, + "orth":"for", + "ner":"O" + }, + { + "id":112, + "orth":"looks", + "ner":"O" + }, + { + "id":113, + "orth":"but", + "ner":"O" + }, + { + "id":114, + "orth":"there", + "ner":"O" + }, + { + "id":115, + "orth":"must", + "ner":"O" + }, + { + "id":116, + "orth":"be", + "ner":"O" + }, + { + "id":117, + "orth":"something", + "ner":"O" + }, + { + "id":118, + "orth":"about", + "ner":"O" + }, + { + "id":119, + "orth":"the", + "ner":"O" + }, + { + "id":120, + "orth":"taste", + "ner":"O" + }, + { + "id":121, + "orth":"otherwise", + "ner":"O" + }, + { + "id":122, + "orth":"it", + "ner":"O" + }, + { + "id":123, + "orth":"would", + "ner":"O" + }, + { + "id":124, + "orth":"call", + "ner":"O" + }, + { + "id":125, + "orth":"for", + "ner":"O" + }, + { + "id":126, + "orth":"parsley", + "ner":"O" + }, + { + "id":127, + "orth":"within", + "ner":"O" + }, + { + "id":128, + "orth":"the", + "ner":"O" + }, + { + "id":129, + "orth":"sauce", + "ner":"O" + }, + { + "id":130, + "orth":"as", + "ner":"O" + }, + { + "id":131, + "orth":"well", + "ner":"O" + }, + { + "id":132, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":133, + "orth":"\n", + "ner":"O" + }, + { + "id":134, + "orth":"I", + "ner":"O" + }, + { + "id":135, + "orth":"would", + "ner":"O" + }, + { + "id":136, + "orth":"especially", + "ner":"O" + }, + { + "id":137, + "orth":"like", + "ner":"O" + }, + { + "id":138, + "orth":"to", + "ner":"O" + }, + { + "id":139, + "orth":"hear", + "ner":"O" + }, + { + "id":140, + "orth":"about", + "ner":"O" + }, + { + "id":141, + "orth":"substitutes", + "ner":"O" + }, + { + "id":142, + "orth":"available", + "ner":"O" + }, + { + "id":143, + "orth":"in", + "ner":"O" + }, + { + "id":144, + "orth":"Southeast", + "ner":"O" + }, + { + "id":145, + "orth":"Asia", + "ner":"O" + }, + { + "id":146, + "orth":"and", + "ner":"O" + }, + { + "id":147, + "orth":"other", + "ner":"O" + }, + { + "id":148, + "orth":"parts", + "ner":"O" + }, + { + "id":149, + "orth":"of", + "ner":"O" + }, + { + "id":150, + "orth":"the", + "ner":"O" + }, + { + "id":151, + "orth":"world", + "ner":"O" + }, + { + "id":152, + "orth":"where", + "ner":"O" + }, + { + "id":153, + "orth":"the", + "ner":"O" + }, + { + "id":154, + "orth":"obvious", + "ner":"O" + }, + { + "id":155, + "orth":"answers", + "ner":"O" + }, + { + "id":156, + "orth":"(", + "ner":"O" + }, + { + "id":157, + "orth":"such", + "ner":"O" + }, + { + "id":158, + "orth":"as", + "ner":"O" + }, + { + "id":159, + "orth":"cilantro", + "ner":"O" + }, + { + "id":160, + "orth":")", + "ner":"O" + }, + { + "id":161, + "orth":"are", + "ner":"O" + }, + { + "id":162, + "orth":"not", + "ner":"O" + }, + { + "id":163, + "orth":"widely", + "ner":"O" + }, + { + "id":164, + "orth":"available", + "ner":"O" + }, + { + "id":165, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":166, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"What is the internal temperature a steak should be cooked to for Rare/Medium Rare/Medium/Well?\nI'd like to know when to take my steaks off the grill and please everybody.\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"What", + "ner":"O" + }, + { + "id":1, + "orth":"is", + "ner":"O" + }, + { + "id":2, + "orth":"the", + "ner":"O" + }, + { + "id":3, + "orth":"internal", + "ner":"O" + }, + { + "id":4, + "orth":"temperature", + "ner":"O" + }, + { + "id":5, + "orth":"a", + "ner":"O" + }, + { + "id":6, + "orth":"steak", + "ner":"O" + }, + { + "id":7, + "orth":"should", + "ner":"O" + }, + { + "id":8, + "orth":"be", + "ner":"O" + }, + { + "id":9, + "orth":"cooked", + "ner":"O" + }, + { + "id":10, + "orth":"to", + "ner":"O" + }, + { + "id":11, + "orth":"for", + "ner":"O" + }, + { + "id":12, + "orth":"Rare", + "ner":"O" + }, + { + "id":13, + "orth":"/", + "ner":"O" + }, + { + "id":14, + "orth":"Medium", + "ner":"O" + }, + { + "id":15, + "orth":"Rare", + "ner":"O" + }, + { + "id":16, + "orth":"/", + "ner":"O" + }, + { + "id":17, + "orth":"Medium", + "ner":"O" + }, + { + "id":18, + "orth":"/", + "ner":"O" + }, + { + "id":19, + "orth":"Well", + "ner":"O" + }, + { + "id":20, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":21, + "orth":"\n", + "ner":"O" + }, + { + "id":22, + "orth":"I", + "ner":"O" + }, + { + "id":23, + "orth":"'d", + "ner":"O" + }, + { + "id":24, + "orth":"like", + "ner":"O" + }, + { + "id":25, + "orth":"to", + "ner":"O" + }, + { + "id":26, + "orth":"know", + "ner":"O" + }, + { + "id":27, + "orth":"when", + "ner":"O" + }, + { + "id":28, + "orth":"to", + "ner":"O" + }, + { + "id":29, + "orth":"take", + "ner":"O" + }, + { + "id":30, + "orth":"my", + "ner":"O" + }, + { + "id":31, + "orth":"steaks", + "ner":"O" + }, + { + "id":32, + "orth":"off", + "ner":"O" + }, + { + "id":33, + "orth":"the", + "ner":"O" + }, + { + "id":34, + "orth":"grill", + "ner":"O" + }, + { + "id":35, + "orth":"and", + "ner":"O" + }, + { + "id":36, + "orth":"please", + "ner":"O" + }, + { + "id":37, + "orth":"everybody", + "ner":"O" + }, + { + "id":38, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":39, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"How should I poach an egg?\nWhat's the best method to poach an egg without it turning into an eggy soupy mess?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"How", + "ner":"O" + }, + { + "id":1, + "orth":"should", + "ner":"O" + }, + { + "id":2, + "orth":"I", + "ner":"O" + }, + { + "id":3, + "orth":"poach", + "ner":"O" + }, + { + "id":4, + "orth":"an", + "ner":"O" + }, + { + "id":5, + "orth":"egg", + "ner":"O" + }, + { + "id":6, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":7, + "orth":"\n", + "ner":"O" + }, + { + "id":8, + "orth":"What", + "ner":"O" + }, + { + "id":9, + "orth":"'s", + "ner":"O" + }, + { + "id":10, + "orth":"the", + "ner":"O" + }, + { + "id":11, + "orth":"best", + "ner":"O" + }, + { + "id":12, + "orth":"method", + "ner":"O" + }, + { + "id":13, + "orth":"to", + "ner":"O" + }, + { + "id":14, + "orth":"poach", + "ner":"O" + }, + { + "id":15, + "orth":"an", + "ner":"O" + }, + { + "id":16, + "orth":"egg", + "ner":"O" + }, + { + "id":17, + "orth":"without", + "ner":"O" + }, + { + "id":18, + "orth":"it", + "ner":"O" + }, + { + "id":19, + "orth":"turning", + "ner":"O" + }, + { + "id":20, + "orth":"into", + "ner":"O" + }, + { + "id":21, + "orth":"an", + "ner":"O" + }, + { + "id":22, + "orth":"eggy", + "ner":"O" + }, + { + "id":23, + "orth":"soupy", + "ner":"O" + }, + { + "id":24, + "orth":"mess", + "ner":"O" + }, + { + "id":25, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":26, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"How can I make my Ice Cream \"creamier\"\nMy ice cream doesn't feel creamy enough. I got the recipe from Good Eats, and I can't tell if it's just the recipe or maybe that I'm just not getting my \"batter\" cold enough before I try to make it (I let it chill overnight in the refrigerator, but it doesn't always come out of the machine looking like \"soft serve\" as he said on the show - it's usually a little thinner).\nRecipe: http://www.foodnetwork.com/recipes/alton-brown/serious-vanilla-ice-cream-recipe/index.html\nThanks!\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"How", + "ner":"O" + }, + { + "id":1, + "orth":"can", + "ner":"O" + }, + { + "id":2, + "orth":"I", + "ner":"O" + }, + { + "id":3, + "orth":"make", + "ner":"O" + }, + { + "id":4, + "orth":"my", + "ner":"O" + }, + { + "id":5, + "orth":"Ice", + "ner":"O" + }, + { + "id":6, + "orth":"Cream", + "ner":"O" + }, + { + "id":7, + "orth":"\"", + "ner":"O" + }, + { + "id":8, + "orth":"creamier", + "ner":"O" + }, + { + "id":9, + "orth":"\"", + "ner":"O" + }, + { + "id":10, + "orth":"\n", + "ner":"O" + }, + { + "id":11, + "orth":"My", + "ner":"O" + }, + { + "id":12, + "orth":"ice", + "ner":"O" + }, + { + "id":13, + "orth":"cream", + "ner":"O" + }, + { + "id":14, + "orth":"does", + "ner":"O" + }, + { + "id":15, + "orth":"n't", + "ner":"O" + }, + { + "id":16, + "orth":"feel", + "ner":"O" + }, + { + "id":17, + "orth":"creamy", + "ner":"O" + }, + { + "id":18, + "orth":"enough", + "ner":"O" + }, + { + "id":19, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":20, + "orth":" ", + "ner":"O" + }, + { + "id":21, + "orth":"I", + "ner":"O" + }, + { + "id":22, + "orth":"got", + "ner":"O" + }, + { + "id":23, + "orth":"the", + "ner":"O" + }, + { + "id":24, + "orth":"recipe", + "ner":"O" + }, + { + "id":25, + "orth":"from", + "ner":"O" + }, + { + "id":26, + "orth":"Good", + "ner":"O" + }, + { + "id":27, + "orth":"Eats", + "ner":"O" + }, + { + "id":28, + "orth":",", + "ner":"O" + }, + { + "id":29, + "orth":"and", + "ner":"O" + }, + { + "id":30, + "orth":"I", + "ner":"O" + }, + { + "id":31, + "orth":"ca", + "ner":"O" + }, + { + "id":32, + "orth":"n't", + "ner":"O" + }, + { + "id":33, + "orth":"tell", + "ner":"O" + }, + { + "id":34, + "orth":"if", + "ner":"O" + }, + { + "id":35, + "orth":"it", + "ner":"O" + }, + { + "id":36, + "orth":"'s", + "ner":"O" + }, + { + "id":37, + "orth":"just", + "ner":"O" + }, + { + "id":38, + "orth":"the", + "ner":"O" + }, + { + "id":39, + "orth":"recipe", + "ner":"O" + }, + { + "id":40, + "orth":"or", + "ner":"O" + }, + { + "id":41, + "orth":"maybe", + "ner":"O" + }, + { + "id":42, + "orth":"that", + "ner":"O" + }, + { + "id":43, + "orth":"I", + "ner":"O" + }, + { + "id":44, + "orth":"'m", + "ner":"O" + }, + { + "id":45, + "orth":"just", + "ner":"O" + }, + { + "id":46, + "orth":"not", + "ner":"O" + }, + { + "id":47, + "orth":"getting", + "ner":"O" + }, + { + "id":48, + "orth":"my", + "ner":"O" + }, + { + "id":49, + "orth":"\"", + "ner":"O" + }, + { + "id":50, + "orth":"batter", + "ner":"O" + }, + { + "id":51, + "orth":"\"", + "ner":"O" + }, + { + "id":52, + "orth":"cold", + "ner":"O" + }, + { + "id":53, + "orth":"enough", + "ner":"O" + }, + { + "id":54, + "orth":"before", + "ner":"O" + }, + { + "id":55, + "orth":"I", + "ner":"O" + }, + { + "id":56, + "orth":"try", + "ner":"O" + }, + { + "id":57, + "orth":"to", + "ner":"O" + }, + { + "id":58, + "orth":"make", + "ner":"O" + }, + { + "id":59, + "orth":"it", + "ner":"O" + }, + { + "id":60, + "orth":"(", + "ner":"O" + }, + { + "id":61, + "orth":"I", + "ner":"O" + }, + { + "id":62, + "orth":"let", + "ner":"O" + }, + { + "id":63, + "orth":"it", + "ner":"O" + }, + { + "id":64, + "orth":"chill", + "ner":"O" + }, + { + "id":65, + "orth":"overnight", + "ner":"O" + }, + { + "id":66, + "orth":"in", + "ner":"O" + }, + { + "id":67, + "orth":"the", + "ner":"O" + }, + { + "id":68, + "orth":"refrigerator", + "ner":"O" + }, + { + "id":69, + "orth":",", + "ner":"O" + }, + { + "id":70, + "orth":"but", + "ner":"O" + }, + { + "id":71, + "orth":"it", + "ner":"O" + }, + { + "id":72, + "orth":"does", + "ner":"O" + }, + { + "id":73, + "orth":"n't", + "ner":"O" + }, + { + "id":74, + "orth":"always", + "ner":"O" + }, + { + "id":75, + "orth":"come", + "ner":"O" + }, + { + "id":76, + "orth":"out", + "ner":"O" + }, + { + "id":77, + "orth":"of", + "ner":"O" + }, + { + "id":78, + "orth":"the", + "ner":"O" + }, + { + "id":79, + "orth":"machine", + "ner":"O" + }, + { + "id":80, + "orth":"looking", + "ner":"O" + }, + { + "id":81, + "orth":"like", + "ner":"O" + }, + { + "id":82, + "orth":"\"", + "ner":"O" + }, + { + "id":83, + "orth":"soft", + "ner":"O" + }, + { + "id":84, + "orth":"serve", + "ner":"O" + }, + { + "id":85, + "orth":"\"", + "ner":"O" + }, + { + "id":86, + "orth":"as", + "ner":"O" + }, + { + "id":87, + "orth":"he", + "ner":"O" + }, + { + "id":88, + "orth":"said", + "ner":"O" + }, + { + "id":89, + "orth":"on", + "ner":"O" + }, + { + "id":90, + "orth":"the", + "ner":"O" + }, + { + "id":91, + "orth":"show", + "ner":"O" + }, + { + "id":92, + "orth":"-", + "ner":"O" + }, + { + "id":93, + "orth":"it", + "ner":"O" + }, + { + "id":94, + "orth":"'s", + "ner":"O" + }, + { + "id":95, + "orth":"usually", + "ner":"O" + }, + { + "id":96, + "orth":"a", + "ner":"O" + }, + { + "id":97, + "orth":"little", + "ner":"O" + }, + { + "id":98, + "orth":"thinner", + "ner":"O" + }, + { + "id":99, + "orth":")", + "ner":"O" + }, + { + "id":100, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":101, + "orth":"\n", + "ner":"O" + }, + { + "id":102, + "orth":"Recipe", + "ner":"O" + }, + { + "id":103, + "orth":":", + "ner":"O" + }, + { + "id":104, + "orth":"http://www.foodnetwork.com/recipes/alton-brown/serious-vanilla-ice-cream-recipe/index.html", + "ner":"O" + }, + { + "id":105, + "orth":"\n", + "ner":"O" + }, + { + "id":106, + "orth":"Thanks", + "ner":"O" + }, + { + "id":107, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":108, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":0.0 + }, + { + "label":"not_baking", + "value":1.0 + } + ] + }, + { + "raw":"How long and at what temperature do the various parts of a chicken need to be cooked?\nI'm interested in baking thighs, legs, breasts and wings. How long do each of these items need to bake and at what temperature?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"How", + "ner":"O" + }, + { + "id":1, + "orth":"long", + "ner":"O" + }, + { + "id":2, + "orth":"and", + "ner":"O" + }, + { + "id":3, + "orth":"at", + "ner":"O" + }, + { + "id":4, + "orth":"what", + "ner":"O" + }, + { + "id":5, + "orth":"temperature", + "ner":"O" + }, + { + "id":6, + "orth":"do", + "ner":"O" + }, + { + "id":7, + "orth":"the", + "ner":"O" + }, + { + "id":8, + "orth":"various", + "ner":"O" + }, + { + "id":9, + "orth":"parts", + "ner":"O" + }, + { + "id":10, + "orth":"of", + "ner":"O" + }, + { + "id":11, + "orth":"a", + "ner":"O" + }, + { + "id":12, + "orth":"chicken", + "ner":"O" + }, + { + "id":13, + "orth":"need", + "ner":"O" + }, + { + "id":14, + "orth":"to", + "ner":"O" + }, + { + "id":15, + "orth":"be", + "ner":"O" + }, + { + "id":16, + "orth":"cooked", + "ner":"O" + }, + { + "id":17, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":18, + "orth":"\n", + "ner":"O" + }, + { + "id":19, + "orth":"I", + "ner":"O" + }, + { + "id":20, + "orth":"'m", + "ner":"O" + }, + { + "id":21, + "orth":"interested", + "ner":"O" + }, + { + "id":22, + "orth":"in", + "ner":"O" + }, + { + "id":23, + "orth":"baking", + "ner":"O" + }, + { + "id":24, + "orth":"thighs", + "ner":"O" + }, + { + "id":25, + "orth":",", + "ner":"O" + }, + { + "id":26, + "orth":"legs", + "ner":"O" + }, + { + "id":27, + "orth":",", + "ner":"O" + }, + { + "id":28, + "orth":"breasts", + "ner":"O" + }, + { + "id":29, + "orth":"and", + "ner":"O" + }, + { + "id":30, + "orth":"wings", + "ner":"O" + }, + { + "id":31, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":32, + "orth":" ", + "ner":"O" + }, + { + "id":33, + "orth":"How", + "ner":"O" + }, + { + "id":34, + "orth":"long", + "ner":"O" + }, + { + "id":35, + "orth":"do", + "ner":"O" + }, + { + "id":36, + "orth":"each", + "ner":"O" + }, + { + "id":37, + "orth":"of", + "ner":"O" + }, + { + "id":38, + "orth":"these", + "ner":"O" + }, + { + "id":39, + "orth":"items", + "ner":"O" + }, + { + "id":40, + "orth":"need", + "ner":"O" + }, + { + "id":41, + "orth":"to", + "ner":"O" + }, + { + "id":42, + "orth":"bake", + "ner":"O" + }, + { + "id":43, + "orth":"and", + "ner":"O" + }, + { + "id":44, + "orth":"at", + "ner":"O" + }, + { + "id":45, + "orth":"what", + "ner":"O" + }, + { + "id":46, + "orth":"temperature", + "ner":"O" + }, + { + "id":47, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":48, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":1.0 + }, + { + "label":"not_baking", + "value":0.0 + } + ] + }, + { + "raw":"Do I need to sift flour that is labeled sifted?\nIs there really an advantage to sifting flour that I bought that was labeled 'sifted'?\n", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"Do", + "ner":"O" + }, + { + "id":1, + "orth":"I", + "ner":"O" + }, + { + "id":2, + "orth":"need", + "ner":"O" + }, + { + "id":3, + "orth":"to", + "ner":"O" + }, + { + "id":4, + "orth":"sift", + "ner":"O" + }, + { + "id":5, + "orth":"flour", + "ner":"O" + }, + { + "id":6, + "orth":"that", + "ner":"O" + }, + { + "id":7, + "orth":"is", + "ner":"O" + }, + { + "id":8, + "orth":"labeled", + "ner":"O" + }, + { + "id":9, + "orth":"sifted", + "ner":"O" + }, + { + "id":10, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":11, + "orth":"\n", + "ner":"O" + }, + { + "id":12, + "orth":"Is", + "ner":"O" + }, + { + "id":13, + "orth":"there", + "ner":"O" + }, + { + "id":14, + "orth":"really", + "ner":"O" + }, + { + "id":15, + "orth":"an", + "ner":"O" + }, + { + "id":16, + "orth":"advantage", + "ner":"O" + }, + { + "id":17, + "orth":"to", + "ner":"O" + }, + { + "id":18, + "orth":"sifting", + "ner":"O" + }, + { + "id":19, + "orth":"flour", + "ner":"O" + }, + { + "id":20, + "orth":"that", + "ner":"O" + }, + { + "id":21, + "orth":"I", + "ner":"O" + }, + { + "id":22, + "orth":"bought", + "ner":"O" + }, + { + "id":23, + "orth":"that", + "ner":"O" + }, + { + "id":24, + "orth":"was", + "ner":"O" + }, + { + "id":25, + "orth":"labeled", + "ner":"O" + }, + { + "id":26, + "orth":"'", + "ner":"O" + }, + { + "id":27, + "orth":"sifted", + "ner":"O" + }, + { + "id":28, + "orth":"'", + "ner":"O" + }, + { + "id":29, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":30, + "orth":"\n", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"baking", + "value":1.0 + }, + { + "label":"not_baking", + "value":0.0 + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/textcat_example_data/cooking.jsonl b/extra/example_data/textcat_example_data/cooking.jsonl new file mode 100644 index 0000000..cfdc9be --- /dev/null +++ b/extra/example_data/textcat_example_data/cooking.jsonl @@ -0,0 +1,10 @@ +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "2"}, "text": "How should I cook bacon in an oven?\nI've heard of people cooking bacon in an oven by laying the strips out on a cookie sheet. When using this method, how long should I cook the bacon for, and at what temperature?\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "3"}, "text": "What is the difference between white and brown eggs?\nI always use brown extra large eggs, but I can't honestly say why I do this other than habit at this point. Are there any distinct advantages or disadvantages like flavor, shelf life, etc?\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "4"}, "text": "What is the difference between baking soda and baking powder?\nAnd can I use one in place of the other in certain recipes?\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "5"}, "text": "In a tomato sauce recipe, how can I cut the acidity?\nIt seems that every time I make a tomato sauce for pasta, the sauce is a little bit too acid for my taste. I've tried using sugar or sodium bicarbonate, but I'm not satisfied with the results.\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "6"}, "text": "What ingredients (available in specific regions) can I substitute for parsley?\nI have a recipe that calls for fresh parsley. I have substituted other fresh herbs for their dried equivalents but I don't have fresh or dried parsley. Is there something else (ex another dried herb) that I can use instead of parsley?\nI know it is used mainly for looks rather than taste but I have a pasta recipe that calls for 2 tablespoons of parsley in the sauce and then another 2 tablespoons on top when it is done. I know the parsley on top is more for looks but there must be something about the taste otherwise it would call for parsley within the sauce as well.\nI would especially like to hear about substitutes available in Southeast Asia and other parts of the world where the obvious answers (such as cilantro) are not widely available.\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "9"}, "text": "What is the internal temperature a steak should be cooked to for Rare/Medium Rare/Medium/Well?\nI'd like to know when to take my steaks off the grill and please everybody.\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "11"}, "text": "How should I poach an egg?\nWhat's the best method to poach an egg without it turning into an eggy soupy mess?\n"} +{"cats": {"baking": 0.0, "not_baking": 1.0}, "meta": {"id": "12"}, "text": "How can I make my Ice Cream \"creamier\"\nMy ice cream doesn't feel creamy enough. I got the recipe from Good Eats, and I can't tell if it's just the recipe or maybe that I'm just not getting my \"batter\" cold enough before I try to make it (I let it chill overnight in the refrigerator, but it doesn't always come out of the machine looking like \"soft serve\" as he said on the show - it's usually a little thinner).\nRecipe: http://www.foodnetwork.com/recipes/alton-brown/serious-vanilla-ice-cream-recipe/index.html\nThanks!\n"} +{"cats": {"baking": 1.0, "not_baking": 0.0}, "meta": {"id": "17"}, "text": "How long and at what temperature do the various parts of a chicken need to be cooked?\nI'm interested in baking thighs, legs, breasts and wings. How long do each of these items need to bake and at what temperature?\n"} +{"cats": {"baking": 1.0, "not_baking": 0.0}, "meta": {"id": "27"}, "text": "Do I need to sift flour that is labeled sifted?\nIs there really an advantage to sifting flour that I bought that was labeled 'sifted'?\n"} diff --git a/extra/example_data/textcat_example_data/jigsaw-toxic-comment.json b/extra/example_data/textcat_example_data/jigsaw-toxic-comment.json new file mode 100644 index 0000000..0c8d8f8 --- /dev/null +++ b/extra/example_data/textcat_example_data/jigsaw-toxic-comment.json @@ -0,0 +1,2987 @@ +[ + { + "id":0, + "paragraphs":[ + { + "raw":"Explanation\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"Explanation", + "ner":"O" + }, + { + "id":1, + "orth":"\n", + "ner":"O" + }, + { + "id":2, + "orth":"Why", + "ner":"O" + }, + { + "id":3, + "orth":"the", + "ner":"O" + }, + { + "id":4, + "orth":"edits", + "ner":"O" + }, + { + "id":5, + "orth":"made", + "ner":"O" + }, + { + "id":6, + "orth":"under", + "ner":"O" + }, + { + "id":7, + "orth":"my", + "ner":"O" + }, + { + "id":8, + "orth":"username", + "ner":"O" + }, + { + "id":9, + "orth":"Hardcore", + "ner":"O" + }, + { + "id":10, + "orth":"Metallica", + "ner":"O" + }, + { + "id":11, + "orth":"Fan", + "ner":"O" + }, + { + "id":12, + "orth":"were", + "ner":"O" + }, + { + "id":13, + "orth":"reverted", + "ner":"O" + }, + { + "id":14, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":15, + "orth":"They", + "ner":"O" + }, + { + "id":16, + "orth":"were", + "ner":"O" + }, + { + "id":17, + "orth":"n't", + "ner":"O" + }, + { + "id":18, + "orth":"vandalisms", + "ner":"O" + }, + { + "id":19, + "orth":",", + "ner":"O" + }, + { + "id":20, + "orth":"just", + "ner":"O" + }, + { + "id":21, + "orth":"closure", + "ner":"O" + }, + { + "id":22, + "orth":"on", + "ner":"O" + }, + { + "id":23, + "orth":"some", + "ner":"O" + }, + { + "id":24, + "orth":"GAs", + "ner":"O" + }, + { + "id":25, + "orth":"after", + "ner":"O" + }, + { + "id":26, + "orth":"I", + "ner":"O" + }, + { + "id":27, + "orth":"voted", + "ner":"O" + }, + { + "id":28, + "orth":"at", + "ner":"O" + }, + { + "id":29, + "orth":"New", + "ner":"O" + }, + { + "id":30, + "orth":"York", + "ner":"O" + }, + { + "id":31, + "orth":"Dolls", + "ner":"O" + }, + { + "id":32, + "orth":"FAC", + "ner":"O" + }, + { + "id":33, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":34, + "orth":"And", + "ner":"O" + }, + { + "id":35, + "orth":"please", + "ner":"O" + }, + { + "id":36, + "orth":"do", + "ner":"O" + }, + { + "id":37, + "orth":"n't", + "ner":"O" + }, + { + "id":38, + "orth":"remove", + "ner":"O" + }, + { + "id":39, + "orth":"the", + "ner":"O" + }, + { + "id":40, + "orth":"template", + "ner":"O" + }, + { + "id":41, + "orth":"from", + "ner":"O" + }, + { + "id":42, + "orth":"the", + "ner":"O" + }, + { + "id":43, + "orth":"talk", + "ner":"O" + }, + { + "id":44, + "orth":"page", + "ner":"O" + }, + { + "id":45, + "orth":"since", + "ner":"O" + }, + { + "id":46, + "orth":"I", + "ner":"O" + }, + { + "id":47, + "orth":"'m", + "ner":"O" + }, + { + "id":48, + "orth":"retired", + "ner":"O" + }, + { + "id":49, + "orth":"now.89.205.38.27", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"I'm Sorry \n\nI'm sorry I screwed around with someones talk page. It was very bad to do. I know how having the templates on their talk page helps you assert your dominance over them. I know I should bow down to the almighty administrators. But then again, I'm going to go play outside....with your mom. 76.122.79.82", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"I", + "ner":"O" + }, + { + "id":1, + "orth":"'m", + "ner":"O" + }, + { + "id":2, + "orth":"Sorry", + "ner":"O" + }, + { + "id":3, + "orth":"\n\n", + "ner":"O" + }, + { + "id":4, + "orth":"I", + "ner":"O" + }, + { + "id":5, + "orth":"'m", + "ner":"O" + }, + { + "id":6, + "orth":"sorry", + "ner":"O" + }, + { + "id":7, + "orth":"I", + "ner":"O" + }, + { + "id":8, + "orth":"screwed", + "ner":"O" + }, + { + "id":9, + "orth":"around", + "ner":"O" + }, + { + "id":10, + "orth":"with", + "ner":"O" + }, + { + "id":11, + "orth":"someones", + "ner":"O" + }, + { + "id":12, + "orth":"talk", + "ner":"O" + }, + { + "id":13, + "orth":"page", + "ner":"O" + }, + { + "id":14, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":15, + "orth":" ", + "ner":"O" + }, + { + "id":16, + "orth":"It", + "ner":"O" + }, + { + "id":17, + "orth":"was", + "ner":"O" + }, + { + "id":18, + "orth":"very", + "ner":"O" + }, + { + "id":19, + "orth":"bad", + "ner":"O" + }, + { + "id":20, + "orth":"to", + "ner":"O" + }, + { + "id":21, + "orth":"do", + "ner":"O" + }, + { + "id":22, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":23, + "orth":" ", + "ner":"O" + }, + { + "id":24, + "orth":"I", + "ner":"O" + }, + { + "id":25, + "orth":"know", + "ner":"O" + }, + { + "id":26, + "orth":"how", + "ner":"O" + }, + { + "id":27, + "orth":"having", + "ner":"O" + }, + { + "id":28, + "orth":"the", + "ner":"O" + }, + { + "id":29, + "orth":"templates", + "ner":"O" + }, + { + "id":30, + "orth":"on", + "ner":"O" + }, + { + "id":31, + "orth":"their", + "ner":"O" + }, + { + "id":32, + "orth":"talk", + "ner":"O" + }, + { + "id":33, + "orth":"page", + "ner":"O" + }, + { + "id":34, + "orth":"helps", + "ner":"O" + }, + { + "id":35, + "orth":"you", + "ner":"O" + }, + { + "id":36, + "orth":"assert", + "ner":"O" + }, + { + "id":37, + "orth":"your", + "ner":"O" + }, + { + "id":38, + "orth":"dominance", + "ner":"O" + }, + { + "id":39, + "orth":"over", + "ner":"O" + }, + { + "id":40, + "orth":"them", + "ner":"O" + }, + { + "id":41, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":42, + "orth":" ", + "ner":"O" + }, + { + "id":43, + "orth":"I", + "ner":"O" + }, + { + "id":44, + "orth":"know", + "ner":"O" + }, + { + "id":45, + "orth":"I", + "ner":"O" + }, + { + "id":46, + "orth":"should", + "ner":"O" + }, + { + "id":47, + "orth":"bow", + "ner":"O" + }, + { + "id":48, + "orth":"down", + "ner":"O" + }, + { + "id":49, + "orth":"to", + "ner":"O" + }, + { + "id":50, + "orth":"the", + "ner":"O" + }, + { + "id":51, + "orth":"almighty", + "ner":"O" + }, + { + "id":52, + "orth":"administrators", + "ner":"O" + }, + { + "id":53, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":54, + "orth":" ", + "ner":"O" + }, + { + "id":55, + "orth":"But", + "ner":"O" + }, + { + "id":56, + "orth":"then", + "ner":"O" + }, + { + "id":57, + "orth":"again", + "ner":"O" + }, + { + "id":58, + "orth":",", + "ner":"O" + }, + { + "id":59, + "orth":"I", + "ner":"O" + }, + { + "id":60, + "orth":"'m", + "ner":"O" + }, + { + "id":61, + "orth":"going", + "ner":"O" + }, + { + "id":62, + "orth":"to", + "ner":"O" + }, + { + "id":63, + "orth":"go", + "ner":"O" + }, + { + "id":64, + "orth":"play", + "ner":"O" + }, + { + "id":65, + "orth":"outside", + "ner":"O" + }, + { + "id":66, + "orth":"....", + "ner":"O" + }, + { + "id":67, + "orth":"with", + "ner":"O" + }, + { + "id":68, + "orth":"your", + "ner":"O" + }, + { + "id":69, + "orth":"mom", + "ner":"O" + }, + { + "id":70, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":71, + "orth":" ", + "ner":"O" + }, + { + "id":72, + "orth":"76.122.79.82", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":1 + } + ] + }, + { + "raw":"Stupid peace of shit stop deleting my stuff asshole go die and fall in a hole go to hell!", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"Stupid", + "ner":"O" + }, + { + "id":1, + "orth":"peace", + "ner":"O" + }, + { + "id":2, + "orth":"of", + "ner":"O" + }, + { + "id":3, + "orth":"shit", + "ner":"O" + }, + { + "id":4, + "orth":"stop", + "ner":"O" + }, + { + "id":5, + "orth":"deleting", + "ner":"O" + }, + { + "id":6, + "orth":"my", + "ner":"O" + }, + { + "id":7, + "orth":"stuff", + "ner":"O" + }, + { + "id":8, + "orth":"asshole", + "ner":"O" + }, + { + "id":9, + "orth":"go", + "ner":"O" + }, + { + "id":10, + "orth":"die", + "ner":"O" + }, + { + "id":11, + "orth":"and", + "ner":"O" + }, + { + "id":12, + "orth":"fall", + "ner":"O" + }, + { + "id":13, + "orth":"in", + "ner":"O" + }, + { + "id":14, + "orth":"a", + "ner":"O" + }, + { + "id":15, + "orth":"hole", + "ner":"O" + }, + { + "id":16, + "orth":"go", + "ner":"O" + }, + { + "id":17, + "orth":"to", + "ner":"O" + }, + { + "id":18, + "orth":"hell", + "ner":"O" + }, + { + "id":19, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":1 + }, + { + "label":"obscene", + "value":1 + }, + { + "label":"severe_toxic", + "value":1 + }, + { + "label":"toxic", + "value":1 + } + ] + }, + { + "raw":"D'aww! He matches this background colour I'm seemingly stuck with. Thanks. (talk) 21:51, January 11, 2016 (UTC)", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"D'aww", + "ner":"O" + }, + { + "id":1, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":2, + "orth":"He", + "ner":"O" + }, + { + "id":3, + "orth":"matches", + "ner":"O" + }, + { + "id":4, + "orth":"this", + "ner":"O" + }, + { + "id":5, + "orth":"background", + "ner":"O" + }, + { + "id":6, + "orth":"colour", + "ner":"O" + }, + { + "id":7, + "orth":"I", + "ner":"O" + }, + { + "id":8, + "orth":"'m", + "ner":"O" + }, + { + "id":9, + "orth":"seemingly", + "ner":"O" + }, + { + "id":10, + "orth":"stuck", + "ner":"O" + }, + { + "id":11, + "orth":"with", + "ner":"O" + }, + { + "id":12, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":13, + "orth":"Thanks", + "ner":"O" + }, + { + "id":14, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":15, + "orth":" ", + "ner":"O" + }, + { + "id":16, + "orth":"(", + "ner":"O" + }, + { + "id":17, + "orth":"talk", + "ner":"O" + }, + { + "id":18, + "orth":")", + "ner":"O" + }, + { + "id":19, + "orth":"21:51", + "ner":"O" + }, + { + "id":20, + "orth":",", + "ner":"O" + }, + { + "id":21, + "orth":"January", + "ner":"O" + }, + { + "id":22, + "orth":"11", + "ner":"O" + }, + { + "id":23, + "orth":",", + "ner":"O" + }, + { + "id":24, + "orth":"2016", + "ner":"O" + }, + { + "id":25, + "orth":"(", + "ner":"O" + }, + { + "id":26, + "orth":"UTC", + "ner":"O" + }, + { + "id":27, + "orth":")", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"GET FUCKED UP. GET FUCKEEED UP. GOT A DRINK THAT YOU CANT PUT DOWN???/ GET FUCK UP GET FUCKED UP. I'M FUCKED UP RIGHT NOW!", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"GET", + "ner":"O" + }, + { + "id":1, + "orth":"FUCKED", + "ner":"O" + }, + { + "id":2, + "orth":"UP", + "ner":"O" + }, + { + "id":3, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":4, + "orth":"GET", + "ner":"O" + }, + { + "id":5, + "orth":"FUCKEEED", + "ner":"O" + }, + { + "id":6, + "orth":"UP", + "ner":"O" + }, + { + "id":7, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":8, + "orth":" ", + "ner":"O" + }, + { + "id":9, + "orth":"GOT", + "ner":"O" + }, + { + "id":10, + "orth":"A", + "ner":"O" + }, + { + "id":11, + "orth":"DRINK", + "ner":"O" + }, + { + "id":12, + "orth":"THAT", + "ner":"O" + }, + { + "id":13, + "orth":"YOU", + "ner":"O" + }, + { + "id":14, + "orth":"CANT", + "ner":"O" + }, + { + "id":15, + "orth":"PUT", + "ner":"O" + }, + { + "id":16, + "orth":"DOWN???/", + "ner":"O" + }, + { + "id":17, + "orth":"GET", + "ner":"O" + }, + { + "id":18, + "orth":"FUCK", + "ner":"O" + }, + { + "id":19, + "orth":"UP", + "ner":"O" + }, + { + "id":20, + "orth":"GET", + "ner":"O" + }, + { + "id":21, + "orth":"FUCKED", + "ner":"O" + }, + { + "id":22, + "orth":"UP", + "ner":"O" + }, + { + "id":23, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":24, + "orth":" ", + "ner":"O" + }, + { + "id":25, + "orth":"I'M", + "ner":"O" + }, + { + "id":26, + "orth":"FUCKED", + "ner":"O" + }, + { + "id":27, + "orth":"UP", + "ner":"O" + }, + { + "id":28, + "orth":"RIGHT", + "ner":"O" + }, + { + "id":29, + "orth":"NOW", + "ner":"O" + }, + { + "id":30, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":1 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":1 + } + ] + }, + { + "raw":"Hey man, I'm really not trying to edit war. It's just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. He seems to care more about the formatting than the actual info.", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"Hey", + "ner":"O" + }, + { + "id":1, + "orth":"man", + "ner":"O" + }, + { + "id":2, + "orth":",", + "ner":"O" + }, + { + "id":3, + "orth":"I", + "ner":"O" + }, + { + "id":4, + "orth":"'m", + "ner":"O" + }, + { + "id":5, + "orth":"really", + "ner":"O" + }, + { + "id":6, + "orth":"not", + "ner":"O" + }, + { + "id":7, + "orth":"trying", + "ner":"O" + }, + { + "id":8, + "orth":"to", + "ner":"O" + }, + { + "id":9, + "orth":"edit", + "ner":"O" + }, + { + "id":10, + "orth":"war", + "ner":"O" + }, + { + "id":11, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":12, + "orth":"It", + "ner":"O" + }, + { + "id":13, + "orth":"'s", + "ner":"O" + }, + { + "id":14, + "orth":"just", + "ner":"O" + }, + { + "id":15, + "orth":"that", + "ner":"O" + }, + { + "id":16, + "orth":"this", + "ner":"O" + }, + { + "id":17, + "orth":"guy", + "ner":"O" + }, + { + "id":18, + "orth":"is", + "ner":"O" + }, + { + "id":19, + "orth":"constantly", + "ner":"O" + }, + { + "id":20, + "orth":"removing", + "ner":"O" + }, + { + "id":21, + "orth":"relevant", + "ner":"O" + }, + { + "id":22, + "orth":"information", + "ner":"O" + }, + { + "id":23, + "orth":"and", + "ner":"O" + }, + { + "id":24, + "orth":"talking", + "ner":"O" + }, + { + "id":25, + "orth":"to", + "ner":"O" + }, + { + "id":26, + "orth":"me", + "ner":"O" + }, + { + "id":27, + "orth":"through", + "ner":"O" + }, + { + "id":28, + "orth":"edits", + "ner":"O" + }, + { + "id":29, + "orth":"instead", + "ner":"O" + }, + { + "id":30, + "orth":"of", + "ner":"O" + }, + { + "id":31, + "orth":"my", + "ner":"O" + }, + { + "id":32, + "orth":"talk", + "ner":"O" + }, + { + "id":33, + "orth":"page", + "ner":"O" + }, + { + "id":34, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":35, + "orth":"He", + "ner":"O" + }, + { + "id":36, + "orth":"seems", + "ner":"O" + }, + { + "id":37, + "orth":"to", + "ner":"O" + }, + { + "id":38, + "orth":"care", + "ner":"O" + }, + { + "id":39, + "orth":"more", + "ner":"O" + }, + { + "id":40, + "orth":"about", + "ner":"O" + }, + { + "id":41, + "orth":"the", + "ner":"O" + }, + { + "id":42, + "orth":"formatting", + "ner":"O" + }, + { + "id":43, + "orth":"than", + "ner":"O" + }, + { + "id":44, + "orth":"the", + "ner":"O" + }, + { + "id":45, + "orth":"actual", + "ner":"O" + }, + { + "id":46, + "orth":"info", + "ner":"O" + }, + { + "id":47, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"\"\nMore\nI can't make any real suggestions on improvement - I wondered if the section statistics should be later on, or a subsection of \"\"types of accidents\"\" -I think the references may need tidying so that they are all in the exact same format ie date format etc. I can do that later on, if no-one else does first - if you have any preferences for formatting style on references or want to do it yourself please let me know.\n\nThere appears to be a backlog on articles for review so I guess there may be a delay until a reviewer turns up. It's listed in the relevant form eg Wikipedia:Good_article_nominations#Transport \"", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"\"", + "ner":"O" + }, + { + "id":1, + "orth":"\n", + "ner":"O" + }, + { + "id":2, + "orth":"More", + "ner":"O" + }, + { + "id":3, + "orth":"\n", + "ner":"O" + }, + { + "id":4, + "orth":"I", + "ner":"O" + }, + { + "id":5, + "orth":"ca", + "ner":"O" + }, + { + "id":6, + "orth":"n't", + "ner":"O" + }, + { + "id":7, + "orth":"make", + "ner":"O" + }, + { + "id":8, + "orth":"any", + "ner":"O" + }, + { + "id":9, + "orth":"real", + "ner":"O" + }, + { + "id":10, + "orth":"suggestions", + "ner":"O" + }, + { + "id":11, + "orth":"on", + "ner":"O" + }, + { + "id":12, + "orth":"improvement", + "ner":"O" + }, + { + "id":13, + "orth":"-", + "ner":"O" + }, + { + "id":14, + "orth":"I", + "ner":"O" + }, + { + "id":15, + "orth":"wondered", + "ner":"O" + }, + { + "id":16, + "orth":"if", + "ner":"O" + }, + { + "id":17, + "orth":"the", + "ner":"O" + }, + { + "id":18, + "orth":"section", + "ner":"O" + }, + { + "id":19, + "orth":"statistics", + "ner":"O" + }, + { + "id":20, + "orth":"should", + "ner":"O" + }, + { + "id":21, + "orth":"be", + "ner":"O" + }, + { + "id":22, + "orth":"later", + "ner":"O" + }, + { + "id":23, + "orth":"on", + "ner":"O" + }, + { + "id":24, + "orth":",", + "ner":"O" + }, + { + "id":25, + "orth":"or", + "ner":"O" + }, + { + "id":26, + "orth":"a", + "ner":"O" + }, + { + "id":27, + "orth":"subsection", + "ner":"O" + }, + { + "id":28, + "orth":"of", + "ner":"O" + }, + { + "id":29, + "orth":"\"", + "ner":"O" + }, + { + "id":30, + "orth":"\"", + "ner":"O" + }, + { + "id":31, + "orth":"types", + "ner":"O" + }, + { + "id":32, + "orth":"of", + "ner":"O" + }, + { + "id":33, + "orth":"accidents", + "ner":"O" + }, + { + "id":34, + "orth":"\"", + "ner":"O" + }, + { + "id":35, + "orth":"\"", + "ner":"O" + }, + { + "id":36, + "orth":" ", + "ner":"O" + }, + { + "id":37, + "orth":"-I", + "ner":"O" + }, + { + "id":38, + "orth":"think", + "ner":"O" + }, + { + "id":39, + "orth":"the", + "ner":"O" + }, + { + "id":40, + "orth":"references", + "ner":"O" + }, + { + "id":41, + "orth":"may", + "ner":"O" + }, + { + "id":42, + "orth":"need", + "ner":"O" + }, + { + "id":43, + "orth":"tidying", + "ner":"O" + }, + { + "id":44, + "orth":"so", + "ner":"O" + }, + { + "id":45, + "orth":"that", + "ner":"O" + }, + { + "id":46, + "orth":"they", + "ner":"O" + }, + { + "id":47, + "orth":"are", + "ner":"O" + }, + { + "id":48, + "orth":"all", + "ner":"O" + }, + { + "id":49, + "orth":"in", + "ner":"O" + }, + { + "id":50, + "orth":"the", + "ner":"O" + }, + { + "id":51, + "orth":"exact", + "ner":"O" + }, + { + "id":52, + "orth":"same", + "ner":"O" + }, + { + "id":53, + "orth":"format", + "ner":"O" + }, + { + "id":54, + "orth":"ie", + "ner":"O" + }, + { + "id":55, + "orth":"date", + "ner":"O" + }, + { + "id":56, + "orth":"format", + "ner":"O" + }, + { + "id":57, + "orth":"etc", + "ner":"O" + }, + { + "id":58, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":59, + "orth":"I", + "ner":"O" + }, + { + "id":60, + "orth":"can", + "ner":"O" + }, + { + "id":61, + "orth":"do", + "ner":"O" + }, + { + "id":62, + "orth":"that", + "ner":"O" + }, + { + "id":63, + "orth":"later", + "ner":"O" + }, + { + "id":64, + "orth":"on", + "ner":"O" + }, + { + "id":65, + "orth":",", + "ner":"O" + }, + { + "id":66, + "orth":"if", + "ner":"O" + }, + { + "id":67, + "orth":"no", + "ner":"O" + }, + { + "id":68, + "orth":"-", + "ner":"O" + }, + { + "id":69, + "orth":"one", + "ner":"O" + }, + { + "id":70, + "orth":"else", + "ner":"O" + }, + { + "id":71, + "orth":"does", + "ner":"O" + }, + { + "id":72, + "orth":"first", + "ner":"O" + }, + { + "id":73, + "orth":"-", + "ner":"O" + }, + { + "id":74, + "orth":"if", + "ner":"O" + }, + { + "id":75, + "orth":"you", + "ner":"O" + }, + { + "id":76, + "orth":"have", + "ner":"O" + }, + { + "id":77, + "orth":"any", + "ner":"O" + }, + { + "id":78, + "orth":"preferences", + "ner":"O" + }, + { + "id":79, + "orth":"for", + "ner":"O" + }, + { + "id":80, + "orth":"formatting", + "ner":"O" + }, + { + "id":81, + "orth":"style", + "ner":"O" + }, + { + "id":82, + "orth":"on", + "ner":"O" + }, + { + "id":83, + "orth":"references", + "ner":"O" + }, + { + "id":84, + "orth":"or", + "ner":"O" + }, + { + "id":85, + "orth":"want", + "ner":"O" + }, + { + "id":86, + "orth":"to", + "ner":"O" + }, + { + "id":87, + "orth":"do", + "ner":"O" + }, + { + "id":88, + "orth":"it", + "ner":"O" + }, + { + "id":89, + "orth":"yourself", + "ner":"O" + }, + { + "id":90, + "orth":"please", + "ner":"O" + }, + { + "id":91, + "orth":"let", + "ner":"O" + }, + { + "id":92, + "orth":"me", + "ner":"O" + }, + { + "id":93, + "orth":"know", + "ner":"O" + }, + { + "id":94, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":95, + "orth":"\n\n", + "ner":"O" + }, + { + "id":96, + "orth":"There", + "ner":"O" + }, + { + "id":97, + "orth":"appears", + "ner":"O" + }, + { + "id":98, + "orth":"to", + "ner":"O" + }, + { + "id":99, + "orth":"be", + "ner":"O" + }, + { + "id":100, + "orth":"a", + "ner":"O" + }, + { + "id":101, + "orth":"backlog", + "ner":"O" + }, + { + "id":102, + "orth":"on", + "ner":"O" + }, + { + "id":103, + "orth":"articles", + "ner":"O" + }, + { + "id":104, + "orth":"for", + "ner":"O" + }, + { + "id":105, + "orth":"review", + "ner":"O" + }, + { + "id":106, + "orth":"so", + "ner":"O" + }, + { + "id":107, + "orth":"I", + "ner":"O" + }, + { + "id":108, + "orth":"guess", + "ner":"O" + }, + { + "id":109, + "orth":"there", + "ner":"O" + }, + { + "id":110, + "orth":"may", + "ner":"O" + }, + { + "id":111, + "orth":"be", + "ner":"O" + }, + { + "id":112, + "orth":"a", + "ner":"O" + }, + { + "id":113, + "orth":"delay", + "ner":"O" + }, + { + "id":114, + "orth":"until", + "ner":"O" + }, + { + "id":115, + "orth":"a", + "ner":"O" + }, + { + "id":116, + "orth":"reviewer", + "ner":"O" + }, + { + "id":117, + "orth":"turns", + "ner":"O" + }, + { + "id":118, + "orth":"up", + "ner":"O" + }, + { + "id":119, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":120, + "orth":"It", + "ner":"O" + }, + { + "id":121, + "orth":"'s", + "ner":"O" + }, + { + "id":122, + "orth":"listed", + "ner":"O" + }, + { + "id":123, + "orth":"in", + "ner":"O" + }, + { + "id":124, + "orth":"the", + "ner":"O" + }, + { + "id":125, + "orth":"relevant", + "ner":"O" + }, + { + "id":126, + "orth":"form", + "ner":"O" + }, + { + "id":127, + "orth":"eg", + "ner":"O" + }, + { + "id":128, + "orth":"Wikipedia", + "ner":"O" + }, + { + "id":129, + "orth":":", + "ner":"O" + }, + { + "id":130, + "orth":"Good_article_nominations#Transport", + "ner":"O" + }, + { + "id":131, + "orth":" ", + "ner":"O" + }, + { + "id":132, + "orth":"\"", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"You, sir, are my hero. Any chance you remember what page that's on?", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"You", + "ner":"O" + }, + { + "id":1, + "orth":",", + "ner":"O" + }, + { + "id":2, + "orth":"sir", + "ner":"O" + }, + { + "id":3, + "orth":",", + "ner":"O" + }, + { + "id":4, + "orth":"are", + "ner":"O" + }, + { + "id":5, + "orth":"my", + "ner":"O" + }, + { + "id":6, + "orth":"hero", + "ner":"O" + }, + { + "id":7, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":8, + "orth":"Any", + "ner":"O" + }, + { + "id":9, + "orth":"chance", + "ner":"O" + }, + { + "id":10, + "orth":"you", + "ner":"O" + }, + { + "id":11, + "orth":"remember", + "ner":"O" + }, + { + "id":12, + "orth":"what", + "ner":"O" + }, + { + "id":13, + "orth":"page", + "ner":"O" + }, + { + "id":14, + "orth":"that", + "ner":"O" + }, + { + "id":15, + "orth":"'s", + "ner":"O" + }, + { + "id":16, + "orth":"on", + "ner":"O" + }, + { + "id":17, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"\"\n\nCongratulations from me as well, use the tools well. \u00a0\u00b7 talk \"", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"\"", + "ner":"O" + }, + { + "id":1, + "orth":"\n\n", + "ner":"O" + }, + { + "id":2, + "orth":"Congratulations", + "ner":"O" + }, + { + "id":3, + "orth":"from", + "ner":"O" + }, + { + "id":4, + "orth":"me", + "ner":"O" + }, + { + "id":5, + "orth":"as", + "ner":"O" + }, + { + "id":6, + "orth":"well", + "ner":"O" + }, + { + "id":7, + "orth":",", + "ner":"O" + }, + { + "id":8, + "orth":"use", + "ner":"O" + }, + { + "id":9, + "orth":"the", + "ner":"O" + }, + { + "id":10, + "orth":"tools", + "ner":"O" + }, + { + "id":11, + "orth":"well", + "ner":"O" + }, + { + "id":12, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":13, + "orth":"\u00a0", + "ner":"O" + }, + { + "id":14, + "orth":"\u00b7", + "ner":"O" + }, + { + "id":15, + "orth":"talk", + "ner":"O" + }, + { + "id":16, + "orth":"\"", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":0 + } + ] + }, + { + "raw":"Why can't you believe how fat Artie is? Did you see him on his recent appearence on the Tonight Show with Jay Leno? He looks absolutely AWFUL! If I had to put money on it, I'd say that Artie Lange is a can't miss candidate for the 2007 Dead pool! \n\n \nKindly keep your malicious fingers off of my above comment, . Everytime you remove it, I will repost it!!!", + "sentences":[ + { + "tokens":[ + { + "id":0, + "orth":"Why", + "ner":"O" + }, + { + "id":1, + "orth":"ca", + "ner":"O" + }, + { + "id":2, + "orth":"n't", + "ner":"O" + }, + { + "id":3, + "orth":"you", + "ner":"O" + }, + { + "id":4, + "orth":"believe", + "ner":"O" + }, + { + "id":5, + "orth":"how", + "ner":"O" + }, + { + "id":6, + "orth":"fat", + "ner":"O" + }, + { + "id":7, + "orth":"Artie", + "ner":"O" + }, + { + "id":8, + "orth":"is", + "ner":"O" + }, + { + "id":9, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":10, + "orth":"Did", + "ner":"O" + }, + { + "id":11, + "orth":"you", + "ner":"O" + }, + { + "id":12, + "orth":"see", + "ner":"O" + }, + { + "id":13, + "orth":"him", + "ner":"O" + }, + { + "id":14, + "orth":"on", + "ner":"O" + }, + { + "id":15, + "orth":"his", + "ner":"O" + }, + { + "id":16, + "orth":"recent", + "ner":"O" + }, + { + "id":17, + "orth":"appearence", + "ner":"O" + }, + { + "id":18, + "orth":"on", + "ner":"O" + }, + { + "id":19, + "orth":"the", + "ner":"O" + }, + { + "id":20, + "orth":"Tonight", + "ner":"O" + }, + { + "id":21, + "orth":"Show", + "ner":"O" + }, + { + "id":22, + "orth":"with", + "ner":"O" + }, + { + "id":23, + "orth":"Jay", + "ner":"O" + }, + { + "id":24, + "orth":"Leno", + "ner":"O" + }, + { + "id":25, + "orth":"?", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":26, + "orth":"He", + "ner":"O" + }, + { + "id":27, + "orth":"looks", + "ner":"O" + }, + { + "id":28, + "orth":"absolutely", + "ner":"O" + }, + { + "id":29, + "orth":"AWFUL", + "ner":"O" + }, + { + "id":30, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":31, + "orth":"If", + "ner":"O" + }, + { + "id":32, + "orth":"I", + "ner":"O" + }, + { + "id":33, + "orth":"had", + "ner":"O" + }, + { + "id":34, + "orth":"to", + "ner":"O" + }, + { + "id":35, + "orth":"put", + "ner":"O" + }, + { + "id":36, + "orth":"money", + "ner":"O" + }, + { + "id":37, + "orth":"on", + "ner":"O" + }, + { + "id":38, + "orth":"it", + "ner":"O" + }, + { + "id":39, + "orth":",", + "ner":"O" + }, + { + "id":40, + "orth":"I", + "ner":"O" + }, + { + "id":41, + "orth":"'d", + "ner":"O" + }, + { + "id":42, + "orth":"say", + "ner":"O" + }, + { + "id":43, + "orth":"that", + "ner":"O" + }, + { + "id":44, + "orth":"Artie", + "ner":"O" + }, + { + "id":45, + "orth":"Lange", + "ner":"O" + }, + { + "id":46, + "orth":"is", + "ner":"O" + }, + { + "id":47, + "orth":"a", + "ner":"O" + }, + { + "id":48, + "orth":"ca", + "ner":"O" + }, + { + "id":49, + "orth":"n't", + "ner":"O" + }, + { + "id":50, + "orth":"miss", + "ner":"O" + }, + { + "id":51, + "orth":"candidate", + "ner":"O" + }, + { + "id":52, + "orth":"for", + "ner":"O" + }, + { + "id":53, + "orth":"the", + "ner":"O" + }, + { + "id":54, + "orth":"2007", + "ner":"O" + }, + { + "id":55, + "orth":"Dead", + "ner":"O" + }, + { + "id":56, + "orth":"pool", + "ner":"O" + }, + { + "id":57, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":58, + "orth":" \n\n \n", + "ner":"O" + }, + { + "id":59, + "orth":"Kindly", + "ner":"O" + }, + { + "id":60, + "orth":"keep", + "ner":"O" + }, + { + "id":61, + "orth":"your", + "ner":"O" + }, + { + "id":62, + "orth":"malicious", + "ner":"O" + }, + { + "id":63, + "orth":"fingers", + "ner":"O" + }, + { + "id":64, + "orth":"off", + "ner":"O" + }, + { + "id":65, + "orth":"of", + "ner":"O" + }, + { + "id":66, + "orth":"my", + "ner":"O" + }, + { + "id":67, + "orth":"above", + "ner":"O" + }, + { + "id":68, + "orth":"comment", + "ner":"O" + }, + { + "id":69, + "orth":",", + "ner":"O" + }, + { + "id":70, + "orth":".", + "ner":"O" + } + ], + "brackets":[ + + ] + }, + { + "tokens":[ + { + "id":71, + "orth":"Everytime", + "ner":"O" + }, + { + "id":72, + "orth":"you", + "ner":"O" + }, + { + "id":73, + "orth":"remove", + "ner":"O" + }, + { + "id":74, + "orth":"it", + "ner":"O" + }, + { + "id":75, + "orth":",", + "ner":"O" + }, + { + "id":76, + "orth":"I", + "ner":"O" + }, + { + "id":77, + "orth":"will", + "ner":"O" + }, + { + "id":78, + "orth":"repost", + "ner":"O" + }, + { + "id":79, + "orth":"it", + "ner":"O" + }, + { + "id":80, + "orth":"!", + "ner":"O" + }, + { + "id":81, + "orth":"!", + "ner":"O" + }, + { + "id":82, + "orth":"!", + "ner":"O" + } + ], + "brackets":[ + + ] + } + ], + "cats":[ + { + "label":"insult", + "value":0 + }, + { + "label":"obscene", + "value":0 + }, + { + "label":"severe_toxic", + "value":0 + }, + { + "label":"toxic", + "value":1 + } + ] + } + ] + } +] \ No newline at end of file diff --git a/extra/example_data/textcat_example_data/jigsaw-toxic-comment.jsonl b/extra/example_data/textcat_example_data/jigsaw-toxic-comment.jsonl new file mode 100644 index 0000000..ac31b62 --- /dev/null +++ b/extra/example_data/textcat_example_data/jigsaw-toxic-comment.jsonl @@ -0,0 +1,10 @@ +{"meta": {"id": "0000997932d777bf"}, "text": "Explanation\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "001956c382006abd"}, "text": "I'm Sorry \n\nI'm sorry I screwed around with someones talk page. It was very bad to do. I know how having the templates on their talk page helps you assert your dominance over them. I know I should bow down to the almighty administrators. But then again, I'm going to go play outside....with your mom. 76.122.79.82", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 1}} +{"meta": {"id": "0020e7119b96eeeb"}, "text": "Stupid peace of shit stop deleting my stuff asshole go die and fall in a hole go to hell!", "cats": {"insult": 1, "obscene": 1, "severe_toxic": 1, "toxic": 1}} +{"meta": {"id": "000103f0d9cfb60f"}, "text": "D'aww! He matches this background colour I'm seemingly stuck with. Thanks. (talk) 21:51, January 11, 2016 (UTC)", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "001dc38a83d420cf"}, "text": "GET FUCKED UP. GET FUCKEEED UP. GOT A DRINK THAT YOU CANT PUT DOWN???/ GET FUCK UP GET FUCKED UP. I'M FUCKED UP RIGHT NOW!", "cats": {"insult": 0, "obscene": 1, "severe_toxic": 0, "toxic": 1}} +{"meta": {"id": "000113f07ec002fd"}, "text": "Hey man, I'm really not trying to edit war. It's just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. He seems to care more about the formatting than the actual info.", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "0001b41b1c6bb37e"}, "text": "\"\nMore\nI can't make any real suggestions on improvement - I wondered if the section statistics should be later on, or a subsection of \"\"types of accidents\"\" -I think the references may need tidying so that they are all in the exact same format ie date format etc. I can do that later on, if no-one else does first - if you have any preferences for formatting style on references or want to do it yourself please let me know.\n\nThere appears to be a backlog on articles for review so I guess there may be a delay until a reviewer turns up. It's listed in the relevant form eg Wikipedia:Good_article_nominations#Transport \"", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "0001d958c54c6e35"}, "text": "You, sir, are my hero. Any chance you remember what page that's on?", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "00025465d4725e87"}, "text": "\"\n\nCongratulations from me as well, use the tools well.  · talk \"", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 0}} +{"meta": {"id": "002264ea4d5f2887"}, "text": "Why can't you believe how fat Artie is? Did you see him on his recent appearence on the Tonight Show with Jay Leno? He looks absolutely AWFUL! If I had to put money on it, I'd say that Artie Lange is a can't miss candidate for the 2007 Dead pool! \n\n \nKindly keep your malicious fingers off of my above comment, . Everytime you remove it, I will repost it!!!", "cats": {"insult": 0, "obscene": 0, "severe_toxic": 0, "toxic": 1}} diff --git a/extra/example_data/textcat_example_data/textcatjsonl_to_trainjson.py b/extra/example_data/textcat_example_data/textcatjsonl_to_trainjson.py new file mode 100644 index 0000000..41b6a70 --- /dev/null +++ b/extra/example_data/textcat_example_data/textcatjsonl_to_trainjson.py @@ -0,0 +1,55 @@ +from pathlib import Path +import plac +import spacy +from spacy.training import docs_to_json +import srsly +import sys + + +@plac.annotations( + model=("Model name. Defaults to 'en'.", "option", "m", str), + input_file=("Input file (jsonl)", "positional", None, Path), + output_dir=("Output directory", "positional", None, Path), + n_texts=("Number of texts to convert", "option", "t", int), +) +def convert(model="en", input_file=None, output_dir=None, n_texts=0): + # Load model with tokenizer + sentencizer only + nlp = spacy.load(model) + nlp.select_pipes(disable=nlp.pipe_names) + sentencizer = nlp.create_pipe("sentencizer") + nlp.add_pipe(sentencizer, first=True) + + texts = [] + cats = [] + count = 0 + + if not input_file.exists(): + print("Input file not found:", input_file) + sys.exit(1) + else: + with open(input_file) as fileh: + for line in fileh: + data = srsly.json_loads(line) + texts.append(data["text"]) + cats.append(data["cats"]) + + if output_dir is not None: + output_dir = Path(output_dir) + if not output_dir.exists(): + output_dir.mkdir() + else: + output_dir = Path(".") + + docs = [] + for i, doc in enumerate(nlp.pipe(texts)): + doc.cats = cats[i] + docs.append(doc) + if n_texts > 0 and count == n_texts: + break + count += 1 + + srsly.write_json(output_dir / input_file.with_suffix(".json"), [docs_to_json(docs)]) + + +if __name__ == "__main__": + plac.call(convert) diff --git a/extra/example_data/training-data.json b/extra/example_data/training-data.json new file mode 100644 index 0000000..2565ce1 --- /dev/null +++ b/extra/example_data/training-data.json @@ -0,0 +1,641 @@ +[ + { + "id": 42, + "paragraphs": [ + { + "raw": "In an Oct. 19 review of \"The Misanthrope\" at Chicago's Goodman Theatre (\"Revitalized Classics Take the Stage in Windy City,\" Leisure & Arts), the role of Celimene, played by Kim Cattrall, was mistakenly attributed to Christina Haag. Ms. Haag plays Elianti.", + "sentences": [ + { + "tokens": [ + { + "head": 44, + "dep": "prep", + "tag": "IN", + "orth": "In", + "ner": "O", + "id": 0 + }, + { + "head": 3, + "dep": "det", + "tag": "DT", + "orth": "an", + "ner": "O", + "id": 1 + }, + { + "head": 2, + "dep": "nmod", + "tag": "NNP", + "orth": "Oct.", + "ner": "B-DATE", + "id": 2 + }, + { + "head": -1, + "dep": "nummod", + "tag": "CD", + "orth": "19", + "ner": "L-DATE", + "id": 3 + }, + { + "head": -4, + "dep": "pobj", + "tag": "NN", + "orth": "review", + "ner": "O", + "id": 4 + }, + { + "head": -1, + "dep": "prep", + "tag": "IN", + "orth": "of", + "ner": "O", + "id": 5 + }, + { + "head": 2, + "dep": "punct", + "tag": "``", + "orth": "``", + "ner": "O", + "id": 6 + }, + { + "head": 1, + "dep": "det", + "tag": "DT", + "orth": "The", + "ner": "B-WORK_OF_ART", + "id": 7 + }, + { + "head": -3, + "dep": "pobj", + "tag": "NN", + "orth": "Misanthrope", + "ner": "L-WORK_OF_ART", + "id": 8 + }, + { + "head": -1, + "dep": "punct", + "tag": "''", + "orth": "''", + "ner": "O", + "id": 9 + }, + { + "head": -2, + "dep": "prep", + "tag": "IN", + "orth": "at", + "ner": "O", + "id": 10 + }, + { + "head": 3, + "dep": "poss", + "tag": "NNP", + "orth": "Chicago", + "ner": "U-GPE", + "id": 11 + }, + { + "head": -1, + "dep": "case", + "tag": "POS", + "orth": "'s", + "ner": "O", + "id": 12 + }, + { + "head": 1, + "dep": "compound", + "tag": "NNP", + "orth": "Goodman", + "ner": "B-FAC", + "id": 13 + }, + { + "head": -4, + "dep": "pobj", + "tag": "NNP", + "orth": "Theatre", + "ner": "L-FAC", + "id": 14 + }, + { + "head": 4, + "dep": "punct", + "tag": "-LRB-", + "orth": "(", + "ner": "O", + "id": 15 + }, + { + "head": 3, + "dep": "punct", + "tag": "``", + "orth": "``", + "ner": "O", + "id": 16 + }, + { + "head": 1, + "dep": "amod", + "tag": "VBN", + "orth": "Revitalized", + "ner": "B-WORK_OF_ART", + "id": 17 + }, + { + "head": 1, + "dep": "nsubj", + "tag": "NNS", + "orth": "Classics", + "ner": "I-WORK_OF_ART", + "id": 18 + }, + { + "head": -15, + "dep": "appos", + "tag": "VBP", + "orth": "Take", + "ner": "I-WORK_OF_ART", + "id": 19 + }, + { + "head": 1, + "dep": "det", + "tag": "DT", + "orth": "the", + "ner": "I-WORK_OF_ART", + "id": 20 + }, + { + "head": -2, + "dep": "dobj", + "tag": "NN", + "orth": "Stage", + "ner": "I-WORK_OF_ART", + "id": 21 + }, + { + "head": -3, + "dep": "prep", + "tag": "IN", + "orth": "in", + "ner": "I-WORK_OF_ART", + "id": 22 + }, + { + "head": 1, + "dep": "compound", + "tag": "NNP", + "orth": "Windy", + "ner": "I-WORK_OF_ART", + "id": 23 + }, + { + "head": -2, + "dep": "pobj", + "tag": "NNP", + "orth": "City", + "ner": "L-WORK_OF_ART", + "id": 24 + }, + { + "head": -6, + "dep": "punct", + "tag": ",", + "orth": ",", + "ner": "O", + "id": 25 + }, + { + "head": -7, + "dep": "punct", + "tag": "''", + "orth": "''", + "ner": "O", + "id": 26 + }, + { + "head": -8, + "dep": "npadvmod", + "tag": "NN", + "orth": "Leisure", + "ner": "B-ORG", + "id": 27 + }, + { + "head": -1, + "dep": "cc", + "tag": "CC", + "orth": "&", + "ner": "I-ORG", + "id": 28 + }, + { + "head": -2, + "dep": "conj", + "tag": "NNS", + "orth": "Arts", + "ner": "L-ORG", + "id": 29 + }, + { + "head": -11, + "dep": "punct", + "tag": "-RRB-", + "orth": ")", + "ner": "O", + "id": 30 + }, + { + "head": 13, + "dep": "punct", + "tag": ",", + "orth": ",", + "ner": "O", + "id": 31 + }, + { + "head": 1, + "dep": "det", + "tag": "DT", + "orth": "the", + "ner": "O", + "id": 32 + }, + { + "head": 11, + "dep": "nsubjpass", + "tag": "NN", + "orth": "role", + "ner": "O", + "id": 33 + }, + { + "head": -1, + "dep": "prep", + "tag": "IN", + "orth": "of", + "ner": "O", + "id": 34 + }, + { + "head": -1, + "dep": "pobj", + "tag": "NNP", + "orth": "Celimene", + "ner": "U-PERSON", + "id": 35 + }, + { + "head": -3, + "dep": "punct", + "tag": ",", + "orth": ",", + "ner": "O", + "id": 36 + }, + { + "head": -4, + "dep": "acl", + "tag": "VBN", + "orth": "played", + "ner": "O", + "id": 37 + }, + { + "head": -1, + "dep": "agent", + "tag": "IN", + "orth": "by", + "ner": "O", + "id": 38 + }, + { + "head": 1, + "dep": "compound", + "tag": "NNP", + "orth": "Kim", + "ner": "B-PERSON", + "id": 39 + }, + { + "head": -2, + "dep": "pobj", + "tag": "NNP", + "orth": "Cattrall", + "ner": "L-PERSON", + "id": 40 + }, + { + "head": -8, + "dep": "punct", + "tag": ",", + "orth": ",", + "ner": "O", + "id": 41 + }, + { + "head": 2, + "dep": "auxpass", + "tag": "VBD", + "orth": "was", + "ner": "O", + "id": 42 + }, + { + "head": 1, + "dep": "advmod", + "tag": "RB", + "orth": "mistakenly", + "ner": "O", + "id": 43 + }, + { + "head": 0, + "dep": "root", + "tag": "VBN", + "orth": "attributed", + "ner": "O", + "id": 44 + }, + { + "head": -1, + "dep": "prep", + "tag": "IN", + "orth": "to", + "ner": "O", + "id": 45 + }, + { + "head": 1, + "dep": "compound", + "tag": "NNP", + "orth": "Christina", + "ner": "B-PERSON", + "id": 46 + }, + { + "head": -2, + "dep": "pobj", + "tag": "NNP", + "orth": "Haag", + "ner": "L-PERSON", + "id": 47 + }, + { + "head": -4, + "dep": "punct", + "tag": ".", + "orth": ".", + "ner": "O", + "id": 48 + } + ], + "brackets": [ + { + "first": 2, + "last": 3, + "label": "NML" + }, + { + "first": 1, + "last": 4, + "label": "NP" + }, + { + "first": 7, + "last": 8, + "label": "NP-TTL" + }, + { + "first": 11, + "last": 12, + "label": "NP" + }, + { + "first": 11, + "last": 14, + "label": "NP" + }, + { + "first": 10, + "last": 14, + "label": "PP-LOC" + }, + { + "first": 6, + "last": 14, + "label": "NP" + }, + { + "first": 5, + "last": 14, + "label": "PP" + }, + { + "first": 1, + "last": 14, + "label": "NP" + }, + { + "first": 17, + "last": 18, + "label": "NP-SBJ" + }, + { + "first": 20, + "last": 21, + "label": "NP" + }, + { + "first": 23, + "last": 24, + "label": "NP" + }, + { + "first": 22, + "last": 24, + "label": "PP-LOC" + }, + { + "first": 19, + "last": 24, + "label": "VP" + }, + { + "first": 17, + "last": 24, + "label": "S-HLN" + }, + { + "first": 27, + "last": 29, + "label": "NP-TMP" + }, + { + "first": 15, + "last": 30, + "label": "NP" + }, + { + "first": 1, + "last": 30, + "label": "NP" + }, + { + "first": 0, + "last": 30, + "label": "PP-LOC" + }, + { + "first": 32, + "last": 33, + "label": "NP" + }, + { + "first": 35, + "last": 35, + "label": "NP" + }, + { + "first": 34, + "last": 35, + "label": "PP" + }, + { + "first": 32, + "last": 35, + "label": "NP" + }, + { + "first": 39, + "last": 40, + "label": "NP-LGS" + }, + { + "first": 38, + "last": 40, + "label": "PP" + }, + { + "first": 37, + "last": 40, + "label": "VP" + }, + { + "first": 32, + "last": 41, + "label": "NP-SBJ-2" + }, + { + "first": 43, + "last": 43, + "label": "ADVP-MNR" + }, + { + "first": 46, + "last": 47, + "label": "NP" + }, + { + "first": 45, + "last": 47, + "label": "PP-CLR" + }, + { + "first": 44, + "last": 47, + "label": "VP" + }, + { + "first": 42, + "last": 47, + "label": "VP" + }, + { + "first": 0, + "last": 48, + "label": "S" + } + ] + }, + { + "tokens": [ + { + "head": 1, + "dep": "compound", + "tag": "NNP", + "orth": "Ms.", + "ner": "O", + "id": 0 + }, + { + "head": 1, + "dep": "nsubj", + "tag": "NNP", + "orth": "Haag", + "ner": "U-PERSON", + "id": 1 + }, + { + "head": 0, + "dep": "root", + "tag": "VBZ", + "orth": "plays", + "ner": "O", + "id": 2 + }, + { + "head": -1, + "dep": "dobj", + "tag": "NNP", + "orth": "Elianti", + "ner": "U-PERSON", + "id": 3 + }, + { + "head": -2, + "dep": "punct", + "tag": ".", + "orth": ".", + "ner": "O", + "id": 4 + } + ], + "brackets": [ + { + "first": 0, + "last": 1, + "label": "NP-SBJ" + }, + { + "first": 3, + "last": 3, + "label": "NP" + }, + { + "first": 2, + "last": 3, + "label": "VP" + }, + { + "first": 0, + "last": 4, + "label": "S" + } + ] + } + ] + } + ] + } + ] diff --git a/extra/example_data/vocab-data.jsonl b/extra/example_data/vocab-data.jsonl new file mode 100644 index 0000000..2f129dd --- /dev/null +++ b/extra/example_data/vocab-data.jsonl @@ -0,0 +1,21 @@ +{"lang": "en", "settings": {"oov_prob": -20.502029418945312}} +{"orth": ".", "id": 1, "lower": ".", "norm": ".", "shape": ".", "prefix": ".", "suffix": ".", "length": 1, "cluster": "8", "prob": -3.0678977966308594, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": true, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": ",", "id": 2, "lower": ",", "norm": ",", "shape": ",", "prefix": ",", "suffix": ",", "length": 1, "cluster": "4", "prob": -3.4549596309661865, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": true, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "the", "id": 3, "lower": "the", "norm": "the", "shape": "xxx", "prefix": "t", "suffix": "the", "length": 3, "cluster": "11", "prob": -3.528766632080078, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "I", "id": 4, "lower": "i", "norm": "I", "shape": "X", "prefix": "I", "suffix": "I", "length": 1, "cluster": "346", "prob": -3.791565179824829, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": false, "is_space": false, "is_title": true, "is_upper": true, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "to", "id": 5, "lower": "to", "norm": "to", "shape": "xx", "prefix": "t", "suffix": "to", "length": 2, "cluster": "12", "prob": -3.8560216426849365, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "a", "id": 6, "lower": "a", "norm": "a", "shape": "x", "prefix": "a", "suffix": "a", "length": 1, "cluster": "19", "prob": -3.92978835105896, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "and", "id": 7, "lower": "and", "norm": "and", "shape": "xxx", "prefix": "a", "suffix": "and", "length": 3, "cluster": "20", "prob": -4.113108158111572, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "of", "id": 8, "lower": "of", "norm": "of", "shape": "xx", "prefix": "o", "suffix": "of", "length": 2, "cluster": "28", "prob": -4.27587366104126, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "you", "id": 9, "lower": "you", "norm": "you", "shape": "xxx", "prefix": "y", "suffix": "you", "length": 3, "cluster": "602", "prob": -4.373791217803955, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "it", "id": 10, "lower": "it", "norm": "it", "shape": "xx", "prefix": "i", "suffix": "it", "length": 2, "cluster": "474", "prob": -4.388050079345703, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "is", "id": 11, "lower": "is", "norm": "is", "shape": "xx", "prefix": "i", "suffix": "is", "length": 2, "cluster": "762", "prob": -4.457748889923096, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "that", "id": 12, "lower": "that", "norm": "that", "shape": "xxxx", "prefix": "t", "suffix": "hat", "length": 4, "cluster": "84", "prob": -4.464504718780518, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "\n\n", "id": 0, "lower": "\n\n", "norm": "\n\n", "shape": "\n\n", "prefix": "\n", "suffix": "\n\n", "length": 2, "cluster": "0", "prob": -4.606560707092285, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": false, "is_space": true, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "in", "id": 13, "lower": "in", "norm": "in", "shape": "xx", "prefix": "i", "suffix": "in", "length": 2, "cluster": "60", "prob": -4.619071960449219, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "'s", "id": 14, "lower": "'s", "norm": "'s", "shape": "'x", "prefix": "'", "suffix": "'s", "length": 2, "cluster": "52", "prob": -4.830559253692627, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "n't", "id": 15, "lower": "n't", "norm": "n't", "shape": "x'x", "prefix": "n", "suffix": "n't", "length": 3, "cluster": "74", "prob": -4.859938621520996, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "for", "id": 16, "lower": "for", "norm": "for", "shape": "xxx", "prefix": "f", "suffix": "for", "length": 3, "cluster": "508", "prob": -4.8801093101501465, "is_alpha": true, "is_ascii": true, "is_digit": false, "is_lower": true, "is_punct": false, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": "\"", "id": 17, "lower": "\"", "norm": "\"", "shape": "\"", "prefix": "\"", "suffix": "\"", "length": 1, "cluster": "0", "prob": -5.02677583694458, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": true, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": true, "is_left_punct": true, "is_right_punct": true} +{"orth": "?", "id": 18, "lower": "?", "norm": "?", "shape": "?", "prefix": "?", "suffix": "?", "length": 1, "cluster": "0", "prob": -5.05924654006958, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": true, "is_space": false, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} +{"orth": " ", "id": 0, "lower": " ", "norm": " ", "shape": " ", "prefix": " ", "suffix": " ", "length": 1, "cluster": "0", "prob": -5.129165172576904, "is_alpha": false, "is_ascii": true, "is_digit": false, "is_lower": false, "is_punct": false, "is_space": true, "is_title": false, "is_upper": false, "like_url": false, "like_num": false, "like_email": false, "is_stop": false, "is_oov": false, "is_quote": false, "is_left_punct": false, "is_right_punct": false} diff --git a/licenses/3rd_party_licenses.txt b/licenses/3rd_party_licenses.txt new file mode 100644 index 0000000..9b037a4 --- /dev/null +++ b/licenses/3rd_party_licenses.txt @@ -0,0 +1,202 @@ +Third Party Licenses for spaCy +============================== + +NumPy +----- + +* Files: setup.py + +Copyright (c) 2005-2020, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +scikit-learn +------------ + +* Files: scorer.py + +The implementation of roc_auc_score() is adapted from scikit-learn, which is +distributed under the following license: + +New BSD License + +Copyright (c) 2007–2019 The scikit-learn developers. +All rights reserved. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of the Scikit-learn Developers nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + + +pyvi +---- + +* Files: lang/vi/__init__.py + +The MIT License (MIT) +Copyright (c) 2016 Viet-Trung Tran + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +importlib_metadata +------------------ + +* Files: util.py + +The implementation of packages_distributions() is adapted from +importlib_metadata, which is distributed under the following license: + +Copyright 2017-2019 Jason R. Coombs, Barry Warsaw + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +polyleven +--------- + +* Files: spacy/matcher/polyleven.c + +MIT License + +Copyright (c) 2021 Fujimoto Seiji +Copyright (c) 2021 Max Bachmann +Copyright (c) 2022 Nick Mazuk +Copyright (c) 2022 Michael Weiss + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +SciPy +----- + +* Files: scorer.py + +The implementation of trapezoid() is adapted from SciPy, which is distributed +under the following license: + +New BSD License + +Copyright (c) 2001-2002 Enthought, Inc. 2003-2023, SciPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 0000000..ddcd0ca --- /dev/null +++ b/netlify.toml @@ -0,0 +1,65 @@ +redirects = [ + # Netlify + {from = "https://spacy.netlify.com/*", to="https://spacy.io/:splat", force = true }, + # Subdomain for branches + {from = "https://nightly.spacy.io/*", to="https://spacy.io/:splat", force = true}, + {from = "https://v2.spacy.io/*", to="https://v2-spacy-io.spacy.io/:splat", force = true, status = 200}, + # Old subdomains + {from = "https://survey.spacy.io/*", to = "https://spacy.io", force = true}, + {from = "http://survey.spacy.io/*", to = "https://spacy.io", force = true}, + {from = "https://alpha.spacy.io/*", to = "https://spacy.io", force = true}, + {from = "http://alpha.spacy.io/*", to = "https://spacy.io", force = true}, + # Old demos + {from = "/demos/*", to = "https://explosion.ai/demos/:splat", force = true}, + # Old blog + {from = "/blog/*", to = "https://explosion.ai/blog/:splat", force = true}, + {from = "/feed", to = "https://explosion.ai/feed", force = true}, + {from = "/feed.xml", to = "https://explosion.ai/feed", force = true}, + # Old documentation pages (1.x) + {from = "/docs/usage/processing-text", to = "/usage/linguistic-features", force = true}, + {from = "/docs/usage/deep-learning", to = "/usage/training", force = true}, + {from = "/docs/usage/pos-tagging", to = "/usage/linguistic-features#pos-tagging", force = true}, + {from = "/docs/usage/dependency-parse", to = "/usage/linguistic-features#dependency-parse", force = true}, + {from = "/docs/usage/entity-recognition", to = "/usage/linguistic-features#named-entities", force = true}, + {from = "/docs/usage/word-vectors-similarities", to = "/usage/vectors-similarity", force = true}, + {from = "/docs/usage/customizing-tokenizer", to = "/usage/linguistic-features#tokenization", force = true}, + {from = "/docs/usage/language-processing-pipeline", to = "/usage/processing-pipelines", force = true}, + {from = "/docs/usage/customizing-pipeline", to = "/usage/processing-pipelines", force = true}, + {from = "/docs/usage/training-ner", to = "/usage/training", force = true}, + {from = "/docs/usage/tutorials", to = "/usage/examples", force = true}, + {from = "/docs/usage/data-model", to = "/api", force = true}, + {from = "/docs/usage/cli", to = "/api/cli", force = true}, + {from = "/docs/usage/lightning-tour", to = "/usage/spacy-101#lightning-tour", force = true}, + {from = "/docs/api/language-models", to = "/usage/models#languages", force = true}, + {from = "/docs/api/spacy", to = "/docs/api/top-level", force = true}, + {from = "/docs/api/displacy", to = "/api/top-level#displacy", force = true}, + {from = "/docs/api/util", to = "/api/top-level#util", force = true}, + {from = "/docs/api/features", to = "/models/#architecture", force = true}, + {from = "/docs/api/philosophy", to = "/usage/spacy-101", force = true}, + {from = "/docs/usage/showcase", to = "/universe", force = true}, + {from = "/tutorials/load-new-word-vectors", to = "/usage/linguistic-features", force = true}, + {from = "/tutorials", to = "/usage/examples", force = true}, + # Old documentation pages (v2.x) + {from = "/usage/adding-languages", to = "/usage/linguistic-features", force = true}, + {from = "/usage/vectors-similarity", to = "/usage/linguistic-features#vectors-similarity", force = true}, + {from = "/api/goldparse", to = "/api/top-level", force = true}, + {from = "/api/goldcorpus", to = "/api/corpus", force = true}, + {from = "/api/annotation", to = "/api/data-formats", force = true}, + {from = "/usage/examples", to = "/usage/projects", force = true}, + # Rewrite all other docs pages to / + {from = "/docs/*", to = "/:splat"}, + # Updated documentation pages + {from = "/usage/resources", to = "/universe", force = true}, + {from = "/usage/lightning-tour", to = "/usage/spacy-101#lightning-tour", force = true}, + {from = "/usage/linguistic-features#rule-based-matching", to = "/usage/rule-based-matching", force = true}, + {from = "/models/comparison", to = "/models", force = true}, + {from = "/api/#section-cython", to = "/api/cython", force = true}, + {from = "/api/#cython", to = "/api/cython", force = true}, + {from = "/api/sentencesegmenter", to="/api/sentencizer"}, + {from = "/universe", to = "/universe/project/:id", query = {id = ":id"}, force = true}, + {from = "/universe", to = "/universe/category/:category", query = {category = ":category"}, force = true}, + # Renamed universe projects + {from = "/universe/project/spacy-pytorch-transformers", to = "/universe/project/spacy-transformers", force = true}, + # Old model pages + {from = "/models/en-starters", to = "/models/en", force = true}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..395c2f7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = [ + "setuptools", + "cython>=3.0,<4.0", + "cymem>=2.0.2,<2.1.0", + "preshed>=3.0.2,<3.1.0", + "murmurhash>=0.28.0,<1.1.0", + "thinc>=8.3.12,<8.4.0", + "numpy>=2.0.0,<3.0.0" +] +build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +build = "*" +skip = "cp39* *-win32 *i686* cp3??t-* *cp310-win_arm64" +test-skip = "" + +archs = ["native"] + +build-frontend = "default" +config-settings = {} +dependency-versions = "pinned" +environment = { PIP_CONSTRAINT = "build-constraints.txt" } + +environment-pass = [] +build-verbosity = 0 + +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable" +before-build = "pip install -r requirements.txt && python setup.py clean" +repair-wheel-command = "" + +test-command = "" +before-test = "" +test-requires = [] +test-extras = [] + +container-engine = "docker" + +manylinux-x86_64-image = "manylinux2014" +manylinux-i686-image = "manylinux2014" +manylinux-aarch64-image = "manylinux2014" +manylinux-ppc64le-image = "manylinux2014" +manylinux-s390x-image = "manylinux2014" +manylinux-pypy_x86_64-image = "manylinux2014" +manylinux-pypy_i686-image = "manylinux2014" +manylinux-pypy_aarch64-image = "manylinux2014" + +musllinux-x86_64-image = "musllinux_1_2" +musllinux-i686-image = "musllinux_1_2" +musllinux-aarch64-image = "musllinux_1_2" +musllinux-ppc64le-image = "musllinux_1_2" +musllinux-s390x-image = "musllinux_1_2" + +[tool.cibuildwheel.linux] +repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}" + +[tool.cibuildwheel.macos] +repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" + +[tool.cibuildwheel.windows] + +[tool.cibuildwheel.pyodide] + + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = ["E", "F", "W", "C", "B", "B9"] +ignore = ["E203", "E266", "E501", "E731", "E741", "F541"] + +[tool.ruff.lint.isort] +combine-as-imports = true +split-on-trailing-comma = true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..50c6382 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,37 @@ +# Our libraries +spacy-legacy>=3.0.11,<3.1.0 +spacy-loggers>=1.0.0,<2.0.0 +cymem>=2.0.2,<2.1.0 +preshed>=3.0.2,<3.1.0 +thinc>=8.3.12,<8.4.0 +ml_datasets>=0.2.1,<0.3.0 +murmurhash>=0.28.0,<1.1.0 +wasabi>=0.9.1,<1.2.0 +srsly>=2.5.3,<3.0.0 +catalogue>=2.0.6,<2.1.0 +typer>=0.3.0,<1.0.0 +weasel>=1.0.0,<2.0.0 +# Third party dependencies +numpy>=2.0.0,<3.0.0 +requests>=2.13.0,<3.0.0 +tqdm>=4.38.0,<5.0.0 +pydantic>=2.0.0,<3.0.0 +jinja2 +# Official Python utilities +setuptools +packaging>=20.0 +# Development dependencies +pre-commit>=2.13.0 +cython>=3.0,<4.0 +pytest>=5.2.0,!=7.1.0 +pytest-timeout>=1.3.0,<2.0.0 +mock>=2.0.0,<3.0.0 +hypothesis>=3.27.0,<7.0.0 +mypy>=1.5.0,<1.6.0; platform_machine != "aarch64" and python_version >= "3.8" +types-mock>=0.1.1 +types-setuptools>=57.0.0 +types-requests +types-setuptools>=57.0.0 +ruff>=0.9.0 +cython-lint>=0.15.0 +confection>=1.1.0,<2.0.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7f9b200 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,147 @@ +[metadata] +description = Industrial-strength Natural Language Processing (NLP) in Python +url = https://spacy.io +author = Explosion +author_email = contact@explosion.ai +license = MIT +long_description = file: README.md +long_description_content_type = text/markdown +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + Intended Audience :: Science/Research + License :: OSI Approved :: MIT License + Operating System :: POSIX :: Linux + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Programming Language :: Cython + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 + Topic :: Scientific/Engineering +project_urls = + Release notes = https://github.com/explosion/spaCy/releases + Source = https://github.com/explosion/spaCy + +[options] +zip_safe = false +include_package_data = true +python_requires = >=3.9,<3.15 +# NOTE: This section is superseded by pyproject.toml and will be removed in +# spaCy v4 +setup_requires = + cython>=3.0,<4.0 + numpy>=2.0.0,<3.0.0; python_version < "3.9" + numpy>=2.0.0,<3.0.0; python_version >= "3.9" + # We also need our Cython packages here to compile against + cymem>=2.0.2,<2.1.0 + preshed>=3.0.2,<3.1.0 + murmurhash>=0.28.0,<1.1.0 + thinc>=8.3.12,<8.4.0 +install_requires = + # Our libraries + spacy-legacy>=3.0.11,<3.1.0 + spacy-loggers>=1.0.0,<2.0.0 + murmurhash>=0.28.0,<1.1.0 + cymem>=2.0.2,<2.1.0 + preshed>=3.0.2,<3.1.0 + thinc>=8.3.12,<8.4.0 + wasabi>=0.9.1,<1.2.0 + srsly>=2.5.3,<3.0.0 + catalogue>=2.0.6,<2.1.0 + weasel>=1.0.0,<2.0.0 + confection>=1.3.2,<2.0.0 + # Third-party dependencies + typer>=0.3.0,<1.0.0 + tqdm>=4.38.0,<5.0.0 + numpy>=1.15.0; python_version < "3.9" + numpy>=1.19.0; python_version >= "3.9" + requests>=2.13.0,<3.0.0 + pydantic>=2.0.0,<3.0.0 + jinja2 + # Official Python utilities + setuptools + packaging>=20.0 + +[options.entry_points] +console_scripts = + spacy = spacy.cli:setup_cli + +[options.extras_require] +lookups = + spacy_lookups_data>=1.0.3,<1.1.0 +transformers = + spacy_transformers>=1.1.2,<1.4.0 +cuda = + cupy>=5.0.0b4,<13.0.0 +cuda80 = + cupy-cuda80>=5.0.0b4,<13.0.0 +cuda90 = + cupy-cuda90>=5.0.0b4,<13.0.0 +cuda91 = + cupy-cuda91>=5.0.0b4,<13.0.0 +cuda92 = + cupy-cuda92>=5.0.0b4,<13.0.0 +cuda100 = + cupy-cuda100>=5.0.0b4,<13.0.0 +cuda101 = + cupy-cuda101>=5.0.0b4,<13.0.0 +cuda102 = + cupy-cuda102>=5.0.0b4,<13.0.0 +cuda110 = + cupy-cuda110>=5.0.0b4,<13.0.0 +cuda111 = + cupy-cuda111>=5.0.0b4,<13.0.0 +cuda112 = + cupy-cuda112>=5.0.0b4,<13.0.0 +cuda113 = + cupy-cuda113>=5.0.0b4,<13.0.0 +cuda114 = + cupy-cuda114>=5.0.0b4,<13.0.0 +cuda115 = + cupy-cuda115>=5.0.0b4,<13.0.0 +cuda116 = + cupy-cuda116>=5.0.0b4,<13.0.0 +cuda117 = + cupy-cuda117>=5.0.0b4,<13.0.0 +cuda11x = + cupy-cuda11x>=11.0.0,<13.0.0 +cuda12x = + cupy-cuda12x>=11.5.0,<13.0.0 +cuda-autodetect = + cupy-wheel>=11.0.0,<13.0.0 +apple = + thinc-apple-ops>=1.0.0,<2.0.0 +# Language tokenizers with external dependencies +ja = + sudachipy>=0.5.2,!=0.6.1 + sudachidict_core>=20211220 +ko = + natto-py>=0.9.0 +th = + pythainlp>=2.0 + +[bdist_wheel] +universal = false + +[sdist] +formats = gztar + +[tool:pytest] +markers = + slow: mark a test as slow + issue: reference specific issue +filterwarnings = + error + ignore:Core Pydantic V1:UserWarning:pydantic + +[mypy] +ignore_missing_imports = True +no_implicit_optional = True +plugins = pydantic.mypy, thinc.mypy +allow_redefinition = True diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..6f28b73 --- /dev/null +++ b/setup.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +from setuptools import Extension, setup, find_packages +import sys +import numpy +from setuptools.command.build_ext import build_ext +from sysconfig import get_path +from pathlib import Path +import shutil +from Cython.Build import cythonize +from Cython.Compiler import Options +import os +import subprocess + + +ROOT = Path(__file__).parent +PACKAGE_ROOT = ROOT / "spacy" + + +# Preserve `__doc__` on functions and classes +# http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options +Options.docstrings = True + +PACKAGES = find_packages() +MOD_NAMES = [ + "spacy.training.alignment_array", + "spacy.training.example", + "spacy.parts_of_speech", + "spacy.strings", + "spacy.lexeme", + "spacy.vocab", + "spacy.attrs", + "spacy.kb.candidate", + "spacy.kb.kb", + "spacy.kb.kb_in_memory", + "spacy.ml.parser_model", + "spacy.morphology", + "spacy.pipeline.dep_parser", + "spacy.pipeline._edit_tree_internals.edit_trees", + "spacy.pipeline.morphologizer", + "spacy.pipeline.multitask", + "spacy.pipeline.ner", + "spacy.pipeline.pipe", + "spacy.pipeline.trainable_pipe", + "spacy.pipeline.sentencizer", + "spacy.pipeline.senter", + "spacy.pipeline.tagger", + "spacy.pipeline.transition_parser", + "spacy.pipeline._parser_internals.arc_eager", + "spacy.pipeline._parser_internals.ner", + "spacy.pipeline._parser_internals.nonproj", + "spacy.pipeline._parser_internals._state", + "spacy.pipeline._parser_internals.stateclass", + "spacy.pipeline._parser_internals.transition_system", + "spacy.pipeline._parser_internals._beam_utils", + "spacy.tokenizer", + "spacy.training.align", + "spacy.training.gold_io", + "spacy.tokens.doc", + "spacy.tokens.span", + "spacy.tokens.token", + "spacy.tokens.span_group", + "spacy.tokens.graph", + "spacy.tokens.morphanalysis", + "spacy.tokens._retokenize", + "spacy.matcher.matcher", + "spacy.matcher.phrasematcher", + "spacy.matcher.dependencymatcher", + "spacy.symbols", + "spacy.vectors", +] +COMPILE_OPTIONS = { + "msvc": ["/Ox", "/EHsc"], + "mingw32": ["-O2", "-Wno-strict-prototypes", "-Wno-unused-function"], + "other": ["-O2", "-Wno-strict-prototypes", "-Wno-unused-function"], +} +LINK_OPTIONS = {"msvc": ["-std=c++11"], "mingw32": ["-std=c++11"], "other": []} +COMPILER_DIRECTIVES = { + "language_level": -3, + "embedsignature": True, + "annotation_typing": False, + "profile": sys.version_info < (3, 12), +} +# Files to copy into the package that are otherwise not included +COPY_FILES = { + ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package" / "test.cfg", + ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package" / "test.toml", + ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package" / "test.txt", +} + + +# By subclassing build_extensions we have the actual compiler that will be used which is really known only after finalize_options +# http://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used +class build_ext_options: + def build_options(self): + for e in self.extensions: + e.extra_compile_args += COMPILE_OPTIONS.get( + self.compiler.compiler_type, COMPILE_OPTIONS["other"] + ) + for e in self.extensions: + e.extra_link_args += LINK_OPTIONS.get( + self.compiler.compiler_type, LINK_OPTIONS["other"] + ) + + +class build_ext_subclass(build_ext, build_ext_options): + def build_extensions(self): + if self.parallel is None and os.environ.get("SPACY_NUM_BUILD_JOBS") is not None: + self.parallel = int(os.environ.get("SPACY_NUM_BUILD_JOBS")) + build_ext_options.build_options(self) + build_ext.build_extensions(self) + + +# Include the git version in the build (adapted from NumPy) +# Copyright (c) 2005-2020, NumPy Developers. +# BSD 3-Clause license, see licenses/3rd_party_licenses.txt +def write_git_info_py(filename="spacy/git_info.py"): + def _minimal_ext_cmd(cmd): + # construct minimal environment + env = {} + for k in ["SYSTEMROOT", "PATH", "HOME"]: + v = os.environ.get(k) + if v is not None: + env[k] = v + # LANGUAGE is used on win32 + env["LANGUAGE"] = "C" + env["LANG"] = "C" + env["LC_ALL"] = "C" + out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env) + return out + + git_version = "Unknown" + if Path(".git").exists(): + try: + out = _minimal_ext_cmd(["git", "rev-parse", "--short", "HEAD"]) + git_version = out.strip().decode("ascii") + except Exception: + pass + elif Path(filename).exists(): + # must be a source distribution, use existing version file + try: + a = open(filename, "r") + lines = a.readlines() + git_version = lines[-1].split('"')[1] + except Exception: + pass + finally: + a.close() + + text = """# THIS FILE IS GENERATED FROM SPACY SETUP.PY +# +GIT_VERSION = "%(git_version)s" +""" + a = open(filename, "w") + try: + a.write(text % {"git_version": git_version}) + finally: + a.close() + + +def clean(path): + for path in path.glob("**/*"): + if path.is_file() and path.suffix in (".so", ".cpp", ".html"): + print(f"Deleting {path.name}") + path.unlink() + + +def setup_package(): + write_git_info_py() + if len(sys.argv) > 1 and sys.argv[1] == "clean": + return clean(PACKAGE_ROOT) + + with (PACKAGE_ROOT / "about.py").open("r") as f: + about = {} + exec(f.read(), about) + + for copy_file, target_file in COPY_FILES.items(): + if copy_file.exists(): + shutil.copyfile(str(copy_file), str(target_file)) + print(f"Copied {copy_file} -> {target_file}") + + include_dirs = [ + numpy.get_include(), + get_path("include"), + ] + ext_modules = [] + ext_modules.append( + Extension( + "spacy.matcher.levenshtein", + [ + "spacy/matcher/levenshtein.pyx", + "spacy/matcher/polyleven.c", + ], + language="c", + include_dirs=include_dirs, + ) + ) + for name in MOD_NAMES: + mod_path = name.replace(".", "/") + ".pyx" + ext = Extension( + name, + [mod_path], + language="c++", + include_dirs=include_dirs, + extra_compile_args=["-std=c++11"], + ) + ext_modules.append(ext) + print("Cythonizing sources") + ext_modules = cythonize(ext_modules, compiler_directives=COMPILER_DIRECTIVES) + + setup( + name="spacy", + packages=PACKAGES, + version=about["__version__"], + ext_modules=ext_modules, + cmdclass={"build_ext": build_ext_subclass}, + package_data={"": ["*.pyx", "*.pxd", "*.pxi"]}, + ) + + +if __name__ == "__main__": + setup_package() diff --git a/spacy/__init__.pxd b/spacy/__init__.pxd new file mode 100644 index 0000000..e69de29 diff --git a/spacy/__init__.py b/spacy/__init__.py new file mode 100644 index 0000000..5b3ff25 --- /dev/null +++ b/spacy/__init__.py @@ -0,0 +1,102 @@ +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, Union + +# set library-specific custom warning handling before doing anything else +from .errors import setup_default_warnings + +setup_default_warnings() # noqa: E402 + +# These are imported as part of the API +from thinc.api import Config, prefer_gpu, require_cpu, require_gpu # noqa: F401 + +from . import ( + pipeline, # noqa: F401 + util, +) +from .about import __version__ # noqa: F401 +from .cli.info import info # noqa: F401 +from .errors import Errors +from .glossary import explain # noqa: F401 +from .language import Language +from .registrations import REGISTRY_POPULATED, populate_registry + +# Rebuild pydantic v2 schemas that use forward references to Language/Vocab +from .schemas import ( # noqa: F401 + ConfigSchema, + ConfigSchemaInit, + ConfigSchemaNlp, + ConfigSchemaPretrain, + ConfigSchemaTraining, +) +from .training import Example # noqa: F401 +from .util import logger, registry # noqa: F401 +from .vocab import Vocab + +_rebuild_ns = {"Language": Language, "Vocab": Vocab, "Example": Example} +for _schema in ( + ConfigSchemaTraining, + ConfigSchemaNlp, + ConfigSchemaPretrain, + ConfigSchemaInit, + ConfigSchema, +): + _schema.model_rebuild(_types_namespace=_rebuild_ns) # type: ignore[attr-defined] + +if sys.maxunicode == 65535: + raise SystemError(Errors.E130) + + +def load( + name: Union[str, Path], + *, + vocab: Union[Vocab, bool] = True, + disable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, + config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(), +) -> Language: + """Load a spaCy model from an installed package or a local path. + + name (str): Package name or model path. + vocab (Vocab): A Vocab object. If True, a vocab is created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled + pipes will be loaded but they won't be run unless you explicitly + enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (but can be enabled later using nlp.enable_pipe). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded + components won't be loaded. + config (Dict[str, Any] / Config): Config overrides as nested dict or dict + keyed by section values in dot notation. + RETURNS (Language): The loaded nlp object. + """ + return util.load_model( + name, + vocab=vocab, + disable=disable, + enable=enable, + exclude=exclude, + config=config, + ) + + +def blank( + name: str, + *, + vocab: Union[Vocab, bool] = True, + config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(), + meta: Dict[str, Any] = util.SimpleFrozenDict(), +) -> Language: + """Create a blank nlp object for a given language code. + + name (str): The language code, e.g. "en". + vocab (Vocab): A Vocab object. If True, a vocab is created. + config (Dict[str, Any] / Config): Optional config overrides. + meta (Dict[str, Any]): Overrides for nlp.meta. + RETURNS (Language): The nlp object. + """ + LangClass = util.get_lang_class(name) + # We should accept both dot notation and nested dict here for consistency + config = util.dot_to_dict(config) + return LangClass.from_config(config, vocab=vocab, meta=meta) diff --git a/spacy/__main__.py b/spacy/__main__.py new file mode 100644 index 0000000..f6b5066 --- /dev/null +++ b/spacy/__main__.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + from spacy.cli import setup_cli + + setup_cli() diff --git a/spacy/about.py b/spacy/about.py new file mode 100644 index 0000000..4ba517b --- /dev/null +++ b/spacy/about.py @@ -0,0 +1,5 @@ +# fmt: off +__title__ = "spacy" +__version__ = "3.8.14" +__download_url__ = "https://github.com/explosion/spacy-models/releases/download" +__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" diff --git a/spacy/attrs.pxd b/spacy/attrs.pxd new file mode 100644 index 0000000..fbbac0e --- /dev/null +++ b/spacy/attrs.pxd @@ -0,0 +1,99 @@ +# Reserve 64 values for flag features +from . cimport symbols + + +cdef enum attr_id_t: + NULL_ATTR + IS_ALPHA + IS_ASCII + IS_DIGIT + IS_LOWER + IS_PUNCT + IS_SPACE + IS_TITLE + IS_UPPER + LIKE_URL + LIKE_NUM + LIKE_EMAIL + IS_STOP + IS_OOV_DEPRECATED + IS_BRACKET + IS_QUOTE + IS_LEFT_PUNCT + IS_RIGHT_PUNCT + IS_CURRENCY + + FLAG19 = 19 + FLAG20 + FLAG21 + FLAG22 + FLAG23 + FLAG24 + FLAG25 + FLAG26 + FLAG27 + FLAG28 + FLAG29 + FLAG30 + FLAG31 + FLAG32 + FLAG33 + FLAG34 + FLAG35 + FLAG36 + FLAG37 + FLAG38 + FLAG39 + FLAG40 + FLAG41 + FLAG42 + FLAG43 + FLAG44 + FLAG45 + FLAG46 + FLAG47 + FLAG48 + FLAG49 + FLAG50 + FLAG51 + FLAG52 + FLAG53 + FLAG54 + FLAG55 + FLAG56 + FLAG57 + FLAG58 + FLAG59 + FLAG60 + FLAG61 + FLAG62 + FLAG63 + + ID + ORTH + LOWER + NORM + SHAPE + PREFIX + SUFFIX + + LENGTH + CLUSTER + LEMMA + POS + TAG + DEP + ENT_IOB + ENT_TYPE + HEAD + SENT_START + SPACY + PROB + + LANG + ENT_KB_ID = symbols.ENT_KB_ID + MORPH + ENT_ID = symbols.ENT_ID + + IDX + SENT_END diff --git a/spacy/attrs.pyx b/spacy/attrs.pyx new file mode 100644 index 0000000..363dd09 --- /dev/null +++ b/spacy/attrs.pyx @@ -0,0 +1,212 @@ +# cython: profile=False +from .errors import Errors + +IOB_STRINGS = ("", "I", "O", "B") + +IDS = { + "": NULL_ATTR, + "IS_ALPHA": IS_ALPHA, + "IS_ASCII": IS_ASCII, + "IS_DIGIT": IS_DIGIT, + "IS_LOWER": IS_LOWER, + "IS_PUNCT": IS_PUNCT, + "IS_SPACE": IS_SPACE, + "IS_TITLE": IS_TITLE, + "IS_UPPER": IS_UPPER, + "LIKE_URL": LIKE_URL, + "LIKE_NUM": LIKE_NUM, + "LIKE_EMAIL": LIKE_EMAIL, + "IS_STOP": IS_STOP, + "IS_OOV_DEPRECATED": IS_OOV_DEPRECATED, + "IS_BRACKET": IS_BRACKET, + "IS_QUOTE": IS_QUOTE, + "IS_LEFT_PUNCT": IS_LEFT_PUNCT, + "IS_RIGHT_PUNCT": IS_RIGHT_PUNCT, + "IS_CURRENCY": IS_CURRENCY, + "FLAG19": FLAG19, + "FLAG20": FLAG20, + "FLAG21": FLAG21, + "FLAG22": FLAG22, + "FLAG23": FLAG23, + "FLAG24": FLAG24, + "FLAG25": FLAG25, + "FLAG26": FLAG26, + "FLAG27": FLAG27, + "FLAG28": FLAG28, + "FLAG29": FLAG29, + "FLAG30": FLAG30, + "FLAG31": FLAG31, + "FLAG32": FLAG32, + "FLAG33": FLAG33, + "FLAG34": FLAG34, + "FLAG35": FLAG35, + "FLAG36": FLAG36, + "FLAG37": FLAG37, + "FLAG38": FLAG38, + "FLAG39": FLAG39, + "FLAG40": FLAG40, + "FLAG41": FLAG41, + "FLAG42": FLAG42, + "FLAG43": FLAG43, + "FLAG44": FLAG44, + "FLAG45": FLAG45, + "FLAG46": FLAG46, + "FLAG47": FLAG47, + "FLAG48": FLAG48, + "FLAG49": FLAG49, + "FLAG50": FLAG50, + "FLAG51": FLAG51, + "FLAG52": FLAG52, + "FLAG53": FLAG53, + "FLAG54": FLAG54, + "FLAG55": FLAG55, + "FLAG56": FLAG56, + "FLAG57": FLAG57, + "FLAG58": FLAG58, + "FLAG59": FLAG59, + "FLAG60": FLAG60, + "FLAG61": FLAG61, + "FLAG62": FLAG62, + "FLAG63": FLAG63, + "ID": ID, + "ORTH": ORTH, + "LOWER": LOWER, + "NORM": NORM, + "SHAPE": SHAPE, + "PREFIX": PREFIX, + "SUFFIX": SUFFIX, + "LENGTH": LENGTH, + "LEMMA": LEMMA, + "POS": POS, + "TAG": TAG, + "DEP": DEP, + "ENT_IOB": ENT_IOB, + "ENT_TYPE": ENT_TYPE, + "ENT_ID": ENT_ID, + "ENT_KB_ID": ENT_KB_ID, + "HEAD": HEAD, + "SENT_START": SENT_START, + "SPACY": SPACY, + "LANG": LANG, + "MORPH": MORPH, + "IDX": IDX, +} + + +# ATTR IDs, in order of the symbol +NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])] +locals().update(IDS) + + +def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False): + """ + Normalize a dictionary of attributes, converting them to ints. + + stringy_attrs (dict): Dictionary keyed by attribute string names. Values + can be ints or strings. + strings_map (StringStore): Defaults to None. If provided, encodes string + values into ints. + RETURNS (dict): Attributes dictionary with keys and optionally values + converted to ints. + """ + inty_attrs = {} + if _do_deprecated: + if "F" in stringy_attrs: + stringy_attrs["ORTH"] = stringy_attrs.pop("F") + if "L" in stringy_attrs: + stringy_attrs["LEMMA"] = stringy_attrs.pop("L") + if "pos" in stringy_attrs: + stringy_attrs["TAG"] = stringy_attrs.pop("pos") + if "morph" in stringy_attrs: + morphs = stringy_attrs.pop("morph") # no-cython-lint + if "number" in stringy_attrs: + stringy_attrs.pop("number") + if "tenspect" in stringy_attrs: + stringy_attrs.pop("tenspect") + morph_keys = [ + "PunctType", + "PunctSide", + "Other", + "Degree", + "AdvType", + "Number", + "VerbForm", + "PronType", + "Aspect", + "Tense", + "PartType", + "Poss", + "Hyph", + "ConjType", + "NumType", + "Foreign", + "VerbType", + "NounType", + "Gender", + "Mood", + "Negative", + "Tense", + "Voice", + "Abbr", + "Derivation", + "Echo", + "Foreign", + "NameType", + "NounType", + "NumForm", + "NumValue", + "PartType", + "Polite", + "StyleVariant", + "PronType", + "AdjType", + "Person", + "Variant", + "AdpType", + "Reflex", + "Negative", + "Mood", + "Aspect", + "Case", + "Polarity", + "PrepCase", + "Animacy", # U20 + ] + for key in morph_keys: + if key in stringy_attrs: + stringy_attrs.pop(key) + elif key.lower() in stringy_attrs: + stringy_attrs.pop(key.lower()) + elif key.upper() in stringy_attrs: + stringy_attrs.pop(key.upper()) + for name, value in stringy_attrs.items(): + int_key = intify_attr(name) + if int_key is not None: + if int_key == ENT_IOB: + if value in IOB_STRINGS: + value = IOB_STRINGS.index(value) + elif isinstance(value, str): + raise ValueError(Errors.E1025.format(value=value)) + if strings_map is not None and isinstance(value, str): + if hasattr(strings_map, "add"): + value = strings_map.add(value) + else: + value = strings_map[value] + inty_attrs[int_key] = value + return inty_attrs + + +def intify_attr(name): + """ + Normalize an attribute name, converting it to int. + + stringy_attr (string): Attribute string name. Can also be int (will then be left unchanged) + RETURNS (int): int representation of the attribute, or None if it couldn't be converted. + """ + if isinstance(name, int): + return name + elif name in IDS: + return IDS[name] + elif name.upper() in IDS: + return IDS[name.upper()] + return None diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py new file mode 100644 index 0000000..3095778 --- /dev/null +++ b/spacy/cli/__init__.py @@ -0,0 +1,47 @@ +from wasabi import msg + +# Needed for testing +from . import download as download_module # noqa: F401 +from ._util import app, setup_cli # noqa: F401 +from .apply import apply # noqa: F401 +from .assemble import assemble_cli # noqa: F401 + +# These are the actual functions, NOT the wrapped CLI commands. The CLI commands +# are registered automatically and won't have to be imported here. +from .benchmark_speed import benchmark_speed_cli # noqa: F401 +from .convert import convert # noqa: F401 +from .debug_config import debug_config # noqa: F401 +from .debug_data import debug_data # noqa: F401 +from .debug_diff import debug_diff # noqa: F401 +from .debug_model import debug_model # noqa: F401 +from .download import download # noqa: F401 +from .evaluate import evaluate # noqa: F401 +from .find_function import find_function # noqa: F401 +from .find_threshold import find_threshold # noqa: F401 +from .info import info # noqa: F401 +from .init_config import fill_config, init_config # noqa: F401 +from .init_pipeline import init_pipeline_cli # noqa: F401 +from .package import package # noqa: F401 +from .pretrain import pretrain # noqa: F401 +from .profile import profile # noqa: F401 +from .project.assets import project_assets # type: ignore[attr-defined] # noqa: F401 +from .project.clone import project_clone # type: ignore[attr-defined] # noqa: F401 +from .project.document import ( # type: ignore[attr-defined] # noqa: F401 + project_document, +) +from .project.dvc import project_update_dvc # type: ignore[attr-defined] # noqa: F401 +from .project.pull import project_pull # type: ignore[attr-defined] # noqa: F401 +from .project.push import project_push # type: ignore[attr-defined] # noqa: F401 +from .project.run import project_run # type: ignore[attr-defined] # noqa: F401 +from .train import train_cli # type: ignore[attr-defined] # noqa: F401 +from .validate import validate # type: ignore[attr-defined] # noqa: F401 + + +@app.command("link", no_args_is_help=True, deprecated=True, hidden=True) +def link(*args, **kwargs): + """As of spaCy v3.0, symlinks like "en" are not supported anymore. You can load trained + pipeline packages using their full names or from a directory path.""" + msg.warn( + "As of spaCy v3.0, model symlinks are not supported anymore. You can load trained " + "pipeline packages using their full names or from a directory path." + ) diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py new file mode 100644 index 0000000..757c418 --- /dev/null +++ b/spacy/cli/_util.py @@ -0,0 +1,301 @@ +import os +import sys +from configparser import InterpolationError +from contextlib import contextmanager +from pathlib import Path +from typing import ( + Any, + Dict, + List, + Optional, + Tuple, + Union, + overload, +) + +import srsly +import typer +from click import NoSuchOption +from click.shell_completion import split_arg_string +from thinc.api import ConfigValidationError, require_gpu +from thinc.util import gpu_is_available +from typer.main import get_command +from wasabi import Printer, msg +from weasel import app as project_cli + +from ..compat import Literal +from ..util import ( + ENV_VARS, + import_file, + logger, + registry, + run_command, +) + +SDIST_SUFFIX = ".tar.gz" +WHEEL_SUFFIX = "-py3-none-any.whl" + +PROJECT_FILE = "project.yml" +COMMAND = "python -m spacy" +NAME = "spacy" +HELP = """spaCy Command-line Interface + +DOCS: https://spacy.io/api/cli +""" +PROJECT_HELP = f"""Command-line interface for spaCy projects and templates. +You'd typically start by cloning a project template to a local directory and +fetching its assets like datasets etc. See the project's {PROJECT_FILE} for the +available commands. +""" +DEBUG_HELP = """Suite of helpful commands for debugging and profiling. Includes +commands to check and validate your config files, training and evaluation data, +and custom model implementations. +""" +BENCHMARK_HELP = """Commands for benchmarking pipelines.""" +INIT_HELP = """Commands for initializing configs and pipeline packages.""" + +# Wrappers for Typer's annotations. Initially created to set defaults and to +# keep the names short, but not needed at the moment. +Arg = typer.Argument +Opt = typer.Option + +app = typer.Typer(name=NAME, help=HELP, rich_markup_mode=None) +benchmark_cli = typer.Typer(name="benchmark", help=BENCHMARK_HELP, no_args_is_help=True) +debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True) +init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True) + +app.add_typer(project_cli, name="project", help=PROJECT_HELP, no_args_is_help=True) +app.add_typer(debug_cli) +app.add_typer(benchmark_cli) +app.add_typer(init_cli) + + +def setup_cli() -> None: + # Make sure the entry-point for CLI runs, so that they get imported. + registry.cli.get_all() + # Ensure that the help messages always display the correct prompt + command = get_command(app) + command(prog_name=COMMAND) + + +def parse_config_overrides( + args: List[str], env_var: Optional[str] = ENV_VARS.CONFIG_OVERRIDES +) -> Dict[str, Any]: + """Generate a dictionary of config overrides based on the extra arguments + provided on the CLI, e.g. --training.batch_size to override + "training.batch_size". Arguments without a "." are considered invalid, + since the config only allows top-level sections to exist. + + env_vars (Optional[str]): Optional environment variable to read from. + RETURNS (Dict[str, Any]): The parsed dict, keyed by nested config setting. + """ + env_string = os.environ.get(env_var, "") if env_var else "" + env_overrides = _parse_overrides(split_arg_string(env_string)) + cli_overrides = _parse_overrides(args, is_cli=True) + if cli_overrides: + keys = [k for k in cli_overrides if k not in env_overrides] + logger.debug("Config overrides from CLI: %s", keys) + if env_overrides: + logger.debug("Config overrides from env variables: %s", list(env_overrides)) + return {**cli_overrides, **env_overrides} + + +def _parse_overrides(args: List[str], is_cli: bool = False) -> Dict[str, Any]: + result = {} + while args: + opt = args.pop(0) + err = f"Invalid config override '{opt}'" + if opt.startswith("--"): # new argument + orig_opt = opt + opt = opt.replace("--", "") + if "." not in opt: + if is_cli: + raise NoSuchOption(orig_opt) + else: + msg.fail(f"{err}: can't override top-level sections", exits=1) + if "=" in opt: # we have --opt=value + opt, value = opt.split("=", 1) + opt = opt.replace("-", "_") + else: + if not args or args[0].startswith("--"): # flag with no value + value = "true" + else: + value = args.pop(0) + result[opt] = _parse_override(value) + else: + msg.fail(f"{err}: name should start with --", exits=1) + return result + + +def _parse_override(value: Any) -> Any: + # Just like we do in the config, we're calling json.loads on the + # values. But since they come from the CLI, it'd be unintuitive to + # explicitly mark strings with escaped quotes. So we're working + # around that here by falling back to a string if parsing fails. + # TODO: improve logic to handle simple types like list of strings? + try: + return srsly.json_loads(value) + except ValueError: + return str(value) + + +@contextmanager +def show_validation_error( + file_path: Optional[Union[str, Path]] = None, + *, + title: Optional[str] = None, + desc: str = "", + show_config: Optional[bool] = None, + hint_fill: bool = True, +): + """Helper to show custom config validation errors on the CLI. + + file_path (str / Path): Optional file path of config file, used in hints. + title (str): Override title of custom formatted error. + desc (str): Override description of custom formatted error. + show_config (bool): Whether to output the config the error refers to. + hint_fill (bool): Show hint about filling config. + """ + try: + yield + except ConfigValidationError as e: + title = title if title is not None else e.title + if e.desc: + desc = f"{e.desc}" if not desc else f"{e.desc}\n\n{desc}" + # Re-generate a new error object with overrides + err = e.from_error(e, title="", desc=desc, show_config=show_config) + msg.fail(title) + print(err.text.strip()) + if hint_fill and "value_error.missing" in err.error_types: + config_path = ( + file_path + if file_path is not None and str(file_path) != "-" + else "config.cfg" + ) + msg.text( + "If your config contains missing values, you can run the 'init " + "fill-config' command to fill in all the defaults, if possible:", + spaced=True, + ) + print(f"{COMMAND} init fill-config {config_path} {config_path} \n") + sys.exit(1) + except InterpolationError as e: + msg.fail("Config validation error", e, exits=1) + + +def import_code(code_path: Optional[Union[Path, str]]) -> None: + """Helper to import Python file provided in training commands / commands + using the config. This makes custom registered functions available. + """ + if code_path is not None: + if not Path(code_path).exists(): + msg.fail("Path to Python code not found", code_path, exits=1) + try: + import_file("python_code", code_path) + except Exception as e: + msg.fail(f"Couldn't load Python code: {code_path}", e, exits=1) + + +def get_git_version( + error: str = "Could not run 'git'. Make sure it's installed and the executable is available.", +) -> Tuple[int, int]: + """Get the version of git and raise an error if calling 'git --version' fails. + error (str): The error message to show. + RETURNS (Tuple[int, int]): The version as a (major, minor) tuple. Returns + (0, 0) if the version couldn't be determined. + """ + try: + ret = run_command("git --version", capture=True) + except: + raise RuntimeError(error) + stdout = ret.stdout.strip() + if not stdout or not stdout.startswith("git version"): + return 0, 0 + version = stdout[11:].strip().split(".") + return int(version[0]), int(version[1]) + + +@overload +def string_to_list(value: str, intify: Literal[False] = ...) -> List[str]: ... + + +@overload +def string_to_list(value: str, intify: Literal[True]) -> List[int]: ... + + +def string_to_list(value: str, intify: bool = False) -> Union[List[str], List[int]]: + """Parse a comma-separated string to a list and account for various + formatting options. Mostly used to handle CLI arguments that take a list of + comma-separated values. + + value (str): The value to parse. + intify (bool): Whether to convert values to ints. + RETURNS (Union[List[str], List[int]]): A list of strings or ints. + """ + if not value: + return [] # type: ignore[return-value] + if value.startswith("[") and value.endswith("]"): + value = value[1:-1] + result = [] + for p in value.split(","): + p = p.strip() + if p.startswith("'") and p.endswith("'"): + p = p[1:-1] + if p.startswith('"') and p.endswith('"'): + p = p[1:-1] + p = p.strip() + if intify: + p = int(p) # type: ignore[assignment] + result.append(p) + return result + + +def setup_gpu(use_gpu: int, silent=None) -> None: + """Configure the GPU and log info.""" + if silent is None: + local_msg = Printer() + else: + local_msg = Printer(no_print=silent, pretty=not silent) + if use_gpu >= 0: + local_msg.info(f"Using GPU: {use_gpu}") + require_gpu(use_gpu) + else: + local_msg.info("Using CPU") + if gpu_is_available(): + local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") + + +def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]: + """Given a directory and a suffix, recursively find all files matching the suffix. + Directories or files with names beginning with a . are ignored, but hidden flags on + filesystems are not checked. + When provided with a suffix `None`, there is no suffix-based filtering.""" + if not path.is_dir(): + return [path] + paths = [path] + locs = [] + seen = set() + for path in paths: + if str(path) in seen: + continue + seen.add(str(path)) + if path.parts[-1].startswith("."): + continue + elif path.is_dir(): + paths.extend(path.iterdir()) + elif suffix is not None and not path.parts[-1].endswith(suffix): + continue + else: + locs.append(path) + # It's good to sort these, in case the ordering messes up cache. + locs.sort() + return locs + + +def _format_number(number: Union[int, float], ndigits: int = 2) -> str: + """Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s, + as happens with `round(number, ndigits)`""" + if isinstance(number, float): + return f"{number:.{ndigits}f}" + else: + return str(number) diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py new file mode 100644 index 0000000..7671026 --- /dev/null +++ b/spacy/cli/apply.py @@ -0,0 +1,145 @@ +from itertools import chain +from pathlib import Path +from typing import Iterable, List, Optional, Union, cast + +import srsly +import tqdm +from wasabi import msg + +from ..tokens import Doc, DocBin +from ..util import ensure_path, load_model +from ..vocab import Vocab +from ._util import Arg, Opt, app, import_code, setup_gpu, walk_directory + +path_help = """Location of the documents to predict on. +Can be a single file in .spacy format or a .jsonl file. +Files with other extensions are treated as single plain text documents. +If a directory is provided it is traversed recursively to grab +all files to be processed. +The files can be a mixture of .spacy, .jsonl and text files. +If .jsonl is provided the specified field is going +to be grabbed ("text" by default).""" + +out_help = "Path to save the resulting .spacy file" +code_help = ( + "Path to Python file with additional code (registered functions) to be imported" +) +gold_help = "Use gold preprocessing provided in the .spacy files" +force_msg = ( + "The provided output file already exists. " + "To force overwriting the output file, set the --force or -F flag." +) + + +DocOrStrStream = Union[Iterable[str], Iterable[Doc]] + + +def _stream_docbin(path: Path, vocab: Vocab) -> Iterable[Doc]: + """ + Stream Doc objects from DocBin. + """ + docbin = DocBin().from_disk(path) + for doc in docbin.get_docs(vocab): + yield doc + + +def _stream_jsonl(path: Path, field: str) -> Iterable[str]: + """ + Stream "text" field from JSONL. If the field "text" is + not found it raises error. + """ + for entry in srsly.read_jsonl(path): + if field not in entry: + msg.fail(f"{path} does not contain the required '{field}' field.", exits=1) + else: + yield entry[field] + + +def _stream_texts(paths: Iterable[Path]) -> Iterable[str]: + """ + Yields strings from text files in paths. + """ + for path in paths: + with open(path, "r") as fin: + text = fin.read() + yield text + + +@app.command("apply") +def apply_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg(..., help=path_help, exists=True), + output_file: Path = Arg(..., help=out_help, dir_okay=False), + code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help), + text_key: str = Opt( + "text", "--text-key", "-tk", help="Key containing text string for JSONL" + ), + force_overwrite: bool = Opt( + False, "--force", "-F", help="Force overwriting the output file" + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."), + batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."), + n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use."), +): + """ + Apply a trained pipeline to documents to get predictions. + Expects a loadable spaCy pipeline and path to the data, which + can be a directory or a file. + The data files can be provided in multiple formats: + 1. .spacy files + 2. .jsonl files with a specified "field" to read the text from. + 3. Files with any other extension are assumed to be containing + a single document. + DOCS: https://spacy.io/api/cli#apply + """ + data_path = ensure_path(data_path) + output_file = ensure_path(output_file) + code_path = ensure_path(code_path) + if output_file.exists() and not force_overwrite: + msg.fail(force_msg, exits=1) + if not data_path.exists(): + msg.fail(f"Couldn't find data path: {data_path}", exits=1) + import_code(code_path) + setup_gpu(use_gpu) + apply(data_path, output_file, model, text_key, batch_size, n_process) + + +def apply( + data_path: Path, + output_file: Path, + model: str, + json_field: str, + batch_size: int, + n_process: int, +): + docbin = DocBin(store_user_data=True) + paths = walk_directory(data_path) + if len(paths) == 0: + docbin.to_disk(output_file) + msg.warn( + f"Did not find data to process, {data_path} seems to be an empty directory." + ) + return + nlp = load_model(model) + msg.good(f"Loaded model {model}") + vocab = nlp.vocab + streams: List[DocOrStrStream] = [] + text_files = [] + for path in paths: + if path.suffix == ".spacy": + streams.append(_stream_docbin(path, vocab)) + elif path.suffix == ".jsonl": + streams.append(_stream_jsonl(path, json_field)) + else: + text_files.append(path) + if len(text_files) > 0: + streams.append(_stream_texts(text_files)) + datagen = cast(DocOrStrStream, chain(*streams)) + for doc in tqdm.tqdm( + nlp.pipe(datagen, batch_size=batch_size, n_process=n_process), disable=None + ): + docbin.add(doc) + if output_file.suffix == "": + output_file = output_file.with_suffix(".spacy") + docbin.to_disk(output_file) diff --git a/spacy/cli/assemble.py b/spacy/cli/assemble.py new file mode 100644 index 0000000..bc97a9d --- /dev/null +++ b/spacy/cli/assemble.py @@ -0,0 +1,80 @@ +import logging +from pathlib import Path +from typing import Optional + +import typer +from wasabi import msg + +from .. import util +from ..util import get_sourced_components, load_model_from_config +from ._util import ( + Arg, + Opt, + app, + import_code, + parse_config_overrides, + show_validation_error, +) + + +@app.command( + "assemble", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def assemble_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + output_path: Path = Arg( + ..., help="Output directory to store assembled pipeline in" + ), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + # fmt: on +): + """ + Assemble a spaCy pipeline from a config file. The config file includes + all settings for initializing the pipeline. To override settings in the + config, e.g. settings that point to local paths or that you want to + experiment with, you can override them as command line options. The + --code argument lets you pass in a Python file that can be used to + register custom functions that are referenced in the config. + + DOCS: https://spacy.io/api/cli#assemble + """ + if verbose: + util.logger.setLevel(logging.DEBUG) + # Make sure all files and paths exists if they are needed + if not config_path or (str(config_path) != "-" and not config_path.exists()): + msg.fail("Config file not found", config_path, exits=1) + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + with show_validation_error(config_path): + config = util.load_config(config_path, overrides=overrides, interpolate=False) + msg.divider("Initializing pipeline") + nlp = load_model_from_config(config, auto_fill=True) + config = config.interpolate() + sourced = get_sourced_components(config) + # Make sure that listeners are defined before initializing further + nlp._link_components() + with nlp.select_pipes(disable=[*sourced]): + nlp.initialize() + msg.good("Initialized pipeline") + msg.divider("Serializing to disk") + if output_path is not None and not output_path.exists(): + output_path.mkdir(parents=True) + msg.good(f"Created output directory: {output_path}") + nlp.to_disk(output_path) diff --git a/spacy/cli/benchmark_speed.py b/spacy/cli/benchmark_speed.py new file mode 100644 index 0000000..052e7d4 --- /dev/null +++ b/spacy/cli/benchmark_speed.py @@ -0,0 +1,193 @@ +import random +import time +from itertools import islice +from pathlib import Path +from typing import Iterable, List, Optional + +import numpy +import typer +from tqdm import tqdm +from wasabi import msg + +from .. import util +from ..language import Language +from ..tokens import Doc +from ..training import Corpus +from ._util import Arg, Opt, benchmark_cli, import_code, setup_gpu + + +@benchmark_cli.command( + "speed", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def benchmark_speed_cli( + # fmt: off + ctx: typer.Context, + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg( + ..., help="Location of binary evaluation data in .spacy format", exists=True + ), + batch_size: Optional[int] = Opt( + None, "--batch-size", "-b", min=1, help="Override the pipeline batch size" + ), + no_shuffle: bool = Opt(False, "--no-shuffle", help="Do not shuffle benchmark data"), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + n_batches: int = Opt( + 50, + "--batches", + help="Minimum number of batches to benchmark", + min=30, + ), + warmup_epochs: int = Opt( + 3, "--warmup", "-w", min=0, help="Number of iterations over the data for warmup" + ), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + # fmt: on +): + """ + Benchmark a pipeline. Expects a loadable spaCy pipeline and benchmark + data in the binary .spacy format. + """ + import_code(code_path) + setup_gpu(use_gpu=use_gpu, silent=False) + + nlp = util.load_model(model) + batch_size = batch_size if batch_size is not None else nlp.batch_size + corpus = Corpus(data_path) + docs = [eg.predicted for eg in corpus(nlp)] + + if len(docs) == 0: + msg.fail("Cannot benchmark speed using an empty corpus.", exits=1) + + print(f"Warming up for {warmup_epochs} epochs...") + warmup(nlp, docs, warmup_epochs, batch_size) + + print() + print(f"Benchmarking {n_batches} batches...") + wps = benchmark(nlp, docs, n_batches, batch_size, not no_shuffle) + + print() + print_outliers(wps) + print_mean_with_ci(wps) + + +# Lowercased, behaves as a context manager function. +class time_context: + """Register the running time of a context.""" + + def __enter__(self): + self.start = time.perf_counter() + return self + + def __exit__(self, type, value, traceback): + self.elapsed = time.perf_counter() - self.start + + +class Quartiles: + """Calculate the q1, q2, q3 quartiles and the inter-quartile range (iqr) + of a sample.""" + + q1: float + q2: float + q3: float + iqr: float + + def __init__(self, sample: numpy.ndarray) -> None: + self.q1 = numpy.quantile(sample, 0.25) + self.q2 = numpy.quantile(sample, 0.5) + self.q3 = numpy.quantile(sample, 0.75) + self.iqr = self.q3 - self.q1 + + +def annotate( + nlp: Language, docs: List[Doc], batch_size: Optional[int] +) -> numpy.ndarray: + docs = nlp.pipe(tqdm(docs, unit="doc", disable=None), batch_size=batch_size) + wps = [] + while True: + with time_context() as elapsed: + batch_docs = list( + islice(docs, batch_size if batch_size else nlp.batch_size) + ) + if len(batch_docs) == 0: + break + n_tokens = count_tokens(batch_docs) + wps.append(n_tokens / elapsed.elapsed) + + return numpy.array(wps) + + +def benchmark( + nlp: Language, + docs: List[Doc], + n_batches: int, + batch_size: int, + shuffle: bool, +) -> numpy.ndarray: + if shuffle: + bench_docs = [ + nlp.make_doc(random.choice(docs).text) + for _ in range(n_batches * batch_size) + ] + else: + bench_docs = [ + nlp.make_doc(docs[i % len(docs)].text) + for i in range(n_batches * batch_size) + ] + + return annotate(nlp, bench_docs, batch_size) + + +def bootstrap(x, statistic=numpy.mean, iterations=10000) -> numpy.ndarray: + """Apply a statistic to repeated random samples of an array.""" + return numpy.fromiter( + ( + statistic(numpy.random.choice(x, len(x), replace=True)) + for _ in range(iterations) + ), + numpy.float64, + ) + + +def count_tokens(docs: Iterable[Doc]) -> int: + return sum(len(doc) for doc in docs) + + +def print_mean_with_ci(sample: numpy.ndarray): + mean = numpy.mean(sample) + bootstrap_means = bootstrap(sample) + bootstrap_means.sort() + + # 95% confidence interval + low = bootstrap_means[int(len(bootstrap_means) * 0.025)] + high = bootstrap_means[int(len(bootstrap_means) * 0.975)] + + print(f"Mean: {mean:.1f} words/s (95% CI: {low - mean:.1f} +{high - mean:.1f})") + + +def print_outliers(sample: numpy.ndarray): + quartiles = Quartiles(sample) + + n_outliers = numpy.sum( + (sample < (quartiles.q1 - 1.5 * quartiles.iqr)) + | (sample > (quartiles.q3 + 1.5 * quartiles.iqr)) + ) + n_extreme_outliers = numpy.sum( + (sample < (quartiles.q1 - 3.0 * quartiles.iqr)) + | (sample > (quartiles.q3 + 3.0 * quartiles.iqr)) + ) + print( + f"Outliers: {(100 * n_outliers) / len(sample):.1f}%, extreme outliers: {(100 * n_extreme_outliers) / len(sample)}%" + ) + + +def warmup( + nlp: Language, docs: List[Doc], warmup_epochs: int, batch_size: Optional[int] +) -> numpy.ndarray: + docs = [doc.copy() for doc in docs * warmup_epochs] + return annotate(nlp, docs, batch_size) diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py new file mode 100644 index 0000000..1409992 --- /dev/null +++ b/spacy/cli/convert.py @@ -0,0 +1,284 @@ +import itertools +import re +import sys +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Optional, Union + +import srsly +from wasabi import Printer + +from ..tokens import Doc, DocBin +from ..training import docs_to_json +from ..training.converters import ( + conll_ner_to_docs, + conllu_to_docs, + iob_to_docs, + json_to_docs, +) +from ._util import Arg, Opt, app, walk_directory + +# Converters are matched by file extension except for ner/iob, which are +# matched by file extension and content. To add a converter, add a new +# entry to this dict with the file extension mapped to the converter function +# imported from /converters. + +CONVERTERS: Mapping[str, Callable[..., Iterable[Doc]]] = { + "conllubio": conllu_to_docs, + "conllu": conllu_to_docs, + "conll": conll_ner_to_docs, + "ner": conll_ner_to_docs, + "iob": iob_to_docs, + "json": json_to_docs, +} + +AUTO = "auto" + + +# File types that can be written to stdout +FILE_TYPES_STDOUT = ("json",) + + +class FileTypes(str, Enum): + json = "json" + spacy = "spacy" + + +@app.command("convert") +def convert_cli( + # fmt: off + input_path: str = Arg(..., help="Input file or directory", exists=True), + output_dir: Path = Arg( + "-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True + ), + file_type: FileTypes = Opt( + "spacy", "--file-type", "-t", help="Type of data to produce" + ), + n_sents: int = Opt( + 1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)" + ), + seg_sents: bool = Opt( + False, "--seg-sents", "-s", help="Segment sentences (for -c ner)" + ), + model: Optional[str] = Opt( + None, + "--model", + "--base", + "-b", + help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)", + ), + morphology: bool = Opt( + False, "--morphology", "-m", help="Enable appending morphology to tags" + ), + merge_subtokens: bool = Opt( + False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens" + ), + converter: str = Opt( + AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}" + ), + ner_map: Optional[Path] = Opt( + None, + "--ner-map", + "-nm", + help="NER tag mapping (as JSON-encoded dict of entity types)", + exists=True, + ), + lang: Optional[str] = Opt( + None, "--lang", "-l", help="Language (if tokenizer required)" + ), + concatenate: bool = Opt( + None, "--concatenate", "-C", help="Concatenate output to a single file" + ), + # fmt: on +): + """ + Convert files into json or DocBin format for training. The resulting .spacy + file can be used with the train command and other experiment management + functions. + + If no output_dir is specified and the output format is JSON, the data + is written to stdout, so you can pipe them forward to a JSON file: + $ spacy convert some_file.conllu --file-type json > some_file.json + + DOCS: https://spacy.io/api/cli#convert + """ + input_path = Path(input_path) + output_dir: Union[str, Path] = "-" if output_dir == Path("-") else output_dir + silent = output_dir == "-" + msg = Printer(no_print=silent) + converter = _get_converter(msg, converter, input_path) + verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map) + convert( + input_path, + output_dir, + file_type=file_type.value, + n_sents=n_sents, + seg_sents=seg_sents, + model=model, + morphology=morphology, + merge_subtokens=merge_subtokens, + converter=converter, + ner_map=ner_map, + lang=lang, + concatenate=concatenate, + silent=silent, + msg=msg, + ) + + +def convert( + input_path: Path, + output_dir: Union[str, Path], + *, + file_type: str = "json", + n_sents: int = 1, + seg_sents: bool = False, + model: Optional[str] = None, + morphology: bool = False, + merge_subtokens: bool = False, + converter: str, + ner_map: Optional[Path] = None, + lang: Optional[str] = None, + concatenate: bool = False, + silent: bool = True, + msg: Optional[Printer] = None, +) -> None: + input_path = Path(input_path) + if not msg: + msg = Printer(no_print=silent) + ner_map = srsly.read_json(ner_map) if ner_map is not None else None + doc_files = [] + for input_loc in walk_directory(input_path, converter): + with input_loc.open("r", encoding="utf-8") as infile: + input_data = infile.read() + # Use converter function to convert data + func = CONVERTERS[converter] + docs = func( + input_data, + n_sents=n_sents, + seg_sents=seg_sents, + append_morphology=morphology, + merge_subtokens=merge_subtokens, + lang=lang, + model=model, + no_print=silent, + ner_map=ner_map, + ) + doc_files.append((input_loc, docs)) + if concatenate: + all_docs = itertools.chain.from_iterable([docs for _, docs in doc_files]) + doc_files = [(input_path, all_docs)] + for input_loc, docs in doc_files: + if file_type == "json": + data = [docs_to_json(docs)] + len_docs = len(data) + else: + db = DocBin(docs=docs, store_user_data=True) + len_docs = len(db) + data = db.to_bytes() # type: ignore[assignment] + if output_dir == "-": + _print_docs_to_stdout(data, file_type) + else: + if input_loc != input_path: + subpath = input_loc.relative_to(input_path) + output_file = Path(output_dir) / subpath.with_suffix(f".{file_type}") + else: + output_file = Path(output_dir) / input_loc.parts[-1] + output_file = output_file.with_suffix(f".{file_type}") + _write_docs_to_file(data, output_file, file_type) + msg.good(f"Generated output file ({len_docs} documents): {output_file}") + + +def _print_docs_to_stdout(data: Any, output_type: str) -> None: + if output_type == "json": + srsly.write_json("-", data) + else: + sys.stdout.buffer.write(data) + + +def _write_docs_to_file(data: Any, output_file: Path, output_type: str) -> None: + if not output_file.parent.exists(): + output_file.parent.mkdir(parents=True) + if output_type == "json": + srsly.write_json(output_file, data) + else: + with output_file.open("wb") as file_: + file_.write(data) + + +def autodetect_ner_format(input_data: str) -> Optional[str]: + # guess format from the first 20 lines + lines = input_data.split("\n")[:20] + format_guesses = {"ner": 0, "iob": 0} + iob_re = re.compile(r"\S+\|(O|[IB]-\S+)") + ner_re = re.compile(r"\S+\s+(O|[IB]-\S+)$") + for line in lines: + line = line.strip() + if iob_re.search(line): + format_guesses["iob"] += 1 + if ner_re.search(line): + format_guesses["ner"] += 1 + if format_guesses["iob"] == 0 and format_guesses["ner"] > 0: + return "ner" + if format_guesses["ner"] == 0 and format_guesses["iob"] > 0: + return "iob" + return None + + +def verify_cli_args( + msg: Printer, + input_path: Path, + output_dir: Union[str, Path], + file_type: str, + converter: str, + ner_map: Optional[Path], +): + if file_type not in FILE_TYPES_STDOUT and output_dir == "-": + msg.fail( + f"Can't write .{file_type} data to stdout. Please specify an output directory.", + exits=1, + ) + if not input_path.exists(): + msg.fail("Input file not found", input_path, exits=1) + if output_dir != "-" and not Path(output_dir).exists(): + msg.fail("Output directory not found", output_dir, exits=1) + if ner_map is not None and not Path(ner_map).exists(): + msg.fail("NER map not found", ner_map, exits=1) + if input_path.is_dir(): + input_locs = walk_directory(input_path, converter) + if len(input_locs) == 0: + msg.fail("No input files in directory", input_path, exits=1) + if converter not in CONVERTERS: + msg.fail(f"Can't find converter for {converter}", exits=1) + + +def _get_converter(msg, converter, input_path: Path): + if input_path.is_dir(): + if converter == AUTO: + input_locs = walk_directory(input_path, suffix=None) + file_types = list(set([loc.suffix[1:] for loc in input_locs])) + if len(file_types) >= 2: + file_types_str = ",".join(file_types) + msg.fail("All input files must be same type", file_types_str, exits=1) + input_path = input_locs[0] + else: + input_path = walk_directory(input_path, suffix=converter)[0] + if converter == AUTO: + converter = input_path.suffix[1:] + if converter == "ner" or converter == "iob": + with input_path.open(encoding="utf8") as file_: + input_data = file_.read() + converter_autodetect = autodetect_ner_format(input_data) + if converter_autodetect == "ner": + msg.info("Auto-detected token-per-line NER format") + converter = converter_autodetect + elif converter_autodetect == "iob": + msg.info("Auto-detected sentence-per-line NER format") + converter = converter_autodetect + else: + msg.warn( + "Can't automatically detect NER format. " + "Conversion may not succeed. " + "See https://spacy.io/api/cli#convert" + ) + return converter diff --git a/spacy/cli/debug_config.py b/spacy/cli/debug_config.py new file mode 100644 index 0000000..4876b6f --- /dev/null +++ b/spacy/cli/debug_config.py @@ -0,0 +1,131 @@ +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import typer +from thinc.api import Config +from thinc.config import VARIABLE_RE +from wasabi import msg, table + +from .. import util +from ..schemas import ConfigSchemaInit, ConfigSchemaTraining +from ..util import registry +from ._util import ( + Arg, + Opt, + debug_cli, + import_code, + parse_config_overrides, + show_validation_error, +) + + +@debug_cli.command( + "config", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def debug_config_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + code_path: Optional[Path] = Opt( + None, + "--code-path", + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + show_funcs: bool = Opt( + False, + "--show-functions", + "-F", + help="Show an overview of all registered functions used in the config and where they come from (modules, files etc.)", + ), + show_vars: bool = Opt( + False, + "--show-variables", + "-V", + help="Show an overview of all variables referenced in the config and their values. This will also reflect variables overwritten on the CLI.", + ), + # fmt: on +): + """Debug a config file and show validation errors. The command will + create all objects in the tree and validate them. Note that some config + validation errors are blocking and will prevent the rest of the config from + being resolved. This means that you may not see all validation errors at + once and some issues are only shown once previous errors have been fixed. + Similar as with the 'train' command, you can override settings from the config + as command line options. For instance, --training.batch_size 128 overrides + the value of "batch_size" in the block "[training]". + + DOCS: https://spacy.io/api/cli#debug-config + """ + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + debug_config( + config_path, overrides=overrides, show_funcs=show_funcs, show_vars=show_vars + ) + + +def debug_config( + config_path: Path, + *, + overrides: Dict[str, Any] = {}, + show_funcs: bool = False, + show_vars: bool = False, +): + msg.divider("Config validation") + with show_validation_error(config_path): + config = util.load_config(config_path, overrides=overrides) + nlp = util.load_model_from_config(config) + config = nlp.config.interpolate() + msg.divider("Config validation for [initialize]") + with show_validation_error(config_path): + T = registry.resolve(config["initialize"], schema=ConfigSchemaInit) # type: ignore[arg-type] + msg.divider("Config validation for [training]") + with show_validation_error(config_path): + T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type] + dot_names = [T["train_corpus"], T["dev_corpus"]] + util.resolve_dot_names(config, dot_names) + msg.good("Config is valid") + if show_vars: + variables = get_variables(config) + msg.divider(f"Variables ({len(variables)})") + head = ("Variable", "Value") + msg.table(variables, header=head, divider=True, widths=(41, 34), spacing=2) + if show_funcs: + funcs = get_registered_funcs(config) + msg.divider(f"Registered functions ({len(funcs)})") + for func in funcs: + func_data = { + "Registry": f"@{func['registry']}", + "Name": func["name"], + "Module": func["module"], + "File": f"{func['file']} (line {func['line_no']})", + } + msg.info(f"[{func['path']}]") + print(table(func_data).strip()) + + +def get_registered_funcs(config: Config) -> List[Dict[str, Optional[Union[str, int]]]]: + result = [] + for key, value in util.walk_dict(config): + if not key[-1].startswith("@"): + continue + # We have a reference to a registered function + reg_name = key[-1][1:] + registry = getattr(util.registry, reg_name) + path = ".".join(key[:-1]) + info = registry.find(value) + result.append({"name": value, "registry": reg_name, "path": path, **info}) + return result + + +def get_variables(config: Config) -> Dict[str, Any]: + result = {} + for variable in sorted(set(VARIABLE_RE.findall(config.to_str()))): + path = variable[2:-1].replace(":", ".") + value = util.dot_to_object(config, path) + result[variable] = repr(value) + return result diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py new file mode 100644 index 0000000..6ba18e7 --- /dev/null +++ b/spacy/cli/debug_data.py @@ -0,0 +1,1247 @@ +import math +import sys +from collections import Counter +from pathlib import Path +from typing import ( + Any, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, + overload, +) + +import numpy +import srsly +import typer +from wasabi import MESSAGES, Printer, msg + +from .. import util +from ..compat import Literal +from ..language import Language +from ..morphology import Morphology +from ..pipeline import Morphologizer, SpanCategorizer, TrainablePipe +from ..pipeline._edit_tree_internals.edit_trees import EditTrees +from ..pipeline._parser_internals import nonproj +from ..pipeline._parser_internals.nonproj import DELIMITER +from ..schemas import ConfigSchemaTraining +from ..training import Example, remove_bilu_prefix +from ..training.initialize import get_sourced_components +from ..util import registry, resolve_dot_names +from ..vectors import Mode as VectorsMode +from ._util import ( + Arg, + Opt, + _format_number, + app, + debug_cli, + import_code, + parse_config_overrides, + show_validation_error, +) + +# Minimum number of expected occurrences of NER label in data to train new label +NEW_LABEL_THRESHOLD = 50 +# Minimum number of expected occurrences of dependency labels +DEP_LABEL_THRESHOLD = 20 +# Minimum number of expected examples to train a new pipeline +BLANK_MODEL_MIN_THRESHOLD = 100 +BLANK_MODEL_THRESHOLD = 2000 +# Arbitrary threshold where SpanCat performs well +SPAN_DISTINCT_THRESHOLD = 1 +# Arbitrary threshold where SpanCat performs well +BOUNDARY_DISTINCT_THRESHOLD = 1 +# Arbitrary threshold for filtering span lengths during reporting (percentage) +SPAN_LENGTH_THRESHOLD_PERCENTAGE = 90 + + +@debug_cli.command( + "data", context_settings={"allow_extra_args": True, "ignore_unknown_options": True} +) +@app.command( + "debug-data", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + hidden=True, # hide this from main CLI help but still allow it to work with warning +) +def debug_data_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + code_path: Optional[Path] = Opt( + None, + "--code-path", + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + ignore_warnings: bool = Opt( + False, + "--ignore-warnings", + "-IW", + help="Ignore warnings, only show stats and errors", + ), + verbose: bool = Opt( + False, "--verbose", "-V", help="Print additional information and explanations" + ), + no_format: bool = Opt( + False, "--no-format", "-NF", help="Don't pretty-print the results" + ), + # fmt: on +): + """ + Analyze, debug and validate your training and development data. Outputs + useful stats, and can help you find problems like invalid entity annotations, + cyclic dependencies, low data labels and more. + + DOCS: https://spacy.io/api/cli#debug-data + """ + if ctx.command.name == "debug-data": + msg.warn( + "The debug-data command is now available via the 'debug data' " + "subcommand (without the hyphen). You can run python -m spacy debug " + "--help for an overview of the other available debugging commands." + ) + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + debug_data( + config_path, + config_overrides=overrides, + ignore_warnings=ignore_warnings, + verbose=verbose, + no_format=no_format, + silent=False, + ) + + +def debug_data( + config_path: Path, + *, + config_overrides: Dict[str, Any] = {}, + ignore_warnings: bool = False, + verbose: bool = False, + no_format: bool = True, + silent: bool = True, +): + msg = Printer( + no_print=silent, pretty=not no_format, ignore_warnings=ignore_warnings + ) + # Make sure all files and paths exists if they are needed + with show_validation_error(config_path): + cfg = util.load_config(config_path, overrides=config_overrides) + nlp = util.load_model_from_config(cfg) + config = nlp.config.interpolate() + T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type] + # Use original config here, not resolved version + sourced_components = get_sourced_components(cfg) + frozen_components = T["frozen_components"] + resume_components = [p for p in sourced_components if p not in frozen_components] + pipeline = nlp.pipe_names + factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names] + msg.divider("Data file validation") + + # Create the gold corpus to be able to better analyze data + dot_names = [T["train_corpus"], T["dev_corpus"]] + train_corpus, dev_corpus = resolve_dot_names(config, dot_names) + + nlp.initialize(lambda: train_corpus(nlp)) + msg.good("Pipeline can be initialized with data") + + train_dataset = list(train_corpus(nlp)) + dev_dataset = list(dev_corpus(nlp)) + msg.good("Corpus is loadable") + + # Create all gold data here to avoid iterating over the train_dataset constantly + gold_train_data = _compile_gold(train_dataset, factory_names, nlp, make_proj=True) + gold_train_unpreprocessed_data = _compile_gold( + train_dataset, factory_names, nlp, make_proj=False + ) + gold_dev_data = _compile_gold(dev_dataset, factory_names, nlp, make_proj=True) + + train_texts = gold_train_data["texts"] + dev_texts = gold_dev_data["texts"] + frozen_components = T["frozen_components"] + + msg.divider("Training stats") + msg.text(f"Language: {nlp.lang}") + msg.text(f"Training pipeline: {', '.join(pipeline)}") + if resume_components: + msg.text(f"Components from other pipelines: {', '.join(resume_components)}") + if frozen_components: + msg.text(f"Frozen components: {', '.join(frozen_components)}") + msg.text(f"{len(train_dataset)} training docs") + msg.text(f"{len(dev_dataset)} evaluation docs") + + if not len(gold_dev_data): + msg.fail("No evaluation docs") + overlap = len(train_texts.intersection(dev_texts)) + if overlap: + msg.warn(f"{overlap} training examples also in evaluation data") + else: + msg.good("No overlap between training and evaluation data") + # TODO: make this feedback more fine-grained and report on updated + # components vs. blank components + if not resume_components and len(train_dataset) < BLANK_MODEL_THRESHOLD: + text = f"Low number of examples to train a new pipeline ({len(train_dataset)})" + if len(train_dataset) < BLANK_MODEL_MIN_THRESHOLD: + msg.fail(text) + else: + msg.warn(text) + msg.text( + f"It's recommended to use at least {BLANK_MODEL_THRESHOLD} examples " + f"(minimum {BLANK_MODEL_MIN_THRESHOLD})", + show=verbose, + ) + + msg.divider("Vocab & Vectors") + n_words = gold_train_data["n_words"] + msg.info( + f"{n_words} total word(s) in the data ({len(gold_train_data['words'])} unique)" + ) + if gold_train_data["n_misaligned_words"] > 0: + n_misaligned = gold_train_data["n_misaligned_words"] + msg.warn(f"{n_misaligned} misaligned tokens in the training data") + if gold_dev_data["n_misaligned_words"] > 0: + n_misaligned = gold_dev_data["n_misaligned_words"] + msg.warn(f"{n_misaligned} misaligned tokens in the dev data") + most_common_words = gold_train_data["words"].most_common(10) + msg.text( + f"10 most common words: {_format_labels(most_common_words, counts=True)}", + show=verbose, + ) + if len(nlp.vocab.vectors): + if nlp.vocab.vectors.mode == VectorsMode.floret: + msg.info( + f"floret vectors with {len(nlp.vocab.vectors)} vectors, " + f"{nlp.vocab.vectors_length} dimensions, " + f"{nlp.vocab.vectors.minn}-{nlp.vocab.vectors.maxn} char " + f"n-gram subwords" + ) + else: + msg.info( + f"{len(nlp.vocab.vectors)} vectors ({nlp.vocab.vectors.n_keys} " + f"unique keys, {nlp.vocab.vectors_length} dimensions)" + ) + n_missing_vectors = sum(gold_train_data["words_missing_vectors"].values()) + msg.warn( + "{} words in training data without vectors ({:.0f}%)".format( + n_missing_vectors, + 100 * (n_missing_vectors / gold_train_data["n_words"]), + ), + ) + msg.text( + "10 most common words without vectors: {}".format( + _format_labels( + gold_train_data["words_missing_vectors"].most_common(10), + counts=True, + ) + ), + show=verbose, + ) + else: + msg.info("No word vectors present in the package") + + if "spancat" in factory_names or "spancat_singlelabel" in factory_names: + model_labels_spancat = _get_labels_from_spancat(nlp) + has_low_data_warning = False + has_no_neg_warning = False + + msg.divider("Span Categorization") + msg.table(model_labels_spancat, header=["Spans Key", "Labels"], divider=True) + + msg.text("Label counts in train data: ", show=verbose) + for spans_key, data_labels in gold_train_data["spancat"].items(): + msg.text( + f"Key: {spans_key}, {_format_labels(data_labels.items(), counts=True)}", + show=verbose, + ) + # Data checks: only take the spans keys in the actual spancat components + data_labels_in_component = { + spans_key: gold_train_data["spancat"][spans_key] + for spans_key in model_labels_spancat.keys() + } + for spans_key, data_labels in data_labels_in_component.items(): + for label, count in data_labels.items(): + # Check for missing labels + spans_key_in_model = spans_key in model_labels_spancat.keys() + if (spans_key_in_model) and ( + label not in model_labels_spancat[spans_key] + ): + msg.warn( + f"Label '{label}' is not present in the model labels of key '{spans_key}'. " + "Performance may degrade after training." + ) + # Check for low number of examples per label + if count <= NEW_LABEL_THRESHOLD: + msg.warn( + f"Low number of examples for label '{label}' in key '{spans_key}' ({count})" + ) + has_low_data_warning = True + # Check for negative examples + with msg.loading("Analyzing label distribution..."): + neg_docs = _get_examples_without_label( + train_dataset, label, "spancat", spans_key + ) + if neg_docs == 0: + msg.warn(f"No examples for texts WITHOUT new label '{label}'") + has_no_neg_warning = True + + with msg.loading("Obtaining span characteristics..."): + span_characteristics = _get_span_characteristics( + train_dataset, gold_train_data, spans_key + ) + + msg.info(f"Span characteristics for spans_key '{spans_key}'") + msg.info("SD = Span Distinctiveness, BD = Boundary Distinctiveness") + _print_span_characteristics(span_characteristics) + + _span_freqs = _get_spans_length_freq_dist( + gold_train_data["spans_length"][spans_key] + ) + _filtered_span_freqs = _filter_spans_length_freq_dist( + _span_freqs, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE + ) + + msg.info( + f"Over {SPAN_LENGTH_THRESHOLD_PERCENTAGE}% of spans have lengths of 1 -- " + f"{max(_filtered_span_freqs.keys())} " + f"(min={span_characteristics['min_length']}, max={span_characteristics['max_length']}). " + f"The most common span lengths are: {_format_freqs(_filtered_span_freqs)}. " + "If you are using the n-gram suggester, note that omitting " + "infrequent n-gram lengths can greatly improve speed and " + "memory usage." + ) + + msg.text( + f"Full distribution of span lengths: {_format_freqs(_span_freqs)}", + show=verbose, + ) + + # Add report regarding span characteristics + if span_characteristics["avg_sd"] < SPAN_DISTINCT_THRESHOLD: + msg.warn("Spans may not be distinct from the rest of the corpus") + else: + msg.good("Spans are distinct from the rest of the corpus") + + p_spans = span_characteristics["p_spans"].values() + all_span_tokens: Counter = sum(p_spans, Counter()) + most_common_spans = [w for w, _ in all_span_tokens.most_common(10)] + msg.text( + "10 most common span tokens: {}".format( + _format_labels(most_common_spans) + ), + show=verbose, + ) + + # Add report regarding span boundary characteristics + if span_characteristics["avg_bd"] < BOUNDARY_DISTINCT_THRESHOLD: + msg.warn("Boundary tokens are not distinct from the rest of the corpus") + else: + msg.good("Boundary tokens are distinct from the rest of the corpus") + + p_bounds = span_characteristics["p_bounds"].values() + all_span_bound_tokens: Counter = sum(p_bounds, Counter()) + most_common_bounds = [w for w, _ in all_span_bound_tokens.most_common(10)] + msg.text( + "10 most common span boundary tokens: {}".format( + _format_labels(most_common_bounds) + ), + show=verbose, + ) + + if has_low_data_warning: + msg.text( + f"To train a new span type, your data should include at " + f"least {NEW_LABEL_THRESHOLD} instances of the new label", + show=verbose, + ) + else: + msg.good("Good amount of examples for all labels") + + if has_no_neg_warning: + msg.text( + "Training data should always include examples of spans " + "in context, as well as examples without a given span " + "type.", + show=verbose, + ) + else: + msg.good("Examples without occurrences available for all labels") + + if "ner" in factory_names: + # Get all unique NER labels present in the data + labels = set( + label for label in gold_train_data["ner"] if label not in ("O", "-", None) + ) + label_counts = gold_train_data["ner"] + model_labels = _get_labels_from_model(nlp, "ner") + has_low_data_warning = False + has_no_neg_warning = False + has_ws_ents_error = False + has_boundary_cross_ents_warning = False + + msg.divider("Named Entity Recognition") + msg.info(f"{len(model_labels)} label(s)") + missing_values = label_counts["-"] + msg.text(f"{missing_values} missing value(s) (tokens with '-' label)") + for label in labels: + if len(label) == 0: + msg.fail("Empty label found in train data") + labels_with_counts = [ + (label, count) + for label, count in label_counts.most_common() + if label != "-" + ] + labels_with_counts = _format_labels(labels_with_counts, counts=True) + msg.text(f"Labels in train data: {labels_with_counts}", show=verbose) + missing_labels = model_labels - labels + if missing_labels: + msg.warn( + "Some model labels are not present in the train data. The " + "model performance may be degraded for these labels after " + f"training: {_format_labels(missing_labels)}." + ) + if gold_train_data["ws_ents"]: + msg.fail(f"{gold_train_data['ws_ents']} invalid whitespace entity spans") + has_ws_ents_error = True + + for label in labels: + if label_counts[label] <= NEW_LABEL_THRESHOLD: + msg.warn( + f"Low number of examples for label '{label}' ({label_counts[label]})" + ) + has_low_data_warning = True + + with msg.loading("Analyzing label distribution..."): + neg_docs = _get_examples_without_label(train_dataset, label, "ner") + if neg_docs == 0: + msg.warn(f"No examples for texts WITHOUT new label '{label}'") + has_no_neg_warning = True + + if gold_train_data["boundary_cross_ents"]: + msg.warn( + f"{gold_train_data['boundary_cross_ents']} entity span(s) crossing sentence boundaries" + ) + has_boundary_cross_ents_warning = True + + if not has_low_data_warning: + msg.good("Good amount of examples for all labels") + if not has_no_neg_warning: + msg.good("Examples without occurrences available for all labels") + if not has_ws_ents_error: + msg.good("No entities consisting of or starting/ending with whitespace") + if not has_boundary_cross_ents_warning: + msg.good("No entities crossing sentence boundaries") + + if has_low_data_warning: + msg.text( + f"To train a new entity type, your data should include at " + f"least {NEW_LABEL_THRESHOLD} instances of the new label", + show=verbose, + ) + if has_no_neg_warning: + msg.text( + "Training data should always include examples of entities " + "in context, as well as examples without a given entity " + "type.", + show=verbose, + ) + if has_ws_ents_error: + msg.text( + "Entity spans consisting of or starting/ending " + "with whitespace characters are considered invalid." + ) + + if "textcat" in factory_names: + msg.divider("Text Classification (Exclusive Classes)") + labels = _get_labels_from_model(nlp, "textcat") + msg.info(f"Text Classification: {len(labels)} label(s)") + msg.text(f"Labels: {_format_labels(labels)}", show=verbose) + missing_labels = labels - set(gold_train_data["cats"]) + if missing_labels: + msg.warn( + "Some model labels are not present in the train data. The " + "model performance may be degraded for these labels after " + f"training: {_format_labels(missing_labels)}." + ) + if set(gold_train_data["cats"]) != set(gold_dev_data["cats"]): + msg.warn( + "Potential train/dev mismatch: the train and dev labels are " + "not the same. " + f"Train labels: {_format_labels(gold_train_data['cats'])}. " + f"Dev labels: {_format_labels(gold_dev_data['cats'])}." + ) + if len(labels) < 2: + msg.fail( + "The model does not have enough labels. 'textcat' requires at " + "least two labels due to mutually-exclusive classes, e.g. " + "LABEL/NOT_LABEL or POSITIVE/NEGATIVE for a binary " + "classification task." + ) + if ( + gold_train_data["n_cats_bad_values"] > 0 + or gold_dev_data["n_cats_bad_values"] > 0 + ): + msg.fail( + "Unsupported values for cats: the supported values are " + "1.0/True and 0.0/False." + ) + if gold_train_data["n_cats_multilabel"] > 0: + # Note: you should never get here because you run into E895 on + # initialization first. + msg.fail( + "The train data contains instances without mutually-exclusive " + "classes. Use the component 'textcat_multilabel' instead of " + "'textcat'." + ) + if gold_dev_data["n_cats_multilabel"] > 0: + msg.fail( + "The dev data contains instances without mutually-exclusive " + "classes. Use the component 'textcat_multilabel' instead of " + "'textcat'." + ) + + if "textcat_multilabel" in factory_names: + msg.divider("Text Classification (Multilabel)") + labels = _get_labels_from_model(nlp, "textcat_multilabel") + msg.info(f"Text Classification: {len(labels)} label(s)") + msg.text(f"Labels: {_format_labels(labels)}", show=verbose) + missing_labels = labels - set(gold_train_data["cats"]) + if missing_labels: + msg.warn( + "Some model labels are not present in the train data. The " + "model performance may be degraded for these labels after " + f"training: {_format_labels(missing_labels)}." + ) + if set(gold_train_data["cats"]) != set(gold_dev_data["cats"]): + msg.warn( + "Potential train/dev mismatch: the train and dev labels are " + "not the same. " + f"Train labels: {_format_labels(gold_train_data['cats'])}. " + f"Dev labels: {_format_labels(gold_dev_data['cats'])}." + ) + if ( + gold_train_data["n_cats_bad_values"] > 0 + or gold_dev_data["n_cats_bad_values"] > 0 + ): + msg.fail( + "Unsupported values for cats: the supported values are " + "1.0/True and 0.0/False." + ) + if gold_train_data["n_cats_multilabel"] > 0: + if gold_dev_data["n_cats_multilabel"] == 0: + msg.warn( + "Potential train/dev mismatch: the train data contains " + "instances without mutually-exclusive classes while the " + "dev data contains only instances with mutually-exclusive " + "classes." + ) + else: + msg.warn( + "The train data contains only instances with " + "mutually-exclusive classes. You can potentially use the " + "component 'textcat' instead of 'textcat_multilabel'." + ) + if gold_dev_data["n_cats_multilabel"] > 0: + msg.fail( + "Train/dev mismatch: the dev data contains instances " + "without mutually-exclusive classes while the train data " + "contains only instances with mutually-exclusive classes." + ) + + if "tagger" in factory_names: + msg.divider("Part-of-speech Tagging") + label_list, counts = zip(*gold_train_data["tags"].items()) + msg.info(f"{len(label_list)} label(s) in train data") + p = numpy.array(counts) + p = p / p.sum() + norm_entropy = (-p * numpy.log2(p)).sum() / numpy.log2(len(label_list)) + msg.info(f"{norm_entropy} is the normalised label entropy") + model_labels = _get_labels_from_model(nlp, "tagger") + labels = set(label_list) + missing_labels = model_labels - labels + if missing_labels: + msg.warn( + "Some model labels are not present in the train data. The " + "model performance may be degraded for these labels after " + f"training: {_format_labels(missing_labels)}." + ) + labels_with_counts = _format_labels( + gold_train_data["tags"].most_common(), counts=True + ) + msg.text(labels_with_counts, show=verbose) + + if "morphologizer" in factory_names: + msg.divider("Morphologizer (POS+Morph)") + label_list = [label for label in gold_train_data["morphs"]] + model_labels = _get_labels_from_model(nlp, "morphologizer") + msg.info(f"{len(label_list)} label(s) in train data") + labels = set(label_list) + missing_labels = model_labels - labels + if missing_labels: + msg.warn( + "Some model labels are not present in the train data. The " + "model performance may be degraded for these labels after " + f"training: {_format_labels(missing_labels)}." + ) + labels_with_counts = _format_labels( + gold_train_data["morphs"].most_common(), counts=True + ) + msg.text(labels_with_counts, show=verbose) + + if "parser" in factory_names: + has_low_data_warning = False + msg.divider("Dependency Parsing") + + # profile sentence length + msg.info( + f"Found {gold_train_data['n_sents']} sentence(s) with an average " + f"length of {gold_train_data['n_words'] / gold_train_data['n_sents']:.1f} words." + ) + + # check for documents with multiple sentences + sents_per_doc = gold_train_data["n_sents"] / len(gold_train_data["texts"]) + if sents_per_doc < 1.1: + msg.warn( + f"The training data contains {sents_per_doc:.2f} sentences per " + f"document. When there are very few documents containing more " + f"than one sentence, the parser will not learn how to segment " + f"longer texts into sentences." + ) + + # profile labels + labels_train = [label for label in gold_train_data["deps"]] + labels_train_unpreprocessed = [ + label for label in gold_train_unpreprocessed_data["deps"] + ] + labels_dev = [label for label in gold_dev_data["deps"]] + + if gold_train_unpreprocessed_data["n_nonproj"] > 0: + n_nonproj = gold_train_unpreprocessed_data["n_nonproj"] + msg.info(f"Found {n_nonproj} nonprojective train sentence(s)") + if gold_dev_data["n_nonproj"] > 0: + n_nonproj = gold_dev_data["n_nonproj"] + msg.info(f"Found {n_nonproj} nonprojective dev sentence(s)") + msg.info(f"{len(labels_train_unpreprocessed)} label(s) in train data") + msg.info(f"{len(labels_train)} label(s) in projectivized train data") + labels_with_counts = _format_labels( + gold_train_unpreprocessed_data["deps"].most_common(), counts=True + ) + msg.text(labels_with_counts, show=verbose) + + # rare labels in train + for label in gold_train_unpreprocessed_data["deps"]: + if gold_train_unpreprocessed_data["deps"][label] <= DEP_LABEL_THRESHOLD: + msg.warn( + f"Low number of examples for label '{label}' " + f"({gold_train_unpreprocessed_data['deps'][label]})" + ) + has_low_data_warning = True + + # rare labels in projectivized train + rare_projectivized_labels = [] + for label in gold_train_data["deps"]: + if ( + gold_train_data["deps"][label] <= DEP_LABEL_THRESHOLD + and DELIMITER in label + ): + rare_projectivized_labels.append( + f"{label}: {gold_train_data['deps'][label]}" + ) + + if len(rare_projectivized_labels) > 0: + msg.warn( + f"Low number of examples for {len(rare_projectivized_labels)} " + "label(s) in the projectivized dependency trees used for " + "training. You may want to projectivize labels such as punct " + "before training in order to improve parser performance." + ) + msg.warn( + f"Projectivized labels with low numbers of examples: ", + ", ".join(rare_projectivized_labels), + show=verbose, + ) + has_low_data_warning = True + + # labels only in train + if set(labels_train) - set(labels_dev): + msg.warn( + "The following labels were found only in the train data:", + ", ".join(set(labels_train) - set(labels_dev)), + show=verbose, + ) + + # labels only in dev + if set(labels_dev) - set(labels_train): + msg.warn( + "The following labels were found only in the dev data:", + ", ".join(set(labels_dev) - set(labels_train)), + show=verbose, + ) + + if has_low_data_warning: + msg.text( + f"To train a parser, your data should include at " + f"least {DEP_LABEL_THRESHOLD} instances of each label.", + show=verbose, + ) + + # multiple root labels + if len(gold_train_unpreprocessed_data["roots"]) > 1: + msg.warn( + f"Multiple root labels " + f"({', '.join(gold_train_unpreprocessed_data['roots'])}) " + f"found in training data. spaCy's parser uses a single root " + f"label ROOT so this distinction will not be available." + ) + + # these should not happen, but just in case + if gold_train_data["n_nonproj"] > 0: + msg.fail( + f"Found {gold_train_data['n_nonproj']} nonprojective " + f"projectivized train sentence(s)" + ) + if gold_train_data["n_cycles"] > 0: + msg.fail( + f"Found {gold_train_data['n_cycles']} projectivized train sentence(s) with cycles" + ) + + if "trainable_lemmatizer" in factory_names: + msg.divider("Trainable Lemmatizer") + trees_train: Set[str] = gold_train_data["lemmatizer_trees"] + trees_dev: Set[str] = gold_dev_data["lemmatizer_trees"] + # This is necessary context when someone is attempting to interpret whether the + # number of trees exclusively in the dev set is meaningful. + msg.info(f"{len(trees_train)} lemmatizer trees generated from training data") + msg.info(f"{len(trees_dev)} lemmatizer trees generated from dev data") + dev_not_train = trees_dev - trees_train + + if len(dev_not_train) != 0: + pct = len(dev_not_train) / len(trees_dev) + msg.info( + f"{len(dev_not_train)} lemmatizer trees ({pct * 100:.1f}% of dev trees)" + " were found exclusively in the dev data." + ) + else: + # Would we ever expect this case? It seems like it would be pretty rare, + # and we might actually want a warning? + msg.info("All trees in dev data present in training data.") + + if gold_train_data["n_low_cardinality_lemmas"] > 0: + n = gold_train_data["n_low_cardinality_lemmas"] + msg.warn(f"{n} training docs with 0 or 1 unique lemmas.") + + if gold_dev_data["n_low_cardinality_lemmas"] > 0: + n = gold_dev_data["n_low_cardinality_lemmas"] + msg.warn(f"{n} dev docs with 0 or 1 unique lemmas.") + + if gold_train_data["no_lemma_annotations"] > 0: + n = gold_train_data["no_lemma_annotations"] + msg.warn(f"{n} training docs with no lemma annotations.") + else: + msg.good("All training docs have lemma annotations.") + + if gold_dev_data["no_lemma_annotations"] > 0: + n = gold_dev_data["no_lemma_annotations"] + msg.warn(f"{n} dev docs with no lemma annotations.") + else: + msg.good("All dev docs have lemma annotations.") + + if gold_train_data["partial_lemma_annotations"] > 0: + n = gold_train_data["partial_lemma_annotations"] + msg.info(f"{n} training docs with partial lemma annotations.") + else: + msg.good("All training docs have complete lemma annotations.") + + if gold_dev_data["partial_lemma_annotations"] > 0: + n = gold_dev_data["partial_lemma_annotations"] + msg.info(f"{n} dev docs with partial lemma annotations.") + else: + msg.good("All dev docs have complete lemma annotations.") + + msg.divider("Summary") + good_counts = msg.counts[MESSAGES.GOOD] + warn_counts = msg.counts[MESSAGES.WARN] + fail_counts = msg.counts[MESSAGES.FAIL] + if good_counts: + msg.good(f"{good_counts} {'check' if good_counts == 1 else 'checks'} passed") + if warn_counts: + msg.warn(f"{warn_counts} {'warning' if warn_counts == 1 else 'warnings'}") + if fail_counts: + msg.fail(f"{fail_counts} {'error' if fail_counts == 1 else 'errors'}") + sys.exit(1) + + +def _load_file(file_path: Path, msg: Printer) -> None: + file_name = file_path.parts[-1] + if file_path.suffix == ".json": + with msg.loading(f"Loading {file_name}..."): + data = srsly.read_json(file_path) + msg.good(f"Loaded {file_name}") + return data + elif file_path.suffix == ".jsonl": + with msg.loading(f"Loading {file_name}..."): + data = srsly.read_jsonl(file_path) + msg.good(f"Loaded {file_name}") + return data + msg.fail( + f"Can't load file extension {file_path.suffix}", + "Expected .json or .jsonl", + exits=1, + ) + + +def _compile_gold( + examples: Sequence[Example], + factory_names: List[str], + nlp: Language, + make_proj: bool, +) -> Dict[str, Any]: + data: Dict[str, Any] = { + "ner": Counter(), + "cats": Counter(), + "tags": Counter(), + "morphs": Counter(), + "deps": Counter(), + "words": Counter(), + "roots": Counter(), + "spancat": dict(), + "spans_length": dict(), + "spans_per_type": dict(), + "sb_per_type": dict(), + "ws_ents": 0, + "boundary_cross_ents": 0, + "n_words": 0, + "n_misaligned_words": 0, + "words_missing_vectors": Counter(), + "n_sents": 0, + "n_nonproj": 0, + "n_cycles": 0, + "n_cats_multilabel": 0, + "n_cats_bad_values": 0, + "texts": set(), + "lemmatizer_trees": set(), + "no_lemma_annotations": 0, + "partial_lemma_annotations": 0, + "n_low_cardinality_lemmas": 0, + } + if "trainable_lemmatizer" in factory_names: + trees = EditTrees(nlp.vocab.strings) + for eg in examples: + gold = eg.reference + doc = eg.predicted + valid_words = [x.text for x in gold] + data["words"].update(valid_words) + data["n_words"] += len(valid_words) + align = eg.alignment + for token in doc: + if token.orth_.isspace(): + continue + if align.x2y.lengths[token.i] != 1: + data["n_misaligned_words"] += 1 + data["texts"].add(doc.text) + if len(nlp.vocab.vectors): + for word in [t.text for t in doc]: + if nlp.vocab.strings[word] not in nlp.vocab.vectors: + data["words_missing_vectors"].update([word]) + if "ner" in factory_names: + sent_starts = eg.get_aligned_sent_starts() + for i, label in enumerate(eg.get_aligned_ner()): + if label is None: + continue + if label.startswith(("B-", "U-", "L-")) and doc[i].is_space: + # "Illegal" whitespace entity + data["ws_ents"] += 1 + if label.startswith(("B-", "U-")): + combined_label = remove_bilu_prefix(label) + data["ner"][combined_label] += 1 + if sent_starts[i] and label.startswith(("I-", "L-")): + data["boundary_cross_ents"] += 1 + elif label == "-": + data["ner"]["-"] += 1 + if "spancat" in factory_names or "spancat_singlelabel" in factory_names: + for spans_key in list(eg.reference.spans.keys()): + # Obtain the span frequency + if spans_key not in data["spancat"]: + data["spancat"][spans_key] = Counter() + for i, span in enumerate(eg.reference.spans[spans_key]): + if span.label_ is None: + continue + else: + data["spancat"][spans_key][span.label_] += 1 + + # Obtain the span length + if spans_key not in data["spans_length"]: + data["spans_length"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ is None: + continue + if span.label_ not in data["spans_length"][spans_key]: + data["spans_length"][spans_key][span.label_] = [] + data["spans_length"][spans_key][span.label_].append(len(span)) + + # Obtain spans per span type + if spans_key not in data["spans_per_type"]: + data["spans_per_type"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ not in data["spans_per_type"][spans_key]: + data["spans_per_type"][spans_key][span.label_] = [] + data["spans_per_type"][spans_key][span.label_].append(span) + + # Obtain boundary tokens per span type + window_size = 1 + if spans_key not in data["sb_per_type"]: + data["sb_per_type"][spans_key] = dict() + for span in gold.spans[spans_key]: + if span.label_ not in data["sb_per_type"][spans_key]: + # Creating a data structure that holds the start and + # end tokens for each span type + data["sb_per_type"][spans_key][span.label_] = { + "start": [], + "end": [], + } + for offset in range(window_size): + sb_start_idx = span.start - (offset + 1) + if sb_start_idx >= 0: + data["sb_per_type"][spans_key][span.label_]["start"].append( + gold[sb_start_idx : sb_start_idx + 1] + ) + + sb_end_idx = span.end + (offset + 1) + if sb_end_idx <= len(gold): + data["sb_per_type"][spans_key][span.label_]["end"].append( + gold[sb_end_idx - 1 : sb_end_idx] + ) + + if "textcat" in factory_names or "textcat_multilabel" in factory_names: + data["cats"].update(gold.cats) + if any(val not in (0, 1) for val in gold.cats.values()): + data["n_cats_bad_values"] += 1 + if list(gold.cats.values()).count(1) != 1: + data["n_cats_multilabel"] += 1 + if "tagger" in factory_names: + tags = eg.get_aligned("TAG", as_string=True) + data["tags"].update([x for x in tags if x is not None]) + if "morphologizer" in factory_names: + pos_tags = eg.get_aligned("POS", as_string=True) + morphs = eg.get_aligned("MORPH", as_string=True) + for pos, morph in zip(pos_tags, morphs): + # POS may align (same value for multiple tokens) when morph + # doesn't, so if either is misaligned (None), treat the + # annotation as missing so that truths doesn't end up with an + # unknown morph+POS combination + if pos is None or morph is None: + pass + # If both are unset, the annotation is missing (empty morph + # converted from int is "_" rather than "") + elif pos == "" and morph == "": + pass + # Otherwise, generate the combined label + else: + label_dict = Morphology.feats_to_dict(morph) + if pos: + label_dict[Morphologizer.POS_FEAT] = pos + label = eg.reference.vocab.strings[ + eg.reference.vocab.morphology.add(label_dict) + ] + data["morphs"].update([label]) + if "parser" in factory_names: + aligned_heads, aligned_deps = eg.get_aligned_parse(projectivize=make_proj) + data["deps"].update([x for x in aligned_deps if x is not None]) + for i, (dep, head) in enumerate(zip(aligned_deps, aligned_heads)): + if head == i: + data["roots"].update([dep]) + data["n_sents"] += 1 + if nonproj.is_nonproj_tree(aligned_heads): + data["n_nonproj"] += 1 + if nonproj.contains_cycle(aligned_heads): + data["n_cycles"] += 1 + if "trainable_lemmatizer" in factory_names: + # from EditTreeLemmatizer._labels_from_data + if all(token.lemma == 0 for token in gold): + data["no_lemma_annotations"] += 1 + continue + if any(token.lemma == 0 for token in gold): + data["partial_lemma_annotations"] += 1 + lemma_set = set() + for token in gold: + if token.lemma != 0: + lemma_set.add(token.lemma) + tree_id = trees.add(token.text, token.lemma_) + tree_str = trees.tree_to_str(tree_id) + data["lemmatizer_trees"].add(tree_str) + # We want to identify cases where lemmas aren't assigned + # or are all assigned the same value, as this would indicate + # an issue since we're expecting a large set of lemmas + if len(lemma_set) < 2 and len(gold) > 1: + data["n_low_cardinality_lemmas"] += 1 + return data + + +@overload +def _format_labels(labels: Iterable[str], counts: Literal[False] = False) -> str: ... + + +@overload +def _format_labels( + labels: Iterable[Tuple[str, int]], + counts: Literal[True], +) -> str: ... + + +def _format_labels( + labels: Union[Iterable[str], Iterable[Tuple[str, int]]], + counts: bool = False, +) -> str: + if counts: + return ", ".join( + [f"'{l}' ({c})" for l, c in cast(Iterable[Tuple[str, int]], labels)] + ) + return ", ".join([f"'{l}'" for l in cast(Iterable[str], labels)]) + + +def _format_freqs(freqs: Dict[int, float], sort: bool = True) -> str: + if sort: + freqs = dict(sorted(freqs.items())) + + _freqs = [(str(k), v) for k, v in freqs.items()] + return ", ".join( + [f"{l} ({c}%)" for l, c in cast(Iterable[Tuple[str, float]], _freqs)] + ) + + +def _get_examples_without_label( + data: Sequence[Example], + label: str, + component: Literal["ner", "spancat"] = "ner", + spans_key: Optional[str] = "sc", +) -> int: + count = 0 + for eg in data: + if component == "ner": + labels = [ + remove_bilu_prefix(label) + for label in eg.get_aligned_ner() + if label not in ("O", "-", None) + ] + + if component == "spancat": + labels = ( + [span.label_ for span in eg.reference.spans[spans_key]] + if spans_key in eg.reference.spans + else [] + ) + + if label not in labels: + count += 1 + return count + + +def _get_labels_from_model(nlp: Language, factory_name: str) -> Set[str]: + pipe_names = [ + pipe_name + for pipe_name in nlp.pipe_names + if nlp.get_pipe_meta(pipe_name).factory == factory_name + ] + labels: Set[str] = set() + for pipe_name in pipe_names: + pipe = nlp.get_pipe(pipe_name) + assert isinstance(pipe, TrainablePipe) + labels.update(pipe.labels) + return labels + + +def _get_labels_from_spancat(nlp: Language) -> Dict[str, Set[str]]: + pipe_names = [ + pipe_name + for pipe_name in nlp.pipe_names + if nlp.get_pipe_meta(pipe_name).factory in ("spancat", "spancat_singlelabel") + ] + labels: Dict[str, Set[str]] = {} + for pipe_name in pipe_names: + pipe = nlp.get_pipe(pipe_name) + assert isinstance(pipe, SpanCategorizer) + if pipe.key not in labels: + labels[pipe.key] = set() + labels[pipe.key].update(pipe.labels) + return labels + + +def _gmean(l: List) -> float: + """Compute geometric mean of a list""" + return math.exp(math.fsum(math.log(i) for i in l) / len(l)) + + +def _wgt_average(metric: Dict[str, float], frequencies: Counter) -> float: + total = sum(value * frequencies[span_type] for span_type, value in metric.items()) + return total / sum(frequencies.values()) + + +def _get_distribution(docs, normalize: bool = True) -> Counter: + """Get the frequency distribution given a set of Docs""" + word_counts: Counter = Counter() + for doc in docs: + for token in doc: + # Normalize the text + t = token.text.lower().replace("``", '"').replace("''", '"') + word_counts[t] += 1 + if normalize: + total = sum(word_counts.values(), 0.0) + word_counts = Counter({k: v / total for k, v in word_counts.items()}) + return word_counts + + +def _get_kl_divergence(p: Counter, q: Counter) -> float: + """Compute the Kullback-Leibler divergence from two frequency distributions""" + total = 0.0 + for word, p_word in p.items(): + total += p_word * math.log(p_word / q[word]) + return total + + +def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]: + """Compile into one list for easier reporting""" + d = { + label: [label] + list(_format_number(d[label]) for d in span_data) + for label in labels + } + return list(d.values()) + + +def _get_span_characteristics( + examples: List[Example], compiled_gold: Dict[str, Any], spans_key: str +) -> Dict[str, Any]: + """Obtain all span characteristics""" + data_labels = compiled_gold["spancat"][spans_key] + # Get lengths + span_length = { + label: _gmean(l) + for label, l in compiled_gold["spans_length"][spans_key].items() + } + spans_per_type = { + label: len(spans) + for label, spans in compiled_gold["spans_per_type"][spans_key].items() + } + min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()] + max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()] + + # Get relevant distributions: corpus, spans, span boundaries + p_corpus = _get_distribution([eg.reference for eg in examples], normalize=True) + p_spans = { + label: _get_distribution(spans, normalize=True) + for label, spans in compiled_gold["spans_per_type"][spans_key].items() + } + p_bounds = { + label: _get_distribution(sb["start"] + sb["end"], normalize=True) + for label, sb in compiled_gold["sb_per_type"][spans_key].items() + } + + # Compute for actual span characteristics + span_distinctiveness = { + label: _get_kl_divergence(freq_dist, p_corpus) + for label, freq_dist in p_spans.items() + } + sb_distinctiveness = { + label: _get_kl_divergence(freq_dist, p_corpus) + for label, freq_dist in p_bounds.items() + } + + return { + "sd": span_distinctiveness, + "bd": sb_distinctiveness, + "spans_per_type": spans_per_type, + "lengths": span_length, + "min_length": min(min_lengths), + "max_length": max(max_lengths), + "avg_sd": _wgt_average(span_distinctiveness, data_labels), + "avg_bd": _wgt_average(sb_distinctiveness, data_labels), + "avg_length": _wgt_average(span_length, data_labels), + "labels": list(data_labels.keys()), + "p_spans": p_spans, + "p_bounds": p_bounds, + } + + +def _print_span_characteristics(span_characteristics: Dict[str, Any]): + """Print all span characteristics into a table""" + headers = ("Span Type", "Length", "SD", "BD", "N") + # Wasabi has this at 30 by default, but we might have some long labels + max_col = max(30, max(len(label) for label in span_characteristics["labels"])) + # Prepare table data with all span characteristics + table_data = [ + span_characteristics["lengths"], + span_characteristics["sd"], + span_characteristics["bd"], + span_characteristics["spans_per_type"], + ] + table = _format_span_row( + span_data=table_data, labels=span_characteristics["labels"] + ) + # Prepare table footer with weighted averages + footer_data = [ + span_characteristics["avg_length"], + span_characteristics["avg_sd"], + span_characteristics["avg_bd"], + ] + + footer = ( + ["Wgt. Average"] + ["{:.2f}".format(round(f, 2)) for f in footer_data] + ["-"] + ) + msg.table( + table, + footer=footer, + header=headers, + divider=True, + aligns=["l"] + ["r"] * (len(footer_data) + 1), + max_col=max_col, + ) + + +def _get_spans_length_freq_dist( + length_dict: Dict, threshold=SPAN_LENGTH_THRESHOLD_PERCENTAGE +) -> Dict[int, float]: + """Get frequency distribution of spans length under a certain threshold""" + all_span_lengths = [] + for _, lengths in length_dict.items(): + all_span_lengths.extend(lengths) + + freq_dist: Counter = Counter() + for i in all_span_lengths: + if freq_dist.get(i): + freq_dist[i] += 1 + else: + freq_dist[i] = 1 + + # We will be working with percentages instead of raw counts + freq_dist_percentage = {} + for span_length, count in freq_dist.most_common(): + percentage = (count / len(all_span_lengths)) * 100.0 + percentage = round(percentage, 2) + freq_dist_percentage[span_length] = percentage + + return freq_dist_percentage + + +def _filter_spans_length_freq_dist( + freq_dist: Dict[int, float], threshold: int +) -> Dict[int, float]: + """Filter frequency distribution with respect to a threshold + + We're going to filter all the span lengths that fall + around a percentage threshold when summed. + """ + total = 0.0 + filtered_freq_dist = {} + for span_length, dist in freq_dist.items(): + if total >= threshold: + break + else: + filtered_freq_dist[span_length] = dist + total += dist + return filtered_freq_dist diff --git a/spacy/cli/debug_diff.py b/spacy/cli/debug_diff.py new file mode 100644 index 0000000..71d8826 --- /dev/null +++ b/spacy/cli/debug_diff.py @@ -0,0 +1,112 @@ +from pathlib import Path +from typing import Optional + +import typer +from wasabi import MarkdownRenderer, Printer, diff_strings + +from ..util import load_config +from ._util import Arg, Opt, debug_cli, show_validation_error +from .init_config import Optimizations, init_config + + +@debug_cli.command( + "diff-config", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def debug_diff_cli( + # fmt: off + ctx: typer.Context, + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + compare_to: Optional[Path] = Opt( + None, + help="Path to a config file to diff against, or `None` to compare against default settings", + exists=True, + allow_dash=True, + ), + optimize: Optimizations = Opt( + Optimizations.efficiency.value, + "--optimize", + "-o", + help="Whether the user config was optimized for efficiency or accuracy. Only relevant when comparing against the default config.", + ), + gpu: bool = Opt( + False, + "--gpu", + "-G", + help="Whether the original config can run on a GPU. Only relevant when comparing against the default config.", + ), + pretraining: bool = Opt( + False, + "--pretraining", + "--pt", + help="Whether to compare on a config with pretraining involved. Only relevant when comparing against the default config.", + ), + markdown: bool = Opt( + False, "--markdown", "-md", help="Generate Markdown for GitHub issues" + ), + # fmt: on +): + """Show a diff of a config file with respect to spaCy's defaults or another config file. If + additional settings were used in the creation of the config file, then you + must supply these as extra parameters to the command when comparing to the default settings. The generated diff + can also be used when posting to the discussion forum to provide more + information for the maintainers. + + The `optimize`, `gpu`, and `pretraining` options are only relevant when + comparing against the default configuration (or specifically when `compare_to` is None). + + DOCS: https://spacy.io/api/cli#debug-diff + """ + debug_diff( + config_path=config_path, + compare_to=compare_to, + gpu=gpu, + optimize=optimize, + pretraining=pretraining, + markdown=markdown, + ) + + +def debug_diff( + config_path: Path, + compare_to: Optional[Path], + gpu: bool, + optimize: Optimizations, + pretraining: bool, + markdown: bool, +): + msg = Printer() + with show_validation_error(hint_fill=False): + user_config = load_config(config_path) + if compare_to: + other_config = load_config(compare_to) + else: + # Recreate a default config based from user's config + lang = user_config["nlp"]["lang"] + pipeline = list(user_config["nlp"]["pipeline"]) + msg.info(f"Found user-defined language: '{lang}'") + msg.info(f"Found user-defined pipelines: {pipeline}") + other_config = init_config( + lang=lang, + pipeline=pipeline, + optimize=optimize.value, + gpu=gpu, + pretraining=pretraining, + silent=True, + ) + + user = user_config.to_str() + other = other_config.to_str() + + if user == other: + msg.warn("No diff to show: configs are identical") + else: + diff_text = diff_strings(other, user, add_symbols=markdown) + if markdown: + md = MarkdownRenderer() + md.add(md.code_block(diff_text, "diff")) + print(md.text) + else: + print(diff_text) diff --git a/spacy/cli/debug_model.py b/spacy/cli/debug_model.py new file mode 100644 index 0000000..dc0de3e --- /dev/null +++ b/spacy/cli/debug_model.py @@ -0,0 +1,250 @@ +import itertools +from pathlib import Path +from typing import Any, Dict, Optional + +import typer +from thinc.api import ( + Model, + data_validation, + fix_random_seed, + set_dropout_rate, + set_gpu_allocator, +) +from wasabi import msg + +from spacy.training import Example +from spacy.util import resolve_dot_names + +from .. import util +from ..schemas import ConfigSchemaTraining +from ..util import registry +from ._util import ( + Arg, + Opt, + debug_cli, + parse_config_overrides, + setup_gpu, + show_validation_error, + string_to_list, +) + + +@debug_cli.command( + "model", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def debug_model_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + component: str = Arg( + ..., help="Name of the pipeline component of which the model should be analysed" + ), + layers: str = Opt( + "", "--layers", "-l", help="Comma-separated names of layer IDs to print" + ), + dimensions: bool = Opt(False, "--dimensions", "-DIM", help="Show dimensions"), + parameters: bool = Opt(False, "--parameters", "-PAR", help="Show parameters"), + gradients: bool = Opt(False, "--gradients", "-GRAD", help="Show gradients"), + attributes: bool = Opt(False, "--attributes", "-ATTR", help="Show attributes"), + P0: bool = Opt(False, "--print-step0", "-P0", help="Print model before training"), + P1: bool = Opt( + False, "--print-step1", "-P1", help="Print model after initialization" + ), + P2: bool = Opt(False, "--print-step2", "-P2", help="Print model after training"), + P3: bool = Opt(False, "--print-step3", "-P3", help="Print final predictions"), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + # fmt: on +): + """ + Analyze a Thinc model implementation. Includes checks for internal structure + and activations during training. + + DOCS: https://spacy.io/api/cli#debug-model + """ + setup_gpu(use_gpu) + layers = string_to_list(layers, intify=True) + print_settings = { + "dimensions": dimensions, + "parameters": parameters, + "gradients": gradients, + "attributes": attributes, + "layers": layers, + "print_before_training": P0, + "print_after_init": P1, + "print_after_training": P2, + "print_prediction": P3, + } + config_overrides = parse_config_overrides(ctx.args) + with show_validation_error(config_path): + raw_config = util.load_config( + config_path, overrides=config_overrides, interpolate=False + ) + config = raw_config.interpolate() + allocator = config["training"]["gpu_allocator"] + if use_gpu >= 0 and allocator: + set_gpu_allocator(allocator) + with show_validation_error(config_path): + nlp = util.load_model_from_config(raw_config) + config = nlp.config.interpolate() + T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type] + seed = T["seed"] + if seed is not None: + msg.info(f"Fixing random seed: {seed}") + fix_random_seed(seed) + pipe = nlp.get_pipe(component) + + debug_model(config, T, nlp, pipe, print_settings=print_settings) + + +def debug_model( + config, + resolved_train_config, + nlp, + pipe, + *, + print_settings: Optional[Dict[str, Any]] = None, +): + if not hasattr(pipe, "model"): + msg.fail( + f"The component '{pipe}' does not specify an object that holds a Model.", + exits=1, + ) + model = pipe.model + if not isinstance(model, Model): + msg.fail( + f"Requires a Thinc Model to be analysed, but found {type(model)} instead.", + exits=1, + ) + if print_settings is None: + print_settings = {} + + # STEP 0: Printing before training + msg.info(f"Analysing model with ID {model.id}") + if print_settings.get("print_before_training"): + msg.divider(f"STEP 0 - before training") + _print_model(model, print_settings) + + # STEP 1: Initializing the model and printing again + with data_validation(False): + try: + dot_names = [resolved_train_config["train_corpus"]] + with show_validation_error(): + (train_corpus,) = resolve_dot_names(config, dot_names) + nlp.initialize(lambda: train_corpus(nlp)) + msg.info("Initialized the model with the training corpus.") + examples = list(itertools.islice(train_corpus(nlp), 5)) + except ValueError: + try: + _set_output_dim(nO=7, model=model) + with show_validation_error(): + examples = [Example.from_dict(x, {}) for x in _get_docs()] + nlp.initialize(lambda: examples) + msg.info("Initialized the model with dummy data.") + except Exception: + msg.fail( + "Could not initialize the model: you'll have to provide a valid 'train_corpus' argument in the config file.", + exits=1, + ) + + if print_settings.get("print_after_init"): + msg.divider(f"STEP 1 - after initialization") + _print_model(model, print_settings) + + # STEP 2: Updating the model and printing again + set_dropout_rate(model, 0.2) + # ugly hack to deal with Tok2Vec/Transformer listeners + upstream_component = None + if model.has_ref("tok2vec") and "tok2vec-listener" in model.get_ref("tok2vec").name: + upstream_component = nlp.get_pipe("tok2vec") + if ( + model.has_ref("tok2vec") + and "transformer-listener" in model.get_ref("tok2vec").name + ): + upstream_component = nlp.get_pipe("transformer") + for e in range(3): + if upstream_component: + upstream_component.update(examples) + pipe.update(examples) + if print_settings.get("print_after_training"): + msg.divider(f"STEP 2 - after training") + _print_model(model, print_settings) + + # STEP 3: the final prediction + prediction = model.predict([ex.predicted for ex in examples]) + if print_settings.get("print_prediction"): + msg.divider(f"STEP 3 - prediction") + msg.info(str(prediction)) + + msg.good(f"Successfully ended analysis - model looks good.") + + +def _sentences(): + return [ + "Apple is looking at buying U.K. startup for $1 billion", + "Autonomous cars shift insurance liability toward manufacturers", + "San Francisco considers banning sidewalk delivery robots", + "London is a big city in the United Kingdom.", + ] + + +def _get_docs(lang: str = "en"): + nlp = util.get_lang_class(lang)() + return list(nlp.pipe(_sentences())) + + +def _set_output_dim(model, nO): + # simulating dim inference by directly setting the nO argument of the model + if model.has_dim("nO") is None: + model.set_dim("nO", nO) + if model.has_ref("output_layer"): + if model.get_ref("output_layer").has_dim("nO") is None: + model.get_ref("output_layer").set_dim("nO", nO) + + +def _print_model(model, print_settings): + layers = print_settings.get("layers", "") + parameters = print_settings.get("parameters", False) + dimensions = print_settings.get("dimensions", False) + gradients = print_settings.get("gradients", False) + attributes = print_settings.get("attributes", False) + + for i, node in enumerate(model.walk()): + if not layers or i in layers: + msg.info(f"Layer {i}: model ID {node.id}: '{node.name}'") + + if dimensions: + for name in node.dim_names: + msg.info(f" - dim {name}: {node.maybe_get_dim(name)}") + if parameters: + for name in node.param_names: + if node.has_param(name): + print_value = _print_matrix(node.get_param(name)) + msg.info(f" - param {name}: {print_value}") + else: + msg.info(f" - param {name}: {node.has_param(name)}") + if gradients: + for name in node.param_names: + if node.has_grad(name): + print_value = _print_matrix(node.get_grad(name)) + msg.info(f" - grad {name}: {print_value}") + else: + msg.info(f" - grad {name}: {node.has_grad(name)}") + if attributes: + attrs = node.attrs + for name, value in attrs.items(): + msg.info(f" - attr {name}: {value}") + + +def _print_matrix(value): + if value is None or isinstance(value, bool): + return value + result = str(value.shape) + " - sample: " + sample_matrix = value + for d in range(value.ndim - 1): + sample_matrix = sample_matrix[0] + sample_matrix = sample_matrix[0:5] + result = result + str(sample_matrix) + return result diff --git a/spacy/cli/download.py b/spacy/cli/download.py new file mode 100644 index 0000000..4690343 --- /dev/null +++ b/spacy/cli/download.py @@ -0,0 +1,203 @@ +import importlib.util +import shutil +import sys +from typing import Optional, Sequence +from urllib.parse import urljoin + +import requests +import typer +from wasabi import msg + +from .. import about +from ..errors import OLD_MODEL_SHORTCUTS +from ..util import ( + get_minor_version, + is_in_interactive, + is_in_jupyter, + is_package, + is_prerelease_version, + run_command, +) +from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app + + +@app.command( + "download", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def download_cli( + # fmt: off + ctx: typer.Context, + model: str = Arg(..., help="Name of pipeline package to download"), + direct: bool = Opt( + False, "--direct", "-d", "-D", help="Force direct download of name + version" + ), + sdist: bool = Opt( + False, + "--sdist", + "-S", + help="Download sdist (.tar.gz) archive instead of pre-built binary wheel", + ), + url: str = Opt(None, "--url", "-U", help="Download from given url"), + # fmt: on +): + """ + Download compatible trained pipeline from the default download path using + pip. If --direct flag is set, the command expects the full package name with + version. For direct downloads, the compatibility check will be skipped. All + additional arguments provided to this command will be passed to `pip install` + on package installation. + + DOCS: https://spacy.io/api/cli#download + AVAILABLE PACKAGES: https://spacy.io/models + """ + download(model, direct, sdist, url, *ctx.args) + + +def download( + model: str, + direct: bool = False, + sdist: bool = False, + custom_url: Optional[str] = None, + *pip_args, +) -> None: + if ( + not (is_package("spacy") or is_package("spacy-nightly")) + and "--no-deps" not in pip_args + ): + msg.warn( + "Skipping pipeline package dependencies and setting `--no-deps`. " + "You don't seem to have the spaCy package itself installed " + "(maybe because you've built from source?), so installing the " + "package dependencies would cause spaCy to be downloaded, which " + "probably isn't what you want. If the pipeline package has other " + "dependencies, you'll have to install them manually." + ) + pip_args = pip_args + ("--no-deps",) + if direct: + # Reject model names with '/', in order to prevent shenanigans. + if "/" in model: + msg.fail( + title="Model download rejected", + text=f"Cannot download model '{model}'. Models are expected to be file names, not URLs or fragments", + exits=True, + ) + components = model.split("-") + model_name = "".join(components[:-1]) + version = components[-1] + else: + model_name = model + if model in OLD_MODEL_SHORTCUTS: + msg.warn( + f"As of spaCy v3.0, shortcuts like '{model}' are deprecated. Please " + f"use the full pipeline package name '{OLD_MODEL_SHORTCUTS[model]}' instead." + ) + model_name = OLD_MODEL_SHORTCUTS[model] + compatibility = get_compatibility() + version = get_version(model_name, compatibility) + + filename = get_model_filename(model_name, version, sdist) + + download_model(filename, pip_args, custom_url) + msg.good( + "Download and installation successful", + f"You can now load the package via spacy.load('{model_name}')", + ) + if is_in_jupyter(): + reload_deps_msg = ( + "If you are in a Jupyter or Colab notebook, you may need to " + "restart Python in order to load all the package's dependencies. " + "You can do this by selecting the 'Restart kernel' or 'Restart " + "runtime' option." + ) + msg.warn( + "Restart to reload dependencies", + reload_deps_msg, + ) + elif is_in_interactive(): + reload_deps_msg = ( + "If you are in an interactive Python session, you may need to " + "exit and restart Python to load all the package's dependencies. " + "You can exit with Ctrl-D (or Ctrl-Z and Enter on Windows)." + ) + msg.warn( + "Restart to reload dependencies", + reload_deps_msg, + ) + + +def get_model_filename(model_name: str, version: str, sdist: bool = False) -> str: + dl_tpl = "{m}-{v}/{m}-{v}{s}" + suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX + filename = dl_tpl.format(m=model_name, v=version, s=suffix) + return filename + + +def get_compatibility() -> dict: + if is_prerelease_version(about.__version__): + version: Optional[str] = about.__version__ + else: + version = get_minor_version(about.__version__) + r = requests.get(about.__compatibility__) + if r.status_code != 200: + msg.fail( + f"Server error ({r.status_code})", + f"Couldn't fetch compatibility table. Please find a package for your spaCy " + f"installation (v{about.__version__}), and download it manually. " + f"For more details, see the documentation: " + f"https://spacy.io/usage/models", + exits=1, + ) + comp_table = r.json() + comp = comp_table["spacy"] + if version not in comp: + msg.fail(f"No compatible packages found for v{version} of spaCy", exits=1) + return comp[version] + + +def get_version(model: str, comp: dict) -> str: + if model not in comp: + msg.fail( + f"No compatible package found for '{model}' (spaCy v{about.__version__})", + exits=1, + ) + return comp[model][0] + + +def get_latest_version(model: str) -> str: + comp = get_compatibility() + return get_version(model, comp) + + +def download_model( + filename: str, + user_pip_args: Optional[Sequence[str]] = None, + custom_url: Optional[str] = None, +) -> None: + # Construct the download URL carefully. We need to make sure we don't + # allow relative paths or other shenanigans to trick us into download + # from outside our own repo. + base_url = custom_url if custom_url else about.__download_url__ + # urljoin requires that the path ends with /, or the last path part will be dropped + if not base_url.endswith("/"): + base_url = about.__download_url__ + "/" + download_url = urljoin(base_url, filename) + if not download_url.startswith(about.__download_url__): + raise ValueError(f"Download from {filename} rejected. Was it a relative path?") + pip_args = list(user_pip_args) if user_pip_args is not None else [] + cmd = _get_pip_install_cmd() + pip_args + [download_url] + run_command(cmd) + + +def _get_pip_install_cmd() -> list: + if importlib.util.find_spec("pip") is not None: + return [sys.executable, "-m", "pip", "install"] + elif shutil.which("uv"): + return ["uv", "pip", "install"] + else: + msg.fail( + "No package installer found", + "spaCy requires either pip or uv to download models. " + "Please install one of them and try again.", + exits=1, + ) diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py new file mode 100644 index 0000000..9704ea4 --- /dev/null +++ b/spacy/cli/evaluate.py @@ -0,0 +1,277 @@ +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +import srsly +from thinc.api import fix_random_seed +from wasabi import Printer + +from .. import displacy, util +from ..tokens import Doc +from ..training import Corpus +from ._util import Arg, Opt, app, benchmark_cli, import_code, setup_gpu + + +@benchmark_cli.command( + "accuracy", +) +@app.command("evaluate") +def evaluate_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg( + ..., help="Location of binary evaluation data in .spacy format", exists=True + ), + output: Optional[Path] = Opt( + None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False + ), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + gold_preproc: bool = Opt( + False, "--gold-preproc", "-G", help="Use gold preprocessing" + ), + displacy_path: Optional[Path] = Opt( + None, + "--displacy-path", + "-dp", + help="Directory to output rendered parses as HTML", + exists=True, + file_okay=False, + ), + displacy_limit: int = Opt( + 25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML" + ), + per_component: bool = Opt( + False, + "--per-component", + "-P", + help="Return scores per component, only applicable when an output JSON file is specified.", + ), + spans_key: str = Opt( + "sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans" + ), + # fmt: on +): + """ + Evaluate a trained pipeline. Expects a loadable spaCy pipeline and evaluation + data in the binary .spacy format. The --gold-preproc option sets up the + evaluation examples with gold-standard sentences and tokens for the + predictions. Gold preprocessing helps the annotations align to the + tokenization, and may result in sequences of more consistent length. However, + it may reduce runtime accuracy due to train/test skew. To render a sample of + dependency parses in a HTML file, set as output directory as the + displacy_path argument. + + DOCS: https://spacy.io/api/cli#benchmark-accuracy + """ + import_code(code_path) + evaluate( + model, + data_path, + output=output, + use_gpu=use_gpu, + gold_preproc=gold_preproc, + displacy_path=displacy_path, + displacy_limit=displacy_limit, + per_component=per_component, + silent=False, + spans_key=spans_key, + ) + + +def evaluate( + model: str, + data_path: Path, + output: Optional[Path] = None, + use_gpu: int = -1, + gold_preproc: bool = False, + displacy_path: Optional[Path] = None, + displacy_limit: int = 25, + silent: bool = True, + spans_key: str = "sc", + per_component: bool = False, +) -> Dict[str, Any]: + msg = Printer(no_print=silent, pretty=not silent) + fix_random_seed() + setup_gpu(use_gpu, silent=silent) + data_path = util.ensure_path(data_path) + output_path = util.ensure_path(output) + displacy_path = util.ensure_path(displacy_path) + if not data_path.exists(): + msg.fail("Evaluation data not found", data_path, exits=1) + if displacy_path and not displacy_path.exists(): + msg.fail("Visualization output directory not found", displacy_path, exits=1) + corpus = Corpus(data_path, gold_preproc=gold_preproc) + nlp = util.load_model(model) + dev_dataset = list(corpus(nlp)) + scores = nlp.evaluate(dev_dataset, per_component=per_component) + if per_component: + data = scores + if output is None: + msg.warn( + "The per-component option is enabled but there is no output JSON file provided to save the scores to." + ) + else: + msg.info("Per-component scores will be saved to output JSON file.") + else: + metrics = { + "TOK": "token_acc", + "TAG": "tag_acc", + "POS": "pos_acc", + "MORPH": "morph_acc", + "LEMMA": "lemma_acc", + "UAS": "dep_uas", + "LAS": "dep_las", + "NER P": "ents_p", + "NER R": "ents_r", + "NER F": "ents_f", + "TEXTCAT": "cats_score", + "SENT P": "sents_p", + "SENT R": "sents_r", + "SENT F": "sents_f", + "SPAN P": f"spans_{spans_key}_p", + "SPAN R": f"spans_{spans_key}_r", + "SPAN F": f"spans_{spans_key}_f", + "SPEED": "speed", + } + results = {} + data = {} + for metric, key in metrics.items(): + if key in scores: + if key == "cats_score": + metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")" + if isinstance(scores[key], (int, float)): + if key == "speed": + results[metric] = f"{scores[key]:.0f}" + else: + results[metric] = f"{scores[key] * 100:.2f}" + else: + results[metric] = "-" + data[re.sub(r"[\s/]", "_", key.lower())] = scores[key] + + msg.table(results, title="Results") + data = handle_scores_per_type(scores, data, spans_key=spans_key, silent=silent) + + if displacy_path: + factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names] + docs = list(nlp.pipe(ex.reference.text for ex in dev_dataset[:displacy_limit])) + render_deps = "parser" in factory_names + render_ents = "ner" in factory_names + render_spans = "spancat" in factory_names + + render_parses( + docs, + displacy_path, + model_name=model, + limit=displacy_limit, + deps=render_deps, + ents=render_ents, + spans=render_spans, + ) + msg.good(f"Generated {displacy_limit} parses as HTML", displacy_path) + + if output_path is not None: + srsly.write_json(output_path, data) + msg.good(f"Saved results to {output_path}") + return data + + +def handle_scores_per_type( + scores: Dict[str, Any], + data: Dict[str, Any] = {}, + *, + spans_key: str = "sc", + silent: bool = False, +) -> Dict[str, Any]: + msg = Printer(no_print=silent, pretty=not silent) + if "morph_per_feat" in scores: + if scores["morph_per_feat"]: + print_prf_per_type(msg, scores["morph_per_feat"], "MORPH", "feat") + data["morph_per_feat"] = scores["morph_per_feat"] + if "dep_las_per_type" in scores: + if scores["dep_las_per_type"]: + print_prf_per_type(msg, scores["dep_las_per_type"], "LAS", "type") + data["dep_las_per_type"] = scores["dep_las_per_type"] + if "ents_per_type" in scores: + if scores["ents_per_type"]: + print_prf_per_type(msg, scores["ents_per_type"], "NER", "type") + data["ents_per_type"] = scores["ents_per_type"] + if f"spans_{spans_key}_per_type" in scores: + if scores[f"spans_{spans_key}_per_type"]: + print_prf_per_type( + msg, scores[f"spans_{spans_key}_per_type"], "SPANS", "type" + ) + data[f"spans_{spans_key}_per_type"] = scores[f"spans_{spans_key}_per_type"] + if "cats_f_per_type" in scores: + if scores["cats_f_per_type"]: + print_prf_per_type(msg, scores["cats_f_per_type"], "Textcat F", "label") + data["cats_f_per_type"] = scores["cats_f_per_type"] + if "cats_auc_per_type" in scores: + if scores["cats_auc_per_type"]: + print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"]) + data["cats_auc_per_type"] = scores["cats_auc_per_type"] + return scores + + +def render_parses( + docs: List[Doc], + output_path: Path, + model_name: str = "", + limit: int = 250, + deps: bool = True, + ents: bool = True, + spans: bool = True, +): + docs[0].user_data["title"] = model_name + if ents: + html = displacy.render(docs[:limit], style="ent", page=True) + with (output_path / "entities.html").open("w", encoding="utf8") as file_: + file_.write(html) + if deps: + html = displacy.render( + docs[:limit], style="dep", page=True, options={"compact": True} + ) + with (output_path / "parses.html").open("w", encoding="utf8") as file_: + file_.write(html) + + if spans: + html = displacy.render(docs[:limit], style="span", page=True) + with (output_path / "spans.html").open("w", encoding="utf8") as file_: + file_.write(html) + + +def print_prf_per_type( + msg: Printer, scores: Dict[str, Dict[str, float]], name: str, type: str +) -> None: + data = [] + for key, value in scores.items(): + row = [key] + for k in ("p", "r", "f"): + v = value[k] + row.append(f"{v * 100:.2f}" if isinstance(v, (int, float)) else v) + data.append(row) + msg.table( + data, + header=("", "P", "R", "F"), + aligns=("l", "r", "r", "r"), + title=f"{name} (per {type})", + ) + + +def print_textcats_auc_per_cat( + msg: Printer, scores: Dict[str, Dict[str, float]] +) -> None: + msg.table( + [ + (k, f"{v:.2f}" if isinstance(v, (float, int)) else v) + for k, v in scores.items() + ], + header=("", "ROC AUC"), + aligns=("l", "r"), + title="Textcat ROC AUC (per label)", + ) diff --git a/spacy/cli/find_function.py b/spacy/cli/find_function.py new file mode 100644 index 0000000..3b3b333 --- /dev/null +++ b/spacy/cli/find_function.py @@ -0,0 +1,71 @@ +from typing import Optional, Tuple + +from catalogue import RegistryError +from wasabi import msg + +from ..util import registry +from ._util import Arg, Opt, app + + +@app.command("find-function") +def find_function_cli( + # fmt: off + func_name: str = Arg(..., help="Name of the registered function."), + registry_name: Optional[str] = Opt( + None, "--registry", "-r", help="Name of the catalogue registry." + ), + # fmt: on +): + """ + Find the module, path and line number to the file the registered + function is defined in, if available. + + func_name (str): Name of the registered function. + registry_name (Optional[str]): Name of the catalogue registry. + + DOCS: https://spacy.io/api/cli#find-function + """ + if not registry_name: + registry_names = registry.get_registry_names() + for name in registry_names: + if registry.has(name, func_name): + registry_name = name + break + + if not registry_name: + msg.fail( + f"Couldn't find registered function: '{func_name}'", + exits=1, + ) + + assert registry_name is not None + find_function(func_name, registry_name) + + +def find_function(func_name: str, registry_name: str) -> Tuple[str, int]: + registry_desc = None + try: + registry_desc = registry.find(registry_name, func_name) + except RegistryError as e: + msg.fail( + f"Couldn't find registered function: '{func_name}' in registry '{registry_name}'", + ) + msg.fail(f"{e}", exits=1) + assert registry_desc is not None + + registry_path = None + line_no = None + if registry_desc["file"]: + registry_path = registry_desc["file"] + line_no = registry_desc["line_no"] + + if not registry_path or not line_no: + msg.fail( + f"Couldn't find path to registered function: '{func_name}' in registry '{registry_name}'", + exits=1, + ) + assert registry_path is not None + assert line_no is not None + + msg.good(f"Found registered function '{func_name}' at {registry_path}:{line_no}") + return str(registry_path), int(line_no) diff --git a/spacy/cli/find_threshold.py b/spacy/cli/find_threshold.py new file mode 100644 index 0000000..1873f47 --- /dev/null +++ b/spacy/cli/find_threshold.py @@ -0,0 +1,261 @@ +import functools +import logging +import operator +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import numpy +import wasabi.tables + +from .. import util +from ..errors import Errors +from ..pipeline import MultiLabel_TextCategorizer, TextCategorizer +from ..training import Corpus +from ._util import Arg, Opt, app, import_code, setup_gpu + +_DEFAULTS = { + "n_trials": 11, + "use_gpu": -1, + "gold_preproc": False, +} + + +@app.command( + "find-threshold", + context_settings={"allow_extra_args": False, "ignore_unknown_options": True}, +) +def find_threshold_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg( + ..., help="Location of binary evaluation data in .spacy format", exists=True + ), + pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"), + threshold_key: str = Arg( + ..., help="Key of threshold attribute in component's configuration" + ), + scores_key: str = Arg(..., help="Metric to optimize"), + n_trials: int = Opt( + _DEFAULTS["n_trials"], + "--n_trials", + "-n", + help="Number of trials to determine optimal thresholds", + ), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + use_gpu: int = Opt( + _DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU" + ), + gold_preproc: bool = Opt( + _DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing" + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + # fmt: on +): + """ + Runs prediction trials for a trained model with varying thresholds to maximize + the specified metric. The search space for the threshold is traversed linearly + from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout` + (the corresponding API call to `spacy.cli.find_threshold.find_threshold()` + returns all results). + + This is applicable only for components whose predictions are influenced by + thresholds - e.g. `textcat_multilabel` and `spancat`, but not `textcat`. Note + that the full path to the corresponding threshold attribute in the config has to + be provided. + + DOCS: https://spacy.io/api/cli#find-threshold + """ + if verbose: + util.logger.setLevel(logging.DEBUG) + import_code(code_path) + find_threshold( + model=model, + data_path=data_path, + pipe_name=pipe_name, + threshold_key=threshold_key, + scores_key=scores_key, + n_trials=n_trials, + use_gpu=use_gpu, + gold_preproc=gold_preproc, + silent=False, + ) + + +def find_threshold( + model: str, + data_path: Path, + pipe_name: str, + threshold_key: str, + scores_key: str, + *, + n_trials: int = _DEFAULTS["n_trials"], # type: ignore + use_gpu: int = _DEFAULTS["use_gpu"], # type: ignore + gold_preproc: bool = _DEFAULTS["gold_preproc"], # type: ignore + silent: bool = True, +) -> Tuple[float, float, Dict[float, float]]: + """ + Runs prediction trials for models with varying thresholds to maximize the specified metric. + model (Union[str, Path]): Pipeline to evaluate. Can be a package or a path to a data directory. + data_path (Path): Path to file with DocBin with docs to use for threshold search. + pipe_name (str): Name of pipe to examine thresholds for. + threshold_key (str): Key of threshold attribute in component's configuration. + scores_key (str): Name of score to metric to optimize. + n_trials (int): Number of trials to determine optimal thresholds. + use_gpu (int): GPU ID or -1 for CPU. + gold_preproc (bool): Whether to use gold preprocessing. Gold preprocessing helps the annotations align to the + tokenization, and may result in sequences of more consistent length. However, it may reduce runtime accuracy due + to train/test skew. + silent (bool): Whether to print non-error-related output to stdout. + RETURNS (Tuple[float, float, Dict[float, float]]): Best found threshold, the corresponding score, scores for all + evaluated thresholds. + """ + + setup_gpu(use_gpu, silent=silent) + data_path = util.ensure_path(data_path) + if not data_path.exists(): + wasabi.msg.fail("Evaluation data not found", data_path, exits=1) + nlp = util.load_model(model) + + if pipe_name not in nlp.component_names: + raise AttributeError( + Errors.E001.format(name=pipe_name, opts=nlp.component_names) + ) + pipe = nlp.get_pipe(pipe_name) + if not hasattr(pipe, "scorer"): + raise AttributeError(Errors.E1045) + + if type(pipe) == TextCategorizer: + wasabi.msg.warn( + "The `textcat` component doesn't use a threshold as it's not applicable to the concept of " + "exclusive classes. All thresholds will yield the same results." + ) + + if not silent: + wasabi.msg.info( + title=f"Optimizing for {scores_key} for component '{pipe_name}' with {n_trials} " + f"trials." + ) + + # Load evaluation corpus. + corpus = Corpus(data_path, gold_preproc=gold_preproc) + dev_dataset = list(corpus(nlp)) + config_keys = threshold_key.split(".") + + def set_nested_item( + config: Dict[str, Any], keys: List[str], value: float + ) -> Dict[str, Any]: + """Set item in nested dictionary. Adapted from https://stackoverflow.com/a/54138200. + config (Dict[str, Any]): Configuration dictionary. + keys (List[Any]): Path to value to set. + value (float): Value to set. + RETURNS (Dict[str, Any]): Updated dictionary. + """ + functools.reduce(operator.getitem, keys[:-1], config)[keys[-1]] = value + return config + + def filter_config( + config: Dict[str, Any], keys: List[str], full_key: str + ) -> Dict[str, Any]: + """Filters provided config dictionary so that only the specified keys path remains. + config (Dict[str, Any]): Configuration dictionary. + keys (List[Any]): Path to value to set. + full_key (str): Full user-specified key. + RETURNS (Dict[str, Any]): Filtered dictionary. + """ + if keys[0] not in config: + wasabi.msg.fail( + title=f"Failed to look up `{full_key}` in config: sub-key {[keys[0]]} not found.", + text=f"Make sure you specified {[keys[0]]} correctly. The following sub-keys are available instead: " + f"{list(config.keys())}", + exits=1, + ) + return { + keys[0]: ( + filter_config(config[keys[0]], keys[1:], full_key) + if len(keys) > 1 + else config[keys[0]] + ) + } + + # Evaluate with varying threshold values. + scores: Dict[float, float] = {} + config_keys_full = ["components", pipe_name, *config_keys] + table_col_widths = (10, 10) + thresholds = numpy.linspace(0, 1, n_trials) + print(wasabi.tables.row(["Threshold", f"{scores_key}"], widths=table_col_widths)) + for threshold in thresholds: + # Reload pipeline with overrides specifying the new threshold. + nlp = util.load_model( + model, + config=set_nested_item( + filter_config( + nlp.config, config_keys_full, ".".join(config_keys_full) + ).copy(), + config_keys_full, + threshold, + ), + ) + if hasattr(pipe, "cfg"): + nlp.get_pipe(pipe_name).cfg = set_nested_item( # type: ignore[attr-defined] + pipe.cfg, + config_keys, + threshold, # type: ignore[attr-defined] + ) + + eval_scores = nlp.evaluate(dev_dataset) + if scores_key not in eval_scores: + wasabi.msg.fail( + title=f"Failed to look up score `{scores_key}` in evaluation results.", + text=f"Make sure you specified the correct value for `scores_key`. The following scores are " + f"available: {list(eval_scores.keys())}", + exits=1, + ) + scores[threshold] = eval_scores[scores_key] + + if not isinstance(scores[threshold], (float, int)): + wasabi.msg.fail( + f"Returned score for key '{scores_key}' is not numeric. Threshold optimization only works for numeric " + f"scores.", + exits=1, + ) + print( + wasabi.row( + [round(threshold, 3), round(scores[threshold], 3)], + widths=table_col_widths, + ) + ) + + best_threshold = max(scores.keys(), key=(lambda key: scores[key])) + + # If all scores are identical, emit warning. + if len(set(scores.values())) == 1: + wasabi.msg.warn( + title="All scores are identical. Verify that all settings are correct.", + text=( + "" + if ( + not isinstance(pipe, MultiLabel_TextCategorizer) + or scores_key in ("cats_macro_f", "cats_micro_f") + ) + else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`." + ), + ) + + else: + if not silent: + print( + f"\nBest threshold: {round(best_threshold, ndigits=4)} with {scores_key} value of {scores[best_threshold]}." + ) + + return best_threshold, scores[best_threshold], scores diff --git a/spacy/cli/info.py b/spacy/cli/info.py new file mode 100644 index 0000000..ed2394c --- /dev/null +++ b/spacy/cli/info.py @@ -0,0 +1,204 @@ +import json +import platform +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import srsly +from wasabi import MarkdownRenderer, Printer + +from .. import about, util +from ..compat import importlib_metadata +from ._util import Arg, Opt, app, string_to_list +from .download import get_latest_version, get_model_filename + + +@app.command("info") +def info_cli( + # fmt: off + model: Optional[str] = Arg(None, help="Optional loadable spaCy pipeline"), + markdown: bool = Opt( + False, "--markdown", "-md", help="Generate Markdown for GitHub issues" + ), + silent: bool = Opt( + False, "--silent", "-s", "-S", help="Don't print anything (just return)" + ), + exclude: str = Opt( + "labels", + "--exclude", + "-e", + help="Comma-separated keys to exclude from the print-out", + ), + url: bool = Opt( + False, + "--url", + "-u", + help="Print the URL to download the most recent compatible version of the pipeline", + ), + # fmt: on +): + """ + Print info about spaCy installation. If a pipeline is specified as an argument, + print its meta information. Flag --markdown prints details in Markdown for easy + copy-pasting to GitHub issues. + + Flag --url prints only the download URL of the most recent compatible + version of the pipeline. + + DOCS: https://spacy.io/api/cli#info + """ + exclude = string_to_list(exclude) + info( + model, + markdown=markdown, + silent=silent, + exclude=exclude, + url=url, + ) + + +def info( + model: Optional[str] = None, + *, + markdown: bool = False, + silent: bool = True, + exclude: Optional[List[str]] = None, + url: bool = False, +) -> Union[str, dict]: + msg = Printer(no_print=silent, pretty=not silent) + if not exclude: + exclude = [] + if url: + if model is not None: + title = f"Download info for pipeline '{model}'" + data = info_model_url(model) + print(data["download_url"]) + return data + else: + msg.fail("--url option requires a pipeline name", exits=1) + elif model: + title = f"Info about pipeline '{model}'" + data = info_model(model, silent=silent) + else: + title = "Info about spaCy" + data = info_spacy() + raw_data = {k.lower().replace(" ", "_"): v for k, v in data.items()} + if "Pipelines" in data and isinstance(data["Pipelines"], dict): + data["Pipelines"] = ", ".join( + f"{n} ({v})" for n, v in data["Pipelines"].items() + ) + markdown_data = get_markdown(data, title=title, exclude=exclude) + if markdown: + if not silent: + print(markdown_data) + return markdown_data + if not silent: + table_data = {k: v for k, v in data.items() if k not in exclude} + msg.table(table_data, title=title) + return raw_data + + +def info_spacy() -> Dict[str, Any]: + """Generate info about the current spaCy intallation. + + RETURNS (dict): The spaCy info. + """ + all_models = {} + for pkg_name in util.get_installed_models(): + package = pkg_name.replace("-", "_") + all_models[package] = util.get_package_version(pkg_name) + return { + "spaCy version": about.__version__, + "Location": str(Path(__file__).parent.parent), + "Platform": platform.platform(), + "Python version": platform.python_version(), + "Pipelines": all_models, + } + + +def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]: + """Generate info about a specific model. + + model (str): Model name of path. + silent (bool): Don't print anything, just return. + RETURNS (dict): The model meta. + """ + msg = Printer(no_print=silent, pretty=not silent) + if util.is_package(model): + model_path = util.get_package_path(model) + else: + model_path = Path(model) + meta_path = model_path / "meta.json" + if not meta_path.is_file(): + msg.fail("Can't find pipeline meta.json", meta_path, exits=1) + meta = srsly.read_json(meta_path) + if model_path.resolve() != model_path: + meta["source"] = str(model_path.resolve()) + else: + meta["source"] = str(model_path) + download_url = info_installed_model_url(model) + if download_url: + meta["download_url"] = download_url + return { + k: v for k, v in meta.items() if k not in ("accuracy", "performance", "speed") + } + + +def info_installed_model_url(model: str) -> Optional[str]: + """Given a pipeline name, get the download URL if available, otherwise + return None. + + This is only available for pipelines installed as modules that have + dist-info available. + """ + try: + dist = importlib_metadata.distribution(model) + text = dist.read_text("direct_url.json") + if isinstance(text, str): + data = json.loads(text) + return data["url"] + except Exception: + pass + return None + + +def info_model_url(model: str) -> Dict[str, Any]: + """Return the download URL for the latest version of a pipeline.""" + version = get_latest_version(model) + + filename = get_model_filename(model, version) + download_url = about.__download_url__ + "/" + filename + release_tpl = "https://github.com/explosion/spacy-models/releases/tag/{m}-{v}" + release_url = release_tpl.format(m=model, v=version) + return {"download_url": download_url, "release_url": release_url} + + +def get_markdown( + data: Dict[str, Any], + title: Optional[str] = None, + exclude: Optional[List[str]] = None, +) -> str: + """Get data in GitHub-flavoured Markdown format for issues etc. + + data (Dict[str, Any]): Label/value pairs. + title (str): Optional title, will be rendered as headline 2. + exclude (List[str]): Names of keys to exclude. + RETURNS (str): The Markdown string. + """ + md = MarkdownRenderer() + if title: + md.add(md.title(2, title)) + items = [] + for key, value in data.items(): + if exclude and key in exclude: + continue + if isinstance(value, str): + try: + existing_path = Path(value).exists() + except Exception: + # invalid Path, like a URL string + existing_path = False + if existing_path: + continue + items.append(f"{md.bold(f'{key}:')} {value}") + md.add(md.list(items)) + return f"\n{md.text}\n" diff --git a/spacy/cli/init_config.py b/spacy/cli/init_config.py new file mode 100644 index 0000000..c708104 --- /dev/null +++ b/spacy/cli/init_config.py @@ -0,0 +1,303 @@ +import re +from enum import Enum +from pathlib import Path +from typing import List, Optional, Tuple + +import srsly +from jinja2 import Template +from thinc.api import Config +from wasabi import Printer, diff_strings + +from .. import util +from ..language import DEFAULT_CONFIG_PRETRAIN_PATH +from ..schemas import RecommendationSchema +from ..util import SimpleFrozenList +from ._util import ( + COMMAND, + Arg, + Opt, + import_code, + init_cli, + show_validation_error, + string_to_list, +) + +ROOT = Path(__file__).parent / "templates" +TEMPLATE_PATH = ROOT / "quickstart_training.jinja" +RECOMMENDATIONS = srsly.read_yaml(ROOT / "quickstart_training_recommendations.yml") + + +class Optimizations(str, Enum): + efficiency = "efficiency" + accuracy = "accuracy" + + +class InitValues: + """ + Default values for initialization. Dedicated class to allow synchronized default values for init_config_cli() and + init_config(), i.e. initialization calls via CLI respectively Python. + """ + + lang = "en" + pipeline = SimpleFrozenList(["tagger", "parser", "ner"]) + optimize = Optimizations.efficiency + gpu = False + pretraining = False + force_overwrite = False + + +@init_cli.command("config") +def init_config_cli( + # fmt: off + output_file: Path = Arg( + ..., + help="File to save the config to or - for stdout (will only output config and no additional logging info)", + allow_dash=True, + ), + lang: str = Opt( + InitValues.lang, "--lang", "-l", help="Two-letter code of the language to use" + ), + pipeline: str = Opt( + ",".join(InitValues.pipeline), + "--pipeline", + "-p", + help="Comma-separated names of trainable pipeline components to include (without 'tok2vec' or 'transformer')", + ), + optimize: Optimizations = Opt( + InitValues.optimize, + "--optimize", + "-o", + help="Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters.", + ), + gpu: bool = Opt( + InitValues.gpu, + "--gpu", + "-G", + help="Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters.", + ), + pretraining: bool = Opt( + InitValues.pretraining, + "--pretraining", + "-pt", + help="Include config for pretraining (with 'spacy pretrain')", + ), + force_overwrite: bool = Opt( + InitValues.force_overwrite, + "--force", + "-F", + help="Force overwriting the output file", + ), + # fmt: on +): + """ + Generate a starter config file for training. Based on your requirements + specified via the CLI arguments, this command generates a config with the + optimal settings for your use case. This includes the choice of architecture, + pretrained weights and related hyperparameters. + + DOCS: https://spacy.io/api/cli#init-config + """ + pipeline = string_to_list(pipeline) + is_stdout = str(output_file) == "-" + if not is_stdout and output_file.exists() and not force_overwrite: + msg = Printer() + msg.fail( + "The provided output file already exists. To force overwriting the config file, set the --force or -F flag.", + exits=1, + ) + config = init_config( + lang=lang, + pipeline=pipeline, + optimize=optimize.value, + gpu=gpu, + pretraining=pretraining, + silent=is_stdout, + ) + save_config(config, output_file, is_stdout=is_stdout) + + +@init_cli.command("fill-config") +def init_fill_config_cli( + # fmt: off + base_path: Path = Arg( + ..., help="Path to base config to fill", exists=True, dir_okay=False + ), + output_file: Path = Arg( + "-", help="Path to output .cfg file (or - for stdout)", allow_dash=True + ), + pretraining: bool = Opt( + False, + "--pretraining", + "-pt", + help="Include config for pretraining (with 'spacy pretrain')", + ), + diff: bool = Opt( + False, "--diff", "-D", help="Print a visual diff highlighting the changes" + ), + code_path: Optional[Path] = Opt( + None, + "--code-path", + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + # fmt: on +): + """ + Fill partial config file with default values. Will add all missing settings + from the default config and will create all objects, check the registered + functions for their default values and update the base config. This command + can be used with a config generated via the training quickstart widget: + https://spacy.io/usage/training#quickstart + + DOCS: https://spacy.io/api/cli#init-fill-config + """ + import_code(code_path) + fill_config(output_file, base_path, pretraining=pretraining, diff=diff) + + +def fill_config( + output_file: Path, + base_path: Path, + *, + pretraining: bool = False, + diff: bool = False, + silent: bool = False, +) -> Tuple[Config, Config]: + is_stdout = str(output_file) == "-" + no_print = is_stdout or silent + msg = Printer(no_print=no_print) + with show_validation_error(hint_fill=False): + config = util.load_config(base_path) + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + # Load a second time with validation to be extra sure that the produced + # config result is a valid config + nlp = util.load_model_from_config(nlp.config) + filled = nlp.config + # If we have sourced components in the base config, those will have been + # replaced with their actual config after loading, so we have to re-add them + sourced = util.get_sourced_components(config) + filled["components"].update(sourced) + if pretraining: + validate_config_for_pretrain(filled, msg) + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + before = config.to_str() + after = filled.to_str() + if before == after: + msg.warn("Nothing to auto-fill: base config is already complete") + else: + msg.good("Auto-filled config with all values") + if diff and not no_print: + if before == after: + msg.warn("No diff to show: nothing was auto-filled") + else: + msg.divider("START CONFIG DIFF") + print("") + print(diff_strings(before, after)) + msg.divider("END CONFIG DIFF") + print("") + save_config(filled, output_file, is_stdout=is_stdout, silent=silent) + return config, filled + + +def init_config( + *, + lang: str = InitValues.lang, + pipeline: List[str] = InitValues.pipeline, + optimize: str = InitValues.optimize, + gpu: bool = InitValues.gpu, + pretraining: bool = InitValues.pretraining, + silent: bool = True, +) -> Config: + msg = Printer(no_print=silent) + with TEMPLATE_PATH.open("r") as f: + template = Template(f.read()) + # Filter out duplicates since tok2vec and transformer are added by template + pipeline = [pipe for pipe in pipeline if pipe not in ("tok2vec", "transformer")] + defaults = RECOMMENDATIONS["__default__"] + reco = RecommendationSchema(**RECOMMENDATIONS.get(lang, defaults)).model_dump() + variables = { + "lang": lang, + "components": pipeline, + "optimize": optimize, + "hardware": "gpu" if gpu else "cpu", + "transformer_data": reco["transformer"], + "word_vectors": reco["word_vectors"], + "has_letters": reco["has_letters"], + } + if variables["transformer_data"] and not has_spacy_transformers(): + msg.warn( + "To generate a more effective transformer-based config (GPU-only), " + "install the spacy-transformers package and re-run this command. " + "The config generated now does not use transformers." + ) + variables["transformer_data"] = None + base_template = template.render(variables).strip() + # Giving up on getting the newlines right in jinja for now + base_template = re.sub(r"\n\n\n+", "\n\n", base_template) + # Access variables declared in templates + template_vars = template.make_module(variables) + use_case = { + "Language": lang, + "Pipeline": ", ".join(pipeline), + "Optimize for": optimize, + "Hardware": variables["hardware"].upper(), + "Transformer": ( + template_vars.transformer.get("name") # type: ignore[attr-defined] + if template_vars.use_transformer # type: ignore[attr-defined] + else None + ), + } + msg.info("Generated config template specific for your use case") + for label, value in use_case.items(): + msg.text(f"- {label}: {value}") + with show_validation_error(hint_fill=False): + config = util.load_config_from_str(base_template) + nlp = util.load_model_from_config(config, auto_fill=True) + config = nlp.config + if pretraining: + validate_config_for_pretrain(config, msg) + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + config = pretrain_config.merge(config) + msg.good("Auto-filled config with all values") + return config + + +def save_config( + config: Config, output_file: Path, is_stdout: bool = False, silent: bool = False +) -> None: + no_print = is_stdout or silent + msg = Printer(no_print=no_print) + if is_stdout: + print(config.to_str()) + else: + if not output_file.parent.exists(): + output_file.parent.mkdir(parents=True) + config.to_disk(output_file, interpolate=False) + msg.good("Saved config", output_file) + msg.text("You can now add your data and train your pipeline:") + variables = ["--paths.train ./train.spacy", "--paths.dev ./dev.spacy"] + if not no_print: + print(f"{COMMAND} train {output_file.parts[-1]} {' '.join(variables)}") + + +def has_spacy_transformers() -> bool: + try: + import spacy_transformers # noqa: F401 + + return True + except ImportError: + return False + + +def validate_config_for_pretrain(config: Config, msg: Printer) -> None: + if "tok2vec" not in config["nlp"]["pipeline"]: + msg.warn( + "No tok2vec component found in the pipeline. If your tok2vec " + "component has a different name, you may need to adjust the " + "tok2vec_model reference in the [pretraining] block. If you don't " + "have a tok2vec component, make sure to add it to your [components] " + "and the pipeline specified in the [nlp] block, so you can pretrain " + "weights for it." + ) diff --git a/spacy/cli/init_pipeline.py b/spacy/cli/init_pipeline.py new file mode 100644 index 0000000..1c0ff52 --- /dev/null +++ b/spacy/cli/init_pipeline.py @@ -0,0 +1,197 @@ +import logging +from pathlib import Path +from typing import Optional + +import srsly +import typer +from wasabi import msg + +from .. import util +from ..language import Language +from ..training.initialize import convert_vectors, init_nlp +from ._util import ( + Arg, + Opt, + import_code, + init_cli, + parse_config_overrides, + setup_gpu, + show_validation_error, +) + + +@init_cli.command("vectors") +def init_vectors_cli( + # fmt: off + lang: str = Arg(..., help="The language of the nlp object to create"), + vectors_loc: Path = Arg(..., help="Vectors file in Word2Vec format", exists=True), + output_dir: Path = Arg(..., help="Pipeline output directory"), + prune: int = Opt( + -1, "--prune", "-p", help="Optional number of vectors to prune to" + ), + truncate: int = Opt( + 0, + "--truncate", + "-t", + help="Optional number of vectors to truncate to when reading in vectors file", + ), + mode: str = Opt("default", "--mode", "-m", help="Vectors mode: default or floret"), + name: Optional[str] = Opt( + None, + "--name", + "-n", + help="Optional name for the word vectors, e.g. en_core_web_lg.vectors", + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + jsonl_loc: Optional[Path] = Opt( + None, + "--lexemes-jsonl", + "-j", + help="Location of JSONL-formatted attributes file", + hidden=True, + ), + attr: str = Opt( + "ORTH", + "--attr", + "-a", + help="Optional token attribute to use for vectors, e.g. LOWER or NORM", + ), + # fmt: on +): + """Convert word vectors for use with spaCy. Will export an nlp object that + you can use in the [initialize] block of your config to initialize + a model with vectors. + """ + if verbose: + util.logger.setLevel(logging.DEBUG) + msg.info(f"Creating blank nlp object for language '{lang}'") + nlp = util.get_lang_class(lang)() + if jsonl_loc is not None: + update_lexemes(nlp, jsonl_loc) + convert_vectors( + nlp, + vectors_loc, + truncate=truncate, + prune=prune, + name=name, + mode=mode, + attr=attr, + ) + msg.good(f"Successfully converted {len(nlp.vocab.vectors)} vectors") + nlp.to_disk(output_dir) + msg.good( + "Saved nlp object with vectors to output directory. You can now use the " + "path to it in your config as the 'vectors' setting in [initialize].", + output_dir.resolve(), + ) + + +def update_lexemes(nlp: Language, jsonl_loc: Path) -> None: + # Mostly used for backwards-compatibility and may be removed in the future + lex_attrs = srsly.read_jsonl(jsonl_loc) + for attrs in lex_attrs: + if "settings" in attrs: + continue + lexeme = nlp.vocab[attrs["orth"]] + lexeme.set_attrs(**attrs) + + +@init_cli.command( + "nlp", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + hidden=True, +) +def init_pipeline_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + output_path: Path = Arg(..., help="Output directory for the prepared data"), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + # fmt: on +): + if verbose: + util.logger.setLevel(logging.DEBUG) + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + setup_gpu(use_gpu) + with show_validation_error(config_path): + config = util.load_config(config_path, overrides=overrides) + with show_validation_error(hint_fill=False): + nlp = init_nlp(config, use_gpu=use_gpu) + nlp.to_disk(output_path) + msg.good(f"Saved initialized pipeline to {output_path}") + + +@init_cli.command( + "labels", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def init_labels_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + output_path: Path = Arg(..., help="Output directory for the labels"), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + # fmt: on +): + """Generate JSON files for the labels in the data. This helps speed up the + training process, since spaCy won't have to preprocess the data to + extract the labels.""" + if verbose: + util.logger.setLevel(logging.DEBUG) + if not output_path.exists(): + output_path.mkdir(parents=True) + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + setup_gpu(use_gpu) + with show_validation_error(config_path): + config = util.load_config(config_path, overrides=overrides) + with show_validation_error(hint_fill=False): + nlp = init_nlp(config, use_gpu=use_gpu) + _init_labels(nlp, output_path) + + +def _init_labels(nlp, output_path): + for name, component in nlp.pipeline: + if getattr(component, "label_data", None) is not None: + output_file = output_path / f"{name}.json" + srsly.write_json(output_file, component.label_data) + msg.good(f"Saving label data for component '{name}' to {output_file}") + else: + msg.info(f"No label data found for component '{name}'") diff --git a/spacy/cli/package.py b/spacy/cli/package.py new file mode 100644 index 0000000..9291aae --- /dev/null +++ b/spacy/cli/package.py @@ -0,0 +1,654 @@ +import os +import re +import shutil +import subprocess +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +import srsly +from catalogue import RegistryError +from thinc.api import Config +from wasabi import MarkdownRenderer, Printer, get_raw_input + +from .. import about, util +from ..compat import importlib_metadata +from ..schemas import ModelMetaSchema, validate +from ._util import SDIST_SUFFIX, WHEEL_SUFFIX, Arg, Opt, app, string_to_list + + +@app.command("package") +def package_cli( + # fmt: off + input_dir: Path = Arg( + ..., help="Directory with pipeline data", exists=True, file_okay=False + ), + output_dir: Path = Arg( + ..., help="Output parent directory", exists=True, file_okay=False + ), + code_paths: str = Opt( + "", + "--code", + "-c", + help="Comma-separated paths to Python file with additional code (registered functions) to be included in the package", + ), + meta_path: Optional[Path] = Opt( + None, + "--meta-path", + "--meta", + "-m", + help="Path to meta.json", + exists=True, + dir_okay=False, + ), + create_meta: bool = Opt( + False, "--create-meta", "-C", help="Create meta.json, even if one exists" + ), + name: Optional[str] = Opt( + None, "--name", "-n", help="Package name to override meta" + ), + version: Optional[str] = Opt( + None, "--version", "-v", help="Package version to override meta" + ), + build: str = Opt( + "sdist", + "--build", + "-b", + help="Comma-separated formats to build: sdist and/or wheel, or none.", + ), + force: bool = Opt( + False, + "--force", + "-f", + "-F", + help="Force overwriting existing data in output directory", + ), + require_parent: bool = Opt( + True, + "--require-parent/--no-require-parent", + "-R", + "-R", + help="Include the parent package (e.g. spacy) in the requirements", + ), + # fmt: on +): + """ + Generate an installable Python package for a pipeline. Includes binary data, + meta and required installation files. A new directory will be created in the + specified output directory, and the data will be copied over. If + --create-meta is set and a meta.json already exists in the output directory, + the existing values will be used as the defaults in the command-line prompt. + After packaging, "python -m build --sdist" is run in the package directory, + which will create a .tar.gz archive that can be installed via "pip install". + + If additional code files are provided (e.g. Python files containing custom + registered functions like pipeline components), they are copied into the + package and imported in the __init__.py. + + DOCS: https://spacy.io/api/cli#package + """ + create_sdist, create_wheel = get_build_formats(string_to_list(build)) + code_paths = [Path(p.strip()) for p in string_to_list(code_paths)] + package( + input_dir, + output_dir, + meta_path=meta_path, + code_paths=code_paths, + name=name, + version=version, + create_meta=create_meta, + create_sdist=create_sdist, + create_wheel=create_wheel, + force=force, + require_parent=require_parent, + silent=False, + ) + + +def package( + input_dir: Path, + output_dir: Path, + meta_path: Optional[Path] = None, + code_paths: List[Path] = [], + name: Optional[str] = None, + version: Optional[str] = None, + create_meta: bool = False, + create_sdist: bool = True, + create_wheel: bool = False, + require_parent: bool = False, + force: bool = False, + silent: bool = True, +) -> None: + msg = Printer(no_print=silent, pretty=not silent) + input_path = util.ensure_path(input_dir) + output_path = util.ensure_path(output_dir) + meta_path = util.ensure_path(meta_path) + if create_wheel and not has_wheel() and not has_build(): + err = ( + "Generating wheels requires 'build' or 'wheel' (deprecated) to be installed" + ) + msg.fail(err, "pip install build", exits=1) + if not has_build(): + msg.warn( + "Generating packages without the 'build' package is deprecated and " + "will not be supported in the future. To install 'build': pip " + "install build" + ) + if not input_path or not input_path.exists(): + msg.fail("Can't locate pipeline data", input_path, exits=1) + if not output_path or not output_path.exists(): + msg.fail("Output directory not found", output_path, exits=1) + if create_sdist or create_wheel: + opts = ["sdist" if create_sdist else "", "wheel" if create_wheel else ""] + msg.info(f"Building package artifacts: {', '.join(opt for opt in opts if opt)}") + for code_path in code_paths: + if not code_path.exists(): + msg.fail("Can't find code file", code_path, exits=1) + # Import the code here so it's available when model is loaded (via + # get_meta helper). Also verifies that everything works + util.import_file(code_path.stem, code_path) + if code_paths: + msg.good(f"Including {len(code_paths)} Python module(s) with custom code") + if meta_path and not meta_path.exists(): + msg.fail("Can't find pipeline meta.json", meta_path, exits=1) + meta_path = meta_path or input_dir / "meta.json" + if not meta_path.exists() or not meta_path.is_file(): + msg.fail("Can't load pipeline meta.json", meta_path, exits=1) + meta = srsly.read_json(meta_path) + meta = get_meta(input_dir, meta, require_parent=require_parent) + if meta["requirements"]: + msg.good( + f"Including {len(meta['requirements'])} package requirement(s) from " + f"meta and config", + ", ".join(meta["requirements"]), + ) + if name is not None: + if not name.isidentifier(): + msg.fail( + f"Model name ('{name}') is not a valid module name. " + "This is required so it can be imported as a module.", + "We recommend names that use ASCII A-Z, a-z, _ (underscore), " + "and 0-9. " + "For specific details see: https://docs.python.org/3/reference/lexical_analysis.html#identifiers", + exits=1, + ) + if not _is_permitted_package_name(name): + msg.fail( + f"Model name ('{name}') is not a permitted package name. " + "This is required to correctly load the model with spacy.load.", + "We recommend names that use ASCII A-Z, a-z, _ (underscore), " + "and 0-9. " + "For specific details see: https://www.python.org/dev/peps/pep-0426/#name", + exits=1, + ) + meta["name"] = name + if version is not None: + meta["version"] = version + if not create_meta: # only print if user doesn't want to overwrite + msg.good("Loaded meta.json from file", meta_path) + else: + meta = generate_meta(meta, msg) + errors = validate(ModelMetaSchema, meta) + if errors: + msg.fail("Invalid pipeline meta.json") + print("\n".join(errors)) + sys.exit(1) + model_name = meta["name"] + if not model_name.startswith(meta["lang"] + "_"): + model_name = f"{meta['lang']}_{model_name}" + model_name_v = model_name + "-" + meta["version"] + main_path = output_dir / model_name_v + package_path = main_path / model_name + if package_path.exists(): + if force: + shutil.rmtree(str(package_path)) + else: + msg.fail( + "Package directory already exists", + "Please delete the directory and try again, or use the " + "`--force` flag to overwrite existing directories.", + exits=1, + ) + Path.mkdir(package_path, parents=True) + shutil.copytree(str(input_dir), str(package_path / model_name_v)) + for file_name in FILENAMES_DOCS: + file_path = package_path / model_name_v / file_name + if file_path.exists(): + shutil.copy(str(file_path), str(main_path)) + readme_path = main_path / "README.md" + if not readme_path.exists(): + readme = generate_readme(meta) + create_file(readme_path, readme) + create_file(package_path / model_name_v / "README.md", readme) + msg.good("Generated README.md from meta.json") + else: + msg.info("Using existing README.md from pipeline directory") + imports = [] + for code_path in code_paths: + imports.append(code_path.stem) + shutil.copy(str(code_path), str(package_path)) + create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2)) + + create_file(main_path / "setup.py", TEMPLATE_SETUP) + create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST) + init_py = TEMPLATE_INIT.format( + imports="\n".join(f"from . import {m}" for m in imports) + ) + create_file(package_path / "__init__.py", init_py) + msg.good(f"Successfully created package directory '{model_name_v}'", main_path) + if create_sdist: + with util.working_dir(main_path): + # run directly, since util.run_command is not designed to continue + # after a command fails + ret = subprocess.run( + [sys.executable, "-m", "build", ".", "--sdist"], + env=os.environ.copy(), + ) + if ret.returncode != 0: + msg.warn( + "Creating sdist with 'python -m build' failed. Falling " + "back to deprecated use of 'python setup.py sdist'" + ) + util.run_command([sys.executable, "setup.py", "sdist"], capture=False) + zip_file = main_path / "dist" / f"{model_name_v}{SDIST_SUFFIX}" + msg.good(f"Successfully created zipped Python package", zip_file) + if create_wheel: + with util.working_dir(main_path): + # run directly, since util.run_command is not designed to continue + # after a command fails + ret = subprocess.run( + [sys.executable, "-m", "build", ".", "--wheel"], + env=os.environ.copy(), + ) + if ret.returncode != 0: + msg.warn( + "Creating wheel with 'python -m build' failed. Falling " + "back to deprecated use of 'wheel' with " + "'python setup.py bdist_wheel'" + ) + util.run_command( + [sys.executable, "setup.py", "bdist_wheel"], capture=False + ) + wheel_name_squashed = re.sub("_+", "_", model_name_v) + wheel = main_path / "dist" / f"{wheel_name_squashed}{WHEEL_SUFFIX}" + msg.good(f"Successfully created binary wheel", wheel) + if "__" in model_name: + msg.warn( + f"Model name ('{model_name}') contains a run of underscores. " + "Runs of underscores are not significant in installed package names.", + ) + + +def has_wheel() -> bool: + try: + import wheel # noqa: F401 + + return True + except ImportError: + return False + + +def has_build() -> bool: + # it's very likely that there is a local directory named build/ (especially + # in an editable install), so an import check is not sufficient; instead + # check that there is a package version + try: + importlib_metadata.version("build") + return True + except importlib_metadata.PackageNotFoundError: # type: ignore[attr-defined] + return False + + +def get_third_party_dependencies( + config: Config, exclude: List[str] = util.SimpleFrozenList() +) -> List[str]: + """If the config includes references to registered functions that are + provided by third-party packages (spacy-transformers, other libraries), we + want to include them in meta["requirements"] so that the package specifies + them as dependencies and the user won't have to do it manually. + + We do this by: + - traversing the config to check for registered function (@ keys) + - looking up the functions and getting their module + - looking up the module version and generating an appropriate version range + + config (Config): The pipeline config. + exclude (list): List of packages to exclude (e.g. that already exist in meta). + RETURNS (list): The versioned requirements. + """ + own_packages = ("spacy", "spacy-legacy", "spacy-nightly", "thinc", "srsly") + distributions = util.packages_distributions() + funcs = defaultdict(set) + # We only want to look at runtime-relevant sections, not [training] or [initialize] + for section in ("nlp", "components"): + for path, value in util.walk_dict(config[section]): + if path[-1].startswith("@"): # collect all function references by registry + funcs[path[-1][1:]].add(value) + for component in config.get("components", {}).values(): + if "factory" in component: + funcs["factories"].add(component["factory"]) + modules = set() + lang = config["nlp"]["lang"] + for reg_name, func_names in funcs.items(): + for func_name in func_names: + # Try the lang-specific version and fall back + try: + func_info = util.registry.find(reg_name, lang + "." + func_name) + except RegistryError: + try: + func_info = util.registry.find(reg_name, func_name) + except RegistryError as regerr: + # lang-specific version being absent is not actually an issue + raise regerr from None + module_name = func_info.get("module") # type: ignore[attr-defined] + if module_name: # the code is part of a module, not a --code file + modules.add(func_info["module"].split(".")[0]) # type: ignore[union-attr] + dependencies = [] + for module_name in modules: + if module_name == about.__title__: + continue + if module_name in distributions: + dist = distributions.get(module_name) + if dist: + pkg = dist[0] + if pkg in own_packages or pkg in exclude: + continue + version = util.get_package_version(pkg) + version_range = util.get_minor_version_range(version) # type: ignore[arg-type] + dependencies.append(f"{pkg}{version_range}") + return dependencies + + +def get_build_formats(formats: List[str]) -> Tuple[bool, bool]: + supported = ["sdist", "wheel", "none"] + for form in formats: + if form not in supported: + msg = Printer() + err = f"Unknown build format: {form}. Supported: {', '.join(supported)}" + msg.fail(err, exits=1) + if not formats or "none" in formats: + return (False, False) + return ("sdist" in formats, "wheel" in formats) + + +def create_file(file_path: Path, contents: str) -> None: + file_path.touch() + file_path.open("w", encoding="utf-8").write(contents) + + +def get_meta( + model_path: Union[str, Path], + existing_meta: Dict[str, Any], + require_parent: bool = False, +) -> Dict[str, Any]: + meta: Dict[str, Any] = { + "lang": "en", + "name": "pipeline", + "version": "0.0.0", + "description": "", + "author": "", + "email": "", + "url": "", + "license": "MIT", + } + nlp = util.load_model_from_path(Path(model_path)) + meta.update(nlp.meta) + meta["spacy_version"] = util.get_minor_version_range(about.__version__) + meta.update(existing_meta) + meta["vectors"] = { + "width": nlp.vocab.vectors_length, + "vectors": len(nlp.vocab.vectors), + "keys": nlp.vocab.vectors.n_keys, + "name": nlp.vocab.vectors.name, + } + if about.__title__ != "spacy": + meta["parent_package"] = about.__title__ + meta.setdefault("requirements", []) + # Update the requirements with all third-party packages in the config + existing_reqs = [util.split_requirement(req)[0] for req in meta["requirements"]] + reqs = get_third_party_dependencies(nlp.config, exclude=existing_reqs) + meta["requirements"].extend(reqs) + if require_parent and about.__title__ not in meta["requirements"]: + meta["requirements"].append(about.__title__ + meta["spacy_version"]) + return meta + + +def generate_meta(existing_meta: Dict[str, Any], msg: Printer) -> Dict[str, Any]: + meta = existing_meta or {} + settings = [ + ("lang", "Pipeline language", meta.get("lang", "en")), + ("name", "Pipeline name", meta.get("name", "pipeline")), + ("version", "Package version", meta.get("version", "0.0.0")), + ("description", "Package description", meta.get("description", None)), + ("author", "Author", meta.get("author", None)), + ("email", "Author email", meta.get("email", None)), + ("url", "Author website", meta.get("url", None)), + ("license", "License", meta.get("license", "MIT")), + ] + msg.divider("Generating meta.json") + msg.text( + "Enter the package settings for your pipeline. The following information " + "will be read from your pipeline data: pipeline, vectors." + ) + for setting, desc, default in settings: + response = get_raw_input(desc, default) + meta[setting] = default if response == "" and default else response + return meta + + +def generate_readme(meta: Dict[str, Any]) -> str: + """ + Generate a Markdown-formatted README text from a model meta.json. Used + within the GitHub release notes and as content for README.md file added + to model packages. + """ + md = MarkdownRenderer() + lang = meta["lang"] + name = f"{lang}_{meta['name']}" + version = meta["version"] + pipeline = ", ".join([md.code(p) for p in meta.get("pipeline", [])]) + components = ", ".join([md.code(p) for p in meta.get("components", [])]) + vecs = meta.get("vectors", {}) + vectors = f"{vecs.get('keys', 0)} keys, {vecs.get('vectors', 0)} unique vectors ({vecs.get('width', 0)} dimensions)" + author = meta.get("author") or "n/a" + notes = meta.get("notes", "") + license_name = meta.get("license") + sources = _format_sources(meta.get("sources")) + description = meta.get("description") + label_scheme = _format_label_scheme(cast(Dict[str, Any], meta.get("labels"))) + accuracy = _format_accuracy(cast(Dict[str, Any], meta.get("performance"))) + table_data = [ + (md.bold("Name"), md.code(name)), + (md.bold("Version"), md.code(version)), + (md.bold("spaCy"), md.code(meta["spacy_version"])), + (md.bold("Default Pipeline"), pipeline), + (md.bold("Components"), components), + (md.bold("Vectors"), vectors), + (md.bold("Sources"), sources or "n/a"), + (md.bold("License"), md.code(license_name) if license_name else "n/a"), + (md.bold("Author"), md.link(author, meta["url"]) if "url" in meta else author), + ] + # Put together Markdown body + if description: + md.add(description) + md.add(md.table(table_data, ["Feature", "Description"])) + if label_scheme: + md.add(md.title(3, "Label Scheme")) + md.add(label_scheme) + if accuracy: + md.add(md.title(3, "Accuracy")) + md.add(accuracy) + if notes: + md.add(notes) + return md.text + + +def _format_sources(data: Any) -> str: + if not data or not isinstance(data, list): + return "n/a" + sources = [] + for source in data: + if not isinstance(source, dict): + source = {"name": source} + name = source.get("name") + if not name: + continue + url = source.get("url") + author = source.get("author") + result = name if not url else "[{}]({})".format(name, url) + if author: + result += " ({})".format(author) + sources.append(result) + return "
".join(sources) + + +def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> str: + if not data: + return "" + md = MarkdownRenderer() + scalars = [(k, v) for k, v in data.items() if isinstance(v, (int, float))] + scores = [ + (md.code(acc.upper()), f"{score * 100:.2f}") + for acc, score in scalars + if acc not in exclude + ] + md.add(md.table(scores, ["Type", "Score"])) + return md.text + + +def _format_label_scheme(data: Dict[str, Any]) -> str: + if not data: + return "" + md = MarkdownRenderer() + n_labels = 0 + n_pipes = 0 + label_data = [] + for pipe, labels in data.items(): + if not labels: + continue + col1 = md.bold(md.code(pipe)) + col2 = ", ".join([md.code(str(label).replace("|", "\\|")) for label in labels]) # noqa: W605 + label_data.append((col1, col2)) + n_labels += len(labels) + n_pipes += 1 + if not label_data: + return "" + label_info = f"View label scheme ({n_labels} labels for {n_pipes} components)" + md.add("
") + md.add(f"{label_info}") + md.add(md.table(label_data, ["Component", "Labels"])) + md.add("
") + return md.text + + +def _is_permitted_package_name(package_name: str) -> bool: + # regex from: https://www.python.org/dev/peps/pep-0426/#name + permitted_match = re.search( + r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", package_name, re.IGNORECASE + ) + return permitted_match is not None + + +TEMPLATE_SETUP = """ +#!/usr/bin/env python +import io +import json +from os import path, walk +from shutil import copy +from setuptools import setup + + +def load_meta(fp): + with io.open(fp, encoding='utf8') as f: + return json.load(f) + + +def load_readme(fp): + if path.exists(fp): + with io.open(fp, encoding='utf8') as f: + return f.read() + return "" + + +def list_files(data_dir): + output = [] + for root, _, filenames in walk(data_dir): + for filename in filenames: + if not filename.startswith('.'): + output.append(path.join(root, filename)) + output = [path.relpath(p, path.dirname(data_dir)) for p in output] + output.append('meta.json') + return output + + +def list_requirements(meta): + # Up to version 3.7, we included the parent package + # in requirements by default. This behaviour is removed + # in 3.8, with a setting to include the parent package in + # the requirements list in the meta if desired. + requirements = [] + if 'setup_requires' in meta: + requirements += meta['setup_requires'] + if 'requirements' in meta: + requirements += meta['requirements'] + return requirements + + +def setup_package(): + root = path.abspath(path.dirname(__file__)) + meta_path = path.join(root, 'meta.json') + meta = load_meta(meta_path) + readme_path = path.join(root, 'README.md') + readme = load_readme(readme_path) + model_name = str(meta['lang'] + '_' + meta['name']) + model_dir = path.join(model_name, model_name + '-' + meta['version']) + + copy(meta_path, path.join(model_name)) + copy(meta_path, model_dir) + + setup( + name=model_name, + description=meta.get('description'), + long_description=readme, + author=meta.get('author'), + author_email=meta.get('email'), + url=meta.get('url'), + version=meta['version'], + license=meta.get('license'), + packages=[model_name], + package_data={model_name: list_files(model_dir)}, + install_requires=list_requirements(meta), + zip_safe=False, + entry_points={'spacy_models': ['{m} = {m}'.format(m=model_name)]} + ) + + +if __name__ == '__main__': + setup_package() +""".lstrip() + + +TEMPLATE_MANIFEST = """ +include meta.json +include LICENSE +include LICENSES_SOURCES +include README.md +""".strip() + + +TEMPLATE_INIT = """ +from pathlib import Path +from spacy.util import load_model_from_init_py, get_model_meta + +{imports} + +__version__ = get_model_meta(Path(__file__).parent)['version'] + + +def load(**overrides): + return load_model_from_init_py(__file__, **overrides) +""".lstrip() + + +FILENAMES_DOCS = ["LICENSE", "LICENSES_SOURCES", "README.md"] diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py new file mode 100644 index 0000000..daea861 --- /dev/null +++ b/spacy/cli/pretrain.py @@ -0,0 +1,144 @@ +import re +from pathlib import Path +from typing import Optional + +import typer +from wasabi import msg + +from ..training.pretrain import pretrain +from ..util import load_config +from ._util import ( + Arg, + Opt, + app, + import_code, + parse_config_overrides, + setup_gpu, + show_validation_error, +) + + +@app.command( + "pretrain", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def pretrain_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, dir_okay=False, allow_dash=True + ), + output_dir: Path = Arg(..., help="Directory to write weights to on each epoch"), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + resume_path: Optional[Path] = Opt( + None, + "--resume-path", + "-r", + help="Path to pretrained weights from which to resume pretraining", + ), + epoch_resume: Optional[int] = Opt( + None, + "--epoch-resume", + "-er", + help="The epoch to resume counting from when using --resume-path. Prevents unintended overwriting of existing weight files.", + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + skip_last: bool = Opt( + False, "--skip-last", "-L", help="Skip saving model-last.bin" + ), + # fmt: on +): + """ + Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components, + using an approximate language-modelling objective. Two objective types + are available, vector-based and character-based. + + In the vector-based objective, we load word vectors that have been trained + using a word2vec-style distributional similarity algorithm, and train a + component like a CNN, BiLSTM, etc to predict vectors which match the + pretrained ones. The weights are saved to a directory after each epoch. You + can then pass a path to one of these pretrained weights files to the + 'spacy train' command. + + This technique may be especially helpful if you have little labelled data. + However, it's still quite experimental, so your mileage may vary. + + To load the weights back in during 'spacy train', you need to ensure + all settings are the same between pretraining and training. Ideally, + this is done by using the same config file for both commands. + + DOCS: https://spacy.io/api/cli#pretrain + """ + config_overrides = parse_config_overrides(ctx.args) + import_code(code_path) + verify_cli_args(config_path, output_dir, resume_path, epoch_resume) + setup_gpu(use_gpu) + msg.info(f"Loading config from: {config_path}") + + with show_validation_error(config_path): + raw_config = load_config( + config_path, overrides=config_overrides, interpolate=False + ) + config = raw_config.interpolate() + if not config.get("pretraining"): + # TODO: What's the solution here? How do we handle optional blocks? + msg.fail("The [pretraining] block in your config is empty", exits=1) + if not output_dir.exists(): + output_dir.mkdir(parents=True) + msg.good(f"Created output directory: {output_dir}") + # Save non-interpolated config + raw_config.to_disk(output_dir / "config.cfg") + msg.good("Saved config file in the output directory") + + pretrain( + config, + output_dir, + resume_path=resume_path, + epoch_resume=epoch_resume, + use_gpu=use_gpu, + silent=False, + skip_last=skip_last, + ) + msg.good("Successfully finished pretrain") + + +def verify_cli_args(config_path, output_dir, resume_path, epoch_resume): + if not config_path or (str(config_path) != "-" and not config_path.exists()): + msg.fail("Config file not found", config_path, exits=1) + if output_dir.exists() and [p for p in output_dir.iterdir()]: + if resume_path: + msg.warn( + "Output directory is not empty.", + "If you're resuming a run in this directory, the old weights " + "for the consecutive epochs will be overwritten with the new ones.", + ) + else: + msg.warn( + "Output directory is not empty. ", + "It is better to use an empty directory or refer to a new output path, " + "then the new directory will be created for you.", + ) + if resume_path is not None: + if resume_path.is_dir(): + # This is necessary because Windows gives a Permission Denied when we + # try to open the directory later, which is confusing. See #7878 + msg.fail( + "--resume-path should be a weights file, but {resume_path} is a directory.", + exits=True, + ) + model_name = re.search(r"model\d+\.bin", str(resume_path)) + if not model_name and not epoch_resume: + msg.fail( + "You have to use the --epoch-resume setting when using a renamed weight file for --resume-path", + exits=True, + ) + elif not model_name and epoch_resume < 0: + msg.fail( + f"The argument --epoch-resume has to be greater or equal to 0. {epoch_resume} is invalid", + exits=True, + ) diff --git a/spacy/cli/profile.py b/spacy/cli/profile.py new file mode 100644 index 0000000..03f7127 --- /dev/null +++ b/spacy/cli/profile.py @@ -0,0 +1,99 @@ +import cProfile +import itertools +import pstats +import sys +from pathlib import Path +from typing import Iterator, Optional, Sequence, Union + +import srsly +import tqdm +import typer +from wasabi import Printer, msg + +from ..language import Language +from ..util import load_model +from ._util import NAME, Arg, Opt, app, debug_cli + + +@debug_cli.command("profile") +@app.command("profile", hidden=True) +def profile_cli( + # fmt: off + ctx: typer.Context, # This is only used to read current calling context + model: str = Arg(..., help="Trained pipeline to load"), + inputs: Optional[Path] = Arg( + None, + help="Location of input file. '-' for stdin.", + exists=True, + allow_dash=True, + ), + n_texts: int = Opt( + 10000, "--n-texts", "-n", help="Maximum number of texts to use if available" + ), + # fmt: on +): + """ + Profile which functions take the most time in a spaCy pipeline. + Input should be formatted as one JSON object per line with a key "text". + It can either be provided as a JSONL file, or be read from sys.sytdin. + If no input file is specified, the IMDB dataset is loaded via Thinc. + + DOCS: https://spacy.io/api/cli#debug-profile + """ + if ctx.parent.command.name == NAME: # type: ignore[union-attr] # called as top-level command + msg.warn( + "The profile command is now available via the 'debug profile' " + "subcommand. You can run python -m spacy debug --help for an " + "overview of the other available debugging commands." + ) + profile(model, inputs=inputs, n_texts=n_texts) + + +def profile(model: str, inputs: Optional[Path] = None, n_texts: int = 10000) -> None: + if inputs is not None: + texts = _read_inputs(inputs, msg) + texts = list(itertools.islice(texts, n_texts)) + if inputs is None: + try: + import ml_datasets + except ImportError: + msg.fail( + "This command, when run without an input file, " + "requires the ml_datasets library to be installed: " + "pip install ml_datasets", + exits=1, + ) + + with msg.loading("Loading IMDB dataset via ml_datasets..."): + imdb_train, _ = ml_datasets.imdb(train_limit=n_texts, dev_limit=0) + texts, _ = zip(*imdb_train) + msg.info(f"Loaded IMDB dataset and using {n_texts} examples") + with msg.loading(f"Loading pipeline '{model}'..."): + nlp = load_model(model) + msg.good(f"Loaded pipeline '{model}'") + cProfile.runctx("parse_texts(nlp, texts)", globals(), locals(), "Profile.prof") + s = pstats.Stats("Profile.prof") + msg.divider("Profile stats") + s.strip_dirs().sort_stats("time").print_stats() + + +def parse_texts(nlp: Language, texts: Sequence[str]) -> None: + for doc in nlp.pipe(tqdm.tqdm(texts, disable=None), batch_size=16): + pass + + +def _read_inputs(loc: Union[Path, str], msg: Printer) -> Iterator[str]: + if loc == "-": + msg.info("Reading input from sys.stdin") + file_ = sys.stdin + file_ = (line.encode("utf8") for line in file_) + else: + input_path = Path(loc) + if not input_path.exists() or not input_path.is_file(): + msg.fail("Not a valid input data file", loc, exits=1) + msg.info(f"Using data from {input_path.parts[-1]}") + file_ = input_path.open() # type: ignore[assignment] + for line in file_: + data = srsly.json_loads(line) + text = data["text"] + yield text diff --git a/spacy/cli/project/__init__.py b/spacy/cli/project/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/cli/project/assets.py b/spacy/cli/project/assets.py new file mode 100644 index 0000000..591d195 --- /dev/null +++ b/spacy/cli/project/assets.py @@ -0,0 +1 @@ +from weasel.cli.assets import * diff --git a/spacy/cli/project/clone.py b/spacy/cli/project/clone.py new file mode 100644 index 0000000..11d2511 --- /dev/null +++ b/spacy/cli/project/clone.py @@ -0,0 +1 @@ +from weasel.cli.clone import * diff --git a/spacy/cli/project/document.py b/spacy/cli/project/document.py new file mode 100644 index 0000000..1952524 --- /dev/null +++ b/spacy/cli/project/document.py @@ -0,0 +1 @@ +from weasel.cli.document import * diff --git a/spacy/cli/project/dvc.py b/spacy/cli/project/dvc.py new file mode 100644 index 0000000..aa1ae7d --- /dev/null +++ b/spacy/cli/project/dvc.py @@ -0,0 +1 @@ +from weasel.cli.dvc import * diff --git a/spacy/cli/project/pull.py b/spacy/cli/project/pull.py new file mode 100644 index 0000000..5e60327 --- /dev/null +++ b/spacy/cli/project/pull.py @@ -0,0 +1 @@ +from weasel.cli.pull import * diff --git a/spacy/cli/project/push.py b/spacy/cli/project/push.py new file mode 100644 index 0000000..3a8e886 --- /dev/null +++ b/spacy/cli/project/push.py @@ -0,0 +1 @@ +from weasel.cli.push import * diff --git a/spacy/cli/project/remote_storage.py b/spacy/cli/project/remote_storage.py new file mode 100644 index 0000000..2940915 --- /dev/null +++ b/spacy/cli/project/remote_storage.py @@ -0,0 +1 @@ +from weasel.cli.remote_storage import * diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py new file mode 100644 index 0000000..cc6a5ac --- /dev/null +++ b/spacy/cli/project/run.py @@ -0,0 +1 @@ +from weasel.cli.run import * diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja new file mode 100644 index 0000000..2817147 --- /dev/null +++ b/spacy/cli/templates/quickstart_training.jinja @@ -0,0 +1,651 @@ +{# This is a template for training configs used for the quickstart widget in +the docs and the init config command. It encodes various best practices and +can help generate the best possible configuration, given a user's requirements. #} +{%- set use_transformer = hardware != "cpu" and transformer_data -%} +{%- set transformer = transformer_data[optimize] if use_transformer else {} -%} +{%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "span_finder", "spancat", "spancat_singlelabel", "trainable_lemmatizer"] -%} +[paths] +train = null +dev = null +{% if use_transformer or optimize == "efficiency" or not word_vectors -%} +vectors = null +{% else -%} +vectors = "{{ word_vectors }}" +{% endif -%} + +[system] +{% if use_transformer -%} +gpu_allocator = "pytorch" +{% else -%} +gpu_allocator = null +{% endif %} + +[nlp] +lang = "{{ lang }}" +{%- set has_textcat = ("textcat" in components or "textcat_multilabel" in components) -%} +{%- set with_accuracy = optimize == "accuracy" -%} +{# The BOW textcat doesn't need a source of features, so it can omit the +tok2vec/transformer. #} +{%- set with_accuracy_or_transformer = (use_transformer or with_accuracy) -%} +{%- set textcat_needs_features = has_textcat and with_accuracy_or_transformer -%} +{%- if ("tagger" in components or "morphologizer" in components or "parser" in components or "ner" in components or "span_finder" in components or "spancat" in components or "spancat_singlelabel" in components or "trainable_lemmatizer" in components or "entity_linker" in components or textcat_needs_features) -%} +{%- set full_pipeline = ["transformer" if use_transformer else "tok2vec"] + components -%} +{%- else -%} +{%- set full_pipeline = components -%} +{%- endif %} +pipeline = {{ full_pipeline|pprint()|replace("'", '"')|safe }} +batch_size = {{ 128 if hardware == "gpu" else 1000 }} + +[components] + +{# TRANSFORMER PIPELINE #} +{%- if use_transformer -%} +[components.transformer] +factory = "transformer" + +[components.transformer.model] +@architectures = "spacy-transformers.TransformerModel.v3" +name = "{{ transformer["name"] }}" +tokenizer_config = {"use_fast": true} + +[components.transformer.model.get_spans] +@span_getters = "spacy-transformers.strided_spans.v1" +window = 128 +stride = 96 + +{% if "morphologizer" in components %} +[components.morphologizer] +factory = "morphologizer" + +[components.morphologizer.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.morphologizer.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.morphologizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{%- endif %} + +{% if "tagger" in components %} +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.tagger.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.tagger.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{%- endif %} + +{% if "parser" in components -%} +[components.parser] +factory = "parser" + +[components.parser.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 128 +maxout_pieces = 3 +use_upper = false +nO = null + +[components.parser.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.parser.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{%- endif %} + +{% if "ner" in components -%} +[components.ner] +factory = "ner" + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = false +nO = null + +[components.ner.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{% endif -%} + +{% if "span_finder" in components -%} +[components.span_finder] +factory = "span_finder" +max_length = 25 +min_length = null +scorer = {"@scorers":"spacy.span_finder_scorer.v1"} +spans_key = "sc" +threshold = 0.5 + +[components.span_finder.model] +@architectures = "spacy.SpanFinder.v1" + +[components.span_finder.model.scorer] +@layers = "spacy.LinearLogistic.v1" +nO = 2 + +[components.span_finder.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.span_finder.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{% endif -%} + +{% if "spancat" in components -%} +[components.spancat] +factory = "spancat" +max_positive = null +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" +threshold = 0.5 + +[components.spancat.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat.model.scorer] +@layers = "spacy.LinearLogistic.v1" +nO = null +nI = null + +[components.spancat.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.spancat.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.spancat.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif -%} + +{% if "spancat_singlelabel" in components %} +[components.spancat_singlelabel] +factory = "spancat_singlelabel" +negative_weight = 1.0 +allow_overlap = true +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" + +[components.spancat_singlelabel.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat_singlelabel.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat_singlelabel.model.scorer] +@layers = "Softmax.v2" + +[components.spancat_singlelabel.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.spancat_singlelabel.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.spancat_singlelabel.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif %} + +{% if "trainable_lemmatizer" in components -%} +[components.trainable_lemmatizer] +factory = "trainable_lemmatizer" +backoff = "orth" +min_tree_freq = 3 +overwrite = false +scorer = {"@scorers":"spacy.lemmatizer_scorer.v1"} +top_k = 1 + +[components.trainable_lemmatizer.model] +@architectures = "spacy.Tagger.v2" +nO = null +normalize = false + +[components.trainable_lemmatizer.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.trainable_lemmatizer.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{% endif -%} + +{% if "entity_linker" in components -%} +[components.entity_linker] +factory = "entity_linker" +get_candidates = {"@misc":"spacy.CandidateGenerator.v1"} +incl_context = true +incl_prior = true + +[components.entity_linker.model] +@architectures = "spacy.EntityLinker.v2" +nO = null + +[components.entity_linker.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.entity_linker.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{% endif -%} + +{% if "textcat" in components %} +[components.textcat] +factory = "textcat" + +{% if optimize == "accuracy" %} +[components.textcat.model] +@architectures = "spacy.TextCatEnsemble.v2" +nO = null + +[components.textcat.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.textcat.model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false + +{% else -%} +[components.textcat.model] +@architectures = "spacy.TextCatCNN.v2" +exclusive_classes = true +nO = null + +[components.textcat.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{%- endif %} +{%- endif %} + +{% if "textcat_multilabel" in components %} +[components.textcat_multilabel] +factory = "textcat_multilabel" + +{% if optimize == "accuracy" %} +[components.textcat_multilabel.model] +@architectures = "spacy.TextCatEnsemble.v2" +nO = null + +[components.textcat_multilabel.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat_multilabel.model.tok2vec.pooling] +@layers = "reduce_mean.v1" + +[components.textcat_multilabel.model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = false +length = 262144 +ngram_size = 1 +no_output_layer = false + +{% else -%} +[components.textcat_multilabel.model] +@architectures = "spacy.TextCatCNN.v2" +exclusive_classes = false +nO = null + +[components.textcat_multilabel.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[components.textcat_multilabel.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +{%- endif %} +{%- endif %} + +{# NON-TRANSFORMER PIPELINE #} +{% else -%} +{% if "tok2vec" in full_pipeline -%} +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v2" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = ${components.tok2vec.model.encode.width} +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +rows = [5000, 1000, 2500, 2500] +include_static_vectors = {{ "true" if optimize == "accuracy" else "false" }} + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = {{ 96 if optimize == "efficiency" else 256 }} +depth = {{ 4 if optimize == "efficiency" else 8 }} +window_size = 1 +maxout_pieces = 3 +{% endif -%} + +{% if "morphologizer" in components %} +[components.morphologizer] +factory = "morphologizer" +label_smoothing = 0.05 + +[components.morphologizer.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.morphologizer.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{%- endif %} + +{% if "tagger" in components %} +[components.tagger] +factory = "tagger" +label_smoothing = 0.05 + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{%- endif %} + +{% if "parser" in components -%} +[components.parser] +factory = "parser" + +[components.parser.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 128 +maxout_pieces = 3 +use_upper = true +nO = null + +[components.parser.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{%- endif %} + +{% if "ner" in components %} +[components.ner] +factory = "ner" + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = true +nO = null + +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{% endif %} + +{% if "span_finder" in components %} +[components.span_finder] +factory = "span_finder" +max_length = 25 +min_length = null +scorer = {"@scorers":"spacy.span_finder_scorer.v1"} +spans_key = "sc" +threshold = 0.5 + +[components.span_finder.model] +@architectures = "spacy.SpanFinder.v1" + +[components.span_finder.model.scorer] +@layers = "spacy.LinearLogistic.v1" +nO = 2 + +[components.span_finder.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{% endif %} + +{% if "spancat" in components %} +[components.spancat] +factory = "spancat" +max_positive = null +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" +threshold = 0.5 + +[components.spancat.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat.model.scorer] +@layers = "spacy.LinearLogistic.v1" +nO = null +nI = null + +[components.spancat.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} + +[components.spancat.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif %} + +{% if "spancat_singlelabel" in components %} +[components.spancat_singlelabel] +factory = "spancat_singlelabel" +negative_weight = 1.0 +allow_overlap = true +scorer = {"@scorers":"spacy.spancat_scorer.v1"} +spans_key = "sc" + +[components.spancat_singlelabel.model] +@architectures = "spacy.SpanCategorizer.v1" + +[components.spancat_singlelabel.model.reducer] +@layers = "spacy.mean_max_reducer.v1" +hidden_size = 128 + +[components.spancat_singlelabel.model.scorer] +@layers = "Softmax.v2" + +[components.spancat_singlelabel.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} + +[components.spancat_singlelabel.suggester] +@misc = "spacy.ngram_suggester.v1" +sizes = [1,2,3] +{% endif %} + +{% if "trainable_lemmatizer" in components -%} +[components.trainable_lemmatizer] +factory = "trainable_lemmatizer" +backoff = "orth" +min_tree_freq = 3 +overwrite = false +scorer = {"@scorers":"spacy.lemmatizer_scorer.v1"} +top_k = 1 + +[components.trainable_lemmatizer.model] +@architectures = "spacy.Tagger.v2" +nO = null +normalize = false + +[components.trainable_lemmatizer.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{% endif -%} + +{% if "entity_linker" in components -%} +[components.entity_linker] +factory = "entity_linker" +get_candidates = {"@misc":"spacy.CandidateGenerator.v1"} +incl_context = true +incl_prior = true + +[components.entity_linker.model] +@architectures = "spacy.EntityLinker.v2" +nO = null + +[components.entity_linker.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +{% endif %} + +{% if "textcat" in components %} +[components.textcat] +factory = "textcat" + +{% if optimize == "accuracy" %} +[components.textcat.model] +@architectures = "spacy.TextCatEnsemble.v2" +nO = null + +[components.textcat.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} + +[components.textcat.model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false + +{% else -%} +[components.textcat.model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +ngram_size = 1 +no_output_layer = false +{%- endif %} +{%- endif %} + +{% if "textcat_multilabel" in components %} +[components.textcat_multilabel] +factory = "textcat_multilabel" + +{% if optimize == "accuracy" %} +[components.textcat_multilabel.model] +@architectures = "spacy.TextCatEnsemble.v2" +nO = null + +[components.textcat_multilabel.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} + +[components.textcat_multilabel.model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = false +length = 262144 +ngram_size = 1 +no_output_layer = false + +{% else -%} +[components.textcat_multilabel.model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = false +length = 262144 +ngram_size = 1 +no_output_layer = false +{%- endif %} +{%- endif %} +{% endif %} + +{% for pipe in components %} +{% if pipe not in listener_components %} +{# Other components defined by the user: we just assume they're factories #} +[components.{{ pipe }}] +factory = "{{ pipe }}" +{% endif %} +{% endfor %} + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +max_length = 0 + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +max_length = 0 + +[training] +{% if use_transformer -%} +accumulate_gradient = {{ transformer["size_factor"] }} +{% endif -%} +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" + +[training.optimizer] +@optimizers = "Adam.v1" + +{% if use_transformer -%} +[training.optimizer.learn_rate] +@schedules = "warmup_linear.v1" +warmup_steps = 250 +total_steps = 20000 +initial_rate = 5e-5 +{% endif %} + +{% if use_transformer %} +[training.batcher] +@batchers = "spacy.batch_by_padded.v1" +discard_oversize = true +size = 2000 +buffer = 256 +{%- else %} +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +{% endif %} + +[initialize] +vectors = ${paths.vectors} diff --git a/spacy/cli/templates/quickstart_training_recommendations.yml b/spacy/cli/templates/quickstart_training_recommendations.yml new file mode 100644 index 0000000..4f214d2 --- /dev/null +++ b/spacy/cli/templates/quickstart_training_recommendations.yml @@ -0,0 +1,282 @@ +# Recommended settings and available resources for each language, if available. +# Not all languages have recommended word vectors or transformers and for some, +# the recommended transformer for efficiency and accuracy may be the same. +__default__: + word_vectors: null + transformer: + efficiency: + name: bert-base-multilingual-uncased + size_factor: 3 + accuracy: + name: bert-base-multilingual-uncased + size_factor: 3 +ar: + word_vectors: null + transformer: + efficiency: + name: asafaya/bert-base-arabic + size_factor: 3 + accuracy: + name: asafaya/bert-base-arabic + size_factor: 3 +bg: + word_vectors: null + transformer: + efficiency: + name: iarfmoose/roberta-base-bulgarian + size_factor: 3 + accuracy: + name: iarfmoose/roberta-base-bulgarian + size_factor: 3 +bn: + word_vectors: null + transformer: + efficiency: + name: sagorsarker/bangla-bert-base + size_factor: 3 + accuracy: + name: sagorsarker/bangla-bert-base + size_factor: 3 +ca: + word_vectors: null + transformer: + efficiency: + name: projecte-aina/roberta-base-ca-v2 + size_factor: 3 + accuracy: + name: projecte-aina/roberta-base-ca-v2 + size_factor: 3 +da: + word_vectors: da_core_news_lg + transformer: + efficiency: + name: Maltehb/danish-bert-botxo + size_factor: 3 + accuracy: + name: Maltehb/danish-bert-botxo + size_factor: 3 +de: + word_vectors: de_core_news_lg + transformer: + efficiency: + name: bert-base-german-cased + size_factor: 3 + accuracy: + name: bert-base-german-cased + size_factor: 3 +el: + word_vectors: el_core_news_lg + transformer: + efficiency: + name: nlpaueb/bert-base-greek-uncased-v1 + size_factor: 3 + accuracy: + name: nlpaueb/bert-base-greek-uncased-v1 + size_factor: 3 +en: + word_vectors: en_core_web_lg + transformer: + efficiency: + name: roberta-base + size_factor: 3 + accuracy: + name: roberta-base + size_factor: 3 +es: + word_vectors: es_core_news_lg + transformer: + efficiency: + name: dccuchile/bert-base-spanish-wwm-cased + size_factor: 3 + accuracy: + name: dccuchile/bert-base-spanish-wwm-cased + size_factor: 3 +eu: + word_vectors: null + transformer: + efficiency: + name: mrm8488/RoBasquERTa + size_factor: 3 + accuracy: + name: mrm8488/RoBasquERTa + size_factor: 3 +fi: + word_vectors: null + transformer: + efficiency: + name: TurkuNLP/bert-base-finnish-cased-v1 + size_factor: 3 + accuracy: + name: TurkuNLP/bert-base-finnish-cased-v1 + size_factor: 3 +fr: + word_vectors: fr_core_news_lg + transformer: + efficiency: + name: camembert-base + size_factor: 3 + accuracy: + name: camembert-base + size_factor: 3 +hi: + word_vectors: null + transformer: + efficiency: + name: ai4bharat/indic-bert + size_factor: 3 + accuracy: + name: ai4bharat/indic-bert + size_factor: 3 +id: + word_vectors: null + transformer: + efficiency: + name: indolem/indobert-base-uncased + size_factor: 3 + accuracy: + name: indolem/indobert-base-uncased + size_factor: 3 +it: + word_vectors: it_core_news_lg + transformer: null +ja: + word_vectors: ja_core_news_lg + transformer: null +lt: + word_vectors: lt_core_news_lg + transformer: null +mk: + word_vectors: null + transformer: + efficiency: + name: anon-submission-mk/bert-base-macedonian-cased + size_factor: 3 + accuracy: + name: anon-submission-mk/bert-base-macedonian-cased + size_factor: 3 +nb: + word_vectors: nb_core_news_lg + transformer: + efficiency: + name: NbAiLab/nb-bert-base + size_factor: 3 + accuracy: + name: NbAiLab/nb-bert-base + size_factor: 3 +nl: + word_vectors: nl_core_news_lg + transformer: + efficiency: + name: GroNLP/bert-base-dutch-cased + size_factor: 3 + accuracy: + name: GroNLP/bert-base-dutch-cased + size_factor: 3 +pl: + word_vectors: pl_core_news_lg + transformer: + efficiency: + name: dkleczek/bert-base-polish-cased-v1 + size_factor: 3 + accuracy: + name: dkleczek/bert-base-polish-cased-v1 + size_factor: 3 +pt: + word_vectors: pt_core_news_lg + transformer: + efficiency: + name: neuralmind/bert-base-portuguese-cased + size_factor: 3 + accuracy: + name: neuralmind/bert-base-portuguese-cased + size_factor: 3 +ro: + word_vectors: ro_core_news_lg + transformer: + efficiency: + name: dumitrescustefan/bert-base-romanian-cased-v1 + size_factor: 3 + accuracy: + name: dumitrescustefan/bert-base-romanian-cased-v1 + size_factor: 3 +si: + word_vectors: null + transformer: + efficiency: + name: setu4993/LaBSE + size_factor: 3 + accuracy: + name: setu4993/LaBSE + size_factor: 3 +sv: + word_vectors: null + transformer: + efficiency: + name: KB/bert-base-swedish-cased + size_factor: 3 + accuracy: + name: KB/bert-base-swedish-cased + size_factor: 3 +ta: + word_vectors: null + transformer: + efficiency: + name: ai4bharat/indic-bert + size_factor: 3 + accuracy: + name: ai4bharat/indic-bert + size_factor: 3 +te: + word_vectors: null + transformer: + efficiency: + name: kuppuluri/telugu_bertu + size_factor: 3 + accuracy: + name: kuppuluri/telugu_bertu + size_factor: 3 +th: + word_vectors: null + transformer: + efficiency: + name: monsoon-nlp/bert-base-thai + size_factor: 3 + accuracy: + name: monsoon-nlp/bert-base-thai + size_factor: 3 +tr: + word_vectors: null + transformer: + efficiency: + name: dbmdz/bert-base-turkish-cased + size_factor: 3 + accuracy: + name: dbmdz/bert-base-turkish-cased + size_factor: 3 +uk: + word_vectors: null + transformer: + efficiency: + name: youscan/ukr-roberta-base + size_factor: 3 + accuracy: + name: youscan/ukr-roberta-base + size_factor: 3 +ur: + word_vectors: null + transformer: + efficiency: + name: urduhack/roberta-urdu-small + size_factor: 3 + accuracy: + name: urduhack/roberta-urdu-small + size_factor: 3 +zh: + word_vectors: zh_core_web_lg + transformer: + efficiency: + name: bert-base-chinese + size_factor: 3 + accuracy: + name: bert-base-chinese + size_factor: 3 diff --git a/spacy/cli/train.py b/spacy/cli/train.py new file mode 100644 index 0000000..3792682 --- /dev/null +++ b/spacy/cli/train.py @@ -0,0 +1,103 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict, Optional, Union + +import typer +from wasabi import msg + +from .. import util +from ..training.initialize import init_nlp +from ..training.loop import train as train_nlp +from ._util import ( + Arg, + Opt, + app, + import_code, + parse_config_overrides, + setup_gpu, + show_validation_error, +) + + +@app.command( + "train", context_settings={"allow_extra_args": True, "ignore_unknown_options": True} +) +def train_cli( + # fmt: off + ctx: typer.Context, # This is only used to read additional arguments + config_path: Path = Arg( + ..., help="Path to config file", exists=True, allow_dash=True + ), + output_path: Optional[Path] = Opt( + None, + "--output", + "--output-path", + "-o", + help="Output directory to store trained pipeline in", + ), + code_path: Optional[Path] = Opt( + None, + "--code", + "-c", + help="Path to Python file with additional code (registered functions) to be imported", + ), + verbose: bool = Opt( + False, + "--verbose", + "-V", + "-VV", + help="Display more information for debugging purposes", + ), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + # fmt: on +): + """ + Train or update a spaCy pipeline. Requires data in spaCy's binary format. To + convert data from other formats, use the `spacy convert` command. The + config file includes all settings and hyperparameters used during training. + To override settings in the config, e.g. settings that point to local + paths or that you want to experiment with, you can override them as + command line options. For instance, --training.batch_size 128 overrides + the value of "batch_size" in the block "[training]". The --code argument + lets you pass in a Python file that's imported before training. It can be + used to register custom functions and architectures that can then be + referenced in the config. + + DOCS: https://spacy.io/api/cli#train + """ + if verbose: + util.logger.setLevel(logging.DEBUG) + overrides = parse_config_overrides(ctx.args) + import_code(code_path) + train(config_path, output_path, use_gpu=use_gpu, overrides=overrides) + + +def train( + config_path: Union[str, Path], + output_path: Optional[Union[str, Path]] = None, + *, + use_gpu: int = -1, + overrides: Dict[str, Any] = util.SimpleFrozenDict(), +): + config_path = util.ensure_path(config_path) + output_path = util.ensure_path(output_path) + # Make sure all files and paths exists if they are needed + if not config_path or (str(config_path) != "-" and not config_path.exists()): + msg.fail("Config file not found", config_path, exits=1) + if not output_path: + msg.info("No output directory provided") + else: + if not output_path.exists(): + output_path.mkdir(parents=True) + msg.good(f"Created output directory: {output_path}") + msg.info(f"Saving to output directory: {output_path}") + setup_gpu(use_gpu) + with show_validation_error(config_path): + config = util.load_config(config_path, overrides=overrides, interpolate=False) + msg.divider("Initializing pipeline") + with show_validation_error(config_path, hint_fill=False): + nlp = init_nlp(config, use_gpu=use_gpu) + msg.good("Initialized pipeline") + msg.divider("Training pipeline") + train_nlp(nlp, output_path, use_gpu=use_gpu, stdout=sys.stdout, stderr=sys.stderr) diff --git a/spacy/cli/validate.py b/spacy/cli/validate.py new file mode 100644 index 0000000..0426f05 --- /dev/null +++ b/spacy/cli/validate.py @@ -0,0 +1,123 @@ +import sys +import warnings +from pathlib import Path +from typing import Tuple + +import requests +from wasabi import Printer, msg + +from .. import about +from ..util import ( + get_installed_models, + get_minor_version, + get_model_meta, + get_package_path, + get_package_version, + is_compatible_version, +) +from ._util import app + + +@app.command("validate") +def validate_cli(): + """ + Validate the currently installed pipeline packages and spaCy version. Checks + if the installed packages are compatible and shows upgrade instructions if + available. Should be run after `pip install -U spacy`. + + DOCS: https://spacy.io/api/cli#validate + """ + validate() + + +def validate() -> None: + model_pkgs, compat = get_model_pkgs() + spacy_version = get_minor_version(about.__version__) + current_compat = compat.get(spacy_version, {}) + if not current_compat: + msg.warn(f"No compatible packages found for v{spacy_version} of spaCy") + incompat_models = {d["name"] for _, d in model_pkgs.items() if not d["compat"]} + na_models = [m for m in incompat_models if m not in current_compat] + update_models = [m for m in incompat_models if m in current_compat] + spacy_dir = Path(__file__).parent.parent + + msg.divider(f"Installed pipeline packages (spaCy v{about.__version__})") + msg.info(f"spaCy installation: {spacy_dir}") + + if model_pkgs: + header = ("NAME", "SPACY", "VERSION", "") + rows = [] + for name, data in model_pkgs.items(): + if data["compat"]: + comp = msg.text("", color="green", icon="good", no_print=True) + version = msg.text(data["version"], color="green", no_print=True) + else: + version = msg.text(data["version"], color="yellow", no_print=True) + comp = f"--> {current_compat.get(data['name'], ['n/a'])[0]}" + rows.append((data["name"], data["spacy"], version, comp)) + msg.table(rows, header=header) + else: + msg.text("No pipeline packages found in your current environment.", exits=0) + if update_models: + msg.divider("Install updates") + msg.text("Use the following commands to update the packages:") + cmd = "python -m spacy download {}" + print("\n".join([cmd.format(pkg) for pkg in update_models]) + "\n") + if na_models: + msg.info( + f"The following packages are custom spaCy pipelines or not " + f"available for spaCy v{about.__version__}:", + ", ".join(na_models), + ) + if incompat_models: + sys.exit(1) + + +def get_model_pkgs(silent: bool = False) -> Tuple[dict, dict]: + msg = Printer(no_print=silent, pretty=not silent) + with msg.loading("Loading compatibility table..."): + r = requests.get(about.__compatibility__) + if r.status_code != 200: + msg.fail( + f"Server error ({r.status_code})", + "Couldn't fetch compatibility table.", + exits=1, + ) + msg.good("Loaded compatibility table") + compat = r.json()["spacy"] + all_models = set() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W09[45]") + installed_models = get_installed_models() + for spacy_v, models in dict(compat).items(): + all_models.update(models.keys()) + for model, model_vs in models.items(): + compat[spacy_v][model] = [reformat_version(v) for v in model_vs] + pkgs = {} + for pkg_name in installed_models: + package = pkg_name.replace("-", "_") + version = get_package_version(pkg_name) + if package in compat: + is_compat = version in compat[package] + spacy_version = about.__version__ + else: + model_path = get_package_path(package) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W09[45]") + model_meta = get_model_meta(model_path) + spacy_version = model_meta.get("spacy_version", "n/a") + is_compat = is_compatible_version(about.__version__, spacy_version) # type: ignore[assignment] + pkgs[pkg_name] = { + "name": package, + "version": version, + "spacy": spacy_version, + "compat": is_compat, + } + return pkgs, compat + + +def reformat_version(version: str) -> str: + """Hack to reformat old versions ending on '-alpha' to match pip format.""" + if version.endswith("-alpha"): + return version.replace("-alpha", "a0") + return version.replace("-alpha", "a") diff --git a/spacy/compat.py b/spacy/compat.py new file mode 100644 index 0000000..828ed1b --- /dev/null +++ b/spacy/compat.py @@ -0,0 +1,52 @@ +"""Helpers for Python and platform compatibility.""" + +import sys + +from thinc.util import copy_array + +try: + import cPickle as pickle +except ImportError: + import pickle # type: ignore[no-redef] + +try: + import copy_reg +except ImportError: + import copyreg as copy_reg # type: ignore[no-redef] + +try: + from cupy.cuda.stream import Stream as CudaStream +except ImportError: + CudaStream = None + +try: + import cupy +except ImportError: + cupy = None + +if sys.version_info[:2] >= (3, 8): # Python 3.8+ + from typing import Literal, Protocol, runtime_checkable +else: + from typing_extensions import Literal, Protocol, runtime_checkable # noqa: F401 + +# Important note: The importlib_metadata "backport" includes functionality +# that's not part of the built-in importlib.metadata. We should treat this +# import like the built-in and only use what's available there. +try: # Python 3.8+ + import importlib.metadata as importlib_metadata +except ImportError: + from catalogue import ( # type: ignore[no-redef] + _importlib_metadata as importlib_metadata, # noqa: F401 + ) + +from thinc.api import Optimizer # noqa: F401 + +pickle = pickle +copy_reg = copy_reg +CudaStream = CudaStream +cupy = cupy +copy_array = copy_array + +is_windows = sys.platform.startswith("win") +is_linux = sys.platform.startswith("linux") +is_osx = sys.platform == "darwin" diff --git a/spacy/default_config.cfg b/spacy/default_config.cfg new file mode 100644 index 0000000..b005eef --- /dev/null +++ b/spacy/default_config.cfg @@ -0,0 +1,140 @@ +[paths] +train = null +dev = null +vectors = null +init_tok2vec = null + +[system] +seed = 0 +gpu_allocator = null + +[nlp] +lang = null +# List of pipeline component names, in order. The names should correspond to +# components defined in the [components block] +pipeline = [] +# Components that are loaded but disabled by default +disabled = [] +# Optional callbacks to modify the nlp object before it's initialized, after +# it's created and after the pipeline has been set up +before_creation = null +after_creation = null +after_pipeline_creation = null +# Default batch size to use with nlp.pipe and nlp.evaluate +batch_size = 1000 + +[nlp.tokenizer] +@tokenizers = "spacy.Tokenizer.v1" + +[nlp.vectors] +@vectors = "spacy.Vectors.v1" + +# The pipeline components and their models +[components] + +# Readers for corpora like dev and train. +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +# Whether to train on sequences with 'gold standard' sentence boundaries +# and tokens. If you set this to true, take care to ensure your run-time +# data is passed in sentence-by-sentence via some prior preprocessing. +gold_preproc = false +# Limitations on training document length +max_length = 0 +# Limitation on number of training examples +limit = 0 +# Apply some simply data augmentation, where we replace tokens with variations. +# This is especially useful for punctuation and case replacement, to help +# generalize beyond corpora that don't/only have smart quotes etc. +augmenter = null + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +# Whether to train on sequences with 'gold standard' sentence boundaries +# and tokens. If you set this to true, take care to ensure your run-time +# data is passed in sentence-by-sentence via some prior preprocessing. +gold_preproc = false +# Limitations on training document length +max_length = 0 +# Limitation on number of training examples +limit = 0 +# Optional callback for data augmentation +augmenter = null + +# Training hyper-parameters and additional features. +[training] +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 1 +# Controls early-stopping, i.e., the number of steps to continue without +# improvement before stopping. 0 disables early stopping. +patience = 1600 +# Number of epochs. 0 means unlimited. If >= 0, train corpus is loaded once in +# memory and shuffled within the training loop. -1 means stream train corpus +# rather than loading in memory with no shuffling within the training loop. +max_epochs = 0 +# Maximum number of update steps to train for. 0 means an unlimited number of steps. +max_steps = 20000 +eval_frequency = 200 +# Control how scores are printed and checkpoints are evaluated. +score_weights = {} +# Names of pipeline components that shouldn't be updated during training +frozen_components = [] +# Names of pipeline components that should set annotations during training +annotating_components = [] +# Location in the config where the dev corpus is defined +dev_corpus = "corpora.dev" +# Location in the config where the train corpus is defined +train_corpus = "corpora.train" +# Optional callback before nlp object is saved to disk after training +before_to_disk = null +# Optional callback that is invoked at the start of each training step +before_update = null + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-8 +learn_rate = 0.001 + +# These settings are used when nlp.initialize() is called (typically before +# training or pretraining). Components and the tokenizer can each define their +# own arguments via their initialize methods that are populated by the config. +# This lets them gather data resources, build label sets etc. +[initialize] +vectors = ${paths.vectors} +# Extra resources for transfer-learning or pseudo-rehearsal +init_tok2vec = ${paths.init_tok2vec} +# Data and lookups for vocabulary +vocab_data = null +lookups = null +# Arguments passed to the tokenizer's initialize method +tokenizer = {} +# Arguments for initialize methods of the components (keyed by component) +components = {} +before_init = null +after_init = null diff --git a/spacy/default_config_pretraining.cfg b/spacy/default_config_pretraining.cfg new file mode 100644 index 0000000..d70ecf0 --- /dev/null +++ b/spacy/default_config_pretraining.cfg @@ -0,0 +1,44 @@ +[paths] +raw_text = null + +[pretraining] +max_epochs = 1000 +dropout = 0.2 +n_save_every = null +n_save_epoch = null +component = "tok2vec" +layer = "" +corpus = "corpora.pretrain" + +[pretraining.batcher] +@batchers = "spacy.batch_by_words.v1" +size = 3000 +discard_oversize = false +tolerance = 0.2 +get_length = null + +[pretraining.objective] +@architectures = "spacy.PretrainCharacters.v1" +maxout_pieces = 3 +hidden_size = 300 +n_characters = 4 + +[pretraining.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = true +eps = 1e-8 +learn_rate = 0.001 + +[corpora] + +[corpora.pretrain] +@readers = "spacy.JsonlCorpus.v1" +path = ${paths.raw_text} +min_length = 5 +max_length = 500 +limit = 0 diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py new file mode 100644 index 0000000..d670eb6 --- /dev/null +++ b/spacy/displacy/__init__.py @@ -0,0 +1,278 @@ +""" +spaCy's built in visualization suite for dependencies and named entities. + +DOCS: https://spacy.io/api/top-level#displacy +USAGE: https://spacy.io/usage/visualizers +""" + +import warnings +from typing import Any, Callable, Dict, Iterable, Optional, Union + +from ..errors import Errors, Warnings +from ..tokens import Doc, Span +from ..util import find_available_port, is_in_jupyter +from .render import DependencyRenderer, EntityRenderer, SpanRenderer + +_html = {} +RENDER_WRAPPER = None + + +def render( + docs: Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict], + style: str = "dep", + page: bool = False, + minify: bool = False, + jupyter: Optional[bool] = None, + options: Dict[str, Any] = {}, + manual: bool = False, +) -> str: + """Render displaCy visualisation. + + docs (Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]]): Document(s) to visualise. + a 'dict' is only allowed here when 'manual' is set to True + style (str): Visualisation style, 'dep' or 'ent'. + page (bool): Render markup as full HTML page. + minify (bool): Minify HTML markup. + jupyter (bool): Override Jupyter auto-detection. + options (dict): Visualiser-specific options, e.g. colors. + manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. + RETURNS (str): Rendered SVG or HTML markup. + + DOCS: https://spacy.io/api/top-level#displacy.render + USAGE: https://spacy.io/usage/visualizers + """ + factories = { + "dep": (DependencyRenderer, parse_deps), + "ent": (EntityRenderer, parse_ents), + "span": (SpanRenderer, parse_spans), + } + if style not in factories: + raise ValueError(Errors.E087.format(style=style)) + if isinstance(docs, (Doc, Span, dict)): + docs = [docs] + docs = [obj if not isinstance(obj, Span) else obj.as_doc() for obj in docs] + if not all(isinstance(obj, (Doc, Span, dict)) for obj in docs): + raise ValueError(Errors.E096) + renderer_func, converter = factories[style] + renderer = renderer_func(options=options) + parsed = [converter(doc, options) for doc in docs] if not manual else docs # type: ignore + if manual: + for doc in docs: + if isinstance(doc, dict) and "ents" in doc: + doc["ents"] = sorted(doc["ents"], key=lambda x: (x["start"], x["end"])) + _html["parsed"] = renderer.render(parsed, page=page, minify=minify).strip() # type: ignore + html = _html["parsed"] + if RENDER_WRAPPER is not None: + html = RENDER_WRAPPER(html) + if jupyter or (jupyter is None and is_in_jupyter()): + # return HTML rendered by IPython display() + # See #4840 for details on span wrapper to disable mathjax + from IPython.display import HTML, display + + return display(HTML('{}'.format(html))) + return html + + +def serve( + docs: Union[Iterable[Doc], Doc], + style: str = "dep", + page: bool = True, + minify: bool = False, + options: Dict[str, Any] = {}, + manual: bool = False, + port: int = 5000, + host: str = "0.0.0.0", + auto_select_port: bool = False, +) -> None: + """Serve displaCy visualisation. + + docs (list or Doc): Document(s) to visualise. + style (str): Visualisation style, 'dep' or 'ent'. + page (bool): Render markup as full HTML page. + minify (bool): Minify HTML markup. + options (dict): Visualiser-specific options, e.g. colors. + manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. + port (int): Port to serve visualisation. + host (str): Host to serve visualisation. + auto_select_port (bool): Automatically select a port if the specified port is in use. + + DOCS: https://spacy.io/api/top-level#displacy.serve + USAGE: https://spacy.io/usage/visualizers + """ + from wsgiref import simple_server + + port = find_available_port(port, host, auto_select_port) + + if is_in_jupyter(): + warnings.warn(Warnings.W011) + render(docs, style=style, page=page, minify=minify, options=options, manual=manual) + httpd = simple_server.make_server(host, port, app) + print(f"\nUsing the '{style}' visualizer") + print(f"Serving on http://{host}:{port} ...\n") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print(f"Shutting down server on port {port}.") + finally: + httpd.server_close() + + +def app(environ, start_response): + headers = [("Content-type", "text/html; charset=utf-8")] + start_response("200 OK", headers) + res = _html["parsed"].encode(encoding="utf-8") + return [res] + + +def parse_deps( + orig_doc: Union[Doc, Span], options: Dict[str, Any] = {} +) -> Dict[str, Any]: + """Generate dependency parse in {'words': [], 'arcs': []} format. + + orig_doc (Union[Doc, Span]): Document to parse. + options (Dict[str, Any]): Dependency parse specific visualisation options. + RETURNS (dict): Generated dependency parse keyed by words and arcs. + """ + if isinstance(orig_doc, Span): + orig_doc = orig_doc.as_doc() + doc = Doc(orig_doc.vocab).from_bytes( + orig_doc.to_bytes(exclude=["user_data", "user_hooks"]) + ) + if not doc.has_annotation("DEP"): + warnings.warn(Warnings.W005) + if options.get("collapse_phrases", False): + with doc.retokenize() as retokenizer: + for np in list(doc.noun_chunks): + attrs = { + "tag": np.root.tag_, + "lemma": np.root.lemma_, + "ent_type": np.root.ent_type_, + } + retokenizer.merge(np, attrs=attrs) # type: ignore[arg-type] + if options.get("collapse_punct", True): + spans = [] + for word in doc[:-1]: + if word.is_punct or not word.nbor(1).is_punct: + continue + start = word.i + end = word.i + 1 + while end < len(doc) and doc[end].is_punct: + end += 1 + span = doc[start:end] + spans.append((span, word.tag_, word.lemma_, word.ent_type_)) + with doc.retokenize() as retokenizer: + for span, tag, lemma, ent_type in spans: + attrs = {"tag": tag, "lemma": lemma, "ent_type": ent_type} + retokenizer.merge(span, attrs=attrs) # type: ignore[arg-type] + fine_grained = options.get("fine_grained") + add_lemma = options.get("add_lemma") + words = [ + { + "text": w.text, + "tag": w.tag_ if fine_grained else w.pos_, + "lemma": w.lemma_ if add_lemma else None, + } + for w in doc + ] + arcs = [] + for word in doc: + if word.i < word.head.i: + arcs.append( + {"start": word.i, "end": word.head.i, "label": word.dep_, "dir": "left"} + ) + elif word.i > word.head.i: + arcs.append( + { + "start": word.head.i, + "end": word.i, + "label": word.dep_, + "dir": "right", + } + ) + return {"words": words, "arcs": arcs, "settings": get_doc_settings(orig_doc)} + + +def parse_ents(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]: + """Generate named entities in [{start: i, end: i, label: 'label'}] format. + + doc (Doc): Document to parse. + options (Dict[str, Any]): NER-specific visualisation options. + RETURNS (dict): Generated entities keyed by text (original text) and ents. + """ + kb_url_template = options.get("kb_url_template", None) + ents = [ + { + "start": ent.start_char, + "end": ent.end_char, + "label": ent.label_, + "kb_id": ent.kb_id_ if ent.kb_id_ else "", + "kb_url": kb_url_template.format(ent.kb_id_) if kb_url_template else "#", + } + for ent in doc.ents + ] + if not ents: + warnings.warn(Warnings.W006) + title = doc.user_data.get("title", None) if hasattr(doc, "user_data") else None + settings = get_doc_settings(doc) + return {"text": doc.text, "ents": ents, "title": title, "settings": settings} + + +def parse_spans(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]: + """Generate spans in [{start_token: i, end_token: i, label: 'label'}] format. + + doc (Doc): Document to parse. + options (Dict[str, any]): Span-specific visualisation options. + RETURNS (dict): Generated span types keyed by text (original text) and spans. + """ + kb_url_template = options.get("kb_url_template", None) + spans_key = options.get("spans_key", "sc") + spans = [ + { + "start": span.start_char, + "end": span.end_char, + "start_token": span.start, + "end_token": span.end, + "label": span.label_, + "kb_id": span.kb_id_ if span.kb_id_ else "", + "kb_url": kb_url_template.format(span.kb_id_) if kb_url_template else "#", + } + for span in doc.spans.get(spans_key, []) + ] + tokens = [token.text for token in doc] + + if not spans: + keys = list(doc.spans.keys()) + warnings.warn(Warnings.W117.format(spans_key=spans_key, keys=keys)) + title = doc.user_data.get("title", None) if hasattr(doc, "user_data") else None + settings = get_doc_settings(doc) + return { + "text": doc.text, + "spans": spans, + "title": title, + "settings": settings, + "tokens": tokens, + } + + +def set_render_wrapper(func: Callable[[str], str]) -> None: + """Set an optional wrapper function that is called around the generated + HTML markup on displacy.render. This can be used to allow integration into + other platforms, similar to Jupyter Notebooks that require functions to be + called around the HTML. It can also be used to implement custom callbacks + on render, or to embed the visualization in a custom page. + + func (callable): Function to call around markup before rendering it. Needs + to take one argument, the HTML markup, and should return the desired + output of displacy.render. + """ + global RENDER_WRAPPER + if not hasattr(func, "__call__"): + raise ValueError(Errors.E110.format(obj=type(func))) + RENDER_WRAPPER = func + + +def get_doc_settings(doc: Doc) -> Dict[str, Any]: + return { + "lang": doc.lang_, + "direction": doc.vocab.writing_system.get("direction", "ltr"), + } diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py new file mode 100644 index 0000000..40b9986 --- /dev/null +++ b/spacy/displacy/render.py @@ -0,0 +1,613 @@ +import uuid +from typing import Any, Dict, List, Optional, Tuple, Union + +from ..errors import Errors +from ..util import escape_html, minify_html, registry +from .templates import ( + TPL_DEP_ARCS, + TPL_DEP_SVG, + TPL_DEP_WORDS, + TPL_DEP_WORDS_LEMMA, + TPL_ENT, + TPL_ENT_RTL, + TPL_ENTS, + TPL_FIGURE, + TPL_KB_LINK, + TPL_PAGE, + TPL_SPAN, + TPL_SPAN_RTL, + TPL_SPAN_SLICE, + TPL_SPAN_SLICE_RTL, + TPL_SPAN_START, + TPL_SPAN_START_RTL, + TPL_SPANS, + TPL_TITLE, +) + +DEFAULT_LANG = "en" +DEFAULT_DIR = "ltr" +DEFAULT_ENTITY_COLOR = "#ddd" +DEFAULT_LABEL_COLORS = { + "ORG": "#7aecec", + "PRODUCT": "#bfeeb7", + "GPE": "#feca74", + "LOC": "#ff9561", + "PERSON": "#aa9cfc", + "NORP": "#c887fb", + "FAC": "#9cc9cc", + "EVENT": "#ffeb80", + "LAW": "#ff8197", + "LANGUAGE": "#ff8197", + "WORK_OF_ART": "#f0d0ff", + "DATE": "#bfe1d9", + "TIME": "#bfe1d9", + "MONEY": "#e4e7d2", + "QUANTITY": "#e4e7d2", + "ORDINAL": "#e4e7d2", + "CARDINAL": "#e4e7d2", + "PERCENT": "#e4e7d2", +} + + +class SpanRenderer: + """Render Spans as SVGs.""" + + style = "span" + + def __init__(self, options: Dict[str, Any] = {}) -> None: + """Initialise span renderer + + options (dict): Visualiser-specific options (colors, spans) + """ + # Set up the colors and overall look + colors = dict(DEFAULT_LABEL_COLORS) + user_colors = registry.displacy_colors.get_all() + for user_color in user_colors.values(): + if callable(user_color): + # Since this comes from the function registry, we want to make + # sure we support functions that *return* a dict of colors + user_color = user_color() + if not isinstance(user_color, dict): + raise ValueError(Errors.E925.format(obj=type(user_color))) + colors.update(user_color) + colors.update(options.get("colors", {})) + self.default_color = DEFAULT_ENTITY_COLOR + self.colors = {label.upper(): color for label, color in colors.items()} + + # Set up how the text and labels will be rendered + self.direction = DEFAULT_DIR + self.lang = DEFAULT_LANG + # These values are in px + self.top_offset = options.get("top_offset", 40) + # This is how far under the top offset the span labels appear + self.span_label_offset = options.get("span_label_offset", 20) + self.offset_step = options.get("top_offset_step", 17) + + # Set up which templates will be used + template = options.get("template") + if template: + self.span_template = template["span"] + self.span_slice_template = template["slice"] + self.span_start_template = template["start"] + else: + if self.direction == "rtl": + self.span_template = TPL_SPAN_RTL + self.span_slice_template = TPL_SPAN_SLICE_RTL + self.span_start_template = TPL_SPAN_START_RTL + else: + self.span_template = TPL_SPAN + self.span_slice_template = TPL_SPAN_SLICE + self.span_start_template = TPL_SPAN_START + + def render( + self, parsed: List[Dict[str, Any]], page: bool = False, minify: bool = False + ) -> str: + """Render complete markup. + + parsed (list): Dependency parses to render. + page (bool): Render parses wrapped as full HTML page. + minify (bool): Minify HTML markup. + RETURNS (str): Rendered SVG or HTML markup. + """ + rendered = [] + for i, p in enumerate(parsed): + if i == 0: + settings = p.get("settings", {}) + self.direction = settings.get("direction", DEFAULT_DIR) + self.lang = settings.get("lang", DEFAULT_LANG) + rendered.append(self.render_spans(p["tokens"], p["spans"], p.get("title"))) + + if page: + docs = "".join([TPL_FIGURE.format(content=doc) for doc in rendered]) + markup = TPL_PAGE.format(content=docs, lang=self.lang, dir=self.direction) + else: + markup = "".join(rendered) + if minify: + return minify_html(markup) + return markup + + def render_spans( + self, + tokens: List[str], + spans: List[Dict[str, Any]], + title: Optional[str], + ) -> str: + """Render span types in text. + + Spans are rendered per-token, this means that for each token, we check if it's part + of a span slice (a member of a span type) or a span start (the starting token of a + given span type). + + tokens (list): Individual tokens in the text + spans (list): Individual entity spans and their start, end, label, kb_id and kb_url. + title (str / None): Document title set in Doc.user_data['title']. + """ + per_token_info = self._assemble_per_token_info(tokens, spans) + markup = self._render_markup(per_token_info) + markup = TPL_SPANS.format(content=markup, dir=self.direction) + if title: + markup = TPL_TITLE.format(title=title) + markup + return markup + + @staticmethod + def _assemble_per_token_info( + tokens: List[str], spans: List[Dict[str, Any]] + ) -> List[Dict[str, List[Dict[str, Any]]]]: + """Assembles token info used to generate markup in render_spans(). + tokens (List[str]): Tokens in text. + spans (List[Dict[str, Any]]): Spans in text. + RETURNS (List[Dict[str, List[Dict, str, Any]]]): Per token info needed to render HTML markup for given tokens + and spans. + """ + per_token_info: List[Dict[str, List[Dict[str, Any]]]] = [] + + # we must sort so that we can correctly describe when spans need to "stack" + # which is determined by their start token, then span length (longer spans on top), + # then break any remaining ties with the span label + spans = sorted( + spans, + key=lambda s: ( + s["start_token"], + -(s["end_token"] - s["start_token"]), + s["label"], + ), + ) + + for s in spans: + # this is the vertical 'slot' that the span will be rendered in + # vertical_position = span_label_offset + (offset_step * (slot - 1)) + s["render_slot"] = 0 + + for idx, token in enumerate(tokens): + # Identify if a token belongs to a Span (and which) and if it's a + # start token of said Span. We'll use this for the final HTML render + token_markup: Dict[str, Any] = {} + token_markup["text"] = token + intersecting_spans: List[Dict[str, Any]] = [] + entities = [] + for span in spans: + ent = {} + if span["start_token"] <= idx < span["end_token"]: + span_start = idx == span["start_token"] + ent["label"] = span["label"] + ent["is_start"] = span_start + if span_start: + # When the span starts, we need to know how many other + # spans are on the 'span stack' and will be rendered. + # This value becomes the vertical render slot for this entire span + span["render_slot"] = ( + intersecting_spans[-1]["render_slot"] + if len(intersecting_spans) + else 0 + ) + 1 + intersecting_spans.append(span) + ent["render_slot"] = span["render_slot"] + kb_id = span.get("kb_id", "") + kb_url = span.get("kb_url", "#") + ent["kb_link"] = ( + TPL_KB_LINK.format(kb_id=kb_id, kb_url=kb_url) if kb_id else "" + ) + entities.append(ent) + else: + # We don't specifically need to do this since we loop + # over tokens and spans sorted by their start_token, + # so we'll never use a span again after the last token it appears in, + # but if we were to use these spans again we'd want to make sure + # this value was reset correctly. + span["render_slot"] = 0 + token_markup["entities"] = entities + per_token_info.append(token_markup) + + return per_token_info + + def _render_markup(self, per_token_info: List[Dict[str, Any]]) -> str: + """Render the markup from per-token information""" + markup = "" + for token in per_token_info: + entities = sorted(token["entities"], key=lambda d: d["render_slot"]) + # Whitespace tokens disrupt the vertical space (no line height) so that the + # span indicators get misaligned. We don't render them as individual + # tokens anyway, so we'll just not display a span indicator either. + is_whitespace = token["text"].strip() == "" + if entities and not is_whitespace: + slices = self._get_span_slices(token["entities"]) + starts = self._get_span_starts(token["entities"]) + total_height = ( + self.top_offset + + self.span_label_offset + + (self.offset_step * (len(entities) - 1)) + ) + markup += self.span_template.format( + text=escape_html(token["text"]), + span_slices=slices, + span_starts=starts, + total_height=total_height, + ) + else: + markup += escape_html(token["text"] + " ") + return markup + + def _get_span_slices(self, entities: List[Dict]) -> str: + """Get the rendered markup of all Span slices""" + span_slices = [] + for entity in entities: + # rather than iterate over multiples of offset_step, we use entity['render_slot'] + # to determine the vertical position, since that tells where + # the span starts vertically so we can extend it horizontally, + # past other spans that might have already ended + color = self.colors.get(entity["label"].upper(), self.default_color) + top_offset = self.top_offset + ( + self.offset_step * (entity["render_slot"] - 1) + ) + span_slice = self.span_slice_template.format( + bg=color, + top_offset=top_offset, + ) + span_slices.append(span_slice) + return "".join(span_slices) + + def _get_span_starts(self, entities: List[Dict]) -> str: + """Get the rendered markup of all Span start tokens""" + span_starts = [] + for entity in entities: + color = self.colors.get(entity["label"].upper(), self.default_color) + top_offset = self.top_offset + ( + self.offset_step * (entity["render_slot"] - 1) + ) + span_start = ( + self.span_start_template.format( + bg=color, + top_offset=top_offset, + label=entity["label"], + kb_link=entity["kb_link"], + ) + if entity["is_start"] + else "" + ) + span_starts.append(span_start) + return "".join(span_starts) + + +class DependencyRenderer: + """Render dependency parses as SVGs.""" + + style = "dep" + + def __init__(self, options: Dict[str, Any] = {}) -> None: + """Initialise dependency renderer. + + options (dict): Visualiser-specific options (compact, word_spacing, + arrow_spacing, arrow_width, arrow_stroke, distance, offset_x, + color, bg, font) + """ + self.compact = options.get("compact", False) + self.word_spacing = options.get("word_spacing", 45) + self.arrow_spacing = options.get("arrow_spacing", 12 if self.compact else 20) + self.arrow_width = options.get("arrow_width", 6 if self.compact else 10) + self.arrow_stroke = options.get("arrow_stroke", 2) + self.distance = options.get("distance", 150 if self.compact else 175) + self.offset_x = options.get("offset_x", 50) + self.color = options.get("color", "#000000") + self.bg = options.get("bg", "#ffffff") + self.font = options.get("font", "Arial") + self.direction = DEFAULT_DIR + self.lang = DEFAULT_LANG + + def render( + self, parsed: List[Dict[str, Any]], page: bool = False, minify: bool = False + ) -> str: + """Render complete markup. + + parsed (list): Dependency parses to render. + page (bool): Render parses wrapped as full HTML page. + minify (bool): Minify HTML markup. + RETURNS (str): Rendered SVG or HTML markup. + """ + # Create a random ID prefix to make sure parses don't receive the + # same ID, even if they're identical + id_prefix = uuid.uuid4().hex + rendered = [] + for i, p in enumerate(parsed): + if i == 0: + settings = p.get("settings", {}) + self.direction = settings.get("direction", DEFAULT_DIR) + self.lang = settings.get("lang", DEFAULT_LANG) + render_id = f"{id_prefix}-{i}" + svg = self.render_svg(render_id, p["words"], p["arcs"]) + if p.get("title"): + svg = TPL_TITLE.format(title=p.get("title")) + svg + rendered.append(svg) + if page: + content = "".join([TPL_FIGURE.format(content=svg) for svg in rendered]) + markup = TPL_PAGE.format( + content=content, lang=self.lang, dir=self.direction + ) + else: + markup = "".join(rendered) + if minify: + return minify_html(markup) + return markup + + def render_svg( + self, + render_id: Union[int, str], + words: List[Dict[str, Any]], + arcs: List[Dict[str, Any]], + ) -> str: + """Render SVG. + + render_id (Union[int, str]): Unique ID, typically index of document. + words (list): Individual words and their tags. + arcs (list): Individual arcs and their start, end, direction and label. + RETURNS (str): Rendered SVG markup. + """ + self.levels = self.get_levels(arcs) + self.highest_level = max(self.levels.values(), default=0) + self.offset_y = self.distance / 2 * self.highest_level + self.arrow_stroke + self.width = self.offset_x + len(words) * self.distance + self.height = self.offset_y + 3 * self.word_spacing + self.id = render_id + words_svg = [ + self.render_word(w["text"], w["tag"], w.get("lemma", None), i) + for i, w in enumerate(words) + ] + arcs_svg = [ + self.render_arrow(a["label"], a["start"], a["end"], a["dir"], i) + for i, a in enumerate(arcs) + ] + content = "".join(words_svg) + "".join(arcs_svg) + return TPL_DEP_SVG.format( + id=self.id, + width=self.width, + height=self.height, + color=self.color, + bg=self.bg, + font=self.font, + content=content, + dir=self.direction, + lang=self.lang, + ) + + def render_word(self, text: str, tag: str, lemma: str, i: int) -> str: + """Render individual word. + + text (str): Word text. + tag (str): Part-of-speech tag. + i (int): Unique ID, typically word index. + RETURNS (str): Rendered SVG markup. + """ + y = self.offset_y + self.word_spacing + x = self.offset_x + i * self.distance + if self.direction == "rtl": + x = self.width - x + html_text = escape_html(text) + if lemma is not None: + return TPL_DEP_WORDS_LEMMA.format( + text=html_text, tag=tag, lemma=lemma, x=x, y=y + ) + return TPL_DEP_WORDS.format(text=html_text, tag=tag, x=x, y=y) + + def render_arrow( + self, label: str, start: int, end: int, direction: str, i: int + ) -> str: + """Render individual arrow. + + label (str): Dependency label. + start (int): Index of start word. + end (int): Index of end word. + direction (str): Arrow direction, 'left' or 'right'. + i (int): Unique ID, typically arrow index. + RETURNS (str): Rendered SVG markup. + """ + if start < 0 or end < 0: + error_args = dict(start=start, end=end, label=label, dir=direction) + raise ValueError(Errors.E157.format(**error_args)) + level = self.levels[(start, end, label)] + x_start = self.offset_x + start * self.distance + self.arrow_spacing + if self.direction == "rtl": + x_start = self.width - x_start + y = self.offset_y + x_end = ( + self.offset_x + + (end - start) * self.distance + + start * self.distance + - self.arrow_spacing * (self.highest_level - level) / 4 + ) + if self.direction == "rtl": + x_end = self.width - x_end + y_curve = self.offset_y - level * self.distance / 2 + if self.compact: + y_curve = self.offset_y - level * self.distance / 6 + if y_curve == 0 and max(self.levels.values(), default=0) > 5: + y_curve = -self.distance + arrowhead = self.get_arrowhead(direction, x_start, y, x_end) + arc = self.get_arc(x_start, y, y_curve, x_end) + label_side = "right" if self.direction == "rtl" else "left" + return TPL_DEP_ARCS.format( + id=self.id, + i=i, + stroke=self.arrow_stroke, + head=arrowhead, + label=label, + label_side=label_side, + arc=arc, + ) + + def get_arc(self, x_start: int, y: int, y_curve: int, x_end: int) -> str: + """Render individual arc. + + x_start (int): X-coordinate of arrow start point. + y (int): Y-coordinate of arrow start and end point. + y_curve (int): Y-corrdinate of Cubic Bézier y_curve point. + x_end (int): X-coordinate of arrow end point. + RETURNS (str): Definition of the arc path ('d' attribute). + """ + template = "M{x},{y} C{x},{c} {e},{c} {e},{y}" + if self.compact: + template = "M{x},{y} {x},{c} {e},{c} {e},{y}" + return template.format(x=x_start, y=y, c=y_curve, e=x_end) + + def get_arrowhead(self, direction: str, x: int, y: int, end: int) -> str: + """Render individual arrow head. + + direction (str): Arrow direction, 'left' or 'right'. + x (int): X-coordinate of arrow start point. + y (int): Y-coordinate of arrow start and end point. + end (int): X-coordinate of arrow end point. + RETURNS (str): Definition of the arrow head path ('d' attribute). + """ + if direction == "left": + p1, p2, p3 = (x, x - self.arrow_width + 2, x + self.arrow_width - 2) + else: + p1, p2, p3 = (end, end + self.arrow_width - 2, end - self.arrow_width + 2) + return f"M{p1},{y + 2} L{p2},{y - self.arrow_width} {p3},{y - self.arrow_width}" + + def get_levels(self, arcs: List[Dict[str, Any]]) -> Dict[Tuple[int, int, str], int]: + """Calculate available arc height "levels". + Used to calculate arrow heights dynamically and without wasting space. + + args (list): Individual arcs and their start, end, direction and label. + RETURNS (dict): Arc levels keyed by (start, end, label). + """ + arcs = [dict(t) for t in {tuple(sorted(arc.items())) for arc in arcs}] + length = max([arc["end"] for arc in arcs], default=0) + max_level = [0] * length + levels = {} + for arc in sorted(arcs, key=lambda arc: arc["end"] - arc["start"]): + level = max(max_level[arc["start"] : arc["end"]]) + 1 + for i in range(arc["start"], arc["end"]): + max_level[i] = level + levels[(arc["start"], arc["end"], arc["label"])] = level + return levels + + +class EntityRenderer: + """Render named entities as HTML.""" + + style = "ent" + + def __init__(self, options: Dict[str, Any] = {}) -> None: + """Initialise entity renderer. + + options (dict): Visualiser-specific options (colors, ents) + """ + colors = dict(DEFAULT_LABEL_COLORS) + user_colors = registry.displacy_colors.get_all() + for user_color in user_colors.values(): + if callable(user_color): + # Since this comes from the function registry, we want to make + # sure we support functions that *return* a dict of colors + user_color = user_color() + if not isinstance(user_color, dict): + raise ValueError(Errors.E925.format(obj=type(user_color))) + colors.update(user_color) + colors.update(options.get("colors", {})) + self.default_color = DEFAULT_ENTITY_COLOR + self.colors = {label.upper(): color for label, color in colors.items()} + self.ents = options.get("ents", None) + if self.ents is not None: + self.ents = [ent.upper() for ent in self.ents] + self.direction = DEFAULT_DIR + self.lang = DEFAULT_LANG + template = options.get("template") + if template: + self.ent_template = template + else: + if self.direction == "rtl": + self.ent_template = TPL_ENT_RTL + else: + self.ent_template = TPL_ENT + + def render( + self, parsed: List[Dict[str, Any]], page: bool = False, minify: bool = False + ) -> str: + """Render complete markup. + + parsed (list): Dependency parses to render. + page (bool): Render parses wrapped as full HTML page. + minify (bool): Minify HTML markup. + RETURNS (str): Rendered SVG or HTML markup. + """ + rendered = [] + for i, p in enumerate(parsed): + if i == 0: + settings = p.get("settings", {}) + self.direction = settings.get("direction", DEFAULT_DIR) + self.lang = settings.get("lang", DEFAULT_LANG) + rendered.append(self.render_ents(p["text"], p["ents"], p.get("title"))) + if page: + docs = "".join([TPL_FIGURE.format(content=doc) for doc in rendered]) + markup = TPL_PAGE.format(content=docs, lang=self.lang, dir=self.direction) + else: + markup = "".join(rendered) + if minify: + return minify_html(markup) + return markup + + def render_ents( + self, text: str, spans: List[Dict[str, Any]], title: Optional[str] + ) -> str: + """Render entities in text. + + text (str): Original text. + spans (list): Individual entity spans and their start, end, label, kb_id and kb_url. + title (str / None): Document title set in Doc.user_data['title']. + """ + markup = "" + offset = 0 + for span in spans: + label = span["label"] + start = span["start"] + end = span["end"] + kb_id = span.get("kb_id", "") + kb_url = span.get("kb_url", "#") + kb_link = TPL_KB_LINK.format(kb_id=kb_id, kb_url=kb_url) if kb_id else "" + additional_params = span.get("params", {}) + entity = escape_html(text[start:end]) + fragments = text[offset:start].split("\n") + for i, fragment in enumerate(fragments): + markup += escape_html(fragment) + if len(fragments) > 1 and i != len(fragments) - 1: + markup += "
" + if self.ents is None or label.upper() in self.ents: + color = self.colors.get(label.upper(), self.default_color) + ent_settings = { + "label": label, + "text": entity, + "bg": color, + "kb_link": kb_link, + } + ent_settings.update(additional_params) + markup += self.ent_template.format(**ent_settings) + else: + markup += entity + offset = end + fragments = text[offset:].split("\n") + for i, fragment in enumerate(fragments): + markup += escape_html(fragment) + if len(fragments) > 1 and i != len(fragments) - 1: + markup += "
" + markup = TPL_ENTS.format(content=markup, dir=self.direction) + if title: + markup = TPL_TITLE.format(title=title) + markup + return markup diff --git a/spacy/displacy/templates.py b/spacy/displacy/templates.py new file mode 100644 index 0000000..40f5376 --- /dev/null +++ b/spacy/displacy/templates.py @@ -0,0 +1,129 @@ +# Setting explicit height and max-width: none on the SVG is required for +# Jupyter to render it properly in a cell + +TPL_DEP_SVG = """ +{content} +""" + + +TPL_DEP_WORDS = """ + + {text} + {tag} + +""" + + +TPL_DEP_WORDS_LEMMA = """ + + {text} + {lemma} + {tag} + +""" + + +TPL_DEP_ARCS = """ + + + + {label} + + + +""" + + +TPL_FIGURE = """ +
{content}
+""" + +TPL_TITLE = """ +

{title}

+""" + + +TPL_ENTS = """ +
{content}
+""" + + +TPL_ENT = """ + + {text} + {label}{kb_link} + +""" + +TPL_ENT_RTL = """ + + {text} + {label}{kb_link} + +""" + +TPL_SPANS = """ +
{content}
+""" + +TPL_SPAN = """ + + {text} + {span_slices} + {span_starts} + +""" + +TPL_SPAN_SLICE = """ + + +""" + + +TPL_SPAN_START = """ + + + {label}{kb_link} + + + +""" + +TPL_SPAN_RTL = """ + + {text} + {span_slices} + {span_starts} + +""" + +TPL_SPAN_SLICE_RTL = """ + + +""" + +TPL_SPAN_START_RTL = """ + + + {label}{kb_link} + + +""" + + +# Important: this needs to start with a space! +TPL_KB_LINK = """ + {kb_id} +""" + + +TPL_PAGE = """ + + + + displaCy + + + {content} + +""" diff --git a/spacy/errors.py b/spacy/errors.py new file mode 100644 index 0000000..cf9a7b7 --- /dev/null +++ b/spacy/errors.py @@ -0,0 +1,1017 @@ +import warnings + +from .compat import Literal + + +class ErrorsWithCodes(type): + def __getattribute__(self, code): + msg = super().__getattribute__(code) + if code.startswith("__"): # python system attributes like __class__ + return msg + else: + return "[{code}] {msg}".format(code=code, msg=msg) + + +def setup_default_warnings(): + # ignore certain numpy warnings + filter_warning("ignore", error_msg="numpy.dtype size changed") # noqa + filter_warning("ignore", error_msg="numpy.ufunc size changed") # noqa + + # warn about entity_ruler, span_ruler & matcher having no patterns only once + for pipe in ["matcher", "entity_ruler", "span_ruler"]: + filter_warning("once", error_msg=Warnings.W036.format(name=pipe)) + + # warn once about lemmatizer without required POS + filter_warning("once", error_msg=Warnings.W108) + + # floret vector table cannot be modified + filter_warning("once", error_msg="[W114]") + + +def filter_warning( + action: Literal["default", "error", "ignore", "always", "module", "once"], + error_msg: str, +): + """Customize how spaCy should handle a certain warning. + + error_msg (str): e.g. "W006", or a full error message + action (str): "default", "error", "ignore", "always", "module" or "once" + """ + warnings.filterwarnings(action, message=_escape_warning_msg(error_msg)) + + +def _escape_warning_msg(msg): + """To filter with warnings.filterwarnings, the [] brackets need to be escaped""" + return msg.replace("[", "\\[").replace("]", "\\]") + + +# fmt: off + +class Warnings(metaclass=ErrorsWithCodes): + W005 = ("Doc object not parsed. This means displaCy won't be able to " + "generate a dependency visualization for it. Make sure the Doc " + "was processed with a model that supports dependency parsing, and " + "not just a language class like `English()`. For more info, see " + "the docs:\nhttps://spacy.io/usage/models") + W006 = ("No entities to visualize found in Doc object. If this is " + "surprising to you, make sure the Doc was processed using a model " + "that supports named entity recognition, and check the `doc.ents` " + "property manually if necessary.") + W007 = ("The model you're using has no word vectors loaded, so the result " + "of the {obj}.similarity method will be based on the tagger, " + "parser and NER, which may not give useful similarity judgements. " + "This may happen if you're using one of the small models, e.g. " + "`en_core_web_sm`, which don't ship with word vectors and only " + "use context-sensitive tensors. You can always add your own word " + "vectors, or use one of the larger models instead if available.") + W008 = ("Evaluating {obj}.similarity based on empty vectors.") + W011 = ("It looks like you're calling displacy.serve from within a " + "Jupyter notebook or a similar environment. This likely means " + "you're already running a local web server, so there's no need to " + "make displaCy start another one. Instead, you should be able to " + "replace displacy.serve with displacy.render to show the " + "visualization.") + W012 = ("A Doc object you're adding to the PhraseMatcher for pattern " + "'{key}' is parsed and/or tagged, but to match on '{attr}', you " + "don't actually need this information. This means that creating " + "the patterns is potentially much slower, because all pipeline " + "components are applied. To only create tokenized Doc objects, " + "try using `nlp.make_doc(text)` or process all texts as a stream " + "using `list(nlp.tokenizer.pipe(all_texts))`.") + W017 = ("Alias '{alias}' already exists in the Knowledge Base.") + W018 = ("Entity '{entity}' already exists in the Knowledge Base - " + "ignoring the duplicate entry.") + W021 = ("Unexpected hash collision in PhraseMatcher. Matches may be " + "incorrect. Modify PhraseMatcher._terminal_hash to fix.") + W024 = ("Entity '{entity}' - Alias '{alias}' combination already exists in " + "the Knowledge Base.") + W026 = ("Unable to set all sentence boundaries from dependency parses. If " + "you are constructing a parse tree incrementally by setting " + "token.head values, you can probably ignore this warning. Consider " + "using Doc(words, ..., heads=heads, deps=deps) instead.") + W027 = ("Found a large training file of {size} bytes. Note that it may " + "be more efficient to split your training data into multiple " + "smaller JSON files instead.") + W028 = ("Doc.from_array was called with a vector of type '{type}', " + "but is expecting one of type uint64 instead. This may result " + "in problems with the vocab further on in the pipeline.") + W030 = ("Some entities could not be aligned in the text \"{text}\" with " + "entities \"{entities}\". Use " + "`spacy.training.offsets_to_biluo_tags(nlp.make_doc(text), entities)`" + " to check the alignment. Misaligned entities ('-') will be " + "ignored during training.") + W033 = ("Training a new {model} using a model with no lexeme normalization " + "table. This may degrade the performance of the model to some " + "degree. If this is intentional or the language you're using " + "doesn't have a normalization table, please ignore this warning. " + "If this is surprising, make sure you have the spacy-lookups-data " + "package installed and load the table in your config. The " + "languages with lexeme normalization tables are currently: " + "{langs}\n\nLoad the table in your config with:\n\n" + "[initialize.lookups]\n" + "@misc = \"spacy.LookupsDataLoader.v1\"\n" + "lang = ${{nlp.lang}}\n" + "tables = [\"lexeme_norm\"]\n") + W035 = ("Discarding subpattern '{pattern}' due to an unrecognized " + "attribute or operator.") + W036 = ("The component '{name}' does not have any patterns defined.") + + # New warnings added in v3.x + W086 = ("Component '{listener}' will be (re)trained, but it needs the component " + "'{name}' which is frozen. If you want to prevent retraining '{name}' " + "but want to train '{listener}' on top of it, you should add '{name}' to the " + "list of 'annotating_components' in the 'training' block in the config. " + "See the documentation for details: " + "https://spacy.io/usage/training#annotating-components") + W087 = ("Component '{name}' will be (re)trained, but the component '{listener}' " + "depends on it via a listener and is frozen. This means that the " + "performance of '{listener}' will be degraded. You can either freeze " + "both, or neither of the two. If you're sourcing the component from " + "an existing pipeline, you can use the `replace_listeners` setting in " + "the config block to replace its token-to-vector listener with a copy " + "and make it independent. For example, `replace_listeners = " + "[\"model.tok2vec\"]` See the documentation for details: " + "https://spacy.io/usage/training#config-components-listeners") + W088 = ("The pipeline component {name} implements a `begin_training` " + "method, which won't be called by spaCy. As of v3.0, `begin_training` " + "has been renamed to `initialize`, so you likely want to rename the " + "component method. See the documentation for details: " + "https://spacy.io/api/language#initialize") + W089 = ("As of spaCy v3.0, the `nlp.begin_training` method has been renamed " + "to `nlp.initialize`.") + W090 = ("Could not locate any {format} files in path '{path}'.") + W091 = ("Could not clean/remove the temp directory at {dir}: {msg}.") + W092 = ("Ignoring annotations for sentence starts, as dependency heads are set.") + W093 = ("Could not find any data to train the {name} on. Is your " + "input data correctly formatted?") + W094 = ("Model '{model}' ({model_version}) specifies an under-constrained " + "spaCy version requirement: {version}. This can lead to compatibility " + "problems with older versions, or as new spaCy versions are " + "released, because the model may say it's compatible when it's " + 'not. Consider changing the "spacy_version" in your meta.json to a ' + "version range, with a lower and upper pin. For example: {example}") + W095 = ("Model '{model}' ({model_version}) was trained with spaCy " + "{version} and may not be 100% compatible with the current version " + "({current}). If you see errors or degraded performance, download " + "a newer compatible model or retrain your custom model with the " + "current spaCy version. For more details and available updates, " + "run: python -m spacy validate") + W096 = ("The method `nlp.disable_pipes` is now deprecated - use " + "`nlp.select_pipes` instead.") + W100 = ("Skipping unsupported morphological feature(s): '{feature}'. " + "Provide features as a dict {{\"Field1\": \"Value1,Value2\"}} or " + "string \"Field1=Value1,Value2|Field2=Value3\".") + W101 = ("Skipping Doc custom extension '{name}' while merging docs.") + W102 = ("Skipping unsupported user data '{key}: {value}' while merging docs.") + W103 = ("Unknown {lang} word segmenter '{segmenter}'. Supported " + "word segmenters: {supported}. Defaulting to {default}.") + W104 = ("Skipping modifications for '{target}' segmenter. The current " + "segmenter is '{current}'.") + W105 = ("As of spaCy v3.0, the `{matcher}.pipe` method is deprecated. If you " + "need to match on a stream of documents, you can use `nlp.pipe` and " + "call the {matcher} on each Doc object.") + W107 = ("The property `Doc.{prop}` is deprecated. Use " + "`Doc.has_annotation(\"{attr}\")` instead.") + W108 = ("The rule-based lemmatizer did not find POS annotation for one or " + "more tokens. Check that your pipeline includes components that " + "assign token.pos, typically 'tagger'+'attribute_ruler' or " + "'morphologizer'.") + W109 = ("Unable to save user hooks while serializing the doc. Re-add any " + "required user hooks to the doc after processing.") + W110 = ("The DependencyMatcher token pattern {pattern} matched a span " + "{tokens} that is 2+ tokens long. Only the first token in the span " + "will be included in the results. For better results, token " + "patterns should return matches that are each exactly one token " + "long.") + W111 = ("Jupyter notebook detected: if using `prefer_gpu()` or " + "`require_gpu()`, include it in the same cell right before " + "`spacy.load()` to ensure that the model is loaded on the correct " + "device. More information: " + "http://spacy.io/usage/v3#jupyter-notebook-gpu") + W112 = ("The model specified to use for initial vectors ({name}) has no " + "vectors. This is almost certainly a mistake.") + W113 = ("Sourced component '{name}' may not work as expected: source " + "vectors are not identical to current pipeline vectors.") + W114 = ("Using multiprocessing with GPU models is not recommended and may " + "lead to errors.") + W115 = ("Skipping {method}: the floret vector table cannot be modified. " + "Vectors are calculated from character ngrams.") + W116 = ("Unable to clean attribute '{attr}'.") + W117 = ("No spans to visualize found in Doc object with spans_key: '{spans_key}'. If this is " + "surprising to you, make sure the Doc was processed using a model " + "that supports span categorization, and check the `doc.spans[spans_key]` " + "property manually if necessary.\n\nAvailable keys: {keys}") + W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation " + "for the corpora used to train the language. Please check " + "`nlp.meta[\"sources\"]` for any relevant links.") + W119 = ("Overriding pipe name in `config` is not supported. Ignoring override '{name_in_config}'.") + W120 = ("Unable to load all spans in Doc.spans: more than one span group " + "with the name '{group_name}' was found in the saved spans data. " + "Only the last span group will be loaded under " + "Doc.spans['{group_name}']. Skipping span group with values: " + "{group_values}") + W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'") + W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class " + "is a Cython extension type.") + W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option " + "`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.") + W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.") + W125 = ("The StaticVectors key_attr is no longer used. To set a custom " + "key attribute for vectors, configure it through Vectors(attr=) or " + "'spacy init vectors --attr'") + W126 = ("These keys are unsupported: {unsupported}") + W127 = ("Not all `Language.pipe` worker processes completed successfully") + + +class Errors(metaclass=ErrorsWithCodes): + E001 = ("No component '{name}' found in pipeline. Available names: {opts}") + E002 = ("Can't find factory for '{name}' for language {lang} ({lang_code}). " + "This usually happens when spaCy calls `nlp.{method}` with a custom " + "component name that's not registered on the current language class. " + "If you're using a custom component, make sure you've added the " + "decorator `@Language.component` (for function components) or " + "`@Language.factory` (for class components).\n\nAvailable " + "factories: {opts}") + E003 = ("Not a valid pipeline component. Expected callable, but " + "got {component} (name: '{name}'). If you're using a custom " + "component factory, double-check that it correctly returns your " + "initialized component.") + E004 = ("Can't set up pipeline component: a factory for '{name}' already " + "exists. Existing factory: {func}. New factory: {new_func}") + E005 = ("Pipeline component '{name}' returned {returned_type} instead of a " + "Doc. If you're using a custom component, maybe you forgot to " + "return the processed Doc?") + E006 = ("Invalid constraints for adding pipeline component. You can only " + "set one of the following: before (component name or index), " + "after (component name or index), first (True) or last (True). " + "Invalid configuration: {args}. Existing components: {opts}") + E007 = ("'{name}' already exists in pipeline. Existing names: {opts}") + E008 = ("Can't restore disabled pipeline component '{name}' because it " + "doesn't exist in the pipeline anymore. If you want to remove " + "components from the pipeline, you should do it before calling " + "`nlp.select_pipes` or after restoring the disabled components.") + E010 = ("Word vectors set to length 0. This may be because you don't have " + "a model installed or loaded, or because your model doesn't " + "include word vectors. For more info, see the docs:\n" + "https://spacy.io/usage/models") + E011 = ("Unknown operator: '{op}'. Options: {opts}") + E012 = ("Cannot add pattern for zero tokens to matcher.\nKey: {key}") + E016 = ("MultitaskObjective target should be function or one of: dep, " + "tag, ent, dep_tag_offset, ent_tag.") + E017 = ("Can only add unicode or bytes. Got type: {value_type}") + E018 = ("Can't retrieve string for hash '{hash_value}'. This usually " + "refers to an issue with the `Vocab` or `StringStore`.") + E019 = ("Can't create transition with unknown action ID: {action}. Action " + "IDs are enumerated in spacy/syntax/{src}.pyx.") + E022 = ("Could not find a transition with the name '{name}' in the NER " + "model.") + E024 = ("Could not find an optimal move to supervise the parser. Usually, " + "this means that the model can't be updated in a way that's valid " + "and satisfies the correct annotations specified in the GoldParse. " + "For example, are all labels added to the model? If you're " + "training a named entity recognizer, also make sure that none of " + "your annotated entity spans have leading or trailing whitespace " + "or punctuation. You can also use the `debug data` command to " + "validate your JSON-formatted training data. For details, run:\n" + "python -m spacy debug data --help") + E025 = ("String is too long: {length} characters. Max is 2**30.") + E026 = ("Error accessing token at position {i}: out of bounds in Doc of " + "length {length}.") + E027 = ("Arguments `words` and `spaces` should be sequences of the same " + "length, or `spaces` should be left default at None. `spaces` " + "should be a sequence of booleans, with True meaning that the " + "word owns a ' ' character following it.") + E028 = ("`words` expects a list of unicode strings, but got bytes instance: {value}") + E029 = ("`noun_chunks` requires the dependency parse, which requires a " + "statistical model to be installed and loaded. For more info, see " + "the documentation:\nhttps://spacy.io/usage/models") + E030 = ("Sentence boundaries unset. You can add the 'sentencizer' " + "component to the pipeline with: `nlp.add_pipe('sentencizer')`. " + "Alternatively, add the dependency parser or sentence recognizer, " + "or set sentence boundaries by setting `doc[i].is_sent_start`.") + E031 = ("Invalid token: empty string ('') at position {i}.") + E033 = ("Cannot load into non-empty Doc of length {length}.") + E035 = ("Error creating span with start {start} and end {end} for Doc of " + "length {length}.") + E036 = ("Error calculating span: Can't find a token starting at character " + "offset {start}.") + E037 = ("Error calculating span: Can't find a token ending at character " + "offset {end}.") + E039 = ("Array bounds exceeded while searching for root word. This likely " + "means the parse tree is in an invalid state. Please report this " + "issue here: http://github.com/explosion/spaCy/issues") + E040 = ("Attempt to access token at {i}, max length {max_length}.") + E041 = ("Invalid comparison operator: {op}. Likely a Cython bug?") + E042 = ("Error accessing `doc[{i}].nbor({j})`, for doc of length {length}.") + E043 = ("Refusing to write to token.sent_start if its document is parsed, " + "because this may cause inconsistent state.") + E044 = ("Invalid value for token.sent_start: {value}. Must be one of: " + "None, True, False") + E045 = ("Possibly infinite loop encountered while looking for {attr}.") + E046 = ("Can't retrieve unregistered extension attribute '{name}'. Did " + "you forget to call the `set_extension` method?") + E047 = ("Can't assign a value to unregistered extension attribute " + "'{name}'. Did you forget to call the `set_extension` method?") + E048 = ("Can't import language {lang} or any matching language from spacy.lang: {err}") + E050 = ("Can't find model '{name}'. It doesn't seem to be a Python " + "package or a valid path to a data directory.") + E052 = ("Can't find model directory: {path}") + E053 = ("Could not read {name} from {path}") + E054 = ("No valid '{setting}' setting found in model meta.json.") + E055 = ("Invalid ORTH value in exception:\nKey: {key}\nOrths: {orths}") + E056 = ("Invalid tokenizer exception: ORTH values combined don't match " + "original string.\nKey: {key}\nOrths: {orths}") + E057 = ("Stepped slices not supported in Span objects. Try: " + "`list(tokens)[start:stop:step]` instead.") + E058 = ("Could not retrieve vector for key {key}.") + E059 = ("One (and only one) keyword arg must be set. Got: {kwargs}") + E060 = ("Cannot add new key to vectors: the table is full. Current shape: " + "({rows}, {cols}).") + E062 = ("Cannot find empty bit for new lexical flag. All bits between 0 " + "and 63 are occupied. You can replace one by specifying the " + "`flag_id` explicitly, e.g. " + "`nlp.vocab.add_flag(your_func, flag_id=IS_ALPHA`.") + E063 = ("Invalid value for `flag_id`: {value}. Flag IDs must be between 1 " + "and 63 (inclusive).") + E064 = ("Error fetching a Lexeme from the Vocab. When looking up a " + "string, the lexeme returned had an orth ID that did not match " + "the query string. This means that the cached lexeme structs are " + "mismatched to the string encoding table. The mismatched:\n" + "Query string: {string}\nOrth cached: {orth}\nOrth ID: {orth_id}") + E065 = ("Only one of the vector table's width and shape can be specified. " + "Got width {width} and shape {shape}.") + E067 = ("Invalid BILUO tag sequence: Got a tag starting with {start} " + "without a preceding 'B' (beginning of an entity). " + "Tag sequence:\n{tags}") + E068 = ("Invalid BILUO tag: '{tag}'.") + E071 = ("Error creating lexeme: specified orth ID ({orth}) does not " + "match the one in the vocab ({vocab_orth}).") + E073 = ("Cannot assign vector of length {new_length}. Existing vectors " + "are of length {length}. You can use `vocab.reset_vectors` to " + "clear the existing vectors and resize the table.") + E074 = ("Error interpreting compiled match pattern: patterns are expected " + "to end with the attribute {attr}. Got: {bad_attr}.") + E079 = ("Error computing states in beam: number of predicted beams " + "({pbeams}) does not equal number of gold beams ({gbeams}).") + E080 = ("Duplicate state found in beam: {key}.") + E081 = ("Error getting gradient in beam: number of histories ({n_hist}) " + "does not equal number of losses ({losses}).") + E082 = ("Error deprojectivizing parse: number of heads ({n_heads}), " + "projective heads ({n_proj_heads}) and labels ({n_labels}) do not " + "match.") + E083 = ("Error setting extension: only one of `default`, `method`, or " + "`getter` (plus optional `setter`) is allowed. Got: {nr_defined}") + E084 = ("Error assigning label ID {label} to span: not in StringStore.") + E085 = ("Can't create lexeme for string '{string}'.") + E087 = ("Unknown displaCy style: {style}.") + E088 = ("Text of length {length} exceeds maximum of {max_length}. The " + "parser and NER models require roughly 1GB of temporary " + "memory per 100,000 characters in the input. This means long " + "texts may cause memory allocation errors. If you're not using " + "the parser or NER, it's probably safe to increase the " + "`nlp.max_length` limit. The limit is in number of characters, so " + "you can check whether your inputs are too long by checking " + "`len(text)`.") + E089 = ("Extensions can't have a setter argument without a getter " + "argument. Check the keyword arguments on `set_extension`.") + E090 = ("Extension '{name}' already exists on {obj}. To overwrite the " + "existing extension, set `force=True` on `{obj}.set_extension`.") + E091 = ("Invalid extension attribute {name}: expected callable or None, " + "but got: {value}") + E093 = ("token.ent_iob values make invalid sequence: I without B\n{seq}") + E094 = ("Error reading line {line_num} in vectors file {loc}.") + E095 = ("Can't write to frozen dictionary. This is likely an internal " + "error. Are you writing to a default function argument?") + E096 = ("Invalid object passed to displaCy: Can only visualize `Doc` or " + "Span objects, or dicts if set to `manual=True`.") + E097 = ("Invalid pattern: expected token pattern (list of dicts) or " + "phrase pattern (string) but got:\n{pattern}") + E098 = ("Invalid pattern: expected both RIGHT_ID and RIGHT_ATTRS.") + E099 = ("Invalid pattern: the first node of pattern should be an anchor " + "node. The node should only contain RIGHT_ID and RIGHT_ATTRS.") + E100 = ("Nodes other than the anchor node should all contain {required}, " + "but these are missing: {missing}") + E101 = ("RIGHT_ID should be a new node and LEFT_ID should already have " + "have been declared in previous edges.") + E102 = ("Can't merge non-disjoint spans. '{token}' is already part of " + "tokens to merge. If you want to find the longest non-overlapping " + "spans, you can use the util.filter_spans helper:\n" + "https://spacy.io/api/top-level#util.filter_spans") + E103 = ("Trying to set conflicting doc.ents: '{span1}' and '{span2}'. A " + "token can only be part of one entity, so make sure the entities " + "you're setting don't overlap. To work with overlapping entities, " + "consider using doc.spans instead.") + E106 = ("Can't find `doc._.{attr}` attribute specified in the underscore " + "settings: {opts}") + E107 = ("Value of custom attribute `{attr}` is not JSON-serializable: {value}") + E109 = ("Component '{name}' could not be run. Did you forget to " + "call `initialize()`?") + E110 = ("Invalid displaCy render wrapper. Expected callable, got: {obj}") + E111 = ("Pickling a token is not supported, because tokens are only views " + "of the parent Doc and can't exist on their own. A pickled token " + "would always have to include its Doc and Vocab, which has " + "practically no advantage over pickling the parent Doc directly. " + "So instead of pickling the token, pickle the Doc it belongs to.") + E112 = ("Pickling a span is not supported, because spans are only views " + "of the parent Doc and can't exist on their own. A pickled span " + "would always have to include its Doc and Vocab, which has " + "practically no advantage over pickling the parent Doc directly. " + "So instead of pickling the span, pickle the Doc it belongs to or " + "use Span.as_doc to convert the span to a standalone Doc object.") + E115 = ("All subtokens must have associated heads.") + E117 = ("The newly split tokens must match the text of the original token. " + "New orths: {new}. Old text: {old}.") + E118 = ("The custom extension attribute '{attr}' is not registered on the " + "`Token` object so it can't be set during retokenization. To " + "register an attribute, use the `Token.set_extension` classmethod.") + E119 = ("Can't set custom extension attribute '{attr}' during " + "retokenization because it's not writable. This usually means it " + "was registered with a getter function (and no setter) or as a " + "method extension, so the value is computed dynamically. To " + "overwrite a custom attribute manually, it should be registered " + "with a default value or with a getter AND setter.") + E120 = ("Can't set custom extension attributes during retokenization. " + "Expected dict mapping attribute names to values, but got: {value}") + E121 = ("Can't bulk merge spans. Attribute length {attr_len} should be " + "equal to span length ({span_len}).") + E122 = ("Cannot find token to be split. Did it get merged?") + E123 = ("Cannot find head of token to be split. Did it get merged?") + E125 = ("Unexpected value: {value}") + E126 = ("Unexpected matcher predicate: '{bad}'. Expected one of: {good}. " + "This is likely a bug in spaCy, so feel free to open an issue.") + E130 = ("You are running a narrow unicode build, which is incompatible " + "with spacy >= 2.1.0. To fix this, reinstall Python and use a wide " + "unicode build instead. You can also rebuild Python and set the " + "`--enable-unicode=ucs4 flag`.") + E132 = ("The vectors for entities and probabilities for alias '{alias}' " + "should have equal length, but found {entities_length} and " + "{probabilities_length} respectively.") + E133 = ("The sum of prior probabilities for alias '{alias}' should not " + "exceed 1, but found {sum}.") + E134 = ("Entity '{entity}' is not defined in the Knowledge Base.") + E139 = ("Knowledge base for component '{name}' is empty.") + E140 = ("The list of entities, prior probabilities and entity vectors " + "should be of equal length.") + E141 = ("Entity vectors should be of length {required} instead of the " + "provided {found}.") + E143 = ("Labels for component '{name}' not initialized. This can be fixed " + "by calling add_label, or by providing a representative batch of " + "examples to the component's `initialize` method.") + E145 = ("Error reading `{param}` from input file.") + E146 = ("Could not access {path}.") + E147 = ("Unexpected error in the {method} functionality of the " + "EntityLinker: {msg}. This is likely a bug in spaCy, so feel free " + "to open an issue: https://github.com/explosion/spaCy/issues") + E148 = ("Expected {ents} KB identifiers but got {ids}. Make sure that " + "each entity in `doc.ents` is assigned to a KB identifier.") + E149 = ("Error deserializing model. Check that the config used to create " + "the component matches the model being loaded.") + E150 = ("The language of the `nlp` object and the `vocab` should be the " + "same, but found '{nlp}' and '{vocab}' respectively.") + E152 = ("The attribute {attr} is not supported for token patterns. " + "Please use the option `validate=True` with the Matcher, PhraseMatcher, " + "EntityRuler or AttributeRuler for more details.") + E153 = ("The value type {vtype} is not supported for token patterns. " + "Please use the option validate=True with Matcher, PhraseMatcher, " + "EntityRuler or AttributeRuler for more details.") + E154 = ("One of the attributes or values is not supported for token " + "patterns. Please use the option `validate=True` with the Matcher, " + "PhraseMatcher, or EntityRuler for more details.") + E155 = ("The pipeline needs to include a {pipe} in order to use " + "Matcher or PhraseMatcher with the attribute {attr}. " + "Try using `nlp()` instead of `nlp.make_doc()` or `list(nlp.pipe())` " + "instead of `list(nlp.tokenizer.pipe())`.") + E157 = ("Can't render negative values for dependency arc start or end. " + "Make sure that you're passing in absolute token indices, not " + "relative token offsets.\nstart: {start}, end: {end}, label: " + "{label}, direction: {dir}") + E158 = ("Can't add table '{name}' to lookups because it already exists.") + E159 = ("Can't find table '{name}' in lookups. Available tables: {tables}") + E160 = ("Can't find language data file: {path}") + E161 = ("Found an internal inconsistency when predicting entity links. " + "This is likely a bug in spaCy, so feel free to open an issue: " + "https://github.com/explosion/spaCy/issues") + E163 = ("cumsum was found to be unstable: its last element does not " + "correspond to sum") + E164 = ("x is neither increasing nor decreasing: {x}.") + E165 = ("Only one class present in the gold labels: {label}. " + "ROC AUC score is not defined in that case.") + E166 = ("Can only merge DocBins with the same value for '{param}'.\n" + "Current DocBin: {current}\nOther DocBin: {other}") + E169 = ("Can't find module: {module}") + E170 = ("Cannot apply transition {name}: invalid for the current state.") + E171 = ("Matcher.add received invalid 'on_match' callback argument: expected " + "callable or None, but got: {arg_type}") + E175 = ("Can't remove rule for unknown match pattern ID: {key}") + E176 = ("Alias '{alias}' is not defined in the Knowledge Base.") + E177 = ("Ill-formed IOB input detected: {tag}") + E178 = ("Each pattern should be a list of dicts, but got: {pat}. Maybe you " + "accidentally passed a single pattern to Matcher.add instead of a " + "list of patterns? If you only want to add one pattern, make sure " + "to wrap it in a list. For example: `matcher.add('{key}', [pattern])`") + E179 = ("Invalid pattern. Expected a list of Doc objects but got a single " + "Doc. If you only want to add one pattern, make sure to wrap it " + "in a list. For example: `matcher.add('{key}', [doc])`") + E180 = ("Span attributes can't be declared as required or assigned by " + "components, since spans are only views of the Doc. Use Doc and " + "Token attributes (or custom extension attributes) only and remove " + "the following: {attrs}") + E181 = ("Received invalid attributes for unknown object {obj}: {attrs}. " + "Only Doc and Token attributes are supported.") + E182 = ("Received invalid attribute declaration: {attr}\nDid you forget " + "to define the attribute? For example: `{attr}.???`") + E183 = ("Received invalid attribute declaration: {attr}\nOnly top-level " + "attributes are supported, for example: {solution}") + E184 = ("Only attributes without underscores are supported in component " + "attribute declarations (because underscore and non-underscore " + "attributes are connected anyways): {attr} -> {solution}") + E185 = ("Received invalid attribute in component attribute declaration: " + "`{obj}.{attr}`\nAttribute '{attr}' does not exist on {obj}.") + E187 = ("Only unicode strings are supported as labels.") + E189 = ("Each argument to `Doc.__init__` should be of equal length.") + E190 = ("Token head out of range in `Doc.from_array()` for token index " + "'{index}' with value '{value}' (equivalent to relative head " + "index: '{rel_head_index}'). The head indices should be relative " + "to the current token index rather than absolute indices in the " + "array.") + E191 = ("Invalid head: the head token must be from the same doc as the " + "token itself.") + E192 = ("Unable to resize vectors in place with cupy.") + E193 = ("Unable to resize vectors in place if the resized vector dimension " + "({new_dim}) is not the same as the current vector dimension " + "({curr_dim}).") + E194 = ("Unable to aligned mismatched text '{text}' and words '{words}'.") + E195 = ("Matcher can be called on {good} only, got {got}.") + E196 = ("Refusing to write to `token.is_sent_end`. Sentence boundaries can " + "only be fixed with `token.is_sent_start`.") + E197 = ("Row out of bounds, unable to add row {row} for key {key}.") + E198 = ("Unable to return {n} most similar vectors for the current vectors " + "table, which contains {n_rows} vectors.") + E199 = ("Unable to merge 0-length span at `doc[{start}:{end}]`.") + E200 = ("Can't set {attr} from Span.") + E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.") + E203 = ("If the {name} embedding layer is not updated " + "during training, make sure to include it in 'annotating components'") + + # New errors added in v3.x + E849 = ("The vocab only supports {method} for vectors of type " + "spacy.vectors.Vectors, not {vectors_type}.") + E850 = ("The PretrainVectors objective currently only supports default or " + "floret vectors, not {mode} vectors.") + E851 = ("The 'textcat' component labels should only have values of 0 or 1, " + "but found value of '{val}'.") + E853 = ("Unsupported component factory name '{name}'. The character '.' is " + "not permitted in factory names.") + E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not " + "permit overlapping spans.") + E855 = ("Invalid {obj}: {obj} is not from the same doc.") + E856 = ("Error accessing span at position {i}: out of bounds in span group " + "of length {length}.") + E857 = ("Entry '{name}' not found in edit tree lemmatizer labels.") + E858 = ("The {mode} vector table does not support this operation. " + "{alternative}") + E859 = ("The floret vector table cannot be modified.") + E860 = ("Can't truncate floret vectors.") + E861 = ("No 'keys' should be provided when initializing floret vectors " + "with 'minn' and 'maxn'.") + E862 = ("'hash_count' must be between 1-4 for floret vectors.") + E863 = ("'maxn' must be greater than or equal to 'minn'.") + E864 = ("The complete vector table 'data' is required to initialize floret " + "vectors.") + E865 = ("A SpanGroup is not functional after the corresponding Doc has " + "been garbage collected. To keep using the spans, make sure that " + "the corresponding Doc object is still available in the scope of " + "your function.") + E866 = ("Expected a string or 'Doc' as input, but got: {type}.") + E867 = ("The 'textcat' component requires at least two labels because it " + "uses mutually exclusive classes where exactly one label is True " + "for each doc. For binary classification tasks, you can use two " + "labels with 'textcat' (LABEL / NOT_LABEL) or alternatively, you " + "can use the 'textcat_multilabel' component with one label.") + E868 = ("Found a conflicting gold annotation in a reference document, " + "with the following char-based span occurring both in the gold ents " + "as well as in the negative spans: {span}.") + E869 = ("The notation '{label}' is not supported anymore. To annotate " + "negative NER samples, use `doc.spans[key]` instead, and " + "specify the key as 'incorrect_spans_key' when constructing " + "the NER component.") + E870 = ("Could not serialize the DocBin because it is too large. Consider " + "splitting up your documents into several doc bins and serializing " + "each separately. spacy.Corpus.v1 will search recursively for all " + "*.spacy files if you provide a directory instead of a filename as " + "the 'path'.") + E871 = ("Error encountered in nlp.pipe with multiprocessing:\n\n{error}") + E872 = ("Unable to copy tokenizer from base model due to different " + 'tokenizer settings: current tokenizer config "{curr_config}" ' + 'vs. base model "{base_config}"') + E873 = ("Unable to merge a span from doc.spans with key '{key}' and text " + "'{text}'. This is likely a bug in spaCy, so feel free to open an " + "issue: https://github.com/explosion/spaCy/issues") + E874 = ("Could not initialize the tok2vec model from component " + "'{component}' and layer '{layer}'.") + E875 = ("To use the PretrainVectors objective, make sure that static vectors are loaded. " + "In the config, these are defined by the initialize.vectors setting.") + E879 = ("Unexpected type for 'spans' data. Provide a dictionary mapping keys to " + "a list of spans, with each span represented by a tuple (start_char, end_char). " + "The tuple can be optionally extended with a label and a KB ID.") + E884 = ("The pipeline could not be initialized because the vectors " + "could not be found at '{vectors}'. If your pipeline was already " + "initialized/trained before, call 'resume_training' instead of 'initialize', " + "or initialize only the components that are new.") + E885 = ("entity_linker.set_kb received an invalid 'kb_loader' argument: expected " + "a callable function, but got: {arg_type}") + E886 = ("Can't replace {name} -> {tok2vec} listeners: path '{path}' not " + "found in config for component '{name}'.") + E887 = ("Can't replace {name} -> {tok2vec} listeners: the paths to replace " + "({paths}) don't match the available listeners in the model ({n_listeners}).") + E888 = ("Can't replace listeners for '{name}' ({pipe}): invalid upstream " + "component that doesn't seem to support listeners. Expected Tok2Vec " + "or Transformer component. If you didn't call nlp.replace_listeners " + "manually, this is likely a bug in spaCy.") + E889 = ("Can't replace '{tok2vec}' listeners of component '{name}' because " + "'{unknown}' is not in the pipeline. Available components: {opts}. " + "If you didn't call nlp.replace_listeners manually, this is likely " + "a bug in spaCy.") + E890 = ("Cannot add the alias '{alias}' to the Knowledge base. " + "Each alias should be a meaningful string.") + E891 = ("Alias '{alias}' could not be added to the Knowledge base. " + "This is likely a bug in spaCy.") + E892 = ("Unknown function registry: '{name}'.\n\nAvailable names: {available}") + E893 = ("Could not find function '{name}' in function registry '{reg_name}'. " + "If you're using a custom function, make sure the code is available. " + "If the function is provided by a third-party package, e.g. " + "spacy-transformers, make sure the package is installed in your " + "environment.\n\nAvailable names: {available}") + E894 = ("The 'noun_chunks' syntax iterator is not implemented for language " + "'{lang}'.") + E895 = ("The 'textcat' component received gold-standard annotations with " + "multiple labels per document. In spaCy 3 you should use the " + "'textcat_multilabel' component for this instead. " + "Example of an offending annotation: {value}") + E896 = ("There was an error using the static vectors. Ensure that the vectors " + "of the vocab are properly initialized, or set 'include_static_vectors' " + "to False.") + E897 = ("Field '{field}' should be a dot-notation string referring to the " + "relevant section in the config, but found type {type} instead.") + E898 = ("Can't serialize trainable pipe '{name}': the `model` attribute " + "is not set or None. If you've implemented a custom component, make " + "sure to store the component model as `self.model` in your " + "component's __init__ method.") + E899 = ("Can't serialize trainable pipe '{name}': the `vocab` attribute " + "is not set or None. If you've implemented a custom component, make " + "sure to store the current `nlp` object's vocab as `self.vocab` in " + "your component's __init__ method.") + E900 = ("Could not run the full pipeline for evaluation. If you specified " + "frozen components, make sure they were already initialized and " + "trained. Full pipeline: {pipeline}") + E901 = ("Failed to remove existing output directory: {path}. If your " + "config and the components you train change between runs, a " + "non-empty output directory can lead to stale pipeline data. To " + "solve this, remove the existing directories in the output directory.") + E902 = ("The sentence-per-line IOB/IOB2 file is not formatted correctly. " + "Try checking whitespace and delimiters. See " + "https://spacy.io/api/cli#convert") + E903 = ("The token-per-line NER file is not formatted correctly. Try checking " + "whitespace and delimiters. See https://spacy.io/api/cli#convert") + E904 = ("Cannot initialize StaticVectors layer: nO dimension unset. This " + "dimension refers to the output width, after the linear projection " + "has been applied.") + E905 = ("Cannot initialize StaticVectors layer: nM dimension unset. This " + "dimension refers to the width of the vectors table.") + E906 = ("Unexpected `loss` value in pretraining objective: '{found}'. Supported values " + "are: {supported}") + E908 = ("Can't set `spaces` without `words` in `Doc.__init__`.") + E909 = ("Expected {name} in parser internals. This is likely a bug in spaCy.") + E910 = ("Encountered NaN value when computing loss for component '{name}'.") + E911 = ("Invalid feature: {feat}. Must be a token attribute.") + E912 = ("Failed to initialize lemmatizer. Missing lemmatizer table(s) found " + "for mode '{mode}'. Required tables: {tables}. Found: {found}.") + E913 = ("Corpus path can't be None. Maybe you forgot to define it in your " + ".cfg file or override it on the CLI?") + E914 = ("Executing {name} callback failed. Expected the function to " + "return the nlp object but got: {value}. Maybe you forgot to return " + "the modified object in your function?") + E915 = ("Can't use score '{name}' to calculate final weighted score. Expected " + "float or int but got: {score_type}. To exclude the score from the " + "final score, set its weight to null in the [training.score_weights] " + "section of your training config.") + E916 = ("Can't log score for '{name}' in table: not a valid score ({score_type})") + E917 = ("Received invalid value {value} for `state_type` in " + "TransitionBasedParser: only 'parser' or 'ner' are valid options.") + E918 = ("Received invalid value for vocab: {vocab} ({vocab_type}). Valid " + "values are an instance of `spacy.vocab.Vocab` or True to create one" + " (default).") + E919 = ("A textcat `positive_label` '{pos_label}' was provided for training " + "data that does not appear to be a binary classification problem " + "with two labels. Labels found: {labels}") + E920 = ("The textcat's `positive_label` setting '{pos_label}' " + "does not match any label in the training data or provided during " + "initialization. Available labels: {labels}") + E921 = ("The method `set_output` can only be called on components that have " + "a Model with a `resize_output` attribute. Otherwise, the output " + "layer can not be dynamically changed.") + E922 = ("Component '{name}' has been initialized with an output dimension of " + "{nO} - cannot add any more labels.") + E923 = ("It looks like there is no proper sample data to initialize the " + "Model of component '{name}'. To check your input data paths and " + "annotation, run: python -m spacy debug data config.cfg " + "and include the same config override values you would specify " + "for the 'spacy train' command.") + E924 = ("The '{name}' component does not seem to be initialized properly. " + "This is likely a bug in spaCy, so feel free to open an issue: " + "https://github.com/explosion/spaCy/issues") + E925 = ("Invalid color values for displaCy visualizer: expected dictionary " + "mapping label names to colors but got: {obj}") + E926 = ("It looks like you're trying to modify `nlp.{attr}` directly. This " + "doesn't work because it's an immutable computed property. If you " + "need to modify the pipeline, use the built-in methods like " + "`nlp.add_pipe`, `nlp.remove_pipe`, `nlp.disable_pipe` or " + "`nlp.enable_pipe` instead.") + E927 = ("Can't write to frozen list. Maybe you're trying to modify a computed " + "property or default function argument?") + E928 = ("An InMemoryLookupKB can only be serialized to/from from a directory, " + "but the provided argument {loc} points to a file.") + E929 = ("Couldn't read InMemoryLookupKB from {loc}. The path does not seem to exist.") + E930 = ("Received invalid get_examples callback in `{method}`. " + "Expected function that returns an iterable of Example objects but " + "got: {obj}") + E931 = ("Encountered {parent} subclass without `{parent}.{method}` " + "method in component '{name}'. If you want to use this " + "method, make sure it's overwritten on the subclass.") + E940 = ("Found NaN values in scores.") + E941 = ("Can't find model '{name}'. It looks like you're trying to load a " + "model from a shortcut, which is obsolete as of spaCy v3.0. To " + "load the model, use its full name instead:\n\n" + "nlp = spacy.load(\"{full}\")\n\nFor more details on the available " + "models, see the models directory: https://spacy.io/models and if " + "you want to create a blank model, use spacy.blank: " + "nlp = spacy.blank(\"{name}\")") + E942 = ("Executing `after_{name}` callback failed. Expected the function to " + "return an initialized nlp object but got: {value}. Maybe " + "you forgot to return the modified object in your function?") + E943 = ("Executing `before_creation` callback failed. Expected the function to " + "return an uninitialized Language subclass but got: {value}. Maybe " + "you forgot to return the modified object in your function or " + "returned the initialized nlp object instead?") + E944 = ("Can't copy pipeline component '{name}' from source '{model}': " + "not found in pipeline. Available components: {opts}") + E945 = ("Can't copy pipeline component '{name}' from source. Expected " + "loaded nlp object, but got: {source}") + E947 = ("`Matcher.add` received invalid `greedy` argument: expected " + "a string value from {expected} but got: '{arg}'") + E948 = ("`Matcher.add` received invalid 'patterns' argument: expected " + "a list, but got: {arg_type}") + E949 = ("Unable to align tokens for the predicted and reference docs. It " + "is only possible to align the docs when both texts are the same " + "except for whitespace and capitalization. The predicted tokens " + "start with: {x}. The reference tokens start with: {y}.") + E952 = ("The section '{name}' is not a valid section in the provided config.") + E953 = ("Mismatched IDs received by the Tok2Vec listener: {id1} vs. {id2}") + E954 = ("The Tok2Vec listener did not receive any valid input from an upstream " + "component.") + E955 = ("Can't find table(s) {table} for language '{lang}' in " + "spacy-lookups-data. Make sure you have the package installed or " + "provide your own lookup tables if no default lookups are available " + "for your language.") + E956 = ("Can't find component '{name}' in [components] block in the config. " + "Available components: {opts}") + E957 = ("Writing directly to `Language.factories` isn't needed anymore in " + "spaCy v3. Instead, you can use the `@Language.factory` decorator " + "to register your custom component factory or `@Language.component` " + "to register a simple stateless function component that just takes " + "a Doc and returns it.") + E958 = ("Language code defined in config ({bad_lang_code}) does not match " + "language code of current Language subclass {lang} ({lang_code}). " + "If you want to create an nlp object from a config, make sure to " + "use the matching subclass with the language-specific settings and " + "data.") + E959 = ("Can't insert component {dir} index {idx}. Existing components: {opts}") + E960 = ("No config data found for component '{name}'. This is likely a bug " + "in spaCy.") + E961 = ("Found non-serializable Python object in config. Configs should " + "only include values that can be serialized to JSON. If you need " + "to pass models or other objects to your component, use a reference " + "to a registered function or initialize the object in your " + "component.\n\n{config}") + E962 = ("Received incorrect {style} for pipe '{name}'. Expected dict, " + "got: {cfg_type}.") + E963 = ("Can't read component info from `@Language.{decorator}` decorator. " + "Maybe you forgot to call it? Make sure you're using " + "`@Language.{decorator}()` instead of `@Language.{decorator}`.") + E964 = ("The pipeline component factory for '{name}' needs to have the " + "following named arguments, which are passed in by spaCy:\n- nlp: " + "receives the current nlp object and lets you access the vocab\n- " + "name: the name of the component instance, can be used to identify " + "the component, output losses etc.") + E965 = ("It looks like you're using the `@Language.component` decorator to " + "register '{name}' on a class instead of a function component. If " + "you need to register a class or function that *returns* a component " + "function, use the `@Language.factory` decorator instead.") + E966 = ("`nlp.add_pipe` now takes the string name of the registered component " + "factory, not a callable component. Expected string, but got " + "{component} (name: '{name}').\n\n- If you created your component " + "with `nlp.create_pipe('name')`: remove nlp.create_pipe and call " + "`nlp.add_pipe('name')` instead.\n\n- If you passed in a component " + "like `TextCategorizer()`: call `nlp.add_pipe` with the string name " + "instead, e.g. `nlp.add_pipe('textcat')`.\n\n- If you're using a custom " + "component: Add the decorator `@Language.component` (for function " + "components) or `@Language.factory` (for class components / factories) " + "to your custom component and assign it a name, e.g. " + "`@Language.component('your_name')`. You can then run " + "`nlp.add_pipe('your_name')` to add it to the pipeline.") + E967 = ("No {meta} meta information found for '{name}'. This is likely a bug in spaCy.") + E968 = ("`nlp.replace_pipe` now takes the string name of the registered component " + "factory, not a callable component. Expected string, but got " + "{component}.\n\n- If you created your component with" + "with `nlp.create_pipe('name')`: remove `nlp.create_pipe` and call " + "`nlp.replace_pipe('{name}', 'name')` instead.\n\n- If you passed in a " + "component like `TextCategorizer()`: call `nlp.replace_pipe` with the " + "string name instead, e.g. `nlp.replace_pipe('{name}', 'textcat')`.\n\n" + "- If you're using a custom component: Add the decorator " + "`@Language.component` (for function components) or `@Language.factory` " + "(for class components / factories) to your custom component and " + "assign it a name, e.g. `@Language.component('your_name')`. You can " + "then run `nlp.replace_pipe('{name}', 'your_name')`.") + E969 = ("Expected string values for field '{field}', but received {types} instead. ") + E970 = ("Can not execute command '{str_command}'. Do you have '{tool}' installed?") + E971 = ("Found incompatible lengths in `Doc.from_array`: {array_length} for the " + "array and {doc_length} for the Doc itself.") + E972 = ("`Example.__init__` got None for '{arg}'. Requires Doc.") + E973 = ("Unexpected type for NER data") + E974 = ("Unknown {obj} attribute: {key}") + E976 = ("The method `Example.from_dict` expects a {type} as {n} argument, " + "but received None.") + E977 = ("Can not compare a MorphAnalysis with a string object. " + "This is likely a bug in spaCy, so feel free to open an issue: " + "https://github.com/explosion/spaCy/issues") + E978 = ("The {name} method takes a list of Example objects, but got: {types}") + E980 = ("Each link annotation should refer to a dictionary with at most one " + "identifier mapping to 1.0, and all others to 0.0.") + E981 = ("The offsets of the annotations for `links` could not be aligned " + "to token boundaries.") + E982 = ("The `Token.ent_iob` attribute should be an integer indexing " + "into {values}, but found {value}.") + E983 = ("Invalid key(s) for '{dict}': {key}. Available keys: " + "{keys}") + E984 = ("Invalid component config for '{name}': component block needs either " + "a key `factory` specifying the registered function used to " + "initialize the component, or a key `source` key specifying a " + "spaCy model to copy the component from. For example, `factory = " + "\"ner\"` will use the 'ner' factory and all other settings in the " + "block will be passed to it as arguments. Alternatively, `source = " + "\"en_core_web_sm\"` will copy the component from that model.\n\n{config}") + E985 = ("Can't load model from config file: no [nlp] section found.\n\n{config}") + E986 = ("Could not create any training batches: check your input. " + "Are the train and dev paths defined? Is `discard_oversize` set appropriately? ") + E989 = ("`nlp.update()` was called with two positional arguments. This " + "may be due to a backwards-incompatible change to the format " + "of the training data in spaCy 3.0 onwards. The 'update' " + "function should now be called with a batch of Example " + "objects, instead of `(text, annotation)` tuples. ") + E991 = ("The function `nlp.select_pipes` should be called with either a " + "`disable` argument to list the names of the pipe components " + "that should be disabled, or with an 'enable' argument that " + "specifies which pipes should not be disabled.") + E992 = ("The function `select_pipes` was called with `enable`={enable} " + "and `disable`={disable} but that information is conflicting " + "for the `nlp` pipeline with components {names}.") + E993 = ("The config for the nlp object needs to include a key `lang` specifying " + "the code of the language to initialize it with (for example " + "'en' for English) - this can't be None.\n\n{config}") + E997 = ("Tokenizer special cases are not allowed to modify the text. " + "This would map '{chunk}' to '{orth}' given token attributes " + "'{token_attrs}'.") + E999 = ("Unable to merge the Doc objects because they do not all share " + "the same `Vocab`.") + E1000 = ("The Chinese word segmenter is pkuseg but no pkuseg model was " + "loaded. Provide the name of a pretrained model or the path to " + "a model and initialize the pipeline:\n\n" + 'nlp.tokenizer.initialize(pkuseg_model="default")') + E1001 = ("Target token outside of matched span for match with tokens " + "'{span}' and offset '{index}' matched by patterns '{patterns}'.") + E1002 = ("Span index out of range.") + E1003 = ("Unsupported lemmatizer mode '{mode}'.") + E1004 = ("Missing lemmatizer table(s) found for lemmatizer mode '{mode}'. " + "Required tables: {tables}. Found: {found}. Maybe you forgot to " + "call `nlp.initialize()` to load in the data?") + E1005 = ("Unable to set attribute '{attr}' in tokenizer exception for " + "'{chunk}'. Tokenizer exceptions are only allowed to specify " + "ORTH and NORM.") + E1007 = ("Unsupported DependencyMatcher operator '{op}'.") + E1008 = ("Invalid pattern: each pattern should be a list of dicts. Check " + "that you are providing a list of patterns as `List[List[dict]]`.") + E1010 = ("Unable to set entity information for token {i} which is included " + "in more than one span in entities, blocked, missing or outside.") + E1011 = ("Unsupported default '{default}' in `doc.set_ents`. Available " + "options: {modes}") + E1012 = ("Entity spans and blocked/missing/outside spans should be " + "provided to `doc.set_ents` as lists of Span objects.") + E1013 = ("Invalid morph: the MorphAnalysis must have the same vocab as the " + "token itself. To set the morph from this MorphAnalysis, set from " + "the string value with: `token.set_morph(str(other_morph))`.") + E1014 = ("Error loading DocBin data. It doesn't look like the data is in " + "DocBin (.spacy) format. If your data is in spaCy v2's JSON " + "training format, convert it using `python -m spacy convert " + "file.json .`.") + E1015 = ("Can't initialize model from config: no {value} found. For more " + "information, run: python -m spacy debug config config.cfg") + E1016 = ("The operators 'OP': '?', '*', and '+' are not supported in " + "DependencyMatcher token patterns. The token pattern in " + "RIGHT_ATTR should return matches that are each exactly one token " + "long. Invalid pattern:\n{node}") + E1017 = ("A Doc object requires both 'deps' and 'heads' for dependency " + "parses. If no dependency labels are available, provide " + "placeholder deps such as `deps=[\"dep\"]*len(heads)`.") + E1018 = ("Knowledge base for component '{name}' is not set. " + "Make sure either `nel.initialize` or `nel.set_kb` " + "is called with a `kb_loader` function.") + E1019 = ("`noun_chunks` requires the pos tagging, which requires a " + "statistical model to be installed and loaded. For more info, see " + "the documentation:\nhttps://spacy.io/usage/models") + E1020 = ("No `epoch_resume` value specified and could not infer one from " + "filename. Specify an epoch to resume from.") + E1021 = ("`pos` value \"{pp}\" is not a valid Universal Dependencies tag. " + "Non-UD tags should use the `tag` property.") + E1022 = ("Words must be of type str or int, but input is of type '{wtype}'") + E1023 = ("Couldn't read EntityRuler from the {path}. This file doesn't " + "exist.") + E1024 = ("A pattern with {attr_type} '{label}' is not present in " + "'{component}' patterns.") + E1025 = ("Cannot intify the value '{value}' as an IOB string. The only " + "supported values are: 'I', 'O', 'B' and ''") + E1026 = ("Edit tree has an invalid format:\n{errors}") + E1027 = ("AlignmentArray only supports slicing with a step of 1.") + E1028 = ("AlignmentArray only supports indexing using an int or a slice.") + E1029 = ("Edit tree cannot be applied to form.") + E1030 = ("Edit tree identifier out of range.") + E1031 = ("Could not find gold transition - see logs above.") + E1032 = ("`{var}` should not be {forbidden}, but received {value}.") + E1033 = ("Dimension {name} invalid -- only nO, nF, nP") + E1034 = ("Node index {i} out of bounds ({length})") + E1035 = ("Token index {i} out of bounds ({length})") + E1036 = ("Cannot index into NoneNode") + E1037 = ("Invalid attribute value '{attr}'.") + E1038 = ("Invalid JSON input: {message}") + E1039 = ("The {obj} start or end annotations (start: {start}, end: {end}) " + "could not be aligned to token boundaries.") + E1040 = ("Doc.from_json requires all tokens to have the same attributes. " + "Some tokens do not contain annotation for: {partial_attrs}") + E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}") + E1042 = ("`enable={enable}` and `disable={disable}` are inconsistent with each other.\nIf you only passed " + "one of `enable` or `disable`, the other argument is specified in your pipeline's configuration.\nIn that " + "case pass an empty list for the previously not specified argument to avoid this error.") + E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got " + "{value}.") + E1044 = ("Expected `candidates_batch_size` to be >= 1, but got: {value}") + E1045 = ("Encountered {parent} subclass without `{parent}.{method}` " + "method in '{name}'. If you want to use this method, make " + "sure it's overwritten on the subclass.") + E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " + "knowledge base, use `InMemoryLookupKB`.") + E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.") + E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}") + E1049 = ("No available port found for displaCy on host {host}. Please specify an available port " + "with `displacy.serve(doc, port=port)`") + E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port=port)` " + "or use `auto_select_port=True` to pick an available port automatically.") + E1051 = ("'allow_overlap' can only be False when max_positive is 1, but found 'max_positive': {max_positive}.") + E1052 = ("Unable to copy spans: the character offsets for the span at " + "index {i} in the span group do not align with the tokenization " + "in the target doc.") + E1053 = ("Both 'min_length' and 'max_length' should be larger than 0, but found" + " 'min_length': {min_length}, 'max_length': {max_length}") + E1054 = ("The text, including whitespace, must match between reference and " + "predicted docs when training {component}.") + E1055 = ("The 'replace_listener' callback expects {num_params} parameters, " + "but only callbacks with one or three parameters are supported") + E1056 = ("The `TextCatBOW` architecture expects a length of at least 1, was {length}.") + E1057 = ("The `TextCatReduce` architecture must be used with at least one " + "reduction. Please enable one of `use_reduce_first`, " + "`use_reduce_last`, `use_reduce_max` or `use_reduce_mean`.") + + +# Deprecated model shortcuts, only used in errors and warnings +OLD_MODEL_SHORTCUTS = { + "en": "en_core_web_sm", "de": "de_core_news_sm", "es": "es_core_news_sm", + "pt": "pt_core_news_sm", "fr": "fr_core_news_sm", "it": "it_core_news_sm", + "nl": "nl_core_news_sm", "el": "el_core_news_sm", "nb": "nb_core_news_sm", + "lt": "lt_core_news_sm", "xx": "xx_ent_wiki_sm" +} + + +# fmt: on + + +class MatchPatternError(ValueError): + def __init__(self, key, errors): + """Custom error for validating match patterns. + + key (str): The name of the matcher rule. + errors (dict): Validation errors (sequence of strings) mapped to pattern + ID, i.e. the index of the added pattern. + """ + msg = f"Invalid token patterns for matcher rule '{key}'\n" + for pattern_idx, error_msgs in errors.items(): + pattern_errors = "\n".join([f"- {e}" for e in error_msgs]) + msg += f"\nPattern {pattern_idx}:\n{pattern_errors}\n" + ValueError.__init__(self, msg) diff --git a/spacy/glossary.py b/spacy/glossary.py new file mode 100644 index 0000000..1f62869 --- /dev/null +++ b/spacy/glossary.py @@ -0,0 +1,360 @@ +import warnings + +from .errors import Warnings + + +def explain(term): + """Get a description for a given POS tag, dependency label or entity type. + + term (str): The term to explain. + RETURNS (str): The explanation, or `None` if not found in the glossary. + + EXAMPLE: + >>> spacy.explain(u'NORP') + >>> doc = nlp(u'Hello world') + >>> print([w.text, w.tag_, spacy.explain(w.tag_) for w in doc]) + """ + if term in GLOSSARY: + return GLOSSARY[term] + else: + warnings.warn(Warnings.W118.format(term=term)) + + +GLOSSARY = { + # POS tags + # Universal POS Tags + # http://universaldependencies.org/u/pos/ + "ADJ": "adjective", + "ADP": "adposition", + "ADV": "adverb", + "AUX": "auxiliary", + "CONJ": "conjunction", + "CCONJ": "coordinating conjunction", + "DET": "determiner", + "INTJ": "interjection", + "NOUN": "noun", + "NUM": "numeral", + "PART": "particle", + "PRON": "pronoun", + "PROPN": "proper noun", + "PUNCT": "punctuation", + "SCONJ": "subordinating conjunction", + "SYM": "symbol", + "VERB": "verb", + "X": "other", + "EOL": "end of line", + "SPACE": "space", + # POS tags (English) + # OntoNotes 5 / Penn Treebank + # https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html + ".": "punctuation mark, sentence closer", + ",": "punctuation mark, comma", + "-LRB-": "left round bracket", + "-RRB-": "right round bracket", + "``": "opening quotation mark", + '""': "closing quotation mark", + "''": "closing quotation mark", + ":": "punctuation mark, colon or ellipsis", + "$": "symbol, currency", + "#": "symbol, number sign", + "AFX": "affix", + "CC": "conjunction, coordinating", + "CD": "cardinal number", + "DT": "determiner", + "EX": "existential there", + "FW": "foreign word", + "HYPH": "punctuation mark, hyphen", + "IN": "conjunction, subordinating or preposition", + "JJ": "adjective (English), other noun-modifier (Chinese)", + "JJR": "adjective, comparative", + "JJS": "adjective, superlative", + "LS": "list item marker", + "MD": "verb, modal auxiliary", + "NIL": "missing tag", + "NN": "noun, singular or mass", + "NNP": "noun, proper singular", + "NNPS": "noun, proper plural", + "NNS": "noun, plural", + "PDT": "predeterminer", + "POS": "possessive ending", + "PRP": "pronoun, personal", + "PRP$": "pronoun, possessive", + "RB": "adverb", + "RBR": "adverb, comparative", + "RBS": "adverb, superlative", + "RP": "adverb, particle", + "TO": 'infinitival "to"', + "UH": "interjection", + "VB": "verb, base form", + "VBD": "verb, past tense", + "VBG": "verb, gerund or present participle", + "VBN": "verb, past participle", + "VBP": "verb, non-3rd person singular present", + "VBZ": "verb, 3rd person singular present", + "WDT": "wh-determiner", + "WP": "wh-pronoun, personal", + "WP$": "wh-pronoun, possessive", + "WRB": "wh-adverb", + "SP": "space (English), sentence-final particle (Chinese)", + "ADD": "email", + "NFP": "superfluous punctuation", + "GW": "additional word in multi-word expression", + "XX": "unknown", + "BES": 'auxiliary "be"', + "HVS": 'forms of "have"', + "_SP": "whitespace", + # POS Tags (German) + # TIGER Treebank + # http://www.ims.uni-stuttgart.de/forschung/ressourcen/korpora/TIGERCorpus/annotation/tiger_introduction.pdf + "$(": "other sentence-internal punctuation mark", + "$,": "comma", + "$.": "sentence-final punctuation mark", + "ADJA": "adjective, attributive", + "ADJD": "adjective, adverbial or predicative", + "APPO": "postposition", + "APPR": "preposition; circumposition left", + "APPRART": "preposition with article", + "APZR": "circumposition right", + "ART": "definite or indefinite article", + "CARD": "cardinal number", + "FM": "foreign language material", + "ITJ": "interjection", + "KOKOM": "comparative conjunction", + "KON": "coordinate conjunction", + "KOUI": 'subordinate conjunction with "zu" and infinitive', + "KOUS": "subordinate conjunction with sentence", + "NE": "proper noun", + "NNE": "proper noun", + "PAV": "pronominal adverb", + "PROAV": "pronominal adverb", + "PDAT": "attributive demonstrative pronoun", + "PDS": "substituting demonstrative pronoun", + "PIAT": "attributive indefinite pronoun without determiner", + "PIDAT": "attributive indefinite pronoun with determiner", + "PIS": "substituting indefinite pronoun", + "PPER": "non-reflexive personal pronoun", + "PPOSAT": "attributive possessive pronoun", + "PPOSS": "substituting possessive pronoun", + "PRELAT": "attributive relative pronoun", + "PRELS": "substituting relative pronoun", + "PRF": "reflexive personal pronoun", + "PTKA": "particle with adjective or adverb", + "PTKANT": "answer particle", + "PTKNEG": "negative particle", + "PTKVZ": "separable verbal particle", + "PTKZU": '"zu" before infinitive', + "PWAT": "attributive interrogative pronoun", + "PWAV": "adverbial interrogative or relative pronoun", + "PWS": "substituting interrogative pronoun", + "TRUNC": "word remnant", + "VAFIN": "finite verb, auxiliary", + "VAIMP": "imperative, auxiliary", + "VAINF": "infinitive, auxiliary", + "VAPP": "perfect participle, auxiliary", + "VMFIN": "finite verb, modal", + "VMINF": "infinitive, modal", + "VMPP": "perfect participle, modal", + "VVFIN": "finite verb, full", + "VVIMP": "imperative, full", + "VVINF": "infinitive, full", + "VVIZU": 'infinitive with "zu", full', + "VVPP": "perfect participle, full", + "XY": "non-word containing non-letter", + # POS Tags (Chinese) + # OntoNotes / Chinese Penn Treebank + # https://repository.upenn.edu/cgi/viewcontent.cgi?article=1039&context=ircs_reports + "AD": "adverb", + "AS": "aspect marker", + "BA": "把 in ba-construction", + # "CD": "cardinal number", + "CS": "subordinating conjunction", + "DEC": "的 in a relative clause", + "DEG": "associative 的", + "DER": "得 in V-de const. and V-de-R", + "DEV": "地 before VP", + "ETC": "for words 等, 等等", + # "FW": "foreign words" + "IJ": "interjection", + # "JJ": "other noun-modifier", + "LB": "被 in long bei-const", + "LC": "localizer", + "M": "measure word", + "MSP": "other particle", + # "NN": "common noun", + "NR": "proper noun", + "NT": "temporal noun", + "OD": "ordinal number", + "ON": "onomatopoeia", + "P": "preposition excluding 把 and 被", + "PN": "pronoun", + "PU": "punctuation", + "SB": "被 in short bei-const", + # "SP": "sentence-final particle", + "VA": "predicative adjective", + "VC": "是 (copula)", + "VE": "有 as the main verb", + "VV": "other verb", + # Noun chunks + "NP": "noun phrase", + "PP": "prepositional phrase", + "VP": "verb phrase", + "ADVP": "adverb phrase", + "ADJP": "adjective phrase", + "SBAR": "subordinating conjunction", + "PRT": "particle", + "PNP": "prepositional noun phrase", + # Dependency Labels (English) + # ClearNLP / Universal Dependencies + # https://github.com/clir/clearnlp-guidelines/blob/master/md/specifications/dependency_labels.md + "acl": "clausal modifier of noun (adjectival clause)", + "acomp": "adjectival complement", + "advcl": "adverbial clause modifier", + "advmod": "adverbial modifier", + "agent": "agent", + "amod": "adjectival modifier", + "appos": "appositional modifier", + "attr": "attribute", + "aux": "auxiliary", + "auxpass": "auxiliary (passive)", + "case": "case marking", + "cc": "coordinating conjunction", + "ccomp": "clausal complement", + "clf": "classifier", + "complm": "complementizer", + "compound": "compound", + "conj": "conjunct", + "cop": "copula", + "csubj": "clausal subject", + "csubjpass": "clausal subject (passive)", + "dative": "dative", + "dep": "unclassified dependent", + "det": "determiner", + "discourse": "discourse element", + "dislocated": "dislocated elements", + "dobj": "direct object", + "expl": "expletive", + "fixed": "fixed multiword expression", + "flat": "flat multiword expression", + "goeswith": "goes with", + "hmod": "modifier in hyphenation", + "hyph": "hyphen", + "infmod": "infinitival modifier", + "intj": "interjection", + "iobj": "indirect object", + "list": "list", + "mark": "marker", + "meta": "meta modifier", + "neg": "negation modifier", + "nmod": "modifier of nominal", + "nn": "noun compound modifier", + "npadvmod": "noun phrase as adverbial modifier", + "nsubj": "nominal subject", + "nsubjpass": "nominal subject (passive)", + "nounmod": "modifier of nominal", + "npmod": "noun phrase as adverbial modifier", + "num": "number modifier", + "number": "number compound modifier", + "nummod": "numeric modifier", + "oprd": "object predicate", + "obj": "object", + "obl": "oblique nominal", + "orphan": "orphan", + "parataxis": "parataxis", + "partmod": "participal modifier", + "pcomp": "complement of preposition", + "pobj": "object of preposition", + "poss": "possession modifier", + "possessive": "possessive modifier", + "preconj": "pre-correlative conjunction", + "prep": "prepositional modifier", + "prt": "particle", + "punct": "punctuation", + "quantmod": "modifier of quantifier", + "rcmod": "relative clause modifier", + "relcl": "relative clause modifier", + "reparandum": "overridden disfluency", + "root": "root", + "ROOT": "root", + "vocative": "vocative", + "xcomp": "open clausal complement", + # Dependency labels (German) + # TIGER Treebank + # http://www.ims.uni-stuttgart.de/forschung/ressourcen/korpora/TIGERCorpus/annotation/tiger_introduction.pdf + # currently missing: 'cc' (comparative complement) because of conflict + # with English labels + "ac": "adpositional case marker", + "adc": "adjective component", + "ag": "genitive attribute", + "ams": "measure argument of adjective", + "app": "apposition", + "avc": "adverbial phrase component", + "cd": "coordinating conjunction", + "cj": "conjunct", + "cm": "comparative conjunction", + "cp": "complementizer", + "cvc": "collocational verb construction", + "da": "dative", + "dh": "discourse-level head", + "dm": "discourse marker", + "ep": "expletive es", + "hd": "head", + "ju": "junctor", + "mnr": "postnominal modifier", + "mo": "modifier", + "ng": "negation", + "nk": "noun kernel element", + "nmc": "numerical component", + "oa": "accusative object", + "oc": "clausal object", + "og": "genitive object", + "op": "prepositional object", + "par": "parenthetical element", + "pd": "predicate", + "pg": "phrasal genitive", + "ph": "placeholder", + "pm": "morphological particle", + "pnc": "proper noun component", + "rc": "relative clause", + "re": "repeated element", + "rs": "reported speech", + "sb": "subject", + "sbp": "passivized subject (PP)", + "sp": "subject or predicate", + "svp": "separable verb prefix", + "uc": "unit component", + "vo": "vocative", + # Named Entity Recognition + # OntoNotes 5 + # https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf + "PERSON": "People, including fictional", + "NORP": "Nationalities or religious or political groups", + "FACILITY": "Buildings, airports, highways, bridges, etc.", + "FAC": "Buildings, airports, highways, bridges, etc.", + "ORG": "Companies, agencies, institutions, etc.", + "GPE": "Countries, cities, states", + "LOC": "Non-GPE locations, mountain ranges, bodies of water", + "PRODUCT": "Objects, vehicles, foods, etc. (not services)", + "EVENT": "Named hurricanes, battles, wars, sports events, etc.", + "WORK_OF_ART": "Titles of books, songs, etc.", + "LAW": "Named documents made into laws.", + "LANGUAGE": "Any named language", + "DATE": "Absolute or relative dates or periods", + "TIME": "Times smaller than a day", + "PERCENT": 'Percentage, including "%"', + "MONEY": "Monetary values, including unit", + "QUANTITY": "Measurements, as of weight or distance", + "ORDINAL": '"first", "second", etc.', + "CARDINAL": "Numerals that do not fall under another type", + # Named Entity Recognition + # Wikipedia + # http://www.sciencedirect.com/science/article/pii/S0004370212000276 + # https://pdfs.semanticscholar.org/5744/578cc243d92287f47448870bb426c66cc941.pdf + "PER": "Named person or family.", + "MISC": "Miscellaneous entities, e.g. events, nationalities, products or works of art", + # https://github.com/ltgoslo/norne + "EVT": "Festivals, cultural events, sports events, weather phenomena, wars, etc.", + "PROD": "Product, i.e. artificially produced entities including speeches, radio shows, programming languages, contracts, laws and ideas", + "DRV": "Words (and phrases?) that are dervied from a name, but not a name in themselves, e.g. 'Oslo-mannen' ('the man from Oslo')", + "GPE_LOC": "Geo-political entity, with a locative sense, e.g. 'John lives in Spain'", + "GPE_ORG": "Geo-political entity, with an organisation sense, e.g. 'Spain declined to meet with Belgium'", +} diff --git a/spacy/kb/__init__.py b/spacy/kb/__init__.py new file mode 100644 index 0000000..93a65ab --- /dev/null +++ b/spacy/kb/__init__.py @@ -0,0 +1,11 @@ +from .candidate import Candidate, get_candidates, get_candidates_batch +from .kb import KnowledgeBase +from .kb_in_memory import InMemoryLookupKB + +__all__ = [ + "Candidate", + "KnowledgeBase", + "InMemoryLookupKB", + "get_candidates", + "get_candidates_batch", +] diff --git a/spacy/kb/candidate.pxd b/spacy/kb/candidate.pxd new file mode 100644 index 0000000..80fcbc4 --- /dev/null +++ b/spacy/kb/candidate.pxd @@ -0,0 +1,15 @@ +from libcpp.vector cimport vector + +from ..typedefs cimport hash_t +from .kb cimport KnowledgeBase + + +# Object used by the Entity Linker that summarizes one entity-alias candidate +# combination. +cdef class Candidate: + cdef readonly KnowledgeBase kb + cdef hash_t entity_hash + cdef float entity_freq + cdef vector[float] entity_vector + cdef hash_t alias_hash + cdef float prior_prob diff --git a/spacy/kb/candidate.pyx b/spacy/kb/candidate.pyx new file mode 100644 index 0000000..4369676 --- /dev/null +++ b/spacy/kb/candidate.pyx @@ -0,0 +1,90 @@ +# cython: infer_types=True + +from typing import Iterable + +from .kb cimport KnowledgeBase + +from ..tokens import Span + + +cdef class Candidate: + """A `Candidate` object refers to a textual mention (`alias`) that may or + may not be resolved to a specific `entity` from a Knowledge Base. This + will be used as input for the entity linking algorithm which will + disambiguate the various candidates to the correct one. + Each candidate (alias, entity) pair is assigned a certain prior probability. + + DOCS: https://spacy.io/api/kb/#candidate-init + """ + + def __init__( + self, + KnowledgeBase kb, + entity_hash, + entity_freq, + entity_vector, + alias_hash, + prior_prob + ): + self.kb = kb + self.entity_hash = entity_hash + self.entity_freq = entity_freq + self.entity_vector = entity_vector + self.alias_hash = alias_hash + self.prior_prob = prior_prob + + @property + def entity(self) -> int: + """RETURNS (uint64): hash of the entity's KB ID/name""" + return self.entity_hash + + @property + def entity_(self) -> str: + """RETURNS (str): ID/name of this entity in the KB""" + return self.kb.vocab.strings[self.entity_hash] + + @property + def alias(self) -> int: + """RETURNS (uint64): hash of the alias""" + return self.alias_hash + + @property + def alias_(self) -> str: + """RETURNS (str): ID of the original alias""" + return self.kb.vocab.strings[self.alias_hash] + + @property + def entity_freq(self) -> float: + return self.entity_freq + + @property + def entity_vector(self) -> Iterable[float]: + return self.entity_vector + + @property + def prior_prob(self) -> float: + return self.prior_prob + + +def get_candidates(kb: KnowledgeBase, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for a given mention and fetching appropriate + entries from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Span): Entity mention for which to identify candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + return kb.get_candidates(mention) + + +def get_candidates_batch( + kb: KnowledgeBase, mentions: Iterable[Span] +) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for the given mentions and fetching appropriate entries + from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Iterable[Span]): Entity mentions for which to identify candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return kb.get_candidates_batch(mentions) diff --git a/spacy/kb/kb.pxd b/spacy/kb/kb.pxd new file mode 100644 index 0000000..2634695 --- /dev/null +++ b/spacy/kb/kb.pxd @@ -0,0 +1,12 @@ +"""Knowledge-base for entity or concept linking.""" + +from cymem.cymem cimport Pool +from libc.stdint cimport int64_t + +from ..vocab cimport Vocab + + +cdef class KnowledgeBase: + cdef Pool mem + cdef readonly Vocab vocab + cdef readonly int64_t entity_vector_length diff --git a/spacy/kb/kb.pyx b/spacy/kb/kb.pyx new file mode 100644 index 0000000..c7db34e --- /dev/null +++ b/spacy/kb/kb.pyx @@ -0,0 +1,130 @@ +# cython: infer_types=True + +from pathlib import Path +from typing import Iterable, Tuple, Union + +from cymem.cymem cimport Pool + +from ..errors import Errors +from ..tokens import Span +from ..util import SimpleFrozenList +from .candidate import Candidate + + +cdef class KnowledgeBase: + """A `KnowledgeBase` instance stores unique identifiers for entities and + their textual aliases, to support entity linking of named entities to + real-world concepts. + This is an abstract class and requires its operations to be implemented. + + DOCS: https://spacy.io/api/kb + """ + + def __init__(self, vocab: Vocab, entity_vector_length: int): + """Create a KnowledgeBase.""" + # Make sure abstract KB is not instantiated. + if self.__class__ == KnowledgeBase: + raise TypeError( + Errors.E1046.format(cls_name=self.__class__.__name__) + ) + + self.vocab = vocab + self.entity_vector_length = entity_vector_length + self.mem = Pool() + + def get_candidates_batch( + self, mentions: Iterable[Span] + ) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for specified texts. Each candidate defines + the entity, the original alias, and the prior probability of that + alias resolving to that entity. + If no candidate is found for a given text, an empty list is returned. + mentions (Iterable[Span]): Mentions for which to get candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return [self.get_candidates(span) for span in mentions] + + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for specified text. Each candidate defines + the entity, the original alias, + and the prior probability of that alias resolving to that entity. + If the no candidate is found for a given text, an empty list is returned. + mention (Span): Mention for which to get candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="get_candidates", name=self.__name__ + ) + ) + + def get_vectors(self, entities: Iterable[str]) -> Iterable[Iterable[float]]: + """ + Return vectors for entities. + entity (str): Entity name/ID. + RETURNS (Iterable[Iterable[float]]): Vectors for specified entities. + """ + return [self.get_vector(entity) for entity in entities] + + def get_vector(self, str entity) -> Iterable[float]: + """ + Return vector for entity. + entity (str): Entity name/ID. + RETURNS (Iterable[float]): Vector for specified entity. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="get_vector", name=self.__name__ + ) + ) + + def to_bytes(self, **kwargs) -> bytes: + """Serialize the current state to a binary string. + RETURNS (bytes): Current state as binary string. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="to_bytes", name=self.__name__ + ) + ) + + def from_bytes(self, bytes_data: bytes, *, exclude: Tuple[str] = tuple()): + """Load state from a binary string. + bytes_data (bytes): KB state. + exclude (Tuple[str]): Properties to exclude when restoring KB. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="from_bytes", name=self.__name__ + ) + ) + + def to_disk( + self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """ + Write KnowledgeBase content to disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="to_disk", name=self.__name__ + ) + ) + + def from_disk( + self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """ + Load KnowledgeBase content from disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format( + parent="KnowledgeBase", method="from_disk", name=self.__name__ + ) + ) diff --git a/spacy/kb/kb_in_memory.pxd b/spacy/kb/kb_in_memory.pxd new file mode 100644 index 0000000..e0e3330 --- /dev/null +++ b/spacy/kb/kb_in_memory.pxd @@ -0,0 +1,178 @@ +"""Knowledge-base for entity or concept linking.""" +from libc.stdint cimport int32_t, int64_t +from libc.stdio cimport FILE +from libcpp.vector cimport vector +from preshed.maps cimport PreshMap + +from ..structs cimport AliasC, KBEntryC +from ..typedefs cimport hash_t +from .kb cimport KnowledgeBase + +ctypedef vector[KBEntryC] entry_vec +ctypedef vector[AliasC] alias_vec +ctypedef vector[float] float_vec +ctypedef vector[float_vec] float_matrix + + +cdef class InMemoryLookupKB(KnowledgeBase): + # This maps 64bit keys (hash of unique entity string) + # to 64bit values (position of the _KBEntryC struct in the _entries vector). + # The PreshMap is pretty space efficient, as it uses open addressing. So + # the only overhead is the vacancy rate, which is approximately 30%. + cdef PreshMap _entry_index + + # Each entry takes 128 bits, and again we'll have a 30% or so overhead for + # over allocation. + # In total we end up with (N*128*1.3)+(N*128*1.3) bits for N entries. + # Storing 1m entries would take 41.6mb under this scheme. + cdef entry_vec _entries + + # This maps 64bit keys (hash of unique alias string) + # to 64bit values (position of the _AliasC struct in the _aliases_table vector). + cdef PreshMap _alias_index + + # This should map mention hashes to (entry_id, prob) tuples. The probability + # should be P(entity | mention), which is pretty important to know. + # We can pack both pieces of information into a 64-bit value, to keep things + # efficient. + cdef alias_vec _aliases_table + + # This is the part which might take more space: storing various + # categorical features for the entries, and storing vectors for disambiguation + # and possibly usage. + # If each entry gets a 300-dimensional vector, for 1m entries we would need + # 1.2gb. That gets expensive fast. What might be better is to avoid learning + # a unique vector for every entity. We could instead have a compositional + # model, that embeds different features of the entities into vectors. We'll + # still want some per-entity features, like the Wikipedia text or entity + # co-occurrence. Hopefully those vectors can be narrow, e.g. 64 dimensions. + cdef float_matrix _vectors_table + + # It's very useful to track categorical features, at least for output, even + # if they're not useful in the model itself. For instance, we should be + # able to track stuff like a person's date of birth or whatever. This can + # easily make the KB bigger, but if this isn't needed by the model, and it's + # optional data, we can let users configure a DB as the backend for this. + cdef object _features_table + + cdef inline int64_t c_add_vector(self, vector[float] entity_vector) nogil: + """Add an entity vector to the vectors table.""" + cdef int64_t new_index = self._vectors_table.size() + self._vectors_table.push_back(entity_vector) + return new_index + + cdef inline int64_t c_add_entity( + self, + hash_t entity_hash, + float freq, + int32_t vector_index, + int feats_row + ) nogil: + """Add an entry to the vector of entries. + After calling this method, make sure to update also the _entry_index + using the return value""" + # This is what we'll map the entity hash key to. It's where the entry will sit + # in the vector of entries, so we can get it later. + cdef int64_t new_index = self._entries.size() + + # Avoid struct initializer to enable nogil, cf. + # https://github.com/cython/cython/issues/1642 + cdef KBEntryC entry + entry.entity_hash = entity_hash + entry.vector_index = vector_index + entry.feats_row = feats_row + entry.freq = freq + + self._entries.push_back(entry) + return new_index + + cdef inline int64_t c_add_aliases( + self, + hash_t alias_hash, + vector[int64_t] entry_indices, + vector[float] probs + ) nogil: + """Connect a mention to a list of potential entities with their prior + probabilities. After calling this method, make sure to update also the + _alias_index using the return value""" + # This is what we'll map the alias hash key to. It's where the alias will be + # defined in the vector of aliases. + cdef int64_t new_index = self._aliases_table.size() + + # Avoid struct initializer to enable nogil + cdef AliasC alias + alias.entry_indices = entry_indices + alias.probs = probs + + self._aliases_table.push_back(alias) + return new_index + + cdef inline void _create_empty_vectors(self, hash_t dummy_hash) nogil: + """ + Initializing the vectors and making sure the first element of each vector is a + dummy, because the PreshMap maps pointing to indices in these vectors can not + contain 0 as value. + cf. https://github.com/explosion/preshed/issues/17 + """ + cdef int32_t dummy_value = 0 + + # Avoid struct initializer to enable nogil + cdef KBEntryC entry + entry.entity_hash = dummy_hash + entry.vector_index = dummy_value + entry.feats_row = dummy_value + entry.freq = dummy_value + + # Avoid struct initializer to enable nogil + cdef vector[int64_t] dummy_entry_indices + dummy_entry_indices.push_back(0) + cdef vector[float] dummy_probs + dummy_probs.push_back(0) + + cdef AliasC alias + alias.entry_indices = dummy_entry_indices + alias.probs = dummy_probs + + self._entries.push_back(entry) + self._aliases_table.push_back(alias) + + cpdef set_entities(self, entity_list, freq_list, vector_list) + + +cdef class Writer: + cdef FILE* _fp + + cdef int write_header( + self, int64_t nr_entries, int64_t entity_vector_length + ) except -1 + cdef int write_vector_element(self, float element) except -1 + cdef int write_entry( + self, hash_t entry_hash, float entry_freq, int32_t vector_index + ) except -1 + + cdef int write_alias_length(self, int64_t alias_length) except -1 + cdef int write_alias_header( + self, hash_t alias_hash, int64_t candidate_length + ) except -1 + cdef int write_alias(self, int64_t entry_index, float prob) except -1 + + cdef int _write(self, void* value, size_t size) except -1 + +cdef class Reader: + cdef FILE* _fp + + cdef int read_header( + self, int64_t* nr_entries, int64_t* entity_vector_length + ) except -1 + cdef int read_vector_element(self, float* element) except -1 + cdef int read_entry( + self, hash_t* entity_hash, float* freq, int32_t* vector_index + ) except -1 + + cdef int read_alias_length(self, int64_t* alias_length) except -1 + cdef int read_alias_header( + self, hash_t* alias_hash, int64_t* candidate_length + ) except -1 + cdef int read_alias(self, int64_t* entry_index, float* prob) except -1 + + cdef int _read(self, void* value, size_t size) except -1 diff --git a/spacy/kb/kb_in_memory.pyx b/spacy/kb/kb_in_memory.pyx new file mode 100644 index 0000000..2b21f24 --- /dev/null +++ b/spacy/kb/kb_in_memory.pyx @@ -0,0 +1,728 @@ +# cython: infer_types=True +from typing import Any, Callable, Dict, Iterable + +import srsly + +from cpython.exc cimport PyErr_SetFromErrno +from libc.stdint cimport int32_t, int64_t +from libc.stdio cimport fclose, feof, fopen, fread, fseek, fwrite +from libcpp.vector cimport vector +from preshed.maps cimport PreshMap + +import warnings +from pathlib import Path + +from ..tokens import Span + +from ..typedefs cimport hash_t + +from .. import util +from ..errors import Errors, Warnings +from ..util import SimpleFrozenList, ensure_path + +from ..vocab cimport Vocab +from .kb cimport KnowledgeBase + +from .candidate import Candidate as Candidate + + +cdef class InMemoryLookupKB(KnowledgeBase): + """An `InMemoryLookupKB` instance stores unique identifiers for entities + and their textual aliases, to support entity linking of named entities to + real-world concepts. + + DOCS: https://spacy.io/api/inmemorylookupkb + """ + + def __init__(self, Vocab vocab, entity_vector_length): + """Create an InMemoryLookupKB.""" + super().__init__(vocab, entity_vector_length) + self._entry_index = PreshMap() + self._alias_index = PreshMap() + self._create_empty_vectors(dummy_hash=self.vocab.strings[""]) + + def _initialize_entities(self, int64_t nr_entities): + self._entry_index = PreshMap(nr_entities + 1) + self._entries = entry_vec(nr_entities + 1) + + def _initialize_vectors(self, int64_t nr_entities): + self._vectors_table = float_matrix(nr_entities + 1) + + def _initialize_aliases(self, int64_t nr_aliases): + self._alias_index = PreshMap(nr_aliases + 1) + self._aliases_table = alias_vec(nr_aliases + 1) + + def is_empty(self): + return len(self) == 0 + + def __len__(self): + return self.get_size_entities() + + def get_size_entities(self): + return len(self._entry_index) + + def get_entity_strings(self): + return [self.vocab.strings[x] for x in self._entry_index] + + def get_size_aliases(self): + return len(self._alias_index) + + def get_alias_strings(self): + return [self.vocab.strings[x] for x in self._alias_index] + + def add_entity(self, str entity, float freq, vector[float] entity_vector): + """ + Add an entity to the KB, optionally specifying its log probability + based on corpus frequency. + Return the hash of the entity ID/name at the end. + """ + cdef hash_t entity_hash = self.vocab.strings.add(entity) + + # Return if this entity was added before + if entity_hash in self._entry_index: + warnings.warn(Warnings.W018.format(entity=entity)) + return + + # Raise an error if the provided entity vector is not of the correct length + if len(entity_vector) != self.entity_vector_length: + raise ValueError( + Errors.E141.format( + found=len(entity_vector), required=self.entity_vector_length + ) + ) + + vector_index = self.c_add_vector(entity_vector=entity_vector) + + new_index = self.c_add_entity( + entity_hash=entity_hash, + freq=freq, + vector_index=vector_index, + feats_row=-1 + ) # Features table currently not implemented + self._entry_index[entity_hash] = new_index + + return entity_hash + + cpdef set_entities(self, entity_list, freq_list, vector_list): + if len(entity_list) != len(freq_list) or len(entity_list) != len(vector_list): + raise ValueError(Errors.E140) + + nr_entities = len(set(entity_list)) + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) + + i = 0 + cdef KBEntryC entry + cdef hash_t entity_hash + while i < len(entity_list): + # only process this entity if its unique ID hadn't been added before + entity_hash = self.vocab.strings.add(entity_list[i]) + if entity_hash in self._entry_index: + warnings.warn(Warnings.W018.format(entity=entity_list[i])) + + else: + entity_vector = vector_list[i] + if len(entity_vector) != self.entity_vector_length: + raise ValueError( + Errors.E141.format( + found=len(entity_vector), + required=self.entity_vector_length + ) + ) + + entry.entity_hash = entity_hash + entry.freq = freq_list[i] + + self._vectors_table[i] = entity_vector + entry.vector_index = i + + entry.feats_row = -1 # Features table currently not implemented + + self._entries[i+1] = entry + self._entry_index[entity_hash] = i+1 + + i += 1 + + def contains_entity(self, str entity): + cdef hash_t entity_hash = self.vocab.strings.add(entity) + return entity_hash in self._entry_index + + def contains_alias(self, str alias): + cdef hash_t alias_hash = self.vocab.strings.add(alias) + return alias_hash in self._alias_index + + def add_alias(self, str alias, entities, probabilities): + """ + For a given alias, add its potential entities and prior probabilies to the KB. + Return the alias_hash at the end + """ + if alias is None or len(alias) == 0: + raise ValueError(Errors.E890.format(alias=alias)) + + previous_alias_nr = self.get_size_aliases() + # Throw an error if the length of entities and probabilities are not the same + if not len(entities) == len(probabilities): + raise ValueError( + Errors.E132.format( + alias=alias, + entities_length=len(entities), + probabilities_length=len(probabilities)) + ) + + # Throw an error if the probabilities sum up to more than 1 (allow for + # some rounding errors) + prob_sum = sum(probabilities) + if prob_sum > 1.00001: + raise ValueError(Errors.E133.format(alias=alias, sum=prob_sum)) + + cdef hash_t alias_hash = self.vocab.strings.add(alias) + + # Check whether this alias was added before + if alias_hash in self._alias_index: + warnings.warn(Warnings.W017.format(alias=alias)) + return + + cdef vector[int64_t] entry_indices + cdef vector[float] probs + + for entity, prob in zip(entities, probabilities): + entity_hash = self.vocab.strings[entity] + if entity_hash not in self._entry_index: + raise ValueError(Errors.E134.format(entity=entity)) + + entry_index = self._entry_index.get(entity_hash) + entry_indices.push_back(int(entry_index)) + probs.push_back(float(prob)) + + new_index = self.c_add_aliases( + alias_hash=alias_hash, entry_indices=entry_indices, probs=probs + ) + self._alias_index[alias_hash] = new_index + + if previous_alias_nr + 1 != self.get_size_aliases(): + raise RuntimeError(Errors.E891.format(alias=alias)) + return alias_hash + + def append_alias( + self, str alias, str entity, float prior_prob, ignore_warnings=False + ): + """ + For an alias already existing in the KB, extend its potential entities + with one more. + Throw a warning if either the alias or the entity is unknown, + or when the combination is already previously recorded. + Throw an error if this entity+prior prob would exceed the sum of 1. + For efficiency, it's best to use the method `add_alias` as much as + possible instead of this one. + """ + # Check if the alias exists in the KB + cdef hash_t alias_hash = self.vocab.strings[alias] + if alias_hash not in self._alias_index: + raise ValueError(Errors.E176.format(alias=alias)) + + # Check if the entity exists in the KB + cdef hash_t entity_hash = self.vocab.strings[entity] + if entity_hash not in self._entry_index: + raise ValueError(Errors.E134.format(entity=entity)) + entry_index = self._entry_index.get(entity_hash) + + # Throw an error if the prior probabilities (including the new one) + # sum up to more than 1 + alias_index = self._alias_index.get(alias_hash) + alias_entry = self._aliases_table[alias_index] + current_sum = sum([p for p in alias_entry.probs]) + new_sum = current_sum + prior_prob + + if new_sum > 1.00001: + raise ValueError(Errors.E133.format(alias=alias, sum=new_sum)) + + entry_indices = alias_entry.entry_indices + + is_present = False + for i in range(entry_indices.size()): + if entry_indices[i] == int(entry_index): + is_present = True + + if is_present: + if not ignore_warnings: + warnings.warn(Warnings.W024.format(entity=entity, alias=alias)) + else: + entry_indices.push_back(int(entry_index)) + alias_entry.entry_indices = entry_indices + + probs = alias_entry.probs + probs.push_back(float(prior_prob)) + alias_entry.probs = probs + self._aliases_table[alias_index] = alias_entry + + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + return self.get_alias_candidates(mention.text) # type: ignore + + def get_alias_candidates(self, str alias) -> Iterable[Candidate]: + """ + Return candidate entities for an alias. Each candidate defines the + entity, the original alias, and the prior probability of that alias + resolving to that entity. + If the alias is not known in the KB, and empty list is returned. + """ + cdef hash_t alias_hash = self.vocab.strings[alias] + if alias_hash not in self._alias_index: + return [] + alias_index = self._alias_index.get(alias_hash) + alias_entry = self._aliases_table[alias_index] + + return [Candidate(kb=self, + entity_hash=self._entries[entry_index].entity_hash, + entity_freq=self._entries[entry_index].freq, + entity_vector=self._vectors_table[ + self._entries[entry_index].vector_index + ], + alias_hash=alias_hash, + prior_prob=prior_prob) + for (entry_index, prior_prob) in zip( + alias_entry.entry_indices, alias_entry.probs + ) + if entry_index != 0] + + def get_vector(self, str entity): + cdef hash_t entity_hash = self.vocab.strings[entity] + + # Return an empty list if this entity is unknown in this KB + if entity_hash not in self._entry_index: + return [0] * self.entity_vector_length + entry_index = self._entry_index[entity_hash] + + return self._vectors_table[self._entries[entry_index].vector_index] + + def get_prior_prob(self, str entity, str alias): + """ Return the prior probability of a given alias being linked to a + given entity, or return 0.0 when this combination is not known in the + knowledge base.""" + cdef hash_t alias_hash = self.vocab.strings[alias] + cdef hash_t entity_hash = self.vocab.strings[entity] + + if entity_hash not in self._entry_index or alias_hash not in self._alias_index: + return 0.0 + + alias_index = self._alias_index.get(alias_hash) + entry_index = self._entry_index[entity_hash] + + alias_entry = self._aliases_table[alias_index] + for (entry_index, prior_prob) in zip( + alias_entry.entry_indices, alias_entry.probs + ): + if self._entries[entry_index].entity_hash == entity_hash: + return prior_prob + + return 0.0 + + def to_bytes(self, **kwargs): + """Serialize the current state to a binary string. + """ + def serialize_header(): + header = ( + self.get_size_entities(), + self.get_size_aliases(), + self.entity_vector_length + ) + return srsly.json_dumps(header) + + def serialize_entries(): + i = 1 + tuples = [] + for entry_hash, entry_index in sorted( + self._entry_index.items(), key=lambda x: x[1] + ): + entry = self._entries[entry_index] + assert entry.entity_hash == entry_hash + assert entry_index == i + tuples.append((entry.entity_hash, entry.freq, entry.vector_index)) + i = i + 1 + return srsly.json_dumps(tuples) + + def serialize_aliases(): + i = 1 + headers = [] + indices_lists = [] + probs_lists = [] + for alias_hash, alias_index in sorted( + self._alias_index.items(), key=lambda x: x[1] + ): + alias = self._aliases_table[alias_index] + assert alias_index == i + candidate_length = len(alias.entry_indices) + headers.append((alias_hash, candidate_length)) + indices_lists.append(alias.entry_indices) + probs_lists.append(alias.probs) + i = i + 1 + headers_dump = srsly.json_dumps(headers) + indices_dump = srsly.json_dumps(indices_lists) + probs_dump = srsly.json_dumps(probs_lists) + return srsly.json_dumps((headers_dump, indices_dump, probs_dump)) + + serializers = { + "header": serialize_header, + "entity_vectors": lambda: srsly.json_dumps(self._vectors_table), + "entries": serialize_entries, + "aliases": serialize_aliases, + } + return util.to_bytes(serializers, []) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load state from a binary string. + """ + def deserialize_header(b): + header = srsly.json_loads(b) + nr_entities = header[0] + nr_aliases = header[1] + entity_vector_length = header[2] + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) + self._initialize_aliases(nr_aliases) + self.entity_vector_length = entity_vector_length + + def deserialize_vectors(b): + self._vectors_table = srsly.json_loads(b) + + def deserialize_entries(b): + cdef KBEntryC entry + tuples = srsly.json_loads(b) + i = 1 + for (entity_hash, freq, vector_index) in tuples: + entry.entity_hash = entity_hash + entry.freq = freq + entry.vector_index = vector_index + entry.feats_row = -1 # Features table currently not implemented + self._entries[i] = entry + self._entry_index[entity_hash] = i + i += 1 + + def deserialize_aliases(b): + cdef AliasC alias + i = 1 + all_data = srsly.json_loads(b) + headers = srsly.json_loads(all_data[0]) + indices = srsly.json_loads(all_data[1]) + probs = srsly.json_loads(all_data[2]) + for header, indices, probs in zip(headers, indices, probs): + alias_hash, _candidate_length = header + alias.entry_indices = indices + alias.probs = probs + self._aliases_table[i] = alias + self._alias_index[alias_hash] = i + i += 1 + + setters = { + "header": deserialize_header, + "entity_vectors": deserialize_vectors, + "entries": deserialize_entries, + "aliases": deserialize_aliases, + } + util.from_bytes(bytes_data, setters, exclude) + return self + + def to_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + path = ensure_path(path) + if not path.exists(): + path.mkdir(parents=True) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + serialize = {} + serialize["contents"] = lambda p: self.write_contents(p) + serialize["strings.json"] = lambda p: self.vocab.strings.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + path = ensure_path(path) + if not path.exists(): + raise ValueError(Errors.E929.format(loc=path)) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + deserialize: Dict[str, Callable[[Any], Any]] = {} + deserialize["contents"] = lambda p: self.read_contents(p) + deserialize["strings.json"] = lambda p: self.vocab.strings.from_disk(p) + util.from_disk(path, deserialize, exclude) + + def write_contents(self, file_path): + cdef Writer writer = Writer(file_path) + writer.write_header(self.get_size_entities(), self.entity_vector_length) + + # dumping the entity vectors in their original order + i = 0 + for entity_vector in self._vectors_table: + for element in entity_vector: + writer.write_vector_element(element) + i = i+1 + + # dumping the entry records in the order in which they are in the + # _entries vector. + # index 0 is a dummy object not stored in the _entry_index and can + # be ignored. + i = 1 + for entry_hash, entry_index in sorted( + self._entry_index.items(), key=lambda x: x[1] + ): + entry = self._entries[entry_index] + assert entry.entity_hash == entry_hash + assert entry_index == i + writer.write_entry(entry.entity_hash, entry.freq, entry.vector_index) + i = i+1 + + writer.write_alias_length(self.get_size_aliases()) + + # dumping the aliases in the order in which they are in the _alias_index vector. + # index 0 is a dummy object not stored in the _aliases_table and can be ignored. + i = 1 + for alias_hash, alias_index in sorted( + self._alias_index.items(), key=lambda x: x[1] + ): + alias = self._aliases_table[alias_index] + assert alias_index == i + + candidate_length = len(alias.entry_indices) + writer.write_alias_header(alias_hash, candidate_length) + + for j in range(0, candidate_length): + writer.write_alias(alias.entry_indices[j], alias.probs[j]) + + i = i+1 + + writer.close() + + def read_contents(self, file_path): + cdef hash_t entity_hash + cdef hash_t alias_hash + cdef int64_t entry_index + cdef float freq, prob + cdef int32_t vector_index + cdef KBEntryC entry + cdef AliasC alias + cdef float vector_element + + cdef Reader reader = Reader(file_path) + + # STEP 0: load header and initialize KB + cdef int64_t nr_entities + cdef int64_t entity_vector_length + reader.read_header(&nr_entities, &entity_vector_length) + + self._initialize_entities(nr_entities) + self._initialize_vectors(nr_entities) + self.entity_vector_length = entity_vector_length + + # STEP 1: load entity vectors + cdef int i = 0 + cdef int j = 0 + while i < nr_entities: + entity_vector = float_vec(entity_vector_length) + j = 0 + while j < entity_vector_length: + reader.read_vector_element(&vector_element) + entity_vector[j] = vector_element + j = j+1 + self._vectors_table[i] = entity_vector + i = i+1 + + # STEP 2: load entities + # we assume that the entity data was written in sequence + # index 0 is a dummy object not stored in the _entry_index and can be ignored. + i = 1 + while i <= nr_entities: + reader.read_entry(&entity_hash, &freq, &vector_index) + + entry.entity_hash = entity_hash + entry.freq = freq + entry.vector_index = vector_index + entry.feats_row = -1 # Features table currently not implemented + + self._entries[i] = entry + self._entry_index[entity_hash] = i + + i += 1 + + # check that all entities were read in properly + assert nr_entities == self.get_size_entities() + + # STEP 3: load aliases + cdef int64_t nr_aliases + reader.read_alias_length(&nr_aliases) + self._initialize_aliases(nr_aliases) + + cdef int64_t nr_candidates + cdef vector[int64_t] entry_indices + cdef vector[float] probs + + i = 1 + # we assume the alias data was written in sequence + # index 0 is a dummy object not stored in the _entry_index and can be ignored. + while i <= nr_aliases: + reader.read_alias_header(&alias_hash, &nr_candidates) + entry_indices = vector[int64_t](nr_candidates) + probs = vector[float](nr_candidates) + + for j in range(0, nr_candidates): + reader.read_alias(&entry_index, &prob) + entry_indices[j] = entry_index + probs[j] = prob + + alias.entry_indices = entry_indices + alias.probs = probs + + self._aliases_table[i] = alias + self._alias_index[alias_hash] = i + + i += 1 + + # check that all aliases were read in properly + assert nr_aliases == self.get_size_aliases() + + +cdef class Writer: + def __init__(self, path): + assert isinstance(path, Path) + content = bytes(path) + cdef bytes bytes_loc = content.encode('utf8') \ + if type(content) == str else content + self._fp = fopen(bytes_loc, 'wb') + if not self._fp: + raise IOError(Errors.E146.format(path=path)) + fseek(self._fp, 0, 0) + + def close(self): + cdef size_t status = fclose(self._fp) + assert status == 0 + + cdef int write_header( + self, int64_t nr_entries, int64_t entity_vector_length + ) except -1: + self._write(&nr_entries, sizeof(nr_entries)) + self._write(&entity_vector_length, sizeof(entity_vector_length)) + + cdef int write_vector_element(self, float element) except -1: + self._write(&element, sizeof(element)) + + cdef int write_entry( + self, hash_t entry_hash, float entry_freq, int32_t vector_index + ) except -1: + self._write(&entry_hash, sizeof(entry_hash)) + self._write(&entry_freq, sizeof(entry_freq)) + self._write(&vector_index, sizeof(vector_index)) + # Features table currently not implemented and not written to file + + cdef int write_alias_length(self, int64_t alias_length) except -1: + self._write(&alias_length, sizeof(alias_length)) + + cdef int write_alias_header( + self, hash_t alias_hash, int64_t candidate_length + ) except -1: + self._write(&alias_hash, sizeof(alias_hash)) + self._write(&candidate_length, sizeof(candidate_length)) + + cdef int write_alias(self, int64_t entry_index, float prob) except -1: + self._write(&entry_index, sizeof(entry_index)) + self._write(&prob, sizeof(prob)) + + cdef int _write(self, void* value, size_t size) except -1: + status = fwrite(value, size, 1, self._fp) + assert status == 1, status + + +cdef class Reader: + def __init__(self, path): + content = bytes(path) + cdef bytes bytes_loc = content.encode('utf8') \ + if type(content) == str else content + self._fp = fopen(bytes_loc, 'rb') + if not self._fp: + PyErr_SetFromErrno(IOError) + fseek(self._fp, 0, 0) # this can be 0 if there is no header + + def __dealloc__(self): + fclose(self._fp) + + cdef int read_header( + self, int64_t* nr_entries, int64_t* entity_vector_length + ) except -1: + status = self._read(nr_entries, sizeof(int64_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="header")) + + status = self._read(entity_vector_length, sizeof(int64_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="vector length")) + + cdef int read_vector_element(self, float* element) except -1: + status = self._read(element, sizeof(float)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="vector element")) + + cdef int read_entry( + self, hash_t* entity_hash, float* freq, int32_t* vector_index + ) except -1: + status = self._read(entity_hash, sizeof(hash_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="entity hash")) + + status = self._read(freq, sizeof(float)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="entity freq")) + + status = self._read(vector_index, sizeof(int32_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="vector index")) + + if feof(self._fp): + return 0 + else: + return 1 + + cdef int read_alias_length(self, int64_t* alias_length) except -1: + status = self._read(alias_length, sizeof(int64_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="alias length")) + + cdef int read_alias_header( + self, hash_t* alias_hash, int64_t* candidate_length + ) except -1: + status = self._read(alias_hash, sizeof(hash_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="alias hash")) + + status = self._read(candidate_length, sizeof(int64_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="candidate length")) + + cdef int read_alias(self, int64_t* entry_index, float* prob) except -1: + status = self._read(entry_index, sizeof(int64_t)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="entry index")) + + status = self._read(prob, sizeof(float)) + if status < 1: + if feof(self._fp): + return 0 # end of file + raise IOError(Errors.E145.format(param="prior probability")) + + cdef int _read(self, void* value, size_t size) except -1: + status = fread(value, size, 1, self._fp) + return status diff --git a/spacy/lang/__init__.py b/spacy/lang/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/lang/af/__init__.py b/spacy/lang/af/__init__.py new file mode 100644 index 0000000..8bd73c7 --- /dev/null +++ b/spacy/lang/af/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class AfrikaansDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Afrikaans(Language): + lang = "af" + Defaults = AfrikaansDefaults + + +__all__ = ["Afrikaans"] diff --git a/spacy/lang/af/stop_words.py b/spacy/lang/af/stop_words.py new file mode 100644 index 0000000..4b5a04a --- /dev/null +++ b/spacy/lang/af/stop_words.py @@ -0,0 +1,57 @@ +# Source: https://github.com/stopwords-iso/stopwords-af + +STOP_WORDS = set( + """ +'n +aan +af +al +as +baie +by +daar +dag +dat +die +dit +een +ek +en +gaan +gesê +haar +het +hom +hulle +hy +in +is +jou +jy +kan +kom +ma +maar +met +my +na +nie +om +ons +op +saam +sal +se +sien +so +sy +te +toe +uit +van +vir +was +wat +ʼn +""".split() +) diff --git a/spacy/lang/am/__init__.py b/spacy/lang/am/__init__.py new file mode 100644 index 0000000..284823e --- /dev/null +++ b/spacy/lang/am/__init__.py @@ -0,0 +1,26 @@ +from ...attrs import LANG +from ...language import BaseDefaults, Language +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class AmharicDefaults(BaseDefaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters.update(LEX_ATTRS) + lex_attr_getters[LANG] = lambda text: "am" + tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) + stop_words = STOP_WORDS + suffixes = TOKENIZER_SUFFIXES + writing_system = {"direction": "ltr", "has_case": False, "has_letters": True} + + +class Amharic(Language): + lang = "am" + Defaults = AmharicDefaults + + +__all__ = ["Amharic"] diff --git a/spacy/lang/am/examples.py b/spacy/lang/am/examples.py new file mode 100644 index 0000000..b156cb8 --- /dev/null +++ b/spacy/lang/am/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.am.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "አፕል የዩኬን ጅምር ድርጅት በ 1 ቢሊዮን ዶላር ለመግዛት አስቧል።", + "የራስ ገዝ መኪኖች የኢንሹራንስ ኃላፊነትን ወደ አምራቾች ያዛውራሉ", + "ሳን ፍራንሲስኮ የእግረኛ መንገድ አቅርቦት ሮቦቶችን ማገድን ይመለከታል", + "ለንደን በእንግሊዝ የምትገኝ ትልቅ ከተማ ናት።", + "የት ነህ?", + "የፈረንሳይ ፕሬዝዳንት ማናቸው?", + "የአሜሪካ ዋና ከተማ ምንድነው?", + "ባራክ ኦባማ መቼ ተወለደ?", +] diff --git a/spacy/lang/am/lex_attrs.py b/spacy/lang/am/lex_attrs.py new file mode 100644 index 0000000..c7b2aab --- /dev/null +++ b/spacy/lang/am/lex_attrs.py @@ -0,0 +1,102 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "ዜሮ", + "አንድ", + "ሁለት", + "ሶስት", + "አራት", + "አምስት", + "ስድስት", + "ሰባት", + "ስምት", + "ዘጠኝ", + "አስር", + "አስራ አንድ", + "አስራ ሁለት", + "አስራ ሶስት", + "አስራ አራት", + "አስራ አምስት", + "አስራ ስድስት", + "አስራ ሰባት", + "አስራ ስምንት", + "አስራ ዘጠኝ", + "ሃያ", + "ሰላሳ", + "አርባ", + "ሃምሳ", + "ስልሳ", + "ሰባ", + "ሰማንያ", + "ዘጠና", + "መቶ", + "ሺህ", + "ሚሊዮን", + "ቢሊዮን", + "ትሪሊዮን", + "ኳድሪሊዮን", + "ገጅሊዮን", + "ባዝሊዮን", +] + +_ordinal_words = [ + "አንደኛ", + "ሁለተኛ", + "ሶስተኛ", + "አራተኛ", + "አምስተኛ", + "ስድስተኛ", + "ሰባተኛ", + "ስምንተኛ", + "ዘጠነኛ", + "አስረኛ", + "አስራ አንደኛ", + "አስራ ሁለተኛ", + "አስራ ሶስተኛ", + "አስራ አራተኛ", + "አስራ አምስተኛ", + "አስራ ስድስተኛ", + "አስራ ሰባተኛ", + "አስራ ስምንተኛ", + "አስራ ዘጠነኛ", + "ሃያኛ", + "ሰላሳኛአርባኛ", + "አምሳኛ", + "ስድሳኛ", + "ሰባኛ", + "ሰማንያኛ", + "ዘጠናኛ", + "መቶኛ", + "ሺኛ", + "ሚሊዮንኛ", + "ቢሊዮንኛ", + "ትሪሊዮንኛ", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + + text_lower = text.lower() + if text_lower in _num_words: + return True + + # Check ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith("ኛ"): + if text_lower[:-2].isdigit(): + return True + + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/am/punctuation.py b/spacy/lang/am/punctuation.py new file mode 100644 index 0000000..87447b0 --- /dev/null +++ b/spacy/lang/am/punctuation.py @@ -0,0 +1,25 @@ +from ..char_classes import ( + ALPHA_UPPER, + CURRENCY, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +_list_punct = LIST_PUNCT + "፡ ። ፣ ፤ ፥ ፦ ፧ ፠ ፨".strip().split() + +_suffixes = ( + _list_punct + + LIST_ELLIPSES + + LIST_QUOTES + + [ + r"(?<=[0-9])\+", + # Amharic is written from Left-To-Right + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/am/stop_words.py b/spacy/lang/am/stop_words.py new file mode 100644 index 0000000..5487ada --- /dev/null +++ b/spacy/lang/am/stop_words.py @@ -0,0 +1,33 @@ +# Stop words by Teshome Kassie http://etd.aau.edu.et/bitstream/handle/123456789/3315/Teshome%20Kassie.pdf?sequence=1&isAllowed=y +# Stop words by Tihitina Petros http://etd.aau.edu.et/bitstream/handle/123456789/3384/Tihitina%20Petros.pdf?sequence=1&isAllowed=y + +STOP_WORDS = set( + """ +ግን አንቺ አንተ እናንተ ያንተ ያንቺ የናንተ ራስህን ራስሽን ራሳችሁን +ሁሉ ኋላ በሰሞኑ አሉ በኋላ ሁኔታ በኩል አስታውቀዋል ሆነ በውስጥ +አስታውሰዋል ሆኑ ባጣም እስካሁን ሆኖም በተለይ አሳሰበ ሁል በተመለከተ +አሳስበዋል ላይ በተመሳሳይ አስፈላጊ ሌላ የተለያየ አስገነዘቡ ሌሎች የተለያዩ +አስገንዝበዋል ልዩ ተባለ አብራርተዋል መሆኑ ተገለጸ አስረድተዋል ተገልጿል +ማለቱ ተጨማሪ እባክህ የሚገኝ ተከናወነ እባክሽ ማድረግ ችግር አንጻር ማን +ትናንት እስኪደርስ ነበረች እንኳ ሰሞኑን ነበሩ እንኳን ሲሆን ነበር እዚሁ ሲል +ነው እንደገለጹት አለ ና እንደተናገሩት ቢሆን ነገር እንዳስረዱት ብለዋል ነገሮች +እንደገና ብዙ ናት ወቅት ቦታ ናቸው እንዲሁም በርካታ አሁን እንጂ እስከ +ማለት የሚሆኑት ስለማናቸውም ውስጥ ይሆናሉ ሲባል ከሆነው ስለዚሁ ከአንድ +ያልሆነ ሳለ የነበረውን ከአንዳንድ በማናቸውም በሙሉ የሆነው ያሉ በእነዚሁ +ወር መሆናቸው ከሌሎች በዋና አንዲት ወይም +በላይ እንደ በማቀድ ለሌሎች በሆኑ ቢሆንም ጊዜና ይሆኑበታል በሆነ አንዱ +ለዚህ ለሆነው ለነዚህ ከዚህ የሌላውን ሶስተኛ አንዳንድ ለማንኛውም የሆነ ከሁለት +የነገሩ ሰኣት አንደኛ እንዲሆን እንደነዚህ ማንኛውም ካልሆነ የሆኑት ጋር ቢያንስ +ይህንንም እነደሆነ እነዚህን ይኸው የማናቸውም +በሙሉም ይህችው በተለይም አንዱን የሚችለውን በነዚህ ከእነዚህ በሌላ +የዚሁ ከእነዚሁ ለዚሁ በሚገባ ለእያንዳንዱ የአንቀጹ ወደ ይህም ስለሆነ ወይ +ማናቸውንም ተብሎ እነዚህ መሆናቸውን የሆነችን ከአስር ሳይሆን ከዚያ የለውም +የማይበልጥ እንደሆነና እንዲሆኑ በሚችሉ ብቻ ብሎ ከሌላ የሌላቸውን +ለሆነ በሌሎች ሁለቱንም በቀር ይህ በታች አንደሆነ በነሱ +ይህን የሌላ እንዲህ ከሆነ ያላቸው በነዚሁ በሚል የዚህ ይህንኑ +በእንደዚህ ቁጥር ማናቸውም ሆነው ባሉ በዚህ በስተቀር ሲሆንና +በዚህም መሆን ምንጊዜም እነዚህም በዚህና ያለ ስም +ሲኖር ከዚህም መሆኑን በሁኔታው የማያንስ እነዚህኑ ማንም ከነዚሁ +ያላቸውን እጅግ ሲሆኑ ለሆኑ ሊሆን ለማናቸውም +""".split() +) diff --git a/spacy/lang/am/tokenizer_exceptions.py b/spacy/lang/am/tokenizer_exceptions.py new file mode 100644 index 0000000..1ccf996 --- /dev/null +++ b/spacy/lang/am/tokenizer_exceptions.py @@ -0,0 +1,20 @@ +from ...symbols import NORM, ORTH + +_exc = {} + + +for exc_data in [ + {ORTH: "ት/ቤት"}, + {ORTH: "ወ/ሮ", NORM: "ወይዘሮ"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "ዓ.ም.", + "ኪ.ሜ.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = _exc diff --git a/spacy/lang/ar/__init__.py b/spacy/lang/ar/__init__.py new file mode 100644 index 0000000..d50b072 --- /dev/null +++ b/spacy/lang/ar/__init__.py @@ -0,0 +1,21 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class ArabicDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + suffixes = TOKENIZER_SUFFIXES + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} + + +class Arabic(Language): + Defaults = ArabicDefaults + lang = "ar" + + +__all__ = ["Arabic"] diff --git a/spacy/lang/ar/examples.py b/spacy/lang/ar/examples.py new file mode 100644 index 0000000..a51bb9d --- /dev/null +++ b/spacy/lang/ar/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ar.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "نال الكاتب خالد توفيق جائزة الرواية العربية في معرض الشارقة الدولي للكتاب", + "أين تقع دمشق ؟", + "كيف حالك ؟", + "هل يمكن ان نلتقي على الساعة الثانية عشرة ظهرا ؟", + "ماهي أبرز التطورات السياسية، الأمنية والاجتماعية في العالم ؟", + "هل بالإمكان أن نلتقي غدا؟", + "هناك نحو 382 مليون شخص مصاب بداء السكَّري في العالم", + "كشفت دراسة حديثة أن الخيل تقرأ تعبيرات الوجه وتستطيع أن تتذكر مشاعر الناس وعواطفهم", +] diff --git a/spacy/lang/ar/lex_attrs.py b/spacy/lang/ar/lex_attrs.py new file mode 100644 index 0000000..54ad7a8 --- /dev/null +++ b/spacy/lang/ar/lex_attrs.py @@ -0,0 +1,97 @@ +from ...attrs import LIKE_NUM + +_num_words = set( + """ +صفر +واحد +إثنان +اثنان +ثلاثة +ثلاثه +أربعة +أربعه +خمسة +خمسه +ستة +سته +سبعة +سبعه +ثمانية +ثمانيه +تسعة +تسعه +ﻋﺸﺮﺓ +ﻋﺸﺮه +عشرون +عشرين +ثلاثون +ثلاثين +اربعون +اربعين +أربعون +أربعين +خمسون +خمسين +ستون +ستين +سبعون +سبعين +ثمانون +ثمانين +تسعون +تسعين +مائتين +مائتان +ثلاثمائة +خمسمائة +سبعمائة +الف +آلاف +ملايين +مليون +مليار +مليارات +""".split() +) + +_ordinal_words = set( + """ +اول +أول +حاد +واحد +ثان +ثاني +ثالث +رابع +خامس +سادس +سابع +ثامن +تاسع +عاشر +""".split() +) + + +def like_num(text): + """ + Check if text resembles a number + """ + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ar/punctuation.py b/spacy/lang/ar/punctuation.py new file mode 100644 index 0000000..cf03fc6 --- /dev/null +++ b/spacy/lang/ar/punctuation.py @@ -0,0 +1,23 @@ +from ..char_classes import ( + ALPHA_UPPER, + CURRENCY, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + [ + r"(?<=[0-9])\+", + # Arabic is written from Right-To-Left + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/ar/stop_words.py b/spacy/lang/ar/stop_words.py new file mode 100644 index 0000000..f4da54d --- /dev/null +++ b/spacy/lang/ar/stop_words.py @@ -0,0 +1,390 @@ +STOP_WORDS = set( + """ +من +نحو +لعل +بما +بين +وبين +ايضا +وبينما +تحت +مثلا +لدي +عنه +مع +هي +وهذا +واذا +هذان +انه +بينما +أمسى +وسوف +ولم +لذلك +إلى +منه +منها +كما +ظل +هنا +به +كذلك +اما +هما +بعد +بينهم +التي +أبو +اذا +بدلا +لها +أمام +يلي +حين +ضد +الذي +قد +صار +إذا +مابرح +قبل +كل +وليست +الذين +لهذا +وثي +انهم +باللتي +مافتئ +ولا +بهذه +بحيث +كيف +وله +علي +بات +لاسيما +حتى +وقد +و +أما +فيها +بهذا +لذا +حيث +لقد +إن +فإن +اول +ليت +فاللتي +ولقد +لسوف +هذه +ولماذا +معه +الحالي +بإن +حول +في +عليه +مايزال +ولعل +أنه +أضحى +اي +ستكون +لن +أن +ضمن +وعلى +امسى +الي +ذات +ولايزال +ذلك +فقد +هم +أي +عند +ابن +أو +فهو +فانه +سوف +ما +آل +كلا +عنها +وكذلك +ليست +لم +وأن +ماذا +لو +وهل +اللتي +ولذا +يمكن +فيه +الا +عليها +وبينهم +يوم +وبما +لما +فكان +اضحى +اصبح +لهم +بها +او +الذى +الى +إلي +قال +والتي +لازال +أصبح +ولهذا +مثل +وكانت +لكنه +بذلك +هذا +لماذا +قالت +فقط +لكن +مما +وكل +وان +وأبو +ومن +كان +مازال +هل +بينهن +هو +وما +على +وهو +لأن +واللتي +والذي +دون +عن +وايضا +هناك +بلا +جدا +ثم +منذ +اللذين +لايزال +بعض +مساء +تكون +فلا +بيننا +لا +ولكن +إذ +وأثناء +ليس +ومع +فيهم +ولسوف +بل +تلك +أحد +وهي +وكان +ومنها +وفي +ماانفك +اليوم +وماذا +هؤلاء +وليس +له +أثناء +بد +اليه +كأن +اليها +بتلك +يكون +ولما +هن +والى +كانت +وقبل +ان +لدى +إذما +إذن +أف +أقل +أكثر +ألا +إلا +اللاتي +اللائي +اللتان +اللتيا +اللتين +اللذان +اللواتي +إليك +إليكم +إليكما +إليكن +أم +أما +إما +إنا +أنا +أنت +أنتم +أنتما +أنتن +إنما +إنه +أنى +أنى +آه +آها +أولاء +أولئك +أوه +آي +أيها +إي +أين +أين +أينما +إيه +بخ +بس +بك +بكم +بكم +بكما +بكن +بلى +بماذا +بمن +بنا +بهم +بهما +بهن +بي +بيد +تلكم +تلكما +ته +تي +تين +تينك +ثمة +حاشا +حبذا +حيثما +خلا +ذا +ذاك +ذان +ذانك +ذلكم +ذلكما +ذلكن +ذه +ذو +ذوا +ذواتا +ذواتي +ذي +ذين +ذينك +ريث +سوى +شتان +عدا +عسى +عل +عليك +عما +غير +فإذا +فمن +فيم +فيما +كأنما +كأي +كأين +كذا +كلاهما +كلتا +كلما +كليكما +كليهما +كم +كم +كي +كيت +كيفما +لست +لستم +لستما +لستن +لسن +لسنا +لك +لكم +لكما +لكنما +لكي +لكيلا +لنا +لهما +لهن +لولا +لوما +لي +لئن +ليسا +ليستا +ليسوا +متى +مذ +ممن +مه +مهما +نحن +نعم +ها +هاتان +هاته +هاتي +هاتين +هاك +هاهنا +هذي +هذين +هكذا +هلا +هنالك +هيا +هيت +هيهات +والذين +وإذ +وإذا +وإن +ولو +يا +""".split() +) diff --git a/spacy/lang/ar/tokenizer_exceptions.py b/spacy/lang/ar/tokenizer_exceptions.py new file mode 100644 index 0000000..eb16876 --- /dev/null +++ b/spacy/lang/ar/tokenizer_exceptions.py @@ -0,0 +1,47 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +# Time +for exc_data in [ + {NORM: "قبل الميلاد", ORTH: "ق.م"}, + {NORM: "بعد الميلاد", ORTH: "ب. م"}, + {NORM: "ميلادي", ORTH: ".م"}, + {NORM: "هجري", ORTH: ".هـ"}, + {NORM: "توفي", ORTH: ".ت"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +# Scientific abv. +for exc_data in [ + {NORM: "صلى الله عليه وسلم", ORTH: "صلعم"}, + {NORM: "الشارح", ORTH: "الشـ"}, + {NORM: "الظاهر", ORTH: "الظـ"}, + {NORM: "أيضًا", ORTH: "أيضـ"}, + {NORM: "إلى آخره", ORTH: "إلخ"}, + {NORM: "انتهى", ORTH: "اهـ"}, + {NORM: "حدّثنا", ORTH: "ثنا"}, + {NORM: "حدثني", ORTH: "ثنى"}, + {NORM: "أنبأنا", ORTH: "أنا"}, + {NORM: "أخبرنا", ORTH: "نا"}, + {NORM: "مصدر سابق", ORTH: "م. س"}, + {NORM: "مصدر نفسه", ORTH: "م. ن"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +# Other abv. +for exc_data in [ + {NORM: "دكتور", ORTH: "د."}, + {NORM: "أستاذ دكتور", ORTH: "أ.د"}, + {NORM: "أستاذ", ORTH: "أ."}, + {NORM: "بروفيسور", ORTH: "ب."}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +for exc_data in [{NORM: "تلفون", ORTH: "ت."}, {NORM: "صندوق بريد", ORTH: "ص.ب"}]: + _exc[exc_data[ORTH]] = [exc_data] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/az/__init__.py b/spacy/lang/az/__init__.py new file mode 100644 index 0000000..32949aa --- /dev/null +++ b/spacy/lang/az/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class AzerbaijaniDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Azerbaijani(Language): + lang = "az" + Defaults = AzerbaijaniDefaults + + +__all__ = ["Azerbaijani"] diff --git a/spacy/lang/az/examples.py b/spacy/lang/az/examples.py new file mode 100644 index 0000000..df5e352 --- /dev/null +++ b/spacy/lang/az/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.az.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Bu bir cümlədir.", + "Necəsən?", + "Qarabağ ordeni vətən müharibəsində qələbə münasibəti ilə təsis edilmişdir.", + "Məktəbimizə Bakıdan bir tarix müəllimi gəlmişdi.", + "Atılan növbəti mərmilər lap yaxınlıqda partladı.", + "Sinqapur koronavirus baxımından ən təhlükəsiz ölkələr sırasındadır.", + "Marsda ilk sınaq uçuşu həyata keçirilib.", + "SSRİ dağılandan bəri 5 sahil dövləti Xəzərin statusunu müəyyən edə bilməyiblər.", + "Videoda beyninə xüsusi çip yerləşdirilmiş meymun əks olunub.", +] diff --git a/spacy/lang/az/lex_attrs.py b/spacy/lang/az/lex_attrs.py new file mode 100644 index 0000000..96fb7f0 --- /dev/null +++ b/spacy/lang/az/lex_attrs.py @@ -0,0 +1,88 @@ +from ...attrs import LIKE_NUM + +# Eleven, twelve etc. are written separate: on bir, on iki + +_num_words = [ + "bir", + "iki", + "üç", + "dörd", + "beş", + "altı", + "yeddi", + "səkkiz", + "doqquz", + "on", + "iyirmi", + "otuz", + "qırx", + "əlli", + "altmış", + "yetmiş", + "səksən", + "doxsan", + "yüz", + "min", + "milyon", + "milyard", + "trilyon", + "kvadrilyon", + "kentilyon", +] + + +_ordinal_words = [ + "birinci", + "ikinci", + "üçüncü", + "dördüncü", + "beşinci", + "altıncı", + "yedinci", + "səkkizinci", + "doqquzuncu", + "onuncu", + "iyirminci", + "otuzuncu", + "qırxıncı", + "əllinci", + "altmışıncı", + "yetmişinci", + "səksəninci", + "doxsanıncı", + "yüzüncü", + "mininci", + "milyonuncu", + "milyardıncı", + "trilyonuncu", + "kvadrilyonuncu", + "kentilyonuncu", +] + +_ordinal_endings = ("inci", "ıncı", "nci", "ncı", "uncu", "üncü") + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + # Check cardinal number + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith(_ordinal_endings): + if text_lower[:-3].isdigit() or text_lower[:-4].isdigit(): + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/az/stop_words.py b/spacy/lang/az/stop_words.py new file mode 100644 index 0000000..2114939 --- /dev/null +++ b/spacy/lang/az/stop_words.py @@ -0,0 +1,145 @@ +# Source: https://github.com/eliasdabbas/advertools/blob/master/advertools/stopwords.py +STOP_WORDS = set( + """ +amma +arasında +artıq +ay +az +bax +belə +beş +bilər +bir +biraz +biri +birşey +biz +bizim +bizlər +bu +buna +bundan +bunların +bunu +bunun +buradan +bütün +bəli +bəlkə +bəy +bəzi +bəzən +daha +dedi +deyil +dir +düz +də +dək +dən +dəqiqə +edir +edən +elə +et +etdi +etmə +etmək +faiz +gilə +görə +ha +haqqında +harada +heç +hə +həm +həmin +həmişə +hər +idi +il +ildə +ilk +ilə +in +indi +istifadə +isə +ki +kim +kimi +kimə +lakin +lap +mirşey +məhz +mən +mənə +niyə +nə +nəhayət +o +obirisi +of +olan +olar +olaraq +oldu +olduğu +olmadı +olmaz +olmuşdur +olsun +olur +on +ona +ondan +onlar +onlardan +onların +onsuzda +onu +onun +oradan +qarşı +qədər +saat +sadəcə +saniyə +siz +sizin +sizlər +sonra +səhv +sən +sənin +sənə +təəssüf +var +və +xan +xanım +xeyr +ya +yalnız +yaxşı +yeddi +yenə +yox +yoxdur +yoxsa +yəni +zaman +çox +çünki +öz +özü +üçün +əgər +əlbəttə +ən +əslində +""".split() +) diff --git a/spacy/lang/bg/__init__.py b/spacy/lang/bg/__init__.py new file mode 100644 index 0000000..acca63b --- /dev/null +++ b/spacy/lang/bg/__init__.py @@ -0,0 +1,31 @@ +from ...attrs import LANG +from ...language import BaseDefaults, Language +from ...util import update_exc +from ..punctuation import ( + COMBINING_DIACRITICS_TOKENIZER_INFIXES, + COMBINING_DIACRITICS_TOKENIZER_SUFFIXES, +) +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class BulgarianDefaults(BaseDefaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters[LANG] = lambda text: "bg" + + lex_attr_getters.update(LEX_ATTRS) + + stop_words = STOP_WORDS + tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES + + +class Bulgarian(Language): + lang = "bg" + Defaults = BulgarianDefaults + + +__all__ = ["Bulgarian"] diff --git a/spacy/lang/bg/examples.py b/spacy/lang/bg/examples.py new file mode 100644 index 0000000..a6d40da --- /dev/null +++ b/spacy/lang/bg/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.bg.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Епъл иска да купи английски стартъп за 1 милиард долара." + "Автономните коли прехвърлят застрахователната отговорност към производителите." + "Сан Франциско обмисля забрана на роботи доставящи по тротоари. " + "Лондон е голям град в Обединеното Кралство." +] diff --git a/spacy/lang/bg/lex_attrs.py b/spacy/lang/bg/lex_attrs.py new file mode 100644 index 0000000..0b7942a --- /dev/null +++ b/spacy/lang/bg/lex_attrs.py @@ -0,0 +1,87 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "нула", + "едно", + "един", + "една", + "две", + "три", + "четири", + "пет", + "шест", + "седем", + "осем", + "девет", + "десет", + "единадесет", + "единайсет", + "дванадесет", + "дванайсет", + "тринадесет", + "тринайсет", + "четиринадесет", + "четиринайсет", + "петнадесет", + "петнайсет", + "шестнадесет", + "шестнайсет", + "седемнадесет", + "седемнайсет", + "осемнадесет", + "осемнайсет", + "деветнадесет", + "деветнайсет", + "двадесет", + "двайсет", + "тридесет", + "трийсет", + "четиридесет", + "четиресет", + "петдесет", + "шестдесет", + "шейсет", + "седемдесет", + "осемдесет", + "деветдесет", + "сто", + "двеста", + "триста", + "четиристотин", + "петстотин", + "шестстотин", + "седемстотин", + "осемстотин", + "деветстотин", + "хиляда", + "милион", + "милиона", + "милиард", + "милиарда", + "трилион", + "трилионa", + "билион", + "билионa", + "квадрилион", + "квадрилионa", + "квинтилион", + "квинтилионa", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/bg/stop_words.py b/spacy/lang/bg/stop_words.py new file mode 100644 index 0000000..061850d --- /dev/null +++ b/spacy/lang/bg/stop_words.py @@ -0,0 +1,80 @@ +""" +References: + https://github.com/Alir3z4/stop-words - Original list, serves as a base. + https://postvai.com/books/stop-dumi.pdf - Additions to the original list in order to improve it. +""" + +STOP_WORDS = set( + """ +а автентичен аз ако ала + +бе без беше би бивш бивша бившо бивши бил била били било благодаря близо бъдат +бъде бъда бяха + +в вас ваш ваша вашата вашият вероятно вече взема ви вие винаги внимава време все +всеки всички вместо всичко вследствие всъщност всяка втори във въпреки върху +вътре веднъж + +г ги главен главна главно глас го годно година години годишен + +д да дали далеч далече два двама двамата две двете ден днес дни до добра добре +добро добър достатъчно докато докога дори досега доста друг друга другаде други + +е евтин едва един една еднаква еднакви еднакъв едно екип ето + +живот жив + +за здравей здрасти знае зная забавям зад зададени заедно заради засега заспал +затова запазва започвам защо защото завинаги + +и из или им има имат иска искам използвайки изглежда изглеждаше изглеждайки +извън имайки + +й йо + +каза казва казвайки казвам как каква какво както какъв като кога кауза каузи +когато когото което които кой който колко която къде където към край кратък +кръгъл + +лесен лесно ли летя летиш летим лош + +м май малко макар малцина междувременно минус ме между мек мен месец ми мис +мисля много мнозина мога могат може мой можем мокър моля момента му + +н на над назад най наш навсякъде навътре нагоре направи напред надолу наистина +например наопаки наполовина напоследък нека независимо нас насам наскоро +настрана необходимо него негов нещо нея ни ние никой нито нищо но нов някак нова +нови новина някои някой някога някъде няколко няма + +о обаче около описан опитах опитва опитвайки опитвам определен определено освен +обикновено осигурява обратно означава особен особено от ох отвъд отгоре отдолу +отново отива отивам отидох отсега отделно отколкото откъдето очевидно оттам +относно още + +п пак по повече повечето под поне просто пряко поради после последен последно +посочен почти прави прав прави правя пред преди през при пък първата първи първо +път пъти плюс + +равен равна различен различни разумен разумно + +с са сам само себе сериозно сигурен сигурно се сега си син скоро скорошен след +следващ следващия следва следното следователно случва сме смях собствен +сравнително смея според сред става срещу съвсем съдържа съдържащ съжалявам +съответен съответно сте съм със също + +т така техен техни такива такъв твърде там трета твой те тези ти то това +тогава този той търси толкова точно три трябва тук тъй тя тях + +у утре ужасно употреба успоредно уточнен уточняване + +харесва харесали хиляди + +ч часа ценя цяло цялостен че често чрез чудя + +ще щеше щом щяха + +юмрук + +я як +""".split() +) diff --git a/spacy/lang/bg/tokenizer_exceptions.py b/spacy/lang/bg/tokenizer_exceptions.py new file mode 100644 index 0000000..89d466d --- /dev/null +++ b/spacy/lang/bg/tokenizer_exceptions.py @@ -0,0 +1,212 @@ +""" +References: + https://slovored.com/bg/abbr/grammar/ - Additional refs for abbreviations + (countries, occupations, fields of studies and more). +""" + +from ...symbols import NORM, ORTH + +_exc = {} + +# measurements +for abbr in [ + {ORTH: "м", NORM: "метър"}, + {ORTH: "мм", NORM: "милиметър"}, + {ORTH: "см", NORM: "сантиметър"}, + {ORTH: "дм", NORM: "дециметър"}, + {ORTH: "км", NORM: "километър"}, + {ORTH: "кг", NORM: "килограм"}, + {ORTH: "мг", NORM: "милиграм"}, + {ORTH: "г", NORM: "грам"}, + {ORTH: "т", NORM: "тон"}, + {ORTH: "хл", NORM: "хектолиър"}, + {ORTH: "дкл", NORM: "декалитър"}, + {ORTH: "л", NORM: "литър"}, +]: + _exc[abbr[ORTH]] = [abbr] + +# line abbreviations +for abbr in [ + {ORTH: "г-жа", NORM: "госпожа"}, + {ORTH: "г-н", NORM: "господин"}, + {ORTH: "г-ца", NORM: "госпожица"}, + {ORTH: "д-р", NORM: "доктор"}, + {ORTH: "о-в", NORM: "остров"}, + {ORTH: "п-в", NORM: "полуостров"}, + {ORTH: "с-у", NORM: "срещу"}, + {ORTH: "в-у", NORM: "върху"}, + {ORTH: "м-у", NORM: "между"}, +]: + _exc[abbr[ORTH]] = [abbr] + +# foreign language related abbreviations +for abbr in [ + {ORTH: "англ.", NORM: "английски"}, + {ORTH: "ан.", NORM: "английски термин"}, + {ORTH: "араб.", NORM: "арабски"}, + {ORTH: "афр.", NORM: "африкански"}, + {ORTH: "гр.", NORM: "гръцки"}, + {ORTH: "лат.", NORM: "латински"}, + {ORTH: "рим.", NORM: "римски"}, + {ORTH: "старогр.", NORM: "старогръцки"}, + {ORTH: "староевр.", NORM: "староеврейски"}, + {ORTH: "фр.", NORM: "френски"}, + {ORTH: "хол.", NORM: "холандски"}, + {ORTH: "швед.", NORM: "шведски"}, + {ORTH: "шотл.", NORM: "шотландски"}, + {ORTH: "яп.", NORM: "японски"}, +]: + _exc[abbr[ORTH]] = [abbr] + +# profession and academic titles abbreviations +for abbr in [ + {ORTH: "акад.", NORM: "академик"}, + {ORTH: "арх.", NORM: "архитект"}, + {ORTH: "инж.", NORM: "инженер"}, + {ORTH: "канц.", NORM: "канцлер"}, + {ORTH: "проф.", NORM: "професор"}, + {ORTH: "св.", NORM: "свети"}, +]: + _exc[abbr[ORTH]] = [abbr] + +# fields of studies +for abbr in [ + {ORTH: "агр.", NORM: "агрономия"}, + {ORTH: "ав.", NORM: "авиация"}, + {ORTH: "агр.", NORM: "агрономия"}, + {ORTH: "археол.", NORM: "археология"}, + {ORTH: "астр.", NORM: "астрономия"}, + {ORTH: "геод.", NORM: "геодезия"}, + {ORTH: "геол.", NORM: "геология"}, + {ORTH: "геом.", NORM: "геометрия"}, + {ORTH: "гимн.", NORM: "гимнастика"}, + {ORTH: "грам.", NORM: "граматика"}, + {ORTH: "жур.", NORM: "журналистика"}, + {ORTH: "журн.", NORM: "журналистика"}, + {ORTH: "зем.", NORM: "земеделие"}, + {ORTH: "икон.", NORM: "икономика"}, + {ORTH: "лит.", NORM: "литература"}, + {ORTH: "мат.", NORM: "математика"}, + {ORTH: "мед.", NORM: "медицина"}, + {ORTH: "муз.", NORM: "музика"}, + {ORTH: "печ.", NORM: "печатарство"}, + {ORTH: "пол.", NORM: "политика"}, + {ORTH: "псих.", NORM: "психология"}, + {ORTH: "соц.", NORM: "социология"}, + {ORTH: "стат.", NORM: "статистика"}, + {ORTH: "стил.", NORM: "стилистика"}, + {ORTH: "топогр.", NORM: "топография"}, + {ORTH: "търг.", NORM: "търговия"}, + {ORTH: "фарм.", NORM: "фармацевтика"}, + {ORTH: "фехт.", NORM: "фехтовка"}, + {ORTH: "физиол.", NORM: "физиология"}, + {ORTH: "физ.", NORM: "физика"}, + {ORTH: "фил.", NORM: "философия"}, + {ORTH: "фин.", NORM: "финанси"}, + {ORTH: "фолкл.", NORM: "фолклор"}, + {ORTH: "фон.", NORM: "фонетика"}, + {ORTH: "фот.", NORM: "фотография"}, + {ORTH: "футб.", NORM: "футбол"}, + {ORTH: "хим.", NORM: "химия"}, + {ORTH: "хир.", NORM: "хирургия"}, + {ORTH: "ел.", NORM: "електротехника"}, +]: + _exc[abbr[ORTH]] = [abbr] + +for abbr in [ + {ORTH: "ал.", NORM: "алинея"}, + {ORTH: "авт.", NORM: "автоматично"}, + {ORTH: "адм.", NORM: "администрация"}, + {ORTH: "арт.", NORM: "артилерия"}, + {ORTH: "бл.", NORM: "блок"}, + {ORTH: "бр.", NORM: "брой"}, + {ORTH: "бул.", NORM: "булевард"}, + {ORTH: "букв.", NORM: "буквално"}, + {ORTH: "в.", NORM: "век"}, + {ORTH: "вр.", NORM: "време"}, + {ORTH: "вм.", NORM: "вместо"}, + {ORTH: "воен.", NORM: "военен термин"}, + {ORTH: "г.", NORM: "година"}, + {ORTH: "гр.", NORM: "град"}, + {ORTH: "гл.", NORM: "глагол"}, + {ORTH: "др.", NORM: "други"}, + {ORTH: "ез.", NORM: "езеро"}, + {ORTH: "ж.р.", NORM: "женски род"}, + {ORTH: "жп.", NORM: "железопът"}, + {ORTH: "застр.", NORM: "застрахователно дело"}, + {ORTH: "знач.", NORM: "значение"}, + {ORTH: "и др.", NORM: "и други"}, + {ORTH: "и под.", NORM: "и подобни"}, + {ORTH: "и пр.", NORM: "и прочие"}, + {ORTH: "изр.", NORM: "изречение"}, + {ORTH: "изт.", NORM: "източен"}, + {ORTH: "конкр.", NORM: "конкретно"}, + {ORTH: "лв.", NORM: "лев"}, + {ORTH: "л.", NORM: "лице"}, + {ORTH: "м.р.", NORM: "мъжки род"}, + {ORTH: "мин.вр.", NORM: "минало време"}, + {ORTH: "мн.ч.", NORM: "множествено число"}, + {ORTH: "напр.", NORM: "например"}, + {ORTH: "нар.", NORM: "наречие"}, + {ORTH: "науч.", NORM: "научен термин"}, + {ORTH: "непр.", NORM: "неправилно"}, + {ORTH: "обик.", NORM: "обикновено"}, + {ORTH: "опред.", NORM: "определение"}, + {ORTH: "особ.", NORM: "особено"}, + {ORTH: "ост.", NORM: "остаряло"}, + {ORTH: "относ.", NORM: "относително"}, + {ORTH: "отр.", NORM: "отрицателно"}, + {ORTH: "пл.", NORM: "площад"}, + {ORTH: "пад.", NORM: "падеж"}, + {ORTH: "парл.", NORM: "парламентарен"}, + {ORTH: "погов.", NORM: "поговорка"}, + {ORTH: "пон.", NORM: "понякога"}, + {ORTH: "правосл.", NORM: "православен"}, + {ORTH: "прибл.", NORM: "приблизително"}, + {ORTH: "прил.", NORM: "прилагателно име"}, + {ORTH: "пр.", NORM: "прочие"}, + {ORTH: "с.", NORM: "село"}, + {ORTH: "с.р.", NORM: "среден род"}, + {ORTH: "сп.", NORM: "списание"}, + {ORTH: "стр.", NORM: "страница"}, + {ORTH: "сз.", NORM: "съюз"}, + {ORTH: "сег.", NORM: "сегашно"}, + {ORTH: "сп.", NORM: "спорт"}, + {ORTH: "срв.", NORM: "сравни"}, + {ORTH: "с.ст.", NORM: "селскостопанска техника"}, + {ORTH: "счет.", NORM: "счетоводство"}, + {ORTH: "съкр.", NORM: "съкратено"}, + {ORTH: "съобщ.", NORM: "съобщение"}, + {ORTH: "същ.", NORM: "съществително"}, + {ORTH: "текст.", NORM: "текстилен"}, + {ORTH: "телев.", NORM: "телевизия"}, + {ORTH: "тел.", NORM: "телефон"}, + {ORTH: "т.е.", NORM: "тоест"}, + {ORTH: "т.н.", NORM: "така нататък"}, + {ORTH: "т.нар.", NORM: "така наречен"}, + {ORTH: "търж.", NORM: "тържествено"}, + {ORTH: "ул.", NORM: "улица"}, + {ORTH: "уч.", NORM: "училище"}, + {ORTH: "унив.", NORM: "университет"}, + {ORTH: "харт.", NORM: "хартия"}, + {ORTH: "хидр.", NORM: "хидравлика"}, + {ORTH: "хран.", NORM: "хранителна"}, + {ORTH: "църк.", NORM: "църковен термин"}, + {ORTH: "числ.", NORM: "числително"}, + {ORTH: "чл.", NORM: "член"}, + {ORTH: "ч.", NORM: "число"}, + {ORTH: "числ.", NORM: "числително"}, + {ORTH: "шахм.", NORM: "шахмат"}, + {ORTH: "шах.", NORM: "шахмат"}, + {ORTH: "юр.", NORM: "юридически"}, +]: + _exc[abbr[ORTH]] = [abbr] + +# slash abbreviations +for abbr in [ + {ORTH: "м/у", NORM: "между"}, + {ORTH: "с/у", NORM: "срещу"}, +]: + _exc[abbr[ORTH]] = [abbr] + +TOKENIZER_EXCEPTIONS = _exc diff --git a/spacy/lang/bn/__init__.py b/spacy/lang/bn/__init__.py new file mode 100644 index 0000000..6a5d37f --- /dev/null +++ b/spacy/lang/bn/__init__.py @@ -0,0 +1,49 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ...pipeline import Lemmatizer +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class BengaliDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + stop_words = STOP_WORDS + + +class Bengali(Language): + lang = "bn" + Defaults = BengaliDefaults + + +@Bengali.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return Lemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Bengali"] diff --git a/spacy/lang/bn/examples.py b/spacy/lang/bn/examples.py new file mode 100644 index 0000000..11a65ac --- /dev/null +++ b/spacy/lang/bn/examples.py @@ -0,0 +1,8 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.bn.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = ["তুই খুব ভালো", "আজ আমরা ডাক্তার দেখতে যাবো", "আমি জানি না "] diff --git a/spacy/lang/bn/punctuation.py b/spacy/lang/bn/punctuation.py new file mode 100644 index 0000000..ddb91ce --- /dev/null +++ b/spacy/lang/bn/punctuation.py @@ -0,0 +1,53 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +_currency = r"\$¢£€¥฿৳" +_quotes = CONCAT_QUOTES.replace("'", "") +_list_punct = LIST_PUNCT + "। ॥".strip().split() + + +_prefixes = [r"\+"] + _list_punct + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS + +_suffixes = ( + _list_punct + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:[{c}])".format(c=_currency), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{q}(?:{c})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, c=_currency + ), + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9{zero}-{nine}])[+\-\*^=](?=[0-9{zero}-{nine}-])".format( + zero="০", nine="৯" + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])({h})(?=[{ae}])".format(a=ALPHA, h=HYPHENS, ae="এ"), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/bn/stop_words.py b/spacy/lang/bn/stop_words.py new file mode 100644 index 0000000..bf38e32 --- /dev/null +++ b/spacy/lang/bn/stop_words.py @@ -0,0 +1,42 @@ +STOP_WORDS = set( + """ +অতএব অথচ অথবা অনুযায়ী অনেক অনেকে অনেকেই অন্তত অবধি অবশ্য অর্থাৎ অন্য অনুযায়ী অর্ধভাগে +আগামী আগে আগেই আছে আজ আদ্যভাগে আপনার আপনি আবার আমরা আমাকে আমাদের আমার আমি আর আরও +ইত্যাদি ইহা +উচিত উনি উপর উপরে উত্তর +এ এঁদের এঁরা এই এক একই একজন একটা একটি একবার একে এখন এখনও এখানে এখানেই এটা এসো +এটাই এটি এত এতটাই এতে এদের এবং এবার এমন এমনি এমনকি এর এরা এলো এস এসে +ঐ +ও ওঁদের ওঁর ওঁরা ওই ওকে ওখানে ওদের ওর ওরা +কখনও কত কথা কবে কয়েক কয়েকটি করছে করছেন করতে করবে করবেন করলে কয়েক কয়েকটি করিয়ে করিয়া করায় +করলেন করা করাই করায় করার করি করিতে করিয়া করিয়ে করে করেই করেছিলেন করেছে করেছেন করেন কাউকে +কাছ কাছে কাজ কাজে কারও কারণ কি কিংবা কিছু কিছুই কিন্তু কী কে কেউ কেউই কেন কোন কোনও কোনো কেমনে কোটি +ক্ষেত্রে খুব +গিয়ে গিয়েছে গুলি গেছে গেল গেলে গোটা গিয়ে গিয়েছে +চলে চান চায় চেয়ে চায় চেয়ে চার চালু চেষ্টা +ছাড়া ছাড়াও ছিল ছিলেন ছাড়া ছাড়াও +জন জনকে জনের জন্য জন্যে জানতে জানা জানানো জানায় জানিয়ে জানিয়েছে জানায় জাানিয়ে জানিয়েছে +টি +ঠিক +তখন তত তথা তবু তবে তা তাঁকে তাঁদের তাঁর তাঁরা তাঁহারা তাই তাও তাকে তাতে তাদের তার তারপর তারা তারই তাহলে তাহা তাহাতে তাহার তিনই +তিনি তিনিও তুমি তুলে তেমন তো তোমার তুই তোরা তোর তোমাদের তোদের +থাকবে থাকবেন থাকা থাকায় থাকে থাকেন থেকে থেকেই থেকেও থাকায় +দিকে দিতে দিয়ে দিয়েছে দিয়েছেন দিলেন দিয়ে দু দুটি দুটো দেওয়া দেওয়ার দেখতে দেখা দেখে দেন দেয় দেশের +দ্বারা দিয়েছে দিয়েছেন দেয় দেওয়া দেওয়ার দিন দুই +ধরা ধরে +নয় না নাই নাকি নাগাদ নানা নিজে নিজেই নিজেদের নিজের নিতে নিয়ে নিয়ে নেই নেওয়া নেওয়ার নয় নতুন +পক্ষে পর পরে পরেই পরেও পর্যন্ত পাওয়া পারি পারে পারেন পেয়ে প্রতি প্রভৃতি প্রায় পাওয়া পেয়ে প্রায় পাঁচ প্রথম প্রাথমিক +ফলে ফিরে ফের +বছর বদলে বরং বলতে বলল বললেন বলা বলে বলেছেন বলেন বসে বহু বা বাদে বার বিনা বিভিন্ন বিশেষ বিষয়টি বেশ ব্যবহার ব্যাপারে বক্তব্য বন বেশি +ভাবে ভাবেই +মত মতো মতোই মধ্যভাগে মধ্যে মধ্যেই মধ্যেও মনে মাত্র মাধ্যমে মানুষ মানুষের মোট মোটেই মোদের মোর +যখন যত যতটা যথেষ্ট যদি যদিও যা যাঁর যাঁরা যাওয়া যাওয়ার যাকে যাচ্ছে যাতে যাদের যান যাবে যায় যার যারা যায় যিনি যে যেখানে যেতে যেন +যেমন +রকম রয়েছে রাখা রেখে রয়েছে +লক্ষ +শুধু শুরু +সাধারণ সামনে সঙ্গে সঙ্গেও সব সবার সমস্ত সম্প্রতি সময় সহ সহিত সাথে সুতরাং সে সেই সেখান সেখানে সেটা সেটাই সেটাও সেটি স্পষ্ট স্বয়ং +হইতে হইবে হইয়া হওয়া হওয়ায় হওয়ার হচ্ছে হত হতে হতেই হন হবে হবেন হয় হয়তো হয়নি হয়ে হয়েই হয়েছিল হয়েছে হাজার +হয়েছেন হল হলে হলেই হলেও হলো হিসাবে হিসেবে হৈলে হোক হয় হয়ে হয়েছে হৈতে হইয়া হয়েছিল হয়েছেন হয়নি হয়েই হয়তো হওয়া হওয়ার হওয়ায় +""".split() +) diff --git a/spacy/lang/bn/tokenizer_exceptions.py b/spacy/lang/bn/tokenizer_exceptions.py new file mode 100644 index 0000000..016bf0f --- /dev/null +++ b/spacy/lang/bn/tokenizer_exceptions.py @@ -0,0 +1,25 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +for exc_data in [ + {ORTH: "ডঃ", NORM: "ডক্টর"}, + {ORTH: "ডাঃ", NORM: "ডাক্তার"}, + {ORTH: "ড.", NORM: "ডক্টর"}, + {ORTH: "ডা.", NORM: "ডাক্তার"}, + {ORTH: "মোঃ", NORM: "মোহাম্মদ"}, + {ORTH: "মো.", NORM: "মোহাম্মদ"}, + {ORTH: "সে.", NORM: "সেলসিয়াস"}, + {ORTH: "কি.মি.", NORM: "কিলোমিটার"}, + {ORTH: "কি.মি", NORM: "কিলোমিটার"}, + {ORTH: "সে.মি.", NORM: "সেন্টিমিটার"}, + {ORTH: "সে.মি", NORM: "সেন্টিমিটার"}, + {ORTH: "মি.লি.", NORM: "মিলিলিটার"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/bo/__init__.py b/spacy/lang/bo/__init__.py new file mode 100644 index 0000000..84ef8c0 --- /dev/null +++ b/spacy/lang/bo/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class TibetanDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Tibetan(Language): + lang = "bo" + Defaults = TibetanDefaults + + +__all__ = ["Tibetan"] diff --git a/spacy/lang/bo/examples.py b/spacy/lang/bo/examples.py new file mode 100644 index 0000000..8655f2d --- /dev/null +++ b/spacy/lang/bo/examples.py @@ -0,0 +1,15 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.bo.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "དོན་དུ་རྒྱ་མཚོ་བླ་མ་ཞེས་བྱ་ཞིང༌།", + "ཏཱ་ལའི་ཞེས་པ་ནི་སོག་སྐད་ཡིན་པ་དེ་བོད་སྐད་དུ་རྒྱ་མཚོའི་དོན་དུ་འཇུག", + "སོག་པོ་ཨལ་ཐན་རྒྱལ་པོས་རྒྱལ་དབང་བསོད་ནམས་རྒྱ་མཚོར་ཆེ་བསྟོད་ཀྱི་མཚན་གསོལ་བ་ཞིག་ཡིན་ཞིང༌།", + "རྗེས་སུ་རྒྱལ་བ་དགེ་འདུན་གྲུབ་དང༌། དགེ་འདུན་རྒྱ་མཚོ་སོ་སོར་ཡང་ཏཱ་ལའི་བླ་མའི་སྐུ་ཕྲེང་དང་པོ་དང༌།", + "གཉིས་པའི་མཚན་དེ་གསོལ་ཞིང༌།༸རྒྱལ་དབང་སྐུ་ཕྲེང་ལྔ་པས་དགའ་ལྡན་ཕོ་བྲང་གི་སྲིད་དབང་བཙུགས་པ་ནས་ཏཱ་ལའི་བླ་མ་ནི་བོད་ཀྱི་ཆོས་སྲིད་གཉིས་ཀྱི་དབུ་ཁྲིད་དུ་གྱུར་ཞིང་།", + "ད་ལྟའི་བར་ཏཱ་ལའི་བླ་མ་སྐུ་ཕྲེང་བཅུ་བཞི་བྱོན་ཡོད།", +] diff --git a/spacy/lang/bo/lex_attrs.py b/spacy/lang/bo/lex_attrs.py new file mode 100644 index 0000000..5535934 --- /dev/null +++ b/spacy/lang/bo/lex_attrs.py @@ -0,0 +1,65 @@ +from ...attrs import LIKE_NUM + +# reference 1: https://en.wikipedia.org/wiki/Tibetan_numerals + +_num_words = [ + "ཀླད་ཀོར་", + "གཅིག་", + "གཉིས་", + "གསུམ་", + "བཞི་", + "ལྔ་", + "དྲུག་", + "བདུན་", + "བརྒྱད་", + "དགུ་", + "བཅུ་", + "བཅུ་གཅིག་", + "བཅུ་གཉིས་", + "བཅུ་གསུམ་", + "བཅུ་བཞི་", + "བཅུ་ལྔ་", + "བཅུ་དྲུག་", + "བཅུ་བདུན་", + "བཅུ་པརྒྱད", + "བཅུ་དགུ་", + "ཉི་ཤུ་", + "སུམ་ཅུ", + "བཞི་བཅུ", + "ལྔ་བཅུ", + "དྲུག་ཅུ", + "བདུན་ཅུ", + "བརྒྱད་ཅུ", + "དགུ་བཅུ", + "བརྒྱ་", + "སྟོང་", + "ཁྲི་", + "ས་ཡ་", + " བྱེ་བ་", + "དུང་ཕྱུར་", + "ཐེར་འབུམ་", + "ཐེར་འབུམ་ཆེན་པོ་", + "ཁྲག་ཁྲིག་", + "ཁྲག་ཁྲིག་ཆེན་པོ་", +] + + +def like_num(text): + """ + Check if text resembles a number + """ + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/bo/stop_words.py b/spacy/lang/bo/stop_words.py new file mode 100644 index 0000000..407242c --- /dev/null +++ b/spacy/lang/bo/stop_words.py @@ -0,0 +1,198 @@ +# Source: https://zenodo.org/records/10148636 + +STOP_WORDS = set( + """ +འི་ +། +དུ་ +གིས་ +སོགས་ +ཏེ +གི་ +རྣམས་ +ནི +ཀུན་ +ཡི་ +འདི +ཀྱི་ +སྙེད་ +པས་ +གཞན་ +ཀྱིས་ +ཡི +ལ +ནི་ +དང་ +སོགས +ཅིང་ +ར +དུ +མི་ +སུ་ +བཅས་ +ཡོངས་ +ལས +ཙམ་ +གྱིས་ +དེ་ +ཡང་ +མཐའ་དག་ +ཏུ་ +ཉིད་ +ས +ཏེ་ +གྱི་ +སྤྱི +དེ +ཀ་ +ཡིན་ +ཞིང་ +འདི་ +རུང་ +རང་ +ཞིག་ +སྟེ +སྟེ་ +ན་རེ +ངམ +ཤིང་ +དག་ +ཏོ +རེ་ +འང་ +ཀྱང་ +ལགས་པ +ཚུ +དོ +ཡིན་པ +རེ +ན་རེ་ +ཨེ་ +ཚང་མ +ཐམས་ཅད་ +དམ་ +འོ་ +ཅིག་ +གྱིན་ +ཡིན +ན +ཁོ་ན་ +འམ་ +ཀྱིན་ +ལོ +ཀྱིས +བས་ +ལགས་ +ཤིག +གིས +ཀི་ +སྣ་ཚོགས་ +རྣམས +སྙེད་པ +ཡིས་ +གྱི +གི +བམ་ +ཤིག་ +རེ་རེ་ +ནམ +མིན་ +ནམ་ +ངམ་ +རུ་ +འགའ་ +ཀུན +ཤས་ +ཏུ +ཡིས +གིན་ +གམ་ +འོ +ཡིན་པ་ +མིན +ལགས +གྱིས +ཅང་ +འགའ +སམ་ +ཞིག +འང +ལས་ཆེ་ +འཕྲལ་ +བར་ +རུ +དང +ཡ +འག +སམ +ཀ +ཅུང་ཟད་ +ཅིག +ཉིད +དུ་མ +མ +ཡིན་བ +འམ +མམ +དམ +དག +ཁོ་ན +ཀྱི +ལམ +ཕྱི་ +ནང་ +ཙམ +ནོ་ +སོ་ +རམ་ +བོ་ +ཨང་ +ཕྱི +ཏོ་ +ཚོ +ལ་ལ་ +ཚོ་ +ཅིང +མ་གི་ +གེ +གོ +ཡིན་ལུགས་ +རོ་ +བོ +ལགས་པ་ +པས +རབ་ +འི +རམ +བས +གཞན +སྙེད་པ་ +འབའ་ +མཾ་ +པོ +ག་ +ག +གམ +སྤྱི་ +བམ +མོ་ +ཙམ་པ་ +ཤ་སྟག་ +མམ་ +རེ་རེ +སྙེད +ཏམ་ +ངོ +གྲང་ +ཏ་རེ +ཏམ +ཁ་ +ངེ་ +ཅོག་ +རིལ་ +ཉུང་ཤས་ +གིང་ +ཚ་ +ཀྱང +""".split() +) diff --git a/spacy/lang/ca/__init__.py b/spacy/lang/ca/__init__.py new file mode 100755 index 0000000..8b2f3e8 --- /dev/null +++ b/spacy/lang/ca/__init__.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import CatalanLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class CatalanDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + prefixes = TOKENIZER_PREFIXES + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + + +class Catalan(Language): + lang = "ca" + Defaults = CatalanDefaults + + +@Catalan.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return CatalanLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Catalan"] diff --git a/spacy/lang/ca/examples.py b/spacy/lang/ca/examples.py new file mode 100644 index 0000000..de54c05 --- /dev/null +++ b/spacy/lang/ca/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ca.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple està buscant comprar una startup del Regne Unit per mil milions de dòlars", + "Els cotxes autònoms deleguen la responsabilitat de l'assegurança als seus fabricants", + "San Francisco analitza prohibir els robots de repartiment", + "Londres és una gran ciutat del Regne Unit", + "El gat menja peix", + "Veig a l'home amb el telescopi", + "L'aranya menja mosques", + "El pingüí incuba en el seu niu", +] diff --git a/spacy/lang/ca/lemmatizer.py b/spacy/lang/ca/lemmatizer.py new file mode 100644 index 0000000..0f15e6e --- /dev/null +++ b/spacy/lang/ca/lemmatizer.py @@ -0,0 +1,81 @@ +from typing import List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class CatalanLemmatizer(Lemmatizer): + """ + Copied from French Lemmatizer + Catalan language lemmatizer applies the default rule based lemmatization + procedure with some modifications for better Catalan language support. + + The parts of speech 'ADV', 'PRON', 'DET', 'ADP' and 'AUX' are added to use + the rule-based lemmatization. As a last resort, the lemmatizer checks in + the lookup table. + """ + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "rule": + required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"] + return (required, []) + else: + return super().get_lookups_config(mode) + + def rule_lemmatize(self, token: Token) -> List[str]: + cache_key = (token.orth, token.pos) + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + univ_pos = token.pos_.lower() + if univ_pos in ("", "eol", "space"): + return [string.lower()] + elif "lemma_rules" not in self.lookups or univ_pos not in ( + "noun", + "verb", + "adj", + "adp", + "adv", + "aux", + "cconj", + "det", + "pron", + "punct", + "sconj", + ): + return self.lookup_lemmatize(token) + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + lookup_table = self.lookups.get_table("lemma_lookup", {}) + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, []) + string = string.lower() + forms = [] + if string in index: + forms.append(string) + self.cache[cache_key] = forms + return forms + forms.extend(exceptions.get(string, [])) + oov_forms = [] + if not forms: + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + pass + elif form in index or not form.isalpha(): + forms.append(form) + else: + oov_forms.append(form) + if not forms: + forms.extend(oov_forms) + + # use lookups, and fall back to the token itself + if not forms: + forms.append(lookup_table.get(string, [string])[0]) + forms = list(dict.fromkeys(forms)) + self.cache[cache_key] = forms + return forms diff --git a/spacy/lang/ca/lex_attrs.py b/spacy/lang/ca/lex_attrs.py new file mode 100644 index 0000000..3e99da0 --- /dev/null +++ b/spacy/lang/ca/lex_attrs.py @@ -0,0 +1,58 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "zero", + "un", + "dos", + "tres", + "quatre", + "cinc", + "sis", + "set", + "vuit", + "nou", + "deu", + "onze", + "dotze", + "tretze", + "catorze", + "quinze", + "setze", + "disset", + "divuit", + "dinou", + "vint", + "trenta", + "quaranta", + "cinquanta", + "seixanta", + "setanta", + "vuitanta", + "noranta", + "cent", + "mil", + "milió", + "bilió", + "trilió", + "quatrilió", + "gazilió", + "bazilió", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ca/punctuation.py b/spacy/lang/ca/punctuation.py new file mode 100755 index 0000000..6914f67 --- /dev/null +++ b/spacy/lang/ca/punctuation.py @@ -0,0 +1,66 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + _units, + merge_chars, +) + +ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "") + +_prefixes = ( + ["§", "%", "=", "—", "–", "-", r"\+(?![0-9])"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}][{el}])(?=[{a}0-9])".format(a=ALPHA, el=ELISION), + r"('ls|'l|'ns|'t|'m|'n|-les|-la|-lo|-li|-los|-me|-nos|-te|-vos|-se|-hi|-ne|-ho)(?![A-Za-z])|(-l'|-m'|-t'|-n')", + ] +) + +_units = _units.replace("% ", "") +UNITS = merge_chars(_units) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [r"-", "—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_PREFIXES = _prefixes diff --git a/spacy/lang/ca/stop_words.py b/spacy/lang/ca/stop_words.py new file mode 100644 index 0000000..1a87b2f --- /dev/null +++ b/spacy/lang/ca/stop_words.py @@ -0,0 +1,52 @@ +STOP_WORDS = set( + """ +a abans ací ah així això al aleshores algun alguna algunes alguns alhora allà allí allò +als altra altre altres amb ambdues ambdós anar ans apa aquell aquella aquelles aquells +aquest aquesta aquestes aquests aquí + +baix bastant bé + +cada cadascuna cadascunes cadascuns cadascú com consegueixo conseguim conseguir +consigueix consigueixen consigueixes contra + +d'un d'una d'unes d'uns dalt de del dels des des de després dins dintre donat doncs durant + +e eh el elles ells els em en encara ens entre era erem eren eres es esta estan estat +estava estaven estem esteu estic està estàvem estàveu et etc ets érem éreu és éssent + +fa faig fan fas fem fer feu fi fins fora + +gairebé + +ha han has haver havia he hem heu hi ho + +i igual iguals inclòs + +ja jo + +l'hi la les li li'n llarg llavors + +m'he ma mal malgrat mateix mateixa mateixes mateixos me mentre meu meus meva +meves mode molt molta moltes molts mon mons més + +n'he n'hi ne ni no nogensmenys només nosaltres nostra nostre nostres + +o oh oi on + +pas pel pels per per que perquè però poc poca pocs podem poden poder +podeu poques potser primer propi puc + +qual quals quan quant que quelcom qui quin quina quines quins què + +s'ha s'han sa sabem saben saber sabeu sap saps semblant semblants sense ser ses +seu seus seva seves si sobre sobretot soc solament sols som son sons sota sou sóc són + +t'ha t'han t'he ta tal també tampoc tan tant tanta tantes te tene tenim tenir teniu +teu teus teva teves tinc ton tons tot tota totes tots + +un una unes uns us últim ús + +va vaig vam van vas veu vosaltres vostra vostre vostres + +""".split() +) diff --git a/spacy/lang/ca/syntax_iterators.py b/spacy/lang/ca/syntax_iterators.py new file mode 100644 index 0000000..16a4c6a --- /dev/null +++ b/spacy/lang/ca/syntax_iterators.py @@ -0,0 +1,49 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + # fmt: off + labels = ["nsubj", "nsubj:pass", "obj", "obl", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings[label] for label in labels] + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + left = word.left_edge.i + right = word.right_edge.i + 1 + # leave prepositions and punctuation out of the left side of the chunk + if word.left_edge.pos_ == "ADP" or word.left_edge.pos_ == "PUNCT": + left = word.left_edge.i + 1 + prev_end = word.right_edge.i + # leave subordinated clauses and appositions out of the chunk + a = word.i + 1 + while a < word.right_edge.i: + paraula = doc[a] + if paraula.pos_ == "VERB": + right = paraula.left_edge.i + prev_end = paraula.left_edge.i - 1 + elif paraula.dep_ == "appos": + right = paraula.left_edge.i + 1 + prev_end = paraula.left_edge.i - 1 + a += 1 + # leave punctuation out of the right side of the chunk + if word.right_edge.pos_ == "PUNCT": + right = right - 1 + yield left, right, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/ca/tokenizer_exceptions.py b/spacy/lang/ca/tokenizer_exceptions.py new file mode 100755 index 0000000..6716578 --- /dev/null +++ b/spacy/lang/ca/tokenizer_exceptions.py @@ -0,0 +1,67 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for exc_data in [ + {ORTH: "aprox.", NORM: "aproximadament"}, + {ORTH: "pàg.", NORM: "pàgina"}, + {ORTH: "p.ex.", NORM: "per exemple"}, + {ORTH: "gen.", NORM: "gener"}, + {ORTH: "feb.", NORM: "febrer"}, + {ORTH: "abr.", NORM: "abril"}, + {ORTH: "jul.", NORM: "juliol"}, + {ORTH: "set.", NORM: "setembre"}, + {ORTH: "oct.", NORM: "octubre"}, + {ORTH: "nov.", NORM: "novembre"}, + {ORTH: "dec.", NORM: "desembre"}, + {ORTH: "Dr.", NORM: "doctor"}, + {ORTH: "Dra.", NORM: "doctora"}, + {ORTH: "Sr.", NORM: "senyor"}, + {ORTH: "Sra.", NORM: "senyora"}, + {ORTH: "Srta.", NORM: "senyoreta"}, + {ORTH: "núm", NORM: "número"}, + {ORTH: "St.", NORM: "sant"}, + {ORTH: "Sta.", NORM: "santa"}, + {ORTH: "pl.", NORM: "plaça"}, + {ORTH: "à."}, + {ORTH: "è."}, + {ORTH: "é."}, + {ORTH: "í."}, + {ORTH: "ò."}, + {ORTH: "ó."}, + {ORTH: "ú."}, + {ORTH: "'l"}, + {ORTH: "'ls"}, + {ORTH: "'m"}, + {ORTH: "'n"}, + {ORTH: "'ns"}, + {ORTH: "'s"}, + {ORTH: "'t"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +_exc["del"] = [{ORTH: "d", NORM: "de"}, {ORTH: "el"}] +_exc["dels"] = [{ORTH: "d", NORM: "de"}, {ORTH: "els"}] + +_exc["al"] = [{ORTH: "a"}, {ORTH: "l", NORM: "el"}] +_exc["als"] = [{ORTH: "a"}, {ORTH: "ls", NORM: "els"}] + +_exc["pel"] = [{ORTH: "p", NORM: "per"}, {ORTH: "el"}] +_exc["pels"] = [{ORTH: "p", NORM: "per"}, {ORTH: "els"}] + +_exc["holahola"] = [{ORTH: "holahola", NORM: "cocacola"}] + + +# Times +_exc["12m."] = [{ORTH: "12"}, {ORTH: "m.", NORM: "p.m."}] + +for h in range(1, 12 + 1): + for period in ["a.m.", "am"]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}"}, {ORTH: period, NORM: "a.m."}] + for period in ["p.m.", "pm"]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}"}, {ORTH: period, NORM: "p.m."}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/char_classes.py b/spacy/lang/char_classes.py new file mode 100644 index 0000000..69e752c --- /dev/null +++ b/spacy/lang/char_classes.py @@ -0,0 +1,333 @@ +split_chars = lambda char: list(char.strip().split(" ")) +merge_chars = lambda char: char.strip().replace(" ", "|") +group_chars = lambda char: char.strip().replace(" ", "") + +_ethiopic = r"\u1200-\u137F" + +_bengali = r"\u0980-\u09FF" + +_hebrew = r"\u0591-\u05F4\uFB1D-\uFB4F" + +_hindi = r"\u0900-\u097F" + +_kannada = r"\u0C80-\u0CFF" + +_tamil = r"\u0B80-\u0BFF" + +_telugu = r"\u0C00-\u0C7F" + +# from the final table in: https://en.wikipedia.org/wiki/CJK_Unified_Ideographs +_cjk = ( + r"\u4E00-\u62FF\u6300-\u77FF\u7800-\u8CFF\u8D00-\u9FFF\u3400-\u4DBF" + r"\U00020000-\U000215FF\U00021600-\U000230FF\U00023100-\U000245FF" + r"\U00024600-\U000260FF\U00026100-\U000275FF\U00027600-\U000290FF" + r"\U00029100-\U0002A6DF\U0002A700-\U0002B73F\U0002B740-\U0002B81F" + r"\U0002B820-\U0002CEAF\U0002CEB0-\U0002EBEF\u2E80-\u2EFF\u2F00-\u2FDF" + r"\u2FF0-\u2FFF\u3000-\u303F\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF" + r"\uF900-\uFAFF\uFE30-\uFE4F\U0001F200-\U0001F2FF\U0002F800-\U0002FA1F" +) + +# Latin standard +_latin_u_standard = r"A-Z" +_latin_l_standard = r"a-z" +_latin_standard = _latin_u_standard + _latin_l_standard + +_latin_u_standard_fullwidth = r"\uFF21-\uFF3A" +_latin_l_standard_fullwidth = r"\uFF41-\uFF5A" +_latin_standard_fullwidth = _latin_u_standard_fullwidth + _latin_l_standard_fullwidth + +# letters with diacritics - French, German, Icelandic, Spanish +_latin_u_supplement = r"\u00C0-\u00D6\u00D8-\u00DE" +_latin_l_supplement = r"\u00DF-\u00F6\u00F8-\u00FF" +_latin_supplement = r"\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF" + +_hangul_syllables = r"\uAC00-\uD7AF" +_hangul_jamo = r"\u1100-\u11FF" +_hangul = _hangul_syllables + _hangul_jamo + +_hiragana = r"\u3040-\u309F" +_katakana = r"\u30A0-\u30FFー" +_kana = _hiragana + _katakana + +# letters with diacritics - Catalan, Czech, Latin, Latvian, Lithuanian, Polish, Slovak, Turkish, Welsh +_latin_u_extendedA = ( + r"\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C" + r"\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B" + r"\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158" + r"\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176" + r"\u0178\u0179\u017B\u017D" +) +_latin_l_extendedA = ( + r"\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D" + r"\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137\u0138\u013A" + r"\u013C\u013E\u0140\u0142\u0144\u0146\u0148\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157" + r"\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175" + r"\u0177\u017A\u017C\u017E\u017F" +) +_latin_extendedA = r"\u0100-\u017F" + +# special characters - Khoisan, Pan-Nigerian, Pinyin, Romanian +# those that are a combination of both upper and lower letters are only included in the group _latin_extendedB +_latin_u_extendedB = ( + r"\u0181\u0182\u0184\u0186\u0187\u0189-\u018B\u018E-\u0191\u0193\u0194\u0196-\u0198\u019C" + r"\u019D\u019F\u01A0\u01A2\u01A4\u01A6\u01A7\u01A9\u01AC\u01AE\u01AF\u01B1-\u01B3\u01B5" + r"\u01B7\u01B8\u01BC\u01C4\u01C7\u01CA\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB" + r"\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1\u01F4\u01F6-\u01F8\u01FA" + r"\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216" + r"\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232" + r"\u023A\u023B\u023D\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E" +) +_latin_l_extendedB = ( + r"\u0180\u0183\u0185\u0188\u018C\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5" + r"\u01A8\u01AA\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9\u01BA\u01BD-\u01BF\u01C6\u01C9\u01CC" + r"\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7" + r"\u01E9\u01EB\u01ED\u01EF\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205" + r"\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221" + r"\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F\u0240\u0242" + r"\u0247\u0249\u024B\u024D\u024F" +) +_latin_extendedB = r"\u0180-\u01BF\u01C4-\u024F" + +# special characters - Uighur, Uralic Phonetic +_latin_u_extendedC = ( + r"\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7E\u2C7F" +) +_latin_l_extendedC = ( + r"\u2C61\u2C65\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73\u2C74\u2C76-\u2C7B" +) +_latin_extendedC = r"\u2C60-\u2C7B\u2C7E\u2C7F" + +# special characters - phonetic, Mayan, Medieval +_latin_u_extendedD = ( + r"\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C" + r"\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758" + r"\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA779\uA77B\uA77D" + r"\uA77E\uA780\uA782\uA784\uA786\uA78B\uA78D\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E" + r"\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8" +) +_latin_l_extendedD = ( + r"\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D" + r"\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759" + r"\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F\uA771-\uA778\uA77A" + r"\uA77C\uA77F\uA781\uA783\uA785\uA787\uA78C\uA78E\uA791\uA793-\uA795\uA797\uA799\uA79B" + r"\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7FA" +) +_latin_extendedD = r"\uA722-\uA76F\uA771-\uA787\uA78B-\uA78E\uA790-\uA7B9\uA7FA" + +# special characters - phonetic Teuthonista and Sakha +_latin_l_extendedE = r"\uAB30-\uAB5A\uAB60-\uAB64" +_latin_extendedE = _latin_l_extendedE + +# phonetic letters - Greek, Latin, Cyrillic +_latin_l_phonetic = r"\u0250-\u02AF\u1D00-\u1D25\u1D6B-\u1D77\u1D79-\u1D9A" +_latin_phonetic = _latin_l_phonetic + +# letters with multiple diacritics - Vietnamese +_latin_u_diacritics = ( + r"\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A" + r"\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36" + r"\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52" + r"\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E" + r"\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A" + r"\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE" + r"\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8" + r"\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4" + r"\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFE" +) +_latin_l_diacritics = ( + r"\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B" + r"\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37" + r"\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53" + r"\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F" + r"\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B" + r"\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD" + r"\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9" + r"\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5" + r"\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFF" +) +_latin_diacritics = r"\u1E00-\u1EFF" + +# all lower latin classes +LATIN_LOWER_BASIC = ( + _latin_l_standard + + _latin_l_standard_fullwidth + + _latin_l_supplement + + _latin_l_extendedA +) +LATIN_LOWER = ( + LATIN_LOWER_BASIC + + _latin_l_extendedB + + _latin_l_extendedC + + _latin_l_extendedD + + _latin_l_extendedE + + _latin_l_phonetic + + _latin_l_diacritics +) + +# all upper latin classes +LATIN_UPPER_BASIC = ( + _latin_u_standard + + _latin_u_standard_fullwidth + + _latin_u_supplement + + _latin_u_extendedA +) +LATIN_UPPER = ( + LATIN_UPPER_BASIC + + _latin_u_extendedB + + _latin_u_extendedC + + _latin_u_extendedD + + _latin_u_diacritics +) + +# all latin classes +LATIN_BASIC = ( + _latin_standard + _latin_standard_fullwidth + _latin_supplement + _latin_extendedA +) +LATIN = ( + LATIN_BASIC + + _latin_extendedB + + _latin_extendedC + + _latin_extendedD + + _latin_extendedE + + _latin_phonetic + + _latin_diacritics +) + +_persian = ( + r"\u0620-\u064A\u066E-\u06D5\u06E5-\u06FF\u0750-\u077F\u08A0-\u08BD" + r"\uFB50-\uFBB1\uFBD3-\uFD3D\uFD50-\uFDC7\uFDF0-\uFDFB\uFE70-\uFEFC\U0001EE00-\U0001EEBB" +) + +_russian_lower = r"ёа-я" +_russian_upper = r"ЁА-Я" +_russian = r"ёа-яЁА-Я" + +_sinhala = r"\u0D80-\u0DFF" + +_tatar_lower = r"әөүҗңһ" +_tatar_upper = r"ӘӨҮҖҢҺ" +_tatar = r"әөүҗңһӘӨҮҖҢҺ" + +_greek_lower = r"α-ωάέίόώήύ" +_greek_upper = r"Α-ΩΆΈΊΌΏΉΎ" +_greek = r"α-ωάέίόώήύΑ-ΩΆΈΊΌΏΉΎ" + +_ukrainian_lower = r"а-щюяіїєґ" +_ukrainian_upper = r"А-ЩЮЯІЇЄҐ" +_ukrainian = r"а-щюяіїєґА-ЩЮЯІЇЄҐ" + +_macedonian_lower = r"ѓѕјљњќѐѝ" +_macedonian_upper = r"ЃЅЈЉЊЌЀЍ" +_macedonian = r"ѓѕјљњќѐѝЃЅЈЉЊЌЀЍ" + +_upper = ( + LATIN_UPPER + + _russian_upper + + _tatar_upper + + _greek_upper + + _ukrainian_upper + + _macedonian_upper +) +_lower = ( + LATIN_LOWER + + _russian_lower + + _tatar_lower + + _greek_lower + + _ukrainian_lower + + _macedonian_lower +) + +_uncased = ( + _ethiopic + + _bengali + + _hebrew + + _persian + + _sinhala + + _hindi + + _kannada + + _tamil + + _telugu + + _hangul + + _kana + + _cjk +) + +ALPHA = group_chars( + LATIN + _russian + _tatar + _greek + _ukrainian + _macedonian + _uncased +) +ALPHA_LOWER = group_chars(_lower + _uncased) +ALPHA_UPPER = group_chars(_upper + _uncased) + +_combining_diacritics = r"\u0300-\u036f" + +COMBINING_DIACRITICS = _combining_diacritics + +_units = ( + "km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft " + "kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb " + "TB T G M K % км км² км³ м м² м³ дм дм² дм³ см см² см³ мм мм² мм³ нм " + "кг г мг м/с км/ч кПа Па мбар Кб КБ кб Мб МБ мб Гб ГБ гб Тб ТБ тб" + "كم كم² كم³ م م² م³ سم سم² سم³ مم مم² مم³ كم غرام جرام جم كغ ملغ كوب اكواب" +) +_currency = ( + r"\$ £ € ¥ ฿ US\$ C\$ A\$ ₽ ﷼ ₴ ₠ ₡ ₢ ₣ ₤ ₥ ₦ ₧ ₨ ₩ ₪ ₫ € ₭ ₮ ₯ ₰ " + r"₱ ₲ ₳ ₴ ₵ ₶ ₷ ₸ ₹ ₺ ₻ ₼ ₽ ₾ ₿" +) + +# These expressions contain various unicode variations, including characters +# used in Chinese (see #1333, #1340, #1351) – unless there are cross-language +# conflicts, spaCy's base tokenizer should handle all of those by default +_punct = r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪" +_quotes = ( + r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉 ⟦ ⟧' +) +_hyphens = "- – — -- --- —— ~" + +# Various symbols like dingbats, but also emoji +# Details: https://www.compart.com/en/unicode/category/So +_other_symbols = ( + r"\u00A6\u00A9\u00AE\u00B0\u0482\u058D\u058E\u060E\u060F\u06DE\u06E9\u06FD\u06FE\u07F6\u09FA\u0B70" + r"\u0BF3-\u0BF8\u0BFA\u0C7F\u0D4F\u0D79\u0F01-\u0F03\u0F13\u0F15-\u0F17\u0F1A-\u0F1F\u0F34" + r"\u0F36\u0F38\u0FBE-\u0FC5\u0FC7-\u0FCC\u0FCE\u0FCF\u0FD5-\u0FD8\u109E\u109F\u1390-\u1399" + r"\u1940\u19DE-\u19FF\u1B61-\u1B6A\u1B74-\u1B7C\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116" + r"\u2117\u211E-\u2123\u2125\u2127\u2129\u212E\u213A\u213B\u214A\u214C\u214D\u214F\u218A\u218B" + r"\u2195-\u2199\u219C-\u219F\u21A1\u21A2\u21A4\u21A5\u21A7-\u21AD\u21AF-\u21CD\u21D0\u21D1\u21D3" + r"\u21D5-\u21F3\u2300-\u2307\u230C-\u231F\u2322-\u2328\u232B-\u237B\u237D-\u239A\u23B4-\u23DB" + r"\u23E2-\u2426\u2440-\u244A\u249C-\u24E9\u2500-\u25B6\u25B8-\u25C0\u25C2-\u25F7\u2600-\u266E" + r"\u2670-\u2767\u2794-\u27BF\u2800-\u28FF\u2B00-\u2B2F\u2B45\u2B46\u2B4D-\u2B73\u2B76-\u2B95" + r"\u2B98-\u2BC8\u2BCA-\u2BFE\u2CE5-\u2CEA\u2E80-\u2E99\u2E9B-\u2EF3\u2F00-\u2FD5\u2FF0-\u2FFB" + r"\u3004\u3012\u3013\u3020\u3036\u3037\u303E\u303F\u3190\u3191\u3196-\u319F\u31C0-\u31E3" + r"\u3200-\u321E\u322A-\u3247\u3250\u3260-\u327F\u328A-\u32B0\u32C0-\u32FE\u3300-\u33FF\u4DC0-\u4DFF" + r"\uA490-\uA4C6\uA828-\uA82B\uA836\uA837\uA839\uAA77-\uAA79\uFDFD\uFFE4\uFFE8\uFFED\uFFEE\uFFFC" + r"\uFFFD\U00010137-\U0001013F\U00010179-\U00010189\U0001018C-\U0001018E\U00010190-\U0001019B" + r"\U000101A0\U000101D0-\U000101FC\U00010877\U00010878\U00010AC8\U0001173F\U00016B3C-\U00016B3F" + r"\U00016B45\U0001BC9C\U0001D000-\U0001D0F5\U0001D100-\U0001D126\U0001D129-\U0001D164" + r"\U0001D16A-\U0001D16C\U0001D183\U0001D184\U0001D18C-\U0001D1A9\U0001D1AE-\U0001D1E8" + r"\U0001D200-\U0001D241\U0001D245\U0001D300-\U0001D356\U0001D800-\U0001D9FF\U0001DA37-\U0001DA3A" + r"\U0001DA6D-\U0001DA74\U0001DA76-\U0001DA83\U0001DA85\U0001DA86\U0001ECAC\U0001F000-\U0001F02B" + r"\U0001F030-\U0001F093\U0001F0A0-\U0001F0AE\U0001F0B1-\U0001F0BF\U0001F0C1-\U0001F0CF" + r"\U0001F0D1-\U0001F0F5\U0001F110-\U0001F16B\U0001F170-\U0001F1AC\U0001F1E6-\U0001F202" + r"\U0001F210-\U0001F23B\U0001F240-\U0001F248\U0001F250\U0001F251\U0001F260-\U0001F265" + r"\U0001F300-\U0001F3FA\U0001F400-\U0001F6D4\U0001F6E0-\U0001F6EC\U0001F6F0-\U0001F6F9" + r"\U0001F700-\U0001F773\U0001F780-\U0001F7D8\U0001F800-\U0001F80B\U0001F810-\U0001F847" + r"\U0001F850-\U0001F859\U0001F860-\U0001F887\U0001F890-\U0001F8AD\U0001F900-\U0001F90B" + r"\U0001F910-\U0001F93E\U0001F940-\U0001F970\U0001F973-\U0001F976\U0001F97A\U0001F97C-\U0001F9A2" + r"\U0001F9B0-\U0001F9B9\U0001F9C0-\U0001F9C2\U0001F9D0-\U0001F9FF\U0001FA60-\U0001FA6D" +) + +UNITS = merge_chars(_units) +CURRENCY = merge_chars(_currency) +PUNCT = merge_chars(_punct) +HYPHENS = merge_chars(_hyphens) +ICONS = _other_symbols + +LIST_UNITS = split_chars(_units) +LIST_CURRENCY = split_chars(_currency) +LIST_QUOTES = split_chars(_quotes) +LIST_PUNCT = split_chars(_punct) +LIST_HYPHENS = split_chars(_hyphens) +LIST_ELLIPSES = [r"\.\.+", "…"] +LIST_ICONS = [r"[{i}]".format(i=_other_symbols)] + +CONCAT_QUOTES = group_chars(_quotes) +CONCAT_ICONS = _other_symbols diff --git a/spacy/lang/cs/__init__.py b/spacy/lang/cs/__init__.py new file mode 100644 index 0000000..9ea60af --- /dev/null +++ b/spacy/lang/cs/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class CzechDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Czech(Language): + lang = "cs" + Defaults = CzechDefaults + + +__all__ = ["Czech"] diff --git a/spacy/lang/cs/examples.py b/spacy/lang/cs/examples.py new file mode 100644 index 0000000..35d86dd --- /dev/null +++ b/spacy/lang/cs/examples.py @@ -0,0 +1,37 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.cs.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Máma mele maso.", + "Příliš žluťoučký kůň úpěl ďábelské ódy.", + "ArcGIS je geografický informační systém určený pro práci s prostorovými daty.", + "Může data vytvářet a spravovat, ale především je dokáže analyzovat, najít v nich nové vztahy a vše přehledně vizualizovat.", + "Dnes je krásné počasí.", + "Nestihl autobus, protože pozdě vstal z postele.", + "Než budeš jíst, jdi si umýt ruce.", + "Dnes je neděle.", + "Škola začíná v 8:00.", + "Poslední autobus jede v jedenáct hodin večer.", + "V roce 2020 se téměř zastavila světová ekonomika.", + "Praha je hlavní město České republiky.", + "Kdy půjdeš ven?", + "Kam pojedete na dovolenou?", + "Kolik stojí iPhone 12?", + "Průměrná mzda je 30000 Kč.", + "1. ledna 1993 byla založena Česká republika.", + "Co se stalo 21.8.1968?", + "Moje telefonní číslo je 712 345 678.", + "Můj pes má blechy.", + "Když bude přes noc více než 20°, tak nás čeká tropická noc.", + "Kolik bylo letos tropických nocí?", + "Jak to mám udělat?", + "Bydlíme ve čtvrtém patře.", + "Vysílají 30. sezonu seriálu Simpsonovi.", + "Adresa ČVUT je Thákurova 7, 166 29, Praha 6.", + "Jaké PSČ má Praha 1?", + "PSČ Prahy 1 je 110 00.", + "Za 20 minut jede vlak.", +] diff --git a/spacy/lang/cs/lex_attrs.py b/spacy/lang/cs/lex_attrs.py new file mode 100644 index 0000000..530d1d5 --- /dev/null +++ b/spacy/lang/cs/lex_attrs.py @@ -0,0 +1,61 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nula", + "jedna", + "dva", + "tři", + "čtyři", + "pět", + "šest", + "sedm", + "osm", + "devět", + "deset", + "jedenáct", + "dvanáct", + "třináct", + "čtrnáct", + "patnáct", + "šestnáct", + "sedmnáct", + "osmnáct", + "devatenáct", + "dvacet", + "třicet", + "čtyřicet", + "padesát", + "šedesát", + "sedmdesát", + "osmdesát", + "devadesát", + "sto", + "tisíc", + "milion", + "miliarda", + "bilion", + "biliarda", + "trilion", + "triliarda", + "kvadrilion", + "kvadriliarda", + "kvintilion", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/cs/stop_words.py b/spacy/lang/cs/stop_words.py new file mode 100644 index 0000000..f61f424 --- /dev/null +++ b/spacy/lang/cs/stop_words.py @@ -0,0 +1,365 @@ +# Source: https://github.com/Alir3z4/stop-words +# Source: https://github.com/stopwords-iso/stopwords-cs/blob/master/stopwords-cs.txt + +STOP_WORDS = set( + """ +a +aby +ahoj +ačkoli +ale +alespoň +anebo +ani +aniž +ano +atd. +atp. +asi +aspoň +až +během +bez +beze +blízko +bohužel +brzo +bude +budeme +budeš +budete +budou +budu +by +byl +byla +byli +bylo +byly +bys +být +čau +chce +chceme +chceš +chcete +chci +chtějí +chtít +chuť +chuti +co +což +cz +či +článek +článku +články +čtrnáct +čtyři +dál +dále +daleko +další +děkovat +děkujeme +děkuji +den +deset +devatenáct +devět +dnes +do +dobrý +docela +dva +dvacet +dvanáct +dvě +email +ho +hodně +i +já +jak +jakmile +jako +jakož +jde +je +jeden +jedenáct +jedna +jedno +jednou +jedou +jeho +jehož +jej +její +jejich +jejichž +jehož +jelikož +jemu +jen +jenom +jenž +jež +ještě +jestli +jestliže +ještě +ji +jí +jich +jím +jim +jimi +jinak +jiné +již +jsi +jsme +jsem +jsou +jste +k +kam +každý +kde +kdo +kdy +když +ke +kolik +kromě +která +kterak +kterou +které +kteří +který +kvůli +ku +má +mají +málo +mám +máme +máš +máte +mé +mě +mezi +mi +mí +mít +mne +mně +mnou +moc +mohl +mohou +moje +moji +možná +můj +musí +může +my +na +nad +nade +nám +námi +naproti +nás +náš +naše +naši +načež +ne +ně +nebo +nebyl +nebyla +nebyli +nebyly +nechť +něco +nedělá +nedělají +nedělám +neděláme +neděláš +neděláte +nějak +nejsi +nejsou +někde +někdo +nemají +nemáme +nemáte +neměl +němu +němuž +není +nestačí +ně +nevadí +nové +nový +noví +než +nic +nich +ní +ním +nimi +nula +o +od +ode +on +ona +oni +ono +ony +osm +osmnáct +pak +patnáct +pět +po +pod +pokud +pořád +pouze +potom +pozdě +pravé +před +přede +přes +přece +pro +proč +prosím +prostě +proto +proti +první +právě +protože +při +přičemž +rovně +s +se +sedm +sedmnáct +si +sice +skoro +sic +šest +šestnáct +skoro +smějí +smí +snad +spolu +sta +svůj +své +svá +svých +svým +svými +svůj +sté +sto +strana +ta +tady +tak +takhle +taky +také +takže +tam +támhle +támhleto +tamto +tě +tebe +tebou +teď +tedy +ten +tento +této +ti +tím +tímto +tisíc +tisíce +to +tobě +tohle +tohoto +tom +tomto +tomu +tomuto +toto +třeba +tři +třináct +trošku +trochu +tu +tuto +tvá +tvé +tvoje +tvůj +ty +tyto +těm +těma +těmi +u +určitě +už +v +vám +vámi +vás +váš +vaše +vaši +ve +večer +vedle +více +vlastně +však +všechen +všechno +všichni +vůbec +vy +vždy +z +zda +za +zde +zač +zatímco +ze +že +""".split() +) diff --git a/spacy/lang/da/__init__.py b/spacy/lang/da/__init__.py new file mode 100644 index 0000000..372f372 --- /dev/null +++ b/spacy/lang/da/__init__.py @@ -0,0 +1,23 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class DanishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + syntax_iterators = SYNTAX_ITERATORS + + +class Danish(Language): + lang = "da" + Defaults = DanishDefaults + + +__all__ = ["Danish"] diff --git a/spacy/lang/da/examples.py b/spacy/lang/da/examples.py new file mode 100644 index 0000000..efa1a7c --- /dev/null +++ b/spacy/lang/da/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.da.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple overvejer at købe et britisk startup for 1 milliard dollar.", + "Selvkørende biler flytter forsikringsansvaret over på producenterne.", + "San Francisco overvejer at forbyde udbringningsrobotter på fortovet.", + "London er en storby i Storbritannien.", + "Hvor er du?", + "Hvem er Frankrings president?", + "Hvad er hovedstaden i USA?", + "Hvornår blev Barack Obama født?", +] diff --git a/spacy/lang/da/lex_attrs.py b/spacy/lang/da/lex_attrs.py new file mode 100644 index 0000000..8e04209 --- /dev/null +++ b/spacy/lang/da/lex_attrs.py @@ -0,0 +1,50 @@ +from ...attrs import LIKE_NUM + +# Source http://fjern-uv.dk/tal.php +_num_words = """nul +en et to tre fire fem seks syv otte ni ti +elleve tolv tretten fjorten femten seksten sytten atten nitten tyve +enogtyve toogtyve treogtyve fireogtyve femogtyve seksogtyve syvogtyve otteogtyve niogtyve tredive +enogtredive toogtredive treogtredive fireogtredive femogtredive seksogtredive syvogtredive otteogtredive niogtredive fyrre +enogfyrre toogfyrre treogfyrre fireogfyrre femgogfyrre seksogfyrre syvogfyrre otteogfyrre niogfyrre halvtreds +enoghalvtreds tooghalvtreds treoghalvtreds fireoghalvtreds femoghalvtreds seksoghalvtreds syvoghalvtreds otteoghalvtreds nioghalvtreds tres +enogtres toogtres treogtres fireogtres femogtres seksogtres syvogtres otteogtres niogtres halvfjerds +enoghalvfjerds tooghalvfjerds treoghalvfjerds fireoghalvfjerds femoghalvfjerds seksoghalvfjerds syvoghalvfjerds otteoghalvfjerds nioghalvfjerds firs +enogfirs toogfirs treogfirs fireogfirs femogfirs seksogfirs syvogfirs otteogfirs niogfirs halvfems +enoghalvfems tooghalvfems treoghalvfems fireoghalvfems femoghalvfems seksoghalvfems syvoghalvfems otteoghalvfems nioghalvfems hundrede +million milliard billion billiard trillion trilliard +""".split() + + +# Source: http://www.duda.dk/video/dansk/grammatik/talord/talord.html +_ordinal_words = """nulte +første anden tredje fjerde femte sjette syvende ottende niende tiende +elfte tolvte trettende fjortende femtende sekstende syttende attende nittende tyvende +enogtyvende toogtyvende treogtyvende fireogtyvende femogtyvende seksogtyvende syvogtyvende otteogtyvende niogtyvende tredivte enogtredivte toogtredivte treogtredivte fireogtredivte femogtredivte seksogtredivte syvogtredivte otteogtredivte niogtredivte fyrretyvende +enogfyrretyvende toogfyrretyvende treogfyrretyvende fireogfyrretyvende femogfyrretyvende seksogfyrretyvende syvogfyrretyvende otteogfyrretyvende niogfyrretyvende halvtredsindstyvende enoghalvtredsindstyvende +tooghalvtredsindstyvende treoghalvtredsindstyvende fireoghalvtredsindstyvende femoghalvtredsindstyvende seksoghalvtredsindstyvende syvoghalvtredsindstyvende otteoghalvtredsindstyvende nioghalvtredsindstyvende +tresindstyvende enogtresindstyvende toogtresindstyvende treogtresindstyvende fireogtresindstyvende femogtresindstyvende seksogtresindstyvende syvogtresindstyvende otteogtresindstyvende niogtresindstyvende halvfjerdsindstyvende +enoghalvfjerdsindstyvende tooghalvfjerdsindstyvende treoghalvfjerdsindstyvende fireoghalvfjerdsindstyvende femoghalvfjerdsindstyvende seksoghalvfjerdsindstyvende syvoghalvfjerdsindstyvende otteoghalvfjerdsindstyvende nioghalvfjerdsindstyvende firsindstyvende +enogfirsindstyvende toogfirsindstyvende treogfirsindstyvende fireogfirsindstyvende femogfirsindstyvende seksogfirsindstyvende syvogfirsindstyvende otteogfirsindstyvende niogfirsindstyvende halvfemsindstyvende +enoghalvfemsindstyvende tooghalvfemsindstyvende treoghalvfemsindstyvende fireoghalvfemsindstyvende femoghalvfemsindstyvende seksoghalvfemsindstyvende syvoghalvfemsindstyvende otteoghalvfemsindstyvende nioghalvfemsindstyvende +""".split() + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/da/punctuation.py b/spacy/lang/da/punctuation.py new file mode 100644 index 0000000..f70fe3d --- /dev/null +++ b/spacy/lang/da/punctuation.py @@ -0,0 +1,36 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + LIST_ELLIPSES, + LIST_ICONS, +) +from ..punctuation import TOKENIZER_SUFFIXES + +_quotes = CONCAT_QUOTES.replace("'", "") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = [ + suffix + for suffix in TOKENIZER_SUFFIXES + if suffix not in ["'s", "'S", "’s", "’S", r"\'"] +] +_suffixes += [r"(?<=[^sSxXzZ])\'"] + + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/da/stop_words.py b/spacy/lang/da/stop_words.py new file mode 100644 index 0000000..05b2084 --- /dev/null +++ b/spacy/lang/da/stop_words.py @@ -0,0 +1,45 @@ +# Source: Handpicked by Jens Dahl Møllerhøj. + +STOP_WORDS = set( + """ +af aldrig alene alle allerede alligevel alt altid anden andet andre at + +bag begge blandt blev blive bliver burde bør + +da de dem den denne dens der derefter deres derfor derfra deri dermed derpå derved det dette dig din dine disse dog du + +efter egen eller ellers en end endnu ene eneste enhver ens enten er et + +flere flest fleste for foran fordi forrige fra få før først + +gennem gjorde gjort god gør gøre gørende + +ham han hans har havde have hel heller hen hende hendes henover her herefter heri hermed herpå hun hvad hvem hver hvilke hvilken hvilkes hvis hvor hvordan hvorefter hvorfor hvorfra hvorhen hvori hvorimod hvornår hvorved + +i igen igennem ikke imellem imens imod ind indtil ingen intet + +jeg jer jeres jo + +kan kom kommer kun kunne + +lad langs lav lave lavet lidt lige ligesom lille længere + +man mange med meget mellem men mens mere mest mig min mindre mindst mine mit må måske + +ned nemlig nogen nogensinde noget nogle nok nu ny nyt nær næste næsten + +og også om omkring op os over overalt + +på + +samme sammen selv selvom senere ses siden sig sige skal skulle som stadig synes syntes så sådan således + +temmelig tidligere til tilbage tit + +ud uden udover under undtagen + +var ved vi via vil ville vore vores vær være været + +øvrigt +""".split() +) diff --git a/spacy/lang/da/syntax_iterators.py b/spacy/lang/da/syntax_iterators.py new file mode 100644 index 0000000..60224f0 --- /dev/null +++ b/spacy/lang/da/syntax_iterators.py @@ -0,0 +1,74 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import AUX, NOUN, PRON, PROPN, VERB +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + def is_verb_token(tok): + return tok.pos in [VERB, AUX] + + def get_left_bound(doc, root): + left_bound = root + for tok in reversed(list(root.lefts)): + if tok.dep in np_left_deps: + left_bound = tok + return left_bound + + def get_right_bound(doc, root): + right_bound = root + for tok in root.rights: + if tok.dep in np_right_deps: + right = get_right_bound(doc, tok) + if list( + filter( + lambda t: is_verb_token(t) or t.dep in stop_deps, + doc[root.i : right.i], + ) + ): + break + else: + right_bound = right + return right_bound + + def get_bounds(doc, root): + return get_left_bound(doc, root), get_right_bound(doc, root) + + doc = doclike.doc # Ensure works on both Doc and Span. + + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + if not len(doc): + return + + left_labels = [ + "det", + "fixed", + "nmod:poss", + "amod", + "flat", + "goeswith", + "nummod", + "appos", + ] + right_labels = ["fixed", "nmod:poss", "amod", "flat", "goeswith", "nummod", "appos"] + stop_labels = ["punct"] + + np_label = doc.vocab.strings.add("NP") + np_left_deps = [doc.vocab.strings.add(label) for label in left_labels] + np_right_deps = [doc.vocab.strings.add(label) for label in right_labels] + stop_deps = [doc.vocab.strings.add(label) for label in stop_labels] + + prev_right = -1 + for token in doclike: + if token.pos in [PROPN, NOUN, PRON]: + left, right = get_bounds(doc, token) + if left.i <= prev_right: + continue + yield left.i, right.i + 1, np_label + prev_right = right.i + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/da/tokenizer_exceptions.py b/spacy/lang/da/tokenizer_exceptions.py new file mode 100644 index 0000000..15a943a --- /dev/null +++ b/spacy/lang/da/tokenizer_exceptions.py @@ -0,0 +1,581 @@ +""" +Tokenizer Exceptions. +Source: https://forkortelse.dk/ and various others. +""" + +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +# Abbreviations for weekdays "søn." (for "søndag") as well as "Tor." and "Tors." +# (for "torsdag") are left out because they are ambiguous. The same is the case +# for abbreviations "jul." and "Jul." ("juli"). +for exc_data in [ + {ORTH: "Kbh.", NORM: "København"}, + {ORTH: "jan.", NORM: "januar"}, + {ORTH: "febr.", NORM: "februar"}, + {ORTH: "feb.", NORM: "februar"}, + {ORTH: "mar.", NORM: "marts"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "jun.", NORM: "juni"}, + {ORTH: "aug.", NORM: "august"}, + {ORTH: "sept.", NORM: "september"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "nov.", NORM: "november"}, + {ORTH: "dec.", NORM: "december"}, + {ORTH: "man.", NORM: "mandag"}, + {ORTH: "tirs.", NORM: "tirsdag"}, + {ORTH: "ons.", NORM: "onsdag"}, + {ORTH: "tor.", NORM: "torsdag"}, + {ORTH: "tors.", NORM: "torsdag"}, + {ORTH: "fre.", NORM: "fredag"}, + {ORTH: "lør.", NORM: "lørdag"}, + {ORTH: "Jan.", NORM: "januar"}, + {ORTH: "Febr.", NORM: "februar"}, + {ORTH: "Feb.", NORM: "februar"}, + {ORTH: "Mar.", NORM: "marts"}, + {ORTH: "Apr.", NORM: "april"}, + {ORTH: "Jun.", NORM: "juni"}, + {ORTH: "Aug.", NORM: "august"}, + {ORTH: "Sept.", NORM: "september"}, + {ORTH: "Sep.", NORM: "september"}, + {ORTH: "Okt.", NORM: "oktober"}, + {ORTH: "Nov.", NORM: "november"}, + {ORTH: "Dec.", NORM: "december"}, + {ORTH: "Man.", NORM: "mandag"}, + {ORTH: "Tirs.", NORM: "tirsdag"}, + {ORTH: "Ons.", NORM: "onsdag"}, + {ORTH: "Fre.", NORM: "fredag"}, + {ORTH: "Lør.", NORM: "lørdag"}, + {ORTH: "og/eller", NORM: "og/eller"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Specified case only +for orth in [ + "diam.", + "ib.", + "mia.", + "mik.", + "pers.", + "A.D.", + "A/B", + "a/s", + "A/S", + "B.C.", + "BK.", + "B.T.", + "Dr.", + "Boul.", + "Chr.", + "Dronn.", + "H.K.H.", + "H.M.", + "Hf.", + "i/s", + "I/S", + "Inc.", + "Kprs.", + "L.A.", + "Ll.", + "m/k", + "m/s", + "m/sek.", + "M/S", + "Mag.", + "Mr.", + "Ndr.", + "Ph.d.", + "Prs.", + "Rcp.", + "Sdr.", + "Skt.", + "Spl.", + "TCP/IP", + "Vg.", +]: + _exc[orth] = [{ORTH: orth}] + + +for orth in [ + "aarh.", + "ac.", + "adj.", + "adr.", + "adsk.", + "adv.", + "afb.", + "afd.", + "afg.", + "afk.", + "afs.", + "aht.", + "alg.", + "alk.", + "alm.", + "amer.", + "ang.", + "ank.", + "anl.", + "anv.", + "arb.", + "arr.", + "att.", + "bd.", + "bdt.", + "beg.", + "begr.", + "beh.", + "bet.", + "bev.", + "bhk.", + "bib.", + "bibl.", + "bidr.", + "bildl.", + "bill.", + "biol.", + "bk.", + "bl.", + "bl.a.", + "borgm.", + "br.", + "brolægn.", + "bto.", + "bygn.", + "c/o", + "ca.", + "cm.", + "cand.", + "d.d.", + "d.m.", + "d.s.", + "d.s.s.", + "d.y.", + "d.å.", + "d.æ.", + "dagl.", + "dat.", + "dav.", + "def.", + "dek.", + "dep.", + "desl.", + "dir.", + "disp.", + "distr.", + "div.", + "dkr.", + "dl.", + "do.", + "dobb.", + "dr.", + "dr.h.c", + "dr.phil.", + "ds.", + "dvs.", + "d.v.s.", + "e.b.", + "e.l.", + "e.o.", + "e.v.t.", + "eftf.", + "eftm.", + "egl.", + "eks.", + "eksam.", + "ekskl.", + "eksp.", + "ekspl.", + "el.lign.", + "emer.", + "endv.", + "eng.", + "enk.", + "etc.", + "etym.", + "eur.", + "evt.", + "exam.", + "f.eks.", + "f.m.", + "f.n.", + "f.o.", + "f.o.m.", + "f.s.v.", + "f.t.", + "f.v.t.", + "f.å.", + "fa.", + "fakt.", + "fam.", + "ff.", + "fg.", + "fhv.", + "fig.", + "filol.", + "filos.", + "fl.", + "flg.", + "fm.", + "fmd.", + "fol.", + "forb.", + "foreg.", + "foren.", + "forf.", + "fork.", + "forr.", + "fors.", + "forsk.", + "forts.", + "fr.", + "fr.u.", + "frk.", + "fsva.", + "fuldm.", + "fung.", + "fx.", + "fys.", + "fær.", + "g.d.", + "g.m.", + "gd.", + "gdr.", + "genuds.", + "gl.", + "gn.", + "gns.", + "gr.", + "grdl.", + "gross.", + "h.a.", + "h.c.", + "hdl.", + "henv.", + "hhv.", + "hj.hj.", + "hj.spl.", + "hort.", + "hosp.", + "hpl.", + "hr.", + "hrs.", + "hum.", + "hvp.", + "i.e.", + "id.", + "if.", + "iflg.", + "ifm.", + "ift.", + "iht.", + "ill.", + "indb.", + "indreg.", + "inf.", + "ing.", + "inh.", + "inj.", + "inkl.", + "insp.", + "instr.", + "isl.", + "istf.", + "it.", + "ital.", + "iv.", + "jap.", + "jf.", + "jfr.", + "jnr.", + "j.nr.", + "jr.", + "jur.", + "jvf.", + "kap.", + "kbh.", + "kem.", + "kg.", + "kgs.", + "kgl.", + "kl.", + "kld.", + "km.", + "km/t", + "km/t.", + "knsp.", + "komm.", + "kons.", + "korr.", + "kp.", + "kr.", + "kst.", + "kt.", + "ktr.", + "kv.", + "kvm.", + "kvt.", + "l.c.", + "lab.", + "lat.", + "lb.m.", + "lb.nr.", + "lejl.", + "lgd.", + "lic.", + "lign.", + "lin.", + "ling.merc.", + "litt.", + "loc.cit.", + "lok.", + "lrs.", + "ltr.", + "m.a.o.", + "m.fl.", + "m.m.", + "m.v.", + "m.v.h.", + "maks.", + "md.", + "mdr.", + "mdtl.", + "mezz.", + "mfl.", + "m.h.p.", + "m.h.t.", + "mht.", + "mill.", + "mio.", + "modt.", + "mrk.", + "mul.", + "mv.", + "n.br.", + "n.f.", + "nb.", + "nedenst.", + "nl.", + "nr.", + "nto.", + "nuv.", + "o/m", + "o/m.", + "o.a.", + "o.fl.", + "o.h.", + "o.l.", + "o.lign.", + "o.m.a.", + "o.s.fr.", + "obl.", + "obs.", + "odont.", + "oecon.", + "off.", + "ofl.", + "omg.", + "omkr.", + "omr.", + "omtr.", + "opg.", + "opl.", + "opr.", + "org.", + "orig.", + "osv.", + "ovenst.", + "overs.", + "ovf.", + "p.a.", + "p.b.a", + "p.b.v", + "p.c.", + "p.m.", + "p.m.v.", + "p.n.", + "p.p.", + "p.p.s.", + "p.s.", + "p.t.", + "p.v.a.", + "p.v.c.", + "pag.", + "pass.", + "pcs.", + "pct.", + "pd.", + "pens.", + "pft.", + "pg.", + "pga.", + "pgl.", + "pinx.", + "pk.", + "pkt.", + "polit.", + "polyt.", + "pos.", + "pp.", + "ppm.", + "pr.", + "prc.", + "priv.", + "prod.", + "prof.", + "pron.", + "præd.", + "præf.", + "præt.", + "psych.", + "pt.", + "pæd.", + "q.e.d.", + "rad.", + "red.", + "ref.", + "reg.", + "regn.", + "rel.", + "rep.", + "repr.", + "resp.", + "rest.", + "rm.", + "rtg.", + "russ.", + "s.br.", + "s.d.", + "s.f.", + "s.m.b.a.", + "s.u.", + "s.å.", + "sa.", + "sb.", + "sc.", + "scient.", + "scil.", + "sek.", + "sekr.", + "self.", + "sem.", + "shj.", + "sign.", + "sing.", + "sj.", + "skr.", + "slutn.", + "sml.", + "smp.", + "snr.", + "soc.", + "soc.dem.", + "sp.", + "spec.", + "spm.", + "spr.", + "spsk.", + "statsaut.", + "st.", + "stk.", + "str.", + "stud.", + "subj.", + "subst.", + "suff.", + "sup.", + "suppl.", + "sv.", + "såk.", + "sædv.", + "t/r", + "t.h.", + "t.o.", + "t.o.m.", + "t.v.", + "tbl.", + "tcp/ip", + "td.", + "tdl.", + "tdr.", + "techn.", + "tekn.", + "temp.", + "th.", + "theol.", + "tidl.", + "tilf.", + "tilh.", + "till.", + "tilsv.", + "tjg.", + "tkr.", + "tlf.", + "tlgr.", + "tr.", + "trp.", + "tsk.", + "tv.", + "ty.", + "u/b", + "udb.", + "udbet.", + "ugtl.", + "undt.", + "v.f.", + "vb.", + "vedk.", + "vedl.", + "vedr.", + "vejl.", + "vh.", + "vha.", + "vind.", + "vs.", + "vsa.", + "vær.", + "zool.", + "ø.lgd.", + "øvr.", + "årg.", + "årh.", +]: + _exc[orth] = [{ORTH: orth}] + capitalized = orth.capitalize() + _exc[capitalized] = [{ORTH: capitalized}] + +for exc_data in [ + {ORTH: "s'gu", NORM: "s'gu"}, + {ORTH: "S'gu", NORM: "s'gu"}, + {ORTH: "sgu'", NORM: "s'gu"}, + {ORTH: "Sgu'", NORM: "s'gu"}, + {ORTH: "sku'", NORM: "skulle"}, + {ORTH: "ku'", NORM: "kunne"}, + {ORTH: "Ku'", NORM: "kunne"}, + {ORTH: "ka'", NORM: "kan"}, + {ORTH: "Ka'", NORM: "kan"}, + {ORTH: "gi'", NORM: "giv"}, + {ORTH: "Gi'", NORM: "giv"}, + {ORTH: "li'", NORM: "lide"}, + {ORTH: "ha'", NORM: "have"}, + {ORTH: "Ha'", NORM: "have"}, + {ORTH: "ik'", NORM: "ikke"}, + {ORTH: "Ik'", NORM: "ikke"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Dates +for h in range(1, 31 + 1): + for period in ["."]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}."}] + +_custom_base_exc = {"i.": [{ORTH: "i", NORM: "i"}, {ORTH: "."}]} +_exc.update(_custom_base_exc) + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/de/__init__.py b/spacy/lang/de/__init__.py new file mode 100644 index 0000000..4f45b23 --- /dev/null +++ b/spacy/lang/de/__init__.py @@ -0,0 +1,22 @@ +from ...language import BaseDefaults, Language +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class GermanDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class German(Language): + lang = "de" + Defaults = GermanDefaults + + +__all__ = ["German"] diff --git a/spacy/lang/de/examples.py b/spacy/lang/de/examples.py new file mode 100644 index 0000000..30b8f19 --- /dev/null +++ b/spacy/lang/de/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.de.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Die ganze Stadt ist ein Startup: Shenzhen ist das Silicon Valley für Hardware-Firmen", + "Wie deutsche Startups die Technologie vorantreiben wollen: Künstliche Intelligenz", + "Trend zum Urlaub in Deutschland beschert Gastwirten mehr Umsatz", + "Bundesanwaltschaft erhebt Anklage gegen mutmaßlichen Schweizer Spion", + "San Francisco erwägt Verbot von Lieferrobotern", + "Autonome Fahrzeuge verlagern Haftpflicht auf Hersteller", + "Wo bist du?", + "Was ist die Hauptstadt von Deutschland?", +] diff --git a/spacy/lang/de/punctuation.py b/spacy/lang/de/punctuation.py new file mode 100644 index 0000000..8622076 --- /dev/null +++ b/spacy/lang/de/punctuation.py @@ -0,0 +1,56 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + +_prefixes = ["``"] + BASE_TOKENIZER_PREFIXES + +_suffixes = ( + ["''", "/"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +_quotes = CONCAT_QUOTES.replace("'", "") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[0-9{a}])\/(?=[0-9{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[0-9])-(?=[0-9])", + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/de/stop_words.py b/spacy/lang/de/stop_words.py new file mode 100644 index 0000000..f52687e --- /dev/null +++ b/spacy/lang/de/stop_words.py @@ -0,0 +1,78 @@ +STOP_WORDS = set( + """ +á a ab aber ach acht achte achten achter achtes ag alle allein allem allen +aller allerdings alles allgemeinen als also am an andere anderen anderem andern +anders auch auf aus ausser außer ausserdem außerdem + +bald bei beide beiden beim beispiel bekannt bereits besonders besser besten bin +bis bisher bist + +da dabei dadurch dafür dagegen daher dahin dahinter damals damit danach daneben +dank dann daran darauf daraus darf darfst darin darüber darum darunter das +dasein daselbst dass daß dasselbe davon davor dazu dazwischen dein deine deinem +deiner dem dementsprechend demgegenüber demgemäss demgemäß demselben demzufolge +den denen denn denselben der deren derjenige derjenigen dermassen dermaßen +derselbe derselben des deshalb desselben dessen deswegen dich die diejenige +diejenigen dies diese dieselbe dieselben diesem diesen dieser dieses dir doch +dort drei drin dritte dritten dritter drittes du durch durchaus dürfen dürft +durfte durften + +eben ebenso ehrlich eigen eigene eigenen eigener eigenes ein einander eine +einem einen einer eines einige einigen einiger einiges einmal einmaleins elf en +ende endlich entweder er erst erste ersten erster erstes es etwa etwas euch + +früher fünf fünfte fünften fünfter fünftes für + +gab ganz ganze ganzen ganzer ganzes gar gedurft gegen gegenüber gehabt gehen +geht gekannt gekonnt gemacht gemocht gemusst genug gerade gern gesagt geschweige +gewesen gewollt geworden gibt ging gleich gross groß grosse große grossen +großen grosser großer grosses großes gut gute guter gutes + +habe haben habt hast hat hatte hätte hatten hätten heisst heißt her heute hier +hin hinter hoch + +ich ihm ihn ihnen ihr ihre ihrem ihren ihrer ihres im immer in indem +infolgedessen ins irgend ist + +ja jahr jahre jahren je jede jedem jeden jeder jedermann jedermanns jedoch +jemand jemandem jemanden jene jenem jenen jener jenes jetzt + +kam kann kannst kaum kein keine keinem keinen keiner kleine kleinen kleiner +kleines kommen kommt können könnt konnte könnte konnten kurz + +lang lange leicht leider lieber los + +machen macht machte mag magst man manche manchem manchen mancher manches mehr +mein meine meinem meinen meiner meines mich mir mit mittel mochte möchte mochten +mögen möglich mögt morgen muss muß müssen musst müsst musste mussten + +na nach nachdem nahm natürlich neben nein neue neuen neun neunte neunten neunter +neuntes nicht nichts nie niemand niemandem niemanden noch nun nur + +ob oben oder offen oft ohne + +recht rechte rechten rechter rechtes richtig rund + +sagt sagte sah satt schlecht schon sechs sechste sechsten sechster sechstes +sehr sei seid seien sein seine seinem seinen seiner seines seit seitdem selbst +selbst sich sie sieben siebente siebenten siebenter siebentes siebte siebten +siebter siebtes sind so solang solche solchem solchen solcher solches soll +sollen sollte sollten sondern sonst sowie später statt + +tag tage tagen tat teil tel trotzdem tun + +über überhaupt übrigens uhr um und uns unser unsere unserer unter + +vergangene vergangenen viel viele vielem vielen vielleicht vier vierte vierten +vierter viertes vom von vor + +wahr während währenddem währenddessen wann war wäre waren wart warum was wegen +weil weit weiter weitere weiteren weiteres welche welchem welchen welcher +welches wem wen wenig wenige weniger weniges wenigstens wenn wer werde werden +werdet wessen wie wieder will willst wir wird wirklich wirst wo wohl wollen +wollt wollte wollten worden wurde würde wurden würden + +zehn zehnte zehnten zehnter zehntes zeit zu zuerst zugleich zum zunächst zur +zurück zusammen zwanzig zwar zwei zweite zweiten zweiter zweites zwischen +""".split() +) diff --git a/spacy/lang/de/syntax_iterators.py b/spacy/lang/de/syntax_iterators.py new file mode 100644 index 0000000..544fe29 --- /dev/null +++ b/spacy/lang/de/syntax_iterators.py @@ -0,0 +1,43 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + # this iterator extracts spans headed by NOUNs starting from the left-most + # syntactic dependent until the NOUN itself for close apposition and + # measurement construction, the span is sometimes extended to the right of + # the NOUN. Example: "eine Tasse Tee" (a cup (of) tea) returns "eine Tasse Tee" + # and not just "eine Tasse", same for "das Thema Familie". + # fmt: off + labels = ["sb", "oa", "da", "nk", "mo", "ag", "ROOT", "root", "cj", "pd", "og", "app"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_label = doc.vocab.strings.add("NP") + np_deps = set(doc.vocab.strings.add(label) for label in labels) + close_app = doc.vocab.strings.add("nk") + rbracket = 0 + prev_end = -1 + for i, word in enumerate(doclike): + if i < rbracket: + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.pos in (NOUN, PROPN, PRON) and word.dep in np_deps: + rbracket = word.i + 1 + # try to extend the span to the right + # to capture close apposition/measurement constructions + for rdep in doc[word.i].rights: + if rdep.pos in (NOUN, PROPN) and rdep.dep == close_app: + rbracket = rdep.i + 1 + prev_end = rbracket - 1 + yield word.left_edge.i, rbracket, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/de/tokenizer_exceptions.py b/spacy/lang/de/tokenizer_exceptions.py new file mode 100644 index 0000000..3f1aeec --- /dev/null +++ b/spacy/lang/de/tokenizer_exceptions.py @@ -0,0 +1,234 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = { + "auf'm": [{ORTH: "auf"}, {ORTH: "'m", NORM: "dem"}], + "du's": [{ORTH: "du"}, {ORTH: "'s", NORM: "es"}], + "er's": [{ORTH: "er"}, {ORTH: "'s", NORM: "es"}], + "hinter'm": [{ORTH: "hinter"}, {ORTH: "'m", NORM: "dem"}], + "ich's": [{ORTH: "ich"}, {ORTH: "'s", NORM: "es"}], + "ihr's": [{ORTH: "ihr"}, {ORTH: "'s", NORM: "es"}], + "sie's": [{ORTH: "sie"}, {ORTH: "'s", NORM: "es"}], + "unter'm": [{ORTH: "unter"}, {ORTH: "'m", NORM: "dem"}], + "vor'm": [{ORTH: "vor"}, {ORTH: "'m", NORM: "dem"}], + "wir's": [{ORTH: "wir"}, {ORTH: "'s", NORM: "es"}], + "über'm": [{ORTH: "über"}, {ORTH: "'m", NORM: "dem"}], +} + + +for exc_data in [ + {ORTH: "'S", NORM: "'s"}, + {ORTH: "'s", NORM: "'s"}, + {ORTH: "S'", NORM: "'s"}, + {ORTH: "s'", NORM: "'s"}, + {ORTH: "'n", NORM: "ein"}, + {ORTH: "'ne", NORM: "eine"}, + {ORTH: "'nen", NORM: "einen"}, + {ORTH: "'nem", NORM: "einem"}, + {ORTH: "Abb.", NORM: "Abbildung"}, + {ORTH: "Abk.", NORM: "Abkürzung"}, + {ORTH: "Abt.", NORM: "Abteilung"}, + {ORTH: "Apr.", NORM: "April"}, + {ORTH: "Aug.", NORM: "August"}, + {ORTH: "Bd.", NORM: "Band"}, + {ORTH: "Betr.", NORM: "Betreff"}, + {ORTH: "Bf.", NORM: "Bahnhof"}, + {ORTH: "Bhf.", NORM: "Bahnhof"}, + {ORTH: "Bsp.", NORM: "Beispiel"}, + {ORTH: "Dez.", NORM: "Dezember"}, + {ORTH: "Di.", NORM: "Dienstag"}, + {ORTH: "Do.", NORM: "Donnerstag"}, + {ORTH: "Fa.", NORM: "Firma"}, + {ORTH: "Fam.", NORM: "Familie"}, + {ORTH: "Feb.", NORM: "Februar"}, + {ORTH: "Fr.", NORM: "Frau"}, + {ORTH: "Frl.", NORM: "Fräulein"}, + {ORTH: "Hbf.", NORM: "Hauptbahnhof"}, + {ORTH: "Hr.", NORM: "Herr"}, + {ORTH: "Hrn.", NORM: "Herrn"}, + {ORTH: "Jan.", NORM: "Januar"}, + {ORTH: "Jh.", NORM: "Jahrhundert"}, + {ORTH: "Jhd.", NORM: "Jahrhundert"}, + {ORTH: "Jul.", NORM: "Juli"}, + {ORTH: "Jun.", NORM: "Juni"}, + {ORTH: "Mi.", NORM: "Mittwoch"}, + {ORTH: "Mio.", NORM: "Million"}, + {ORTH: "Mo.", NORM: "Montag"}, + {ORTH: "Mrd.", NORM: "Milliarde"}, + {ORTH: "Mrz.", NORM: "März"}, + {ORTH: "MwSt.", NORM: "Mehrwertsteuer"}, + {ORTH: "Mär.", NORM: "März"}, + {ORTH: "Nov.", NORM: "November"}, + {ORTH: "Nr.", NORM: "Nummer"}, + {ORTH: "Okt.", NORM: "Oktober"}, + {ORTH: "Orig.", NORM: "Original"}, + {ORTH: "Pkt.", NORM: "Punkt"}, + {ORTH: "Prof.", NORM: "Professor"}, + {ORTH: "Red.", NORM: "Redaktion"}, + {ORTH: "Sa.", NORM: "Samstag"}, + {ORTH: "Sep.", NORM: "September"}, + {ORTH: "Sept.", NORM: "September"}, + {ORTH: "So.", NORM: "Sonntag"}, + {ORTH: "Std.", NORM: "Stunde"}, + {ORTH: "Str.", NORM: "Straße"}, + {ORTH: "Tel.", NORM: "Telefon"}, + {ORTH: "Tsd.", NORM: "Tausend"}, + {ORTH: "Univ.", NORM: "Universität"}, + {ORTH: "abzgl.", NORM: "abzüglich"}, + {ORTH: "allg.", NORM: "allgemein"}, + {ORTH: "bspw.", NORM: "beispielsweise"}, + {ORTH: "bzgl.", NORM: "bezüglich"}, + {ORTH: "bzw.", NORM: "beziehungsweise"}, + {ORTH: "d.h."}, + {ORTH: "dgl.", NORM: "dergleichen"}, + {ORTH: "ebd.", NORM: "ebenda"}, + {ORTH: "eigtl.", NORM: "eigentlich"}, + {ORTH: "engl.", NORM: "englisch"}, + {ORTH: "evtl.", NORM: "eventuell"}, + {ORTH: "frz.", NORM: "französisch"}, + {ORTH: "gegr.", NORM: "gegründet"}, + {ORTH: "ggf.", NORM: "gegebenenfalls"}, + {ORTH: "ggfs.", NORM: "gegebenenfalls"}, + {ORTH: "ggü.", NORM: "gegenüber"}, + {ORTH: "i.O."}, + {ORTH: "i.d.R."}, + {ORTH: "incl.", NORM: "inklusive"}, + {ORTH: "inkl.", NORM: "inklusive"}, + {ORTH: "insb.", NORM: "insbesondere"}, + {ORTH: "kath.", NORM: "katholisch"}, + {ORTH: "lt.", NORM: "laut"}, + {ORTH: "max.", NORM: "maximal"}, + {ORTH: "min.", NORM: "minimal"}, + {ORTH: "mind.", NORM: "mindestens"}, + {ORTH: "mtl.", NORM: "monatlich"}, + {ORTH: "n.Chr."}, + {ORTH: "orig.", NORM: "original"}, + {ORTH: "röm.", NORM: "römisch"}, + {ORTH: "s.o."}, + {ORTH: "sog."}, + {ORTH: "stellv."}, + {ORTH: "tägl.", NORM: "täglich"}, + {ORTH: "u.U."}, + {ORTH: "u.s.w."}, + {ORTH: "u.v.m."}, + {ORTH: "usf."}, + {ORTH: "usw."}, + {ORTH: "uvm."}, + {ORTH: "v.Chr."}, + {ORTH: "v.a."}, + {ORTH: "v.l.n.r."}, + {ORTH: "vgl.", NORM: "vergleiche"}, + {ORTH: "vllt.", NORM: "vielleicht"}, + {ORTH: "vlt.", NORM: "vielleicht"}, + {ORTH: "z.B."}, + {ORTH: "z.Bsp."}, + {ORTH: "z.T."}, + {ORTH: "z.Z."}, + {ORTH: "z.Zt."}, + {ORTH: "z.b."}, + {ORTH: "zzgl."}, + {ORTH: "österr.", NORM: "österreichisch"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "``", + "''", + "A.C.", + "a.D.", + "A.D.", + "A.G.", + "a.M.", + "a.Z.", + "Abs.", + "adv.", + "al.", + "B.A.", + "B.Sc.", + "betr.", + "biol.", + "Biol.", + "ca.", + "CDU/CSU", + "Chr.", + "Cie.", + "c/o", + "co.", + "Co.", + "d'", + "D.C.", + "Dipl.-Ing.", + "Dipl.", + "Dr.", + "e.g.", + "e.V.", + "ehem.", + "entspr.", + "erm.", + "etc.", + "ev.", + "G.m.b.H.", + "geb.", + "Gebr.", + "gem.", + "h.c.", + "Hg.", + "hrsg.", + "Hrsg.", + "i.A.", + "i.e.", + "i.G.", + "i.Tr.", + "i.V.", + "I.", + "II.", + "III.", + "IV.", + "Inc.", + "Ing.", + "jr.", + "Jr.", + "jun.", + "jur.", + "K.O.", + "L'", + "L.A.", + "lat.", + "M.A.", + "m.E.", + "m.M.", + "M.Sc.", + "Mr.", + "N.Y.", + "N.Y.C.", + "nat.", + "o.a.", + "o.ä.", + "o.g.", + "o.k.", + "O.K.", + "p.a.", + "p.s.", + "P.S.", + "pers.", + "phil.", + "q.e.d.", + "R.I.P.", + "rer.", + "sen.", + "St.", + "std.", + "u.a.", + "U.S.", + "U.S.A.", + "U.S.S.", + "Vol.", + "vs.", + "wiss.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/dsb/__init__.py b/spacy/lang/dsb/__init__.py new file mode 100644 index 0000000..096eced --- /dev/null +++ b/spacy/lang/dsb/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class LowerSorbianDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class LowerSorbian(Language): + lang = "dsb" + Defaults = LowerSorbianDefaults + + +__all__ = ["LowerSorbian"] diff --git a/spacy/lang/dsb/examples.py b/spacy/lang/dsb/examples.py new file mode 100644 index 0000000..11ecbdd --- /dev/null +++ b/spacy/lang/dsb/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.dsb.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Z tym stwori so wuměnjenje a zakład za dalše wobdźěłanje přez analyzu tekstoweje struktury a semantisku anotaciju a z tym tež za tu předstajenu digitalnu online-wersiju.", + "Mi so tu jara derje spodoba.", + "Kotre nowniny chceće měć?", + "Tak ako w slědnem lěśe jo teke lětosa jano doma zapustowaś móžno.", + "Zwóstanjo pótakem hyšći wjele źěła.", +] diff --git a/spacy/lang/dsb/lex_attrs.py b/spacy/lang/dsb/lex_attrs.py new file mode 100644 index 0000000..367b3af --- /dev/null +++ b/spacy/lang/dsb/lex_attrs.py @@ -0,0 +1,113 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nul", + "jaden", + "jadna", + "jadno", + "dwa", + "dwě", + "tśi", + "tśo", + "styri", + "styrjo", + "pěś", + "pěśo", + "šesć", + "šesćo", + "sedym", + "sedymjo", + "wósym", + "wósymjo", + "źewjeś", + "źewjeśo", + "źaseś", + "źaseśo", + "jadnassćo", + "dwanassćo", + "tśinasćo", + "styrnasćo", + "pěśnasćo", + "šesnasćo", + "sedymnasćo", + "wósymnasćo", + "źewjeśnasćo", + "dwanasćo", + "dwaźasća", + "tśiźasća", + "styrźasća", + "pěśźaset", + "šesćźaset", + "sedymźaset", + "wósymźaset", + "źewjeśźaset", + "sto", + "tysac", + "milion", + "miliarda", + "bilion", + "biliarda", + "trilion", + "triliarda", +] + +_ordinal_words = [ + "prědny", + "prědna", + "prědne", + "drugi", + "druga", + "druge", + "tśeśi", + "tśeśa", + "tśeśe", + "stwórty", + "stwórta", + "stwórte", + "pêty", + "pěta", + "pête", + "šesty", + "šesta", + "šeste", + "sedymy", + "sedyma", + "sedyme", + "wósymy", + "wósyma", + "wósyme", + "źewjety", + "źewjeta", + "źewjete", + "źasety", + "źaseta", + "źasete", + "jadnasty", + "jadnasta", + "jadnaste", + "dwanasty", + "dwanasta", + "dwanaste", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/dsb/stop_words.py b/spacy/lang/dsb/stop_words.py new file mode 100644 index 0000000..376e04a --- /dev/null +++ b/spacy/lang/dsb/stop_words.py @@ -0,0 +1,15 @@ +STOP_WORDS = set( + """ +a abo aby ako ale až + +daniž dokulaž + +gaž + +jolic + +pak pótom + +teke togodla +""".split() +) diff --git a/spacy/lang/el/__init__.py b/spacy/lang/el/__init__.py new file mode 100644 index 0000000..00e52bd --- /dev/null +++ b/spacy/lang/el/__init__.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import GreekLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class GreekDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + syntax_iterators = SYNTAX_ITERATORS + + +class Greek(Language): + lang = "el" + Defaults = GreekDefaults + + +@Greek.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return GreekLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Greek"] diff --git a/spacy/lang/el/examples.py b/spacy/lang/el/examples.py new file mode 100644 index 0000000..62515c0 --- /dev/null +++ b/spacy/lang/el/examples.py @@ -0,0 +1,24 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.el.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + """Η άνιση κατανομή του πλούτου και του εισοδήματος, η οποία έχει λάβει + τρομερές διαστάσεις, δεν δείχνει τάσεις βελτίωσης.""", + """Ο στόχος της σύντομης αυτής έκθεσης είναι να συνοψίσει τα κυριότερα + συμπεράσματα των επισκοπήσεων κάθε μιας χώρας.""", + """Μέχρι αργά χθες το βράδυ ο πλοιοκτήτης παρέμενε έξω από το γραφείο του + γενικού γραμματέα του υπουργείου, ενώ είχε μόνον τηλεφωνική επικοινωνία με + τον υπουργό.""", + """Σύμφωνα με καλά ενημερωμένη πηγή, από την επεξεργασία του προέκυψε ότι + οι δράστες της επίθεσης ήταν δύο, καθώς και ότι προσέγγισαν και αποχώρησαν + από το σημείο με μοτοσικλέτα.""", + "Η υποδομή καταλυμάτων στην Ελλάδα είναι πλήρης και ανανεώνεται συνεχώς.", + """Το επείγον ταχυδρομείο (ήτοι το παραδοτέο εντός 48 ωρών το πολύ) μπορεί + να μεταφέρεται αεροπορικώς μόνον εφόσον εφαρμόζονται οι κανόνες + ασφαλείας""", + """'Στις ορεινές περιοχές του νησιού οι χιονοπτώσεις και οι παγετοί είναι + περιορισμένοι ενώ στις παραθαλάσσιες περιοχές σημειώνονται σπανίως.""", +] diff --git a/spacy/lang/el/get_pos_from_wiktionary.py b/spacy/lang/el/get_pos_from_wiktionary.py new file mode 100644 index 0000000..10b54d1 --- /dev/null +++ b/spacy/lang/el/get_pos_from_wiktionary.py @@ -0,0 +1,64 @@ +def get_pos_from_wiktionary(): + import re + + from gensim.corpora.wikicorpus import extract_pages + + regex = re.compile(r"==={{(\w+)\|el}}===") + regex2 = re.compile(r"==={{(\w+ \w+)\|el}}===") + + # get words based on the Wiktionary dump + # check only for specific parts + + # ==={{κύριο όνομα|el}}=== + expected_parts = [ + "μετοχή", + "ρήμα", + "επίθετο", + "επίρρημα", + "ουσιαστικό", + "κύριο όνομα", + "άρθρο", + ] + + wiktionary_file_path = ( + "/data/gsoc2018-spacy/spacy/lang/el/res/elwiktionary-latest-pages-articles.xml" + ) + + proper_names_dict = { + "ουσιαστικό": "nouns", + "επίθετο": "adjectives", + "άρθρο": "dets", + "επίρρημα": "adverbs", + "κύριο όνομα": "proper_names", + "μετοχή": "participles", + "ρήμα": "verbs", + } + expected_parts_dict = {} + for expected_part in expected_parts: + expected_parts_dict[expected_part] = [] + + for title, text, pageid in extract_pages(wiktionary_file_path): + if text.startswith("#REDIRECT"): + continue + title = title.lower() + all_regex = regex.findall(text) + all_regex.extend(regex2.findall(text)) + for a in all_regex: + if a in expected_parts: + expected_parts_dict[a].append(title) + + for i in expected_parts_dict: + with open("_{0}.py".format(proper_names_dict[i]), "w") as f: + f.write("from __future__ import unicode_literals\n") + f.write('{} = set("""\n'.format(proper_names_dict[i].upper())) + words = sorted(expected_parts_dict[i]) + line = "" + to_write = [] + for word in words: + if len(line + " " + word) > 79: + to_write.append(line) + line = "" + else: + line = line + " " + word + f.write("\n".join(to_write)) + f.write('\n""".split())') diff --git a/spacy/lang/el/lemmatizer.py b/spacy/lang/el/lemmatizer.py new file mode 100644 index 0000000..631848a --- /dev/null +++ b/spacy/lang/el/lemmatizer.py @@ -0,0 +1,62 @@ +from typing import List + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class GreekLemmatizer(Lemmatizer): + """ + Greek language lemmatizer applies the default rule based lemmatization + procedure with some modifications for better Greek language support. + + The first modification is that it checks if the word for lemmatization is + already a lemma and if yes, it just returns it. + The second modification is about removing the base forms function which is + not applicable for Greek language. + """ + + def rule_lemmatize(self, token: Token) -> List[str]: + """Lemmatize using a rule-based approach. + + token (Token): The token to lemmatize. + RETURNS (list): The available lemmas for the string. + """ + cache_key = (token.lower, token.pos) + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + univ_pos = token.pos_.lower() + if univ_pos in ("", "eol", "space"): + return [string.lower()] + + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, {}) + + string = string.lower() + forms = [] + if string in index: + forms.append(string) + return forms + forms.extend(exceptions.get(string, [])) + oov_forms = [] + if not forms: + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + pass + elif form in index or not form.isalpha(): + forms.append(form) + else: + oov_forms.append(form) + if not forms: + forms.extend(oov_forms) + if not forms: + forms.append(string) + forms = list(dict.fromkeys(forms)) + self.cache[cache_key] = forms + return forms diff --git a/spacy/lang/el/lex_attrs.py b/spacy/lang/el/lex_attrs.py new file mode 100644 index 0000000..5c8f968 --- /dev/null +++ b/spacy/lang/el/lex_attrs.py @@ -0,0 +1,99 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "μηδέν", + "ένας", + "δυο", + "δυό", + "τρεις", + "τέσσερις", + "πέντε", + "έξι", + "εφτά", + "επτά", + "οκτώ", + "οχτώ", + "εννιά", + "εννέα", + "δέκα", + "έντεκα", + "ένδεκα", + "δώδεκα", + "δεκατρείς", + "δεκατέσσερις", + "δεκαπέντε", + "δεκαέξι", + "δεκαεπτά", + "δεκαοχτώ", + "δεκαεννέα", + "δεκαεννεα", + "είκοσι", + "τριάντα", + "σαράντα", + "πενήντα", + "εξήντα", + "εβδομήντα", + "ογδόντα", + "ενενήντα", + "εκατό", + "διακόσιοι", + "διακόσοι", + "τριακόσιοι", + "τριακόσοι", + "τετρακόσιοι", + "τετρακόσοι", + "πεντακόσιοι", + "πεντακόσοι", + "εξακόσιοι", + "εξακόσοι", + "εφτακόσιοι", + "εφτακόσοι", + "επτακόσιοι", + "επτακόσοι", + "οχτακόσιοι", + "οχτακόσοι", + "οκτακόσιοι", + "οκτακόσοι", + "εννιακόσιοι", + "χίλιοι", + "χιλιάδα", + "εκατομμύριο", + "δισεκατομμύριο", + "τρισεκατομμύριο", + "τετράκις", + "πεντάκις", + "εξάκις", + "επτάκις", + "οκτάκις", + "εννεάκις", + "ένα", + "δύο", + "τρία", + "τέσσερα", + "δις", + "χιλιάδες", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.count("^") == 1: + num, denom = text.split("^") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words or text.lower().split(" ")[0] in _num_words: + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/el/punctuation.py b/spacy/lang/el/punctuation.py new file mode 100644 index 0000000..b8b717b --- /dev/null +++ b/spacy/lang/el/punctuation.py @@ -0,0 +1,105 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + HYPHENS, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, +) + +_units = ( + "km km² km³ m m² m³ dm dm² dm³ cm cm² cm³ mm mm² mm³ ha µm nm yd in ft " + "kg g mg µg t lb oz m/s km/h kmh mph hPa Pa mbar mb MB kb KB gb GB tb " + "TB T G M K км км² км³ м м² м³ дм дм² дм³ см см² см³ мм мм² мм³ нм " + "кг г мг м/с км/ч кПа Па мбар Кб КБ кб Мб МБ мб Гб ГБ гб Тб ТБ тб" +) + + +def merge_chars(char): + return char.strip().replace(" ", "|") + + +UNITS = merge_chars(_units) + +_prefixes = ( + [ + "''", + "§", + "%", + "=", + r"\+[0-9]+%", # 90% + r"\'([0-9]){2}([\-]\'([0-9]){2})*", # '12'-13 + r"\-([0-9]){1,9}\.([0-9]){1,9}", # -12.13 + r"\'([Α-Ωα-ωίϊΐόάέύϋΰήώ]+)\'", # 'αβγ' + r"([Α-Ωα-ωίϊΐόάέύϋΰήώ]){1,3}\'", # αβγ' + r"http://www.[A-Za-z]+\-[A-Za-z]+(\.[A-Za-z]+)+(\/[A-Za-z]+)*(\.[A-Za-z]+)*", + r"[ΈΆΊΑ-Ωα-ωίϊΐόάέύϋΰήώ]+\*", # όνομα* + r"\$([0-9])+([\,\.]([0-9])+){0,1}", + ] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", # 12+ + r"([0-9])+\'", # 12' + r"([A-Za-z])?\'", # a' + r"^([0-9]){1,2}\.", # 12. + r" ([0-9]){1,2}\.", # 12. + r"([0-9]){1}\) ", # 12) + r"^([0-9]){1}\)$", # 12) + r"(?<=°[FfCcKk])\.", + r"([0-9])+\&", # 12& + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9{al}{e}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"²\-\+", q=CONCAT_QUOTES + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + r"(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\-", # όνομα- + r"(?<=[Α-Ωα-ωίϊΐόάέύϋΰήώ])\.", + r"^[Α-Ω]{1}\.", + r"\ [Α-Ω]{1}\.", + # πρώτος-δεύτερος , πρώτος-δεύτερος-τρίτος + r"[ΈΆΊΑΌ-Ωα-ωίϊΐόάέύϋΰήώ]+([\-]([ΈΆΊΑΌ-Ωα-ωίϊΐόάέύϋΰήώ]+))+", + r"([0-9]+)mg", # 13mg + r"([0-9]+)\.([0-9]+)m", # 1.2m + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\/\-\*^](?=[0-9])", # 1/2 , 1-2 , 1*2 + r"([a-zA-Z]+)\/([a-zA-Z]+)\/([a-zA-Z]+)", # name1/name2/name3 + r"([0-9])+(\.([0-9]+))*([\-]([0-9])+)+", # 10.9 , 10.9.9 , 10.9-6 + r"([0-9])+[,]([0-9])+[\-]([0-9])+[,]([0-9])+", # 10,11,12 + r"([0-9])+[ης]+([\-]([0-9])+)+", # 1ης-2 + # 15/2 , 15/2/17 , 2017/2/15 + r"([0-9]){1,4}[\/]([0-9]){1,2}([\/]([0-9]){0,4}){0,1}", + r"[A-Za-z]+\@[A-Za-z]+(\-[A-Za-z]+)*\.[A-Za-z]+", # abc@cde-fgh.a + r"([a-zA-Z]+)(\-([a-zA-Z]+))+", # abc-abc + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/el/stop_words.py b/spacy/lang/el/stop_words.py new file mode 100644 index 0000000..7c43621 --- /dev/null +++ b/spacy/lang/el/stop_words.py @@ -0,0 +1,87 @@ +# Stop words +# Link to greek stop words: https://www.translatum.gr/forum/index.php?topic=3550.0?topic=3550.0 +STOP_WORDS = set( + """ +αδιάκοπα αι ακόμα ακόμη ακριβώς άλλα αλλά αλλαχού άλλες άλλη άλλην +άλλης αλλιώς αλλιώτικα άλλο άλλοι αλλοιώς αλλοιώτικα άλλον άλλος άλλοτε αλλού +άλλους άλλων άμα άμεσα αμέσως αν ανά ανάμεσα αναμεταξύ άνευ αντί αντίπερα αντίς +άνω ανωτέρω άξαφνα απ απέναντι από απόψε άρα άραγε αρκετά αρκετές +αρχικά ας αύριο αυτά αυτές αυτή αυτήν αυτής αυτό αυτοί αυτόν αυτός αυτού αυτούς +αυτών αφότου αφού + +βέβαια βεβαιότατα + +γι για γιατί γρήγορα γύρω + +δα δε δείνα δεν δεξιά δήθεν δηλαδή δι δια διαρκώς δικά δικό δικοί δικός δικού +δικούς διόλου δίπλα δίχως + +εάν εαυτό εαυτόν εαυτού εαυτούς εαυτών έγκαιρα εγκαίρως εγώ εδώ ειδεμή είθε είμαι +είμαστε είναι εις είσαι είσαστε είστε είτε είχα είχαμε είχαν είχατε είχε είχες έκαστα +έκαστες έκαστη έκαστην έκαστης έκαστο έκαστοι έκαστον έκαστος εκάστου εκάστους εκάστων +εκεί εκείνα εκείνες εκείνη εκείνην εκείνης εκείνο εκείνοι εκείνον εκείνος εκείνου +εκείνους εκείνων εκτός εμάς εμείς εμένα εμπρός εν ένα έναν ένας ενός εντελώς εντός +εναντίον εξής εξαιτίας επιπλέον επόμενη εντωμεταξύ ενώ εξ έξαφνα εξήσ εξίσου έξω επάνω +επειδή έπειτα επί επίσης επομένως εσάς εσείς εσένα έστω εσύ ετέρα ετέραι ετέρας έτερες +έτερη έτερης έτερο έτεροι έτερον έτερος ετέρου έτερους ετέρων ετούτα ετούτες ετούτη ετούτην +ετούτης ετούτο ετούτοι ετούτον ετούτος ετούτου ετούτους ετούτων έτσι εύγε ευθύς ευτυχώς εφεξής +έχει έχεις έχετε έχομε έχουμε έχουν εχτές έχω έως έγιναν έγινε έκανε έξι έχοντας + +η ήδη ήμασταν ήμαστε ήμουν ήσασταν ήσαστε ήσουν ήταν ήτανε ήτοι ήττον + +θα + +ι ιδία ίδια ίδιαν ιδίας ίδιες ίδιο ίδιοι ίδιον ίδιοσ ίδιος ιδίου ίδιους ίδιων ιδίως ιι ιιι +ίσαμε ίσια ίσως + +κάθε καθεμία καθεμίας καθένα καθένας καθενός καθετί καθόλου καθώς και κακά κακώς καλά +καλώς καμία καμίαν καμίας κάμποσα κάμποσες κάμποση κάμποσην κάμποσης κάμποσο κάμποσοι +κάμποσον κάμποσος κάμποσου κάμποσους κάμποσων κανείς κάνεν κανένα κανέναν κανένας +κανενός κάποια κάποιαν κάποιας κάποιες κάποιο κάποιοι κάποιον κάποιος κάποιου κάποιους +κάποιων κάποτε κάπου κάπως κατ κατά κάτι κατιτί κατόπιν κάτω κιόλας κλπ κοντά κτλ κυρίως + +λιγάκι λίγο λιγότερο λόγω λοιπά λοιπόν + +μα μαζί μακάρι μακρυά μάλιστα μάλλον μας με μεθαύριο μείον μέλει μέλλεται μεμιάς μεν +μερικά μερικές μερικοί μερικούς μερικών μέσα μετ μετά μεταξύ μέχρι μη μήδε μην μήπως +μήτε μια μιαν μιας μόλις μολονότι μονάχα μόνες μόνη μόνην μόνης μόνο μόνοι μονομιάς +μόνος μόνου μόνους μόνων μου μπορεί μπορούν μπρος μέσω μία μεσώ + +να ναι νωρίς + +ξανά ξαφνικά + +ο οι όλα όλες όλη όλην όλης όλο ολόγυρα όλοι όλον ολονέν όλος ολότελα όλου όλους όλων +όλως ολωσδιόλου όμως όποια οποιαδήποτε οποίαν οποιανδήποτε οποίας οποίος οποιασδήποτε οποιδήποτε +όποιες οποιεσδήποτε όποιο οποιοδηήποτε όποιοι όποιον οποιονδήποτε όποιος οποιοσδήποτε +οποίου οποιουδήποτε οποίους οποιουσδήποτε οποίων οποιωνδήποτε όποτε οποτεδήποτε όπου +οπουδήποτε όπως ορισμένα ορισμένες ορισμένων ορισμένως όσα οσαδήποτε όσες οσεσδήποτε +όση οσηδήποτε όσην οσηνδήποτε όσης οσησδήποτε όσο οσοδήποτε όσοι οσοιδήποτε όσον οσονδήποτε +όσος οσοσδήποτε όσου οσουδήποτε όσους οσουσδήποτε όσων οσωνδήποτε όταν ότι οτιδήποτε +ότου ου ουδέ ούτε όχι οποία οποίες οποίο οποίοι οπότε ος + +πάνω παρά περί πολλά πολλές πολλοί πολλούς που πρώτα πρώτες πρώτη πρώτο πρώτος πως +πάλι πάντα πάντοτε παντού πάντως πάρα πέρα πέρι περίπου περισσότερο πέρσι πέρυσι πια πιθανόν +πιο πίσω πλάι πλέον πλην ποιά ποιάν ποιάς ποιές ποιό ποιοί ποιόν ποιός ποιού ποιούς +ποιών πολύ πόσες πόση πόσην πόσης πόσοι πόσος πόσους πότε ποτέ πού πούθε πουθενά πρέπει +πριν προ προκειμένου πρόκειται πρόπερσι προς προτού προχθές προχτές πρωτύτερα πώς + +σαν σας σε σεις σου στα στη στην στης στις στο στον στου στους στων συγχρόνως +συν συνάμα συνεπώς συχνάς συχνές συχνή συχνήν συχνής συχνό συχνοί συχνόν +συχνός συχνού συχνούς συχνών συχνώς σχεδόν + +τα τάδε ταύτα ταύτες ταύτη ταύτην ταύτης ταύτοταύτον ταύτος ταύτου ταύτων τάχα τάχατε +τελευταία τελευταίο τελευταίος τού τρία τρίτη τρεις τελικά τελικώς τες τέτοια τέτοιαν +τέτοιας τέτοιες τέτοιο τέτοιοι τέτοιον τέτοιος τέτοιου +τέτοιους τέτοιων τη την της τι τίποτα τίποτε τις το τοι τον τοσ τόσα τόσες τόση τόσην +τόσης τόσο τόσοι τόσον τόσος τόσου τόσους τόσων τότε του τουλάχιστο τουλάχιστον τους τούς τούτα +τούτες τούτη τούτην τούτης τούτο τούτοι τούτοις τούτον τούτος τούτου τούτους τούτων τυχόν +των τώρα + +υπ υπέρ υπό υπόψη υπόψιν ύστερα + +χωρίς χωριστά + +ω ως ωσάν ωσότου ώσπου ώστε ωστόσο ωχ +""".split() +) diff --git a/spacy/lang/el/syntax_iterators.py b/spacy/lang/el/syntax_iterators.py new file mode 100644 index 0000000..31c7dcc --- /dev/null +++ b/spacy/lang/el/syntax_iterators.py @@ -0,0 +1,53 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + # It follows the logic of the noun chunks finder of English language, + # adjusted to some Greek language special characteristics. + # obj tag corrects some DEP tagger mistakes. + # Further improvement of the models will eliminate the need for this tag. + labels = ["nsubj", "obj", "iobj", "appos", "ROOT", "obl"] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings.add(label) for label in labels] + conj = doc.vocab.strings.add("conj") + nmod = doc.vocab.strings.add("nmod") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + flag = False + if word.pos == NOUN: + # check for patterns such as γραμμή παραγωγής + for potential_nmod in word.rights: + if potential_nmod.dep == nmod: + prev_end = potential_nmod.i + yield word.left_edge.i, potential_nmod.i + 1, np_label + flag = True + break + if flag is False: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + elif word.dep == conj: + # covers the case: έχει όμορφα και έξυπνα παιδιά + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/el/tokenizer_exceptions.py b/spacy/lang/el/tokenizer_exceptions.py new file mode 100644 index 0000000..d88d483 --- /dev/null +++ b/spacy/lang/el/tokenizer_exceptions.py @@ -0,0 +1,393 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for token in ["Απ'", "ΑΠ'", "αφ'", "Αφ'"]: + _exc[token] = [{ORTH: token, NORM: "από"}] + +for token in ["Αλλ'", "αλλ'"]: + _exc[token] = [{ORTH: token, NORM: "αλλά"}] + +for token in ["παρ'", "Παρ'", "ΠΑΡ'"]: + _exc[token] = [{ORTH: token, NORM: "παρά"}] + +for token in ["καθ'", "Καθ'"]: + _exc[token] = [{ORTH: token, NORM: "κάθε"}] + +for token in ["κατ'", "Κατ'"]: + _exc[token] = [{ORTH: token, NORM: "κατά"}] + +for token in ["'ΣΟΥΝ", "'ναι", "'ταν", "'τανε", "'μαστε", "'μουνα", "'μουν"]: + _exc[token] = [{ORTH: token, NORM: "είμαι"}] + +for token in ["Επ'", "επ'", "εφ'", "Εφ'"]: + _exc[token] = [{ORTH: token, NORM: "επί"}] + +for token in ["Δι'", "δι'"]: + _exc[token] = [{ORTH: token, NORM: "δια"}] + +for token in ["'χουν", "'χουμε", "'χαμε", "'χα", "'χε", "'χεις", "'χει"]: + _exc[token] = [{ORTH: token, NORM: "έχω"}] + +for token in ["υπ'", "Υπ'"]: + _exc[token] = [{ORTH: token, NORM: "υπό"}] + +for token in ["Μετ'", "ΜΕΤ'", "'μετ"]: + _exc[token] = [{ORTH: token, NORM: "μετά"}] + +for token in ["Μ'", "μ'"]: + _exc[token] = [{ORTH: token, NORM: "με"}] + +for token in ["Γι'", "ΓΙ'", "γι'"]: + _exc[token] = [{ORTH: token, NORM: "για"}] + +for token in ["Σ'", "σ'"]: + _exc[token] = [{ORTH: token, NORM: "σε"}] + +for token in ["Θ'", "θ'"]: + _exc[token] = [{ORTH: token, NORM: "θα"}] + +for token in ["Ν'", "ν'"]: + _exc[token] = [{ORTH: token, NORM: "να"}] + +for token in ["Τ'", "τ'"]: + _exc[token] = [{ORTH: token, NORM: "να"}] + +for token in ["'γω", "'σένα", "'μεις"]: + _exc[token] = [{ORTH: token, NORM: "εγώ"}] + +for token in ["Τ'", "τ'"]: + _exc[token] = [{ORTH: token, NORM: "το"}] + +for token in ["Φέρ'", "Φερ'", "φέρ'", "φερ'"]: + _exc[token] = [{ORTH: token, NORM: "φέρνω"}] + +for token in ["'ρθούνε", "'ρθουν", "'ρθει", "'ρθεί", "'ρθε", "'ρχεται"]: + _exc[token] = [{ORTH: token, NORM: "έρχομαι"}] + +for token in ["'πανε", "'λεγε", "'λεγαν", "'πε", "'λεγα"]: + _exc[token] = [{ORTH: token, NORM: "λέγω"}] + +for token in ["Πάρ'", "πάρ'"]: + _exc[token] = [{ORTH: token, NORM: "παίρνω"}] + +for token in ["μέσ'", "Μέσ'", "μεσ'"]: + _exc[token] = [{ORTH: token, NORM: "μέσα"}] + +for token in ["Δέσ'", "Δεσ'", "δεσ'"]: + _exc[token] = [{ORTH: token, NORM: "δένω"}] + +for token in ["'κανε", "Κάν'"]: + _exc[token] = [{ORTH: token, NORM: "κάνω"}] + +_other_exc = { + "κι": [{ORTH: "κι", NORM: "και"}], + "Παίξ'": [{ORTH: "Παίξ'", NORM: "παίζω"}], + "Αντ'": [{ORTH: "Αντ'", NORM: "αντί"}], + "ολ'": [{ORTH: "ολ'", NORM: "όλος"}], + "ύστερ'": [{ORTH: "ύστερ'", NORM: "ύστερα"}], + "'πρεπε": [{ORTH: "'πρεπε", NORM: "πρέπει"}], + "Δύσκολ'": [{ORTH: "Δύσκολ'", NORM: "δύσκολος"}], + "'θελα": [{ORTH: "'θελα", NORM: "θέλω"}], + "'γραφα": [{ORTH: "'γραφα", NORM: "γράφω"}], + "'παιρνα": [{ORTH: "'παιρνα", NORM: "παίρνω"}], + "'δειξε": [{ORTH: "'δειξε", NORM: "δείχνω"}], + "όμουρφ'": [{ORTH: "όμουρφ'", NORM: "όμορφος"}], + "κ'τσή": [{ORTH: "κ'τσή", NORM: "κουτσός"}], + "μηδ'": [{ORTH: "μηδ'", NORM: "μήδε"}], + "'ξομολογήθηκε": [{ORTH: "'ξομολογήθηκε", NORM: "εξομολογούμαι"}], + "'μας": [{ORTH: "'μας", NORM: "εμάς"}], + "'ξερες": [{ORTH: "'ξερες", NORM: "ξέρω"}], + "έφθασ'": [{ORTH: "έφθασ'", NORM: "φθάνω"}], + "εξ'": [{ORTH: "εξ'", NORM: "εκ"}], + "δώσ'": [{ORTH: "δώσ'", NORM: "δίνω"}], + "τίποτ'": [{ORTH: "τίποτ'", NORM: "τίποτα"}], + "Λήξ'": [{ORTH: "Λήξ'", NORM: "λήγω"}], + "άσ'": [{ORTH: "άσ'", NORM: "αφήνω"}], + "Στ'": [{ORTH: "Στ'", NORM: "στο"}], + "Δωσ'": [{ORTH: "Δωσ'", NORM: "δίνω"}], + "Βάψ'": [{ORTH: "Βάψ'", NORM: "βάφω"}], + "Αλλ'": [{ORTH: "Αλλ'", NORM: "αλλά"}], + "Αμ'": [{ORTH: "Αμ'", NORM: "άμα"}], + "Αγόρασ'": [{ORTH: "Αγόρασ'", NORM: "αγοράζω"}], + "'φύγε": [{ORTH: "'φύγε", NORM: "φεύγω"}], + "'φερε": [{ORTH: "'φερε", NORM: "φέρνω"}], + "'φαγε": [{ORTH: "'φαγε", NORM: "τρώω"}], + "'σπαγαν": [{ORTH: "'σπαγαν", NORM: "σπάω"}], + "'σκασε": [{ORTH: "'σκασε", NORM: "σκάω"}], + "'σβηνε": [{ORTH: "'σβηνε", NORM: "σβήνω"}], + "'ριξε": [{ORTH: "'ριξε", NORM: "ρίχνω"}], + "'κλεβε": [{ORTH: "'κλεβε", NORM: "κλέβω"}], + "'κει": [{ORTH: "'κει", NORM: "εκεί"}], + "'βλεπε": [{ORTH: "'βλεπε", NORM: "βλέπω"}], + "'βγαινε": [{ORTH: "'βγαινε", NORM: "βγαίνω"}], +} + +_exc.update(_other_exc) + +for h in range(1, 12 + 1): + for period in ["π.μ.", "πμ"]: + _exc[f"{h}{period}"] = [ + {ORTH: f"{h}"}, + {ORTH: period, NORM: "π.μ."}, + ] + + for period in ["μ.μ.", "μμ"]: + _exc[f"{h}{period}"] = [ + {ORTH: f"{h}"}, + {ORTH: period, NORM: "μ.μ."}, + ] + +for exc_data in [ + {ORTH: "ΑΓΡ.", NORM: "Αγροτικός"}, + {ORTH: "Αγ. Γρ.", NORM: "Αγία Γραφή"}, + {ORTH: "Αθ.", NORM: "Αθανάσιος"}, + {ORTH: "Αλεξ.", NORM: "Αλέξανδρος"}, + {ORTH: "Απρ.", NORM: "Απρίλιος"}, + {ORTH: "Αύγ.", NORM: "Αύγουστος"}, + {ORTH: "Δεκ.", NORM: "Δεκέμβριος"}, + {ORTH: "Δημ.", NORM: "Δήμος"}, + {ORTH: "Ιαν.", NORM: "Ιανουάριος"}, + {ORTH: "Ιούλ.", NORM: "Ιούλιος"}, + {ORTH: "Ιούν.", NORM: "Ιούνιος"}, + {ORTH: "Ιωαν.", NORM: "Ιωάννης"}, + {ORTH: "Μ. Ασία", NORM: "Μικρά Ασία"}, + {ORTH: "Μάρτ.", NORM: "Μάρτιος"}, + {ORTH: "Μάρτ'", NORM: "Μάρτιος"}, + {ORTH: "Νοέμβρ.", NORM: "Νοέμβριος"}, + {ORTH: "Οκτ.", NORM: "Οκτώβριος"}, + {ORTH: "Σεπτ.", NORM: "Σεπτέμβριος"}, + {ORTH: "Φεβρ.", NORM: "Φεβρουάριος"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +for orth in [ + "$ΗΠΑ", + "Α'", + "Α.Ε.", + "Α.Ε.Β.Ε.", + "Α.Ε.Ι.", + "Α.Ε.Π.", + "Α.Μ.Α.", + "Α.Π.Θ.", + "Α.Τ.", + "Α.Χ.", + "ΑΝ.", + "Αγ.", + "Αλ.", + "Αν.", + "Αντ.", + "Απ.", + "Β'", + "Β)", + "Β.Ζ.", + "Β.Ι.Ο.", + "Β.Κ.", + "Β.Μ.Α.", + "Βασ.", + "Γ'", + "Γ)", + "Γ.Γ.", + "Γ.Δ.", + "Γκ.", + "Δ.Ε.Η.", + "Δ.Ε.Σ.Ε.", + "Δ.Ν.", + "Δ.Ο.Υ.", + "Δ.Σ.", + "Δ.Υ.", + "ΔΙ.ΚΑ.Τ.Σ.Α.", + "Δηλ.", + "Διον.", + "Ε.Α.", + "Ε.Α.Κ.", + "Ε.Α.Π.", + "Ε.Ε.", + "Ε.Κ.", + "Ε.ΚΕ.ΠΙΣ.", + "Ε.Λ.Α.", + "Ε.Λ.Ι.Α.", + "Ε.Π.Σ.", + "Ε.Π.Τ.Α.", + "Ε.Σ.Ε.Ε.Κ.", + "Ε.Υ.Κ.", + "ΕΕ.", + "ΕΚ.", + "ΕΛ.", + "ΕΛ.ΑΣ.", + "Εθν.", + "Ελ.", + "Εμ.", + "Επ.", + "Ευ.", + "Η'", + "Η.Π.Α.", + "ΘΕ.", + "Θεμ.", + "Θεοδ.", + "Θρ.", + "Ι.Ε.Κ.", + "Ι.Κ.Α.", + "Ι.Κ.Υ.", + "Ι.Σ.Θ.", + "Ι.Χ.", + "ΙΖ'", + "ΙΧ.", + "Κ.Α.Α.", + "Κ.Α.Ε.", + "Κ.Β.Σ.", + "Κ.Δ.", + "Κ.Ε.", + "Κ.Ε.Κ.", + "Κ.Ι.", + "Κ.Κ.", + "Κ.Ι.Θ.", + "Κ.Ι.Θ.", + "Κ.ΚΕΚ.", + "Κ.Ο.", + "Κ.Π.Ρ.", + "ΚΑΤ.", + "ΚΚ.", + "Καν.", + "Καρ.", + "Κατ.", + "Κυρ.", + "Κων.", + "Λ.Α.", + "Λ.χ.", + "Λ.Χ.", + "Λεωφ.", + "Λι.", + "Μ.Δ.Ε.", + "Μ.Ε.Ο.", + "Μ.Ζ.", + "Μ.Μ.Ε.", + "Μ.Ο.", + "Μεγ.", + "Μιλτ.", + "Μιχ.", + "Ν.Δ.", + "Ν.Ε.Α.", + "Ν.Κ.", + "Ν.Ο.", + "Ν.Ο.Θ.", + "Ν.Π.Δ.Δ.", + "Ν.Υ.", + "ΝΔ.", + "Νικ.", + "Ντ'", + "Ντ.", + "Ο'", + "Ο.Α.", + "Ο.Α.Ε.Δ.", + "Ο.Δ.", + "Ο.Ε.Ε.", + "Ο.Ε.Ε.Κ.", + "Ο.Η.Ε.", + "Ο.Κ.", + "Π.Δ.", + "Π.Ε.Κ.Δ.Υ.", + "Π.Ε.Π.", + "Π.Μ.Σ.", + "ΠΟΛ.", + "Π.Χ.", + "Παρ.", + "Πλ.", + "Πρ.", + "Σ.Δ.Ο.Ε.", + "Σ.Ε.", + "Σ.Ε.Κ.", + "Σ.Π.Δ.Ω.Β.", + "Σ.Τ.", + "Σαβ.", + "Στ.", + "ΣτΕ.", + "Στρ.", + "Τ.Α.", + "Τ.Ε.Ε.", + "Τ.Ε.Ι.", + "ΤΡ.", + "Τζ.", + "Τηλ.", + "Υ.Γ.", + "ΥΓ.", + "ΥΠ.Ε.Π.Θ.", + "Φ.Α.Β.Ε.", + "Φ.Κ.", + "Φ.Σ.", + "Φ.Χ.", + "Φ.Π.Α.", + "Φιλ.", + "Χ.Α.Α.", + "ΧΡ.", + "Χ.Χ.", + "Χαρ.", + "Χιλ.", + "Χρ.", + "άγ.", + "άρθρ.", + "αι.", + "αν.", + "απ.", + "αρ.", + "αριθ.", + "αριθμ.", + "β'", + "βλ.", + "γ.γ.", + "γεν.", + "γραμμ.", + "δ.δ.", + "δ.σ.", + "δηλ.", + "δισ.", + "δολ.", + "δρχ.", + "εκ.", + "εκατ.", + "ελ.", + "θιν'", + "κ.", + "κ.ά.", + "κ.α.", + "κ.κ.", + "κ.λπ.", + "κ.ο.κ.", + "κ.τ.λ.", + "κλπ.", + "κτλ.", + "κυβ.", + "λ.χ.", + "μ.", + "μ.Χ.", + "μ.μ.", + "μιλ.", + "ντ'", + "π.Χ.", + "π.β.", + "π.δ.", + "π.μ.", + "π.χ.", + "σ.", + "σ.α.λ.", + "σ.σ.", + "σελ.", + "στρ.", + "τ'ς", + "τ.μ.", + "τετ.", + "τετρ.", + "τηλ.", + "τρισ.", + "τόν.", + "υπ.", + "χ.μ.", + "χγρ.", + "χιλ.", + "χλμ.", +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/en/__init__.py b/spacy/lang/en/__init__.py new file mode 100644 index 0000000..c4bcfb9 --- /dev/null +++ b/spacy/lang/en/__init__.py @@ -0,0 +1,51 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import EnglishLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class EnglishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class English(Language): + lang = "en" + Defaults = EnglishDefaults + + +@English.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return EnglishLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["English"] diff --git a/spacy/lang/en/examples.py b/spacy/lang/en/examples.py new file mode 100644 index 0000000..7ed0ba0 --- /dev/null +++ b/spacy/lang/en/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.en.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple is looking at buying U.K. startup for $1 billion", + "Autonomous cars shift insurance liability toward manufacturers", + "San Francisco considers banning sidewalk delivery robots", + "London is a big city in the United Kingdom.", + "Where are you?", + "Who is the president of France?", + "What is the capital of the United States?", + "When was Barack Obama born?", +] diff --git a/spacy/lang/en/lemmatizer.py b/spacy/lang/en/lemmatizer.py new file mode 100644 index 0000000..c88b69b --- /dev/null +++ b/spacy/lang/en/lemmatizer.py @@ -0,0 +1,40 @@ +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class EnglishLemmatizer(Lemmatizer): + """English lemmatizer. Only overrides is_base_form.""" + + def is_base_form(self, token: Token) -> bool: + """ + Check whether we're dealing with an uninflected paradigm, so we can + avoid lemmatization entirely. + + univ_pos (str / int): The token's universal part-of-speech tag. + morphology (dict): The token's morphological features following the + Universal Dependencies scheme. + """ + univ_pos = token.pos_.lower() + morphology = token.morph.to_dict() + if univ_pos == "noun" and morphology.get("Number") == "Sing": + return True + elif univ_pos == "verb" and morphology.get("VerbForm") == "Inf": + return True + # This maps 'VBP' to base form -- probably just need 'IS_BASE' + # morphology + elif univ_pos == "verb" and ( + morphology.get("VerbForm") == "Fin" + and morphology.get("Tense") == "Pres" + and morphology.get("Number") is None + ): + return True + elif univ_pos == "adj" and morphology.get("Degree") == "Pos": + return True + elif morphology.get("VerbForm") == "Inf": + return True + elif morphology.get("VerbForm") == "None": + return True + elif morphology.get("Degree") == "Pos": + return True + else: + return False diff --git a/spacy/lang/en/lex_attrs.py b/spacy/lang/en/lex_attrs.py new file mode 100644 index 0000000..7f9dce9 --- /dev/null +++ b/spacy/lang/en/lex_attrs.py @@ -0,0 +1,46 @@ +from ...attrs import LIKE_NUM + +# fmt: off +_num_words = [ + "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", + "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", + "fifty", "sixty", "seventy", "eighty", "ninety", "hundred", "thousand", + "million", "billion", "trillion", "quadrillion", "quintillion", "sextillion", + "septillion", "octillion", "nonillion", "decillion", "gajillion", "bazillion" +] +_ordinal_words = [ + "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", + "ninth", "tenth", "eleventh", "twelfth", "thirteenth", "fourteenth", + "fifteenth", "sixteenth", "seventeenth", "eighteenth", "nineteenth", + "twentieth", "thirtieth", "fortieth", "fiftieth", "sixtieth", "seventieth", + "eightieth", "ninetieth", "hundredth", "thousandth", "millionth", "billionth", + "trillionth", "quadrillionth", "quintillionth", "sextillionth", "septillionth", + "octillionth", "nonillionth", "decillionth", "gajillionth", "bazillionth" +] +# fmt: on + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith(("st", "nd", "rd", "th")): + if text_lower[:-2].isdigit(): + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/en/punctuation.py b/spacy/lang/en/punctuation.py new file mode 100644 index 0000000..775c6b0 --- /dev/null +++ b/spacy/lang/en/punctuation.py @@ -0,0 +1,26 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/en/stop_words.py b/spacy/lang/en/stop_words.py new file mode 100644 index 0000000..1ca5cbc --- /dev/null +++ b/spacy/lang/en/stop_words.py @@ -0,0 +1,73 @@ +# Stop words +STOP_WORDS = set( + """ +a about above across after afterwards again against all almost alone along +already also although always am among amongst amount an and another any anyhow +anyone anything anyway anywhere are around as at + +back be became because become becomes becoming been before beforehand behind +being below beside besides between beyond both bottom but by + +call can cannot ca could + +did do does doing done down due during + +each eight either eleven else elsewhere empty enough even ever every +everyone everything everywhere except + +few fifteen fifty first five for former formerly forty four from front full +further + +get give go + +had has have he hence her here hereafter hereby herein hereupon hers herself +him himself his how however hundred + +i if in indeed into is it its itself + +keep + +last latter latterly least less + +just + +made make many may me meanwhile might mine more moreover most mostly move much +must my myself + +name namely neither never nevertheless next nine no nobody none noone nor not +nothing now nowhere + +of off often on once one only onto or other others otherwise our ours ourselves +out over own + +part per perhaps please put + +quite + +rather re really regarding + +same say see seem seemed seeming seems serious several she should show side +since six sixty so some somehow someone something sometime sometimes somewhere +still such + +take ten than that the their them themselves then thence there thereafter +thereby therefore therein thereupon these they third this those though three +through throughout thru thus to together too top toward towards twelve twenty +two + +under until up unless upon us used using + +various very very via was we well were what whatever when whence whenever where +whereafter whereas whereby wherein whereupon wherever whether which while +whither who whoever whole whom whose why will with within without would + +yet you your yours yourself yourselves +""".split() +) + +contractions = ["n't", "'d", "'ll", "'m", "'re", "'s", "'ve"] +STOP_WORDS.update(contractions) + +for apostrophe in ["‘", "’"]: + for stopword in contractions: + STOP_WORDS.add(stopword.replace("'", apostrophe)) diff --git a/spacy/lang/en/syntax_iterators.py b/spacy/lang/en/syntax_iterators.py new file mode 100644 index 0000000..140ae0a --- /dev/null +++ b/spacy/lang/en/syntax_iterators.py @@ -0,0 +1,50 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "oprd", + "nsubj", + "dobj", + "nsubjpass", + "pcomp", + "pobj", + "dative", + "appos", + "attr", + "ROOT", + ] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings.add(label) for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/en/tokenizer_exceptions.py b/spacy/lang/en/tokenizer_exceptions.py new file mode 100644 index 0000000..dd3650c --- /dev/null +++ b/spacy/lang/en/tokenizer_exceptions.py @@ -0,0 +1,526 @@ +from typing import Dict, List + +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc: Dict[str, List[Dict]] = {} +_exclude = [ + "Ill", + "ill", + "Its", + "its", + "Hell", + "hell", + "Shell", + "shell", + "Shed", + "shed", + "were", + "Were", + "Well", + "well", + "Whore", + "whore", +] + + +# Pronouns +for pron in ["i"]: + for orth in [pron, pron.title()]: + _exc[orth + "'m"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'m", NORM: "am"}, + ] + + _exc[orth + "m"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "m"}, + ] + + _exc[orth + "'ma"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'m", NORM: "am"}, + {ORTH: "a", NORM: "gonna"}, + ] + + _exc[orth + "ma"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "m", NORM: "am"}, + {ORTH: "a", NORM: "gonna"}, + ] + + +for pron in ["i", "you", "he", "she", "it", "we", "they"]: + for orth in [pron, pron.title()]: + _exc[orth + "'ll"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'ll", NORM: "will"}, + ] + + _exc[orth + "ll"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "ll", NORM: "will"}, + ] + + _exc[orth + "'ll've"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'ll", NORM: "will"}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[orth + "llve"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "ll", NORM: "will"}, + {ORTH: "ve", NORM: "have"}, + ] + + _exc[orth + "'d"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'d", NORM: "'d"}, + ] + + _exc[orth + "d"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "d", NORM: "'d"}, + ] + + _exc[orth + "'d've"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'d", NORM: "would"}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[orth + "dve"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "d", NORM: "would"}, + {ORTH: "ve", NORM: "have"}, + ] + + +for pron in ["i", "you", "we", "they"]: + for orth in [pron, pron.title()]: + _exc[orth + "'ve"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[orth + "ve"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "ve", NORM: "have"}, + ] + + +for pron in ["you", "we", "they"]: + for orth in [pron, pron.title()]: + _exc[orth + "'re"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'re", NORM: "are"}, + ] + + _exc[orth + "re"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "re", NORM: "are"}, + ] + + +for pron in ["he", "she", "it"]: + for orth in [pron, pron.title()]: + _exc[orth + "'s"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "'s", NORM: "'s"}, + ] + + _exc[orth + "s"] = [ + {ORTH: orth, NORM: pron}, + {ORTH: "s"}, + ] + + +# W-words, relative pronouns, prepositions etc. + +for word, morph in [ + ("who", None), + ("what", None), + ("when", None), + ("where", None), + ("why", None), + ("how", None), + ("there", None), + ("that", "Number=Sing|Person=3"), + ("this", "Number=Sing|Person=3"), + ("these", "Number=Plur|Person=3"), + ("those", "Number=Plur|Person=3"), +]: + for orth in [word, word.title()]: + if morph != "Number=Plur|Person=3": + _exc[orth + "'s"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'s", NORM: "'s"}, + ] + + _exc[orth + "s"] = [{ORTH: orth, NORM: word}, {ORTH: "s"}] + + _exc[orth + "'ll"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'ll", NORM: "will"}, + ] + + _exc[orth + "ll"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "ll", NORM: "will"}, + ] + + _exc[orth + "'ll've"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'ll", NORM: "will"}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[orth + "llve"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "ll", NORM: "will"}, + {ORTH: "ve", NORM: "have"}, + ] + + if morph != "Number=Sing|Person=3": + _exc[orth + "'re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'re", NORM: "are"}, + ] + + _exc[orth + "re"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "re", NORM: "are"}, + ] + + _exc[orth + "'ve"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'ve"}, + ] + + _exc[orth + "ve"] = [ + {ORTH: orth}, + {ORTH: "ve", NORM: "have"}, + ] + + _exc[orth + "'d"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'d", NORM: "'d"}, + ] + + _exc[orth + "d"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "d", NORM: "'d"}, + ] + + _exc[orth + "'d've"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "'d", NORM: "would"}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[orth + "dve"] = [ + {ORTH: orth, NORM: word}, + {ORTH: "d", NORM: "would"}, + {ORTH: "ve", NORM: "have"}, + ] + + +# Verbs + +for verb_data in [ + {ORTH: "ca", NORM: "can"}, + {ORTH: "could", NORM: "could"}, + {ORTH: "do", NORM: "do"}, + {ORTH: "does", NORM: "does"}, + {ORTH: "did", NORM: "do"}, + {ORTH: "had", NORM: "have"}, + {ORTH: "may", NORM: "may"}, + {ORTH: "might", NORM: "might"}, + {ORTH: "must", NORM: "must"}, + {ORTH: "need", NORM: "need"}, + {ORTH: "ought", NORM: "ought"}, + {ORTH: "sha", NORM: "shall"}, + {ORTH: "should", NORM: "should"}, + {ORTH: "wo", NORM: "will"}, + {ORTH: "would", NORM: "would"}, +]: + verb_data_tc = dict(verb_data) + verb_data_tc[ORTH] = verb_data_tc[ORTH].title() + for data in [verb_data, verb_data_tc]: + _exc[data[ORTH] + "n't"] = [ + dict(data), + {ORTH: "n't", NORM: "not"}, + ] + + _exc[data[ORTH] + "nt"] = [ + dict(data), + {ORTH: "nt", NORM: "not"}, + ] + + _exc[data[ORTH] + "n't've"] = [ + dict(data), + {ORTH: "n't", NORM: "not"}, + {ORTH: "'ve", NORM: "have"}, + ] + + _exc[data[ORTH] + "ntve"] = [ + dict(data), + {ORTH: "nt", NORM: "not"}, + {ORTH: "ve", NORM: "have"}, + ] + + +for verb_data in [ + {ORTH: "could", NORM: "could"}, + {ORTH: "might", NORM: "might"}, + {ORTH: "must", NORM: "must"}, + {ORTH: "should", NORM: "should"}, + {ORTH: "would", NORM: "would"}, +]: + verb_data_tc = dict(verb_data) + verb_data_tc[ORTH] = verb_data_tc[ORTH].title() + for data in [verb_data, verb_data_tc]: + _exc[data[ORTH] + "'ve"] = [dict(data), {ORTH: "'ve"}] + + _exc[data[ORTH] + "ve"] = [dict(data), {ORTH: "ve"}] + + +for verb_data in [ + {ORTH: "ai", "number": 2}, + {ORTH: "are", NORM: "are", "number": 2}, + {ORTH: "is", NORM: "is"}, + {ORTH: "was", NORM: "was"}, + {ORTH: "were", NORM: "were"}, + {ORTH: "have", NORM: "have"}, + {ORTH: "has", NORM: "has"}, + {ORTH: "dare", NORM: "dare"}, +]: + verb_data_tc = dict(verb_data) # type: ignore[call-overload] + verb_data_tc[ORTH] = verb_data_tc[ORTH].title() + for data in [verb_data, verb_data_tc]: # type: ignore[assignment] + _exc[data[ORTH] + "n't"] = [ + dict(data), + {ORTH: "n't", NORM: "not"}, + ] + + _exc[data[ORTH] + "nt"] = [ + dict(data), + {ORTH: "nt", NORM: "not"}, + ] + + +# Other contractions with trailing apostrophe + +for exc_data in [ + {ORTH: "doin", NORM: "doing"}, + {ORTH: "goin", NORM: "going"}, + {ORTH: "nothin", NORM: "nothing"}, + {ORTH: "nuthin", NORM: "nothing"}, + {ORTH: "ol", NORM: "old"}, + {ORTH: "somethin", NORM: "something"}, +]: + exc_data_tc = dict(exc_data) + exc_data_tc[ORTH] = exc_data_tc[ORTH].title() + for data in [exc_data, exc_data_tc]: + data_apos = dict(data) + data_apos[ORTH] = data_apos[ORTH] + "'" + _exc[data[ORTH]] = [dict(data)] + _exc[data_apos[ORTH]] = [dict(data_apos)] + + +# Other contractions with leading apostrophe + +for exc_data in [ + {ORTH: "em", NORM: "them"}, + {ORTH: "ll", NORM: "will"}, + {ORTH: "nuff", NORM: "enough"}, +]: + exc_data_apos = dict(exc_data) + exc_data_apos[ORTH] = "'" + exc_data_apos[ORTH] + for data in [exc_data, exc_data_apos]: + _exc[data[ORTH]] = [data] + + +# Times + +for h in range(1, 12 + 1): + for period in ["a.m.", "am"]: + _exc[f"{h}{period}"] = [ + {ORTH: f"{h}"}, + {ORTH: period, NORM: "a.m."}, + ] + for period in ["p.m.", "pm"]: + _exc[f"{h}{period}"] = [ + {ORTH: f"{h}"}, + {ORTH: period, NORM: "p.m."}, + ] + + +# Rest + +_other_exc = { + "y'all": [{ORTH: "y'", NORM: "you"}, {ORTH: "all"}], + "yall": [{ORTH: "y", NORM: "you"}, {ORTH: "all"}], + "how'd'y": [{ORTH: "how"}, {ORTH: "'d"}, {ORTH: "'y", NORM: "you"}], + "How'd'y": [{ORTH: "How", NORM: "how"}, {ORTH: "'d"}, {ORTH: "'y", NORM: "you"}], + "not've": [{ORTH: "not"}, {ORTH: "'ve", NORM: "have"}], + "notve": [{ORTH: "not"}, {ORTH: "ve", NORM: "have"}], + "Not've": [{ORTH: "Not", NORM: "not"}, {ORTH: "'ve", NORM: "have"}], + "Notve": [{ORTH: "Not", NORM: "not"}, {ORTH: "ve", NORM: "have"}], + "cannot": [{ORTH: "can"}, {ORTH: "not"}], + "Cannot": [{ORTH: "Can", NORM: "can"}, {ORTH: "not"}], + "gonna": [{ORTH: "gon", NORM: "going"}, {ORTH: "na", NORM: "to"}], + "Gonna": [{ORTH: "Gon", NORM: "going"}, {ORTH: "na", NORM: "to"}], + "gotta": [{ORTH: "got"}, {ORTH: "ta", NORM: "to"}], + "Gotta": [{ORTH: "Got", NORM: "got"}, {ORTH: "ta", NORM: "to"}], + "let's": [{ORTH: "let"}, {ORTH: "'s", NORM: "us"}], + "Let's": [{ORTH: "Let", NORM: "let"}, {ORTH: "'s", NORM: "us"}], + "c'mon": [{ORTH: "c'm", NORM: "come"}, {ORTH: "on"}], + "C'mon": [{ORTH: "C'm", NORM: "come"}, {ORTH: "on"}], +} + +_exc.update(_other_exc) + + +for exc_data in [ + {ORTH: "'S", NORM: "'s"}, + {ORTH: "'s", NORM: "'s"}, + {ORTH: "\u2018S", NORM: "'s"}, + {ORTH: "\u2018s", NORM: "'s"}, + {ORTH: "and/or", NORM: "and/or"}, + {ORTH: "w/o", NORM: "without"}, + {ORTH: "'re", NORM: "are"}, + {ORTH: "'Cause", NORM: "because"}, + {ORTH: "'cause", NORM: "because"}, + {ORTH: "'cos", NORM: "because"}, + {ORTH: "'Cos", NORM: "because"}, + {ORTH: "'coz", NORM: "because"}, + {ORTH: "'Coz", NORM: "because"}, + {ORTH: "'cuz", NORM: "because"}, + {ORTH: "'Cuz", NORM: "because"}, + {ORTH: "'bout", NORM: "about"}, + {ORTH: "ma'am", NORM: "madam"}, + {ORTH: "Ma'am", NORM: "madam"}, + {ORTH: "o'clock", NORM: "o'clock"}, + {ORTH: "O'clock", NORM: "o'clock"}, + {ORTH: "lovin'", NORM: "loving"}, + {ORTH: "Lovin'", NORM: "loving"}, + {ORTH: "lovin", NORM: "loving"}, + {ORTH: "Lovin", NORM: "loving"}, + {ORTH: "havin'", NORM: "having"}, + {ORTH: "Havin'", NORM: "having"}, + {ORTH: "havin", NORM: "having"}, + {ORTH: "Havin", NORM: "having"}, + {ORTH: "doin'", NORM: "doing"}, + {ORTH: "Doin'", NORM: "doing"}, + {ORTH: "doin", NORM: "doing"}, + {ORTH: "Doin", NORM: "doing"}, + {ORTH: "goin'", NORM: "going"}, + {ORTH: "Goin'", NORM: "going"}, + {ORTH: "goin", NORM: "going"}, + {ORTH: "Goin", NORM: "going"}, + {ORTH: "Mt.", NORM: "Mount"}, + {ORTH: "Ak.", NORM: "Alaska"}, + {ORTH: "Ala.", NORM: "Alabama"}, + {ORTH: "Apr.", NORM: "April"}, + {ORTH: "Ariz.", NORM: "Arizona"}, + {ORTH: "Ark.", NORM: "Arkansas"}, + {ORTH: "Aug.", NORM: "August"}, + {ORTH: "Calif.", NORM: "California"}, + {ORTH: "Colo.", NORM: "Colorado"}, + {ORTH: "Conn.", NORM: "Connecticut"}, + {ORTH: "Dec.", NORM: "December"}, + {ORTH: "Del.", NORM: "Delaware"}, + {ORTH: "Feb.", NORM: "February"}, + {ORTH: "Fla.", NORM: "Florida"}, + {ORTH: "Ga.", NORM: "Georgia"}, + {ORTH: "Ia.", NORM: "Iowa"}, + {ORTH: "Id.", NORM: "Idaho"}, + {ORTH: "Ill.", NORM: "Illinois"}, + {ORTH: "Ind.", NORM: "Indiana"}, + {ORTH: "Jan.", NORM: "January"}, + {ORTH: "Jul.", NORM: "July"}, + {ORTH: "Jun.", NORM: "June"}, + {ORTH: "Kan.", NORM: "Kansas"}, + {ORTH: "Kans.", NORM: "Kansas"}, + {ORTH: "Ky.", NORM: "Kentucky"}, + {ORTH: "La.", NORM: "Louisiana"}, + {ORTH: "Mar.", NORM: "March"}, + {ORTH: "Mass.", NORM: "Massachusetts"}, + {ORTH: "Mich.", NORM: "Michigan"}, + {ORTH: "Minn.", NORM: "Minnesota"}, + {ORTH: "Miss.", NORM: "Mississippi"}, + {ORTH: "N.C.", NORM: "North Carolina"}, + {ORTH: "N.D.", NORM: "North Dakota"}, + {ORTH: "N.H.", NORM: "New Hampshire"}, + {ORTH: "N.J.", NORM: "New Jersey"}, + {ORTH: "N.M.", NORM: "New Mexico"}, + {ORTH: "N.Y.", NORM: "New York"}, + {ORTH: "Neb.", NORM: "Nebraska"}, + {ORTH: "Nebr.", NORM: "Nebraska"}, + {ORTH: "Nev.", NORM: "Nevada"}, + {ORTH: "Nov.", NORM: "November"}, + {ORTH: "Oct.", NORM: "October"}, + {ORTH: "Okla.", NORM: "Oklahoma"}, + {ORTH: "Ore.", NORM: "Oregon"}, + {ORTH: "Pa.", NORM: "Pennsylvania"}, + {ORTH: "S.C.", NORM: "South Carolina"}, + {ORTH: "Sep.", NORM: "September"}, + {ORTH: "Sept.", NORM: "September"}, + {ORTH: "Tenn.", NORM: "Tennessee"}, + {ORTH: "Va.", NORM: "Virginia"}, + {ORTH: "Wash.", NORM: "Washington"}, + {ORTH: "Wis.", NORM: "Wisconsin"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "'d", + "a.m.", + "Adm.", + "Bros.", + "co.", + "Co.", + "Corp.", + "D.C.", + "Dr.", + "e.g.", + "E.g.", + "E.G.", + "Gen.", + "Gov.", + "i.e.", + "I.e.", + "I.E.", + "Inc.", + "Jr.", + "Ltd.", + "Md.", + "Messrs.", + "Mo.", + "Mont.", + "Mr.", + "Mrs.", + "Ms.", + "p.m.", + "Ph.D.", + "Prof.", + "Rep.", + "Rev.", + "Sen.", + "St.", + "vs.", + "v.s.", +]: + _exc[orth] = [{ORTH: orth}] + + +for string in _exclude: + if string in _exc: + _exc.pop(string) + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/es/__init__.py b/spacy/lang/es/__init__.py new file mode 100644 index 0000000..bcaed86 --- /dev/null +++ b/spacy/lang/es/__init__.py @@ -0,0 +1,52 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import SpanishLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class SpanishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class Spanish(Language): + lang = "es" + Defaults = SpanishDefaults + + +@Spanish.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return SpanishLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Spanish"] diff --git a/spacy/lang/es/examples.py b/spacy/lang/es/examples.py new file mode 100644 index 0000000..653a38b --- /dev/null +++ b/spacy/lang/es/examples.py @@ -0,0 +1,21 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.es.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple está buscando comprar una startup del Reino Unido por mil millones de dólares.", + "Los coches autónomos delegan la responsabilidad del seguro en sus fabricantes.", + "San Francisco analiza prohibir los robots de reparto.", + "Londres es una gran ciudad del Reino Unido.", + "El gato come pescado.", + "Veo al hombre con el telescopio.", + "La araña come moscas.", + "El pingüino incuba en su nido sobre el hielo.", + "¿Dónde estáis?", + "¿Quién es el presidente francés?", + "¿Dónde se encuentra la capital de Argentina?", + "¿Cuándo nació José de San Martín?", +] diff --git a/spacy/lang/es/lemmatizer.py b/spacy/lang/es/lemmatizer.py new file mode 100644 index 0000000..3102f3b --- /dev/null +++ b/spacy/lang/es/lemmatizer.py @@ -0,0 +1,431 @@ +import re +from typing import List, Optional, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class SpanishLemmatizer(Lemmatizer): + """ + Spanish rule-based lemmatizer with morph-based rule selection. + """ + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "rule": + required = ["lemma_rules", "lemma_rules_groups", "lemma_index", "lemma_exc"] + return (required, []) + else: + return super().get_lookups_config(mode) + + def rule_lemmatize(self, token: Token) -> List[str]: + cache_key = (token.orth, token.pos, str(token.morph)) + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + pos = token.pos_.lower() + features = set(token.morph) + if pos in ("", "eol", "space"): + return [string.lower()] + if pos in ( + "adp", + "cconj", + "intj", + "part", + "propn", + "punct", + "sconj", + "sym", + "x", + ): + if token.is_sent_start and pos != "propn": + return [string.lower()] + else: + return [string] + + string = string.lower() + exc = self.lookups.get_table("lemma_exc").get(pos, {}).get(string) + if exc is not None: + lemmas = list(exc) + else: + if pos == "aux": + rule_pos = "verb" + else: + rule_pos = pos + rule = self.select_rule(rule_pos, list(features)) + index = self.lookups.get_table("lemma_index").get(rule_pos, []) + lemmas = getattr(self, "lemmatize_" + rule_pos)( + string, features, rule, index + ) + # Remove duplicates but preserve the ordering + lemmas = list(dict.fromkeys(lemmas)) + + self.cache[cache_key] = lemmas + return lemmas + + def select_rule(self, pos: str, features: List[str]) -> Optional[str]: + groups = self.lookups.get_table("lemma_rules_groups") + if pos in groups: + for group in groups[pos]: + if set(group[1]).issubset(features): + return group[0] + return None + + def lemmatize_adj( + self, word: str, features: List[str], rule: str, index: List[str] + ) -> List[str]: + """ + Lemmatize an adjective. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # Initialize empty lists for the generated lemmas + possible_lemmas = [] + selected_lemmas = [] + + # Apply lemmatization rules + for old, new in self.lookups.get_table("lemma_rules").get(rule, []): + possible_lemma = re.sub(old + "$", new, word) + if possible_lemma != word: + possible_lemmas.append(possible_lemma) + + # Additional rule for plurals that go from esdrújula to grave and end in + # 'n' or 's', e.g., jóvenes -> joven + additional_lemmas = [] + if "Number=Plur" in features: + for possible_lemma in possible_lemmas: + if possible_lemma.endswith("n") or possible_lemma.endswith("s"): + for old, new in self.lookups.get_table("lemma_rules").get( + "accents", [] + ): + additional_lemmas.append(re.sub(old, new, possible_lemma)) + possible_lemmas.extend(additional_lemmas) + + for lemma in possible_lemmas: + if lemma in index: + selected_lemmas.append(lemma) + # If one or more of the created possible lemmas are in the lookup list, + # return all of them + if len(selected_lemmas) > 0: + return selected_lemmas + elif len(possible_lemmas) > 0: + return possible_lemmas + else: + return [word] + + def lemmatize_adv( + self, word: str, features: List[str], rule: str, index: List[str] + ) -> List[str]: + """ + Lemmatize an adverb. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # Apply lemmatization rules + for old, new in self.lookups.get_table("lemma_rules").get("adverbs", []): + if word == old: + return [new] + + # If none of the rules applies, return the original word + return [word] + + def lemmatize_det( + self, word: str, features: List[str], rule: str, index: List[str] + ) -> List[str]: + """ + Lemmatize a determiner. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # Initialize empty lists for the generated lemmas + possible_lemmas = [] + selected_lemmas = [] + + # First, search in rules specific to determiners + for old, new in self.lookups.get_table("lemma_rules").get("det", []): + if word == old: + return [new] + # If none of the specific rules apply, search in the common rules for + # determiners and pronouns that follow a unique pattern for + # lemmatization. If the word is in the list, return the corresponding + # lemma. + for old, new in self.lookups.get_table("lemma_rules").get( + "det_and_pron_fixed", [] + ): + if word == old: + return [new] + # If the word is not in the list of unique determiners and pronouns, + # apply general rules of lemmatization. Include the original word in the # list of possible lemmas. + for old, new in self.lookups.get_table("lemma_rules").get( + "det_and_pron_general", [] + ): + possible_lemma = re.sub(old + "$", new, word) + possible_lemmas.append(possible_lemma) + possible_lemmas.append(word) + + if len(possible_lemmas) == 1: + return possible_lemmas + elif len(possible_lemmas) > 1: + for lemma in possible_lemmas: + if lemma in index: + selected_lemmas.append(lemma) + if len(selected_lemmas) >= 1: + return selected_lemmas + else: + return possible_lemmas + else: + return [] + + def lemmatize_noun( + self, word: str, features: List[str], rule: str, index: List[str] + ) -> List[str]: + """ + Lemmatize a noun. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # Initialize empty lists for the generated lemmas + possible_lemmas = [] + selected_lemmas = [] + + # Apply lemmatization rules + for old, new in self.lookups.get_table("lemma_rules").get(rule, []): + possible_lemma = re.sub(old + "$", new, word) + if possible_lemma != word: + possible_lemmas.append(possible_lemma) + + # Additional rule for plurals that go from esdrújula to grave and end in + # 'n' or 's', e.g., órdenes -> orden, exámenes -> examen + additional_lemmas = [] + if "Number=Plur" in features: + for possible_lemma in possible_lemmas: + if possible_lemma.endswith("n") or possible_lemma.endswith("s"): + for old, new in self.lookups.get_table("lemma_rules").get( + "accents", [] + ): + additional_lemmas.append(re.sub(old, new, possible_lemma)) + possible_lemmas.extend(additional_lemmas) + + for lemma in possible_lemmas: + if lemma in index: + selected_lemmas.append(lemma) + # If one or more of the created possible lemmas are in the lookup list, + # return all of them + if len(selected_lemmas) > 0: + return selected_lemmas + elif len(possible_lemmas) > 0: + return possible_lemmas + else: + return [word] + + def lemmatize_num( + self, word: str, features: List[str], rule: str, index: List[str] + ) -> List[str]: + """ + Lemmatize a numeral. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # If the word is in the list of rules for numerals, return the + # corresponding lemma + for old, new in self.lookups.get_table("lemma_rules").get("num", []): + if word == old: + return [new] + + # Normalize punctuation + splitted_word = word.split(",") + if re.search(r"(\.)([0-9]{3})$", splitted_word[0]): + word = re.sub(r"\.", r"", word) + word = re.sub(r",", r".", word) + return [word] + + def lemmatize_pron( + self, word: str, features: List[str], rule: Optional[str], index: List[str] + ) -> List[str]: + """ + Lemmatize a pronoun. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + + # Initialize empty lists for the generated lemmas + possible_lemmas = [] + selected_lemmas = [] + + # First, search in rules specific to pronouns + for old, new in self.lookups.get_table("lemma_rules").get("pron", []): + if word == old: + return [new] + # If none of the specific rules apply, search in the common rules for + # determiners and pronouns that follow a unique pattern for + # lemmatization. If the word is in the list, return the corresponding + # lemma. + for old, new in self.lookups.get_table("lemma_rules").get( + "det_and_pron_fixed", [] + ): + if word == old: + return [new] + # If the word is not in the list of unique determiners and pronouns, + # apply general rules of lemmatization. Include the original word in the + # list of possible lemmas. + for old, new in self.lookups.get_table("lemma_rules").get( + "det_and_pron_general", [] + ): + possible_lemma = re.sub(old + "$", new, word) + if possible_lemma != word: + possible_lemmas.append(possible_lemma) + possible_lemmas.append(word) + + if len(possible_lemmas) == 1: + return possible_lemmas + elif len(possible_lemmas) > 1: + for lemma in possible_lemmas: + if lemma in index: + selected_lemmas.append(lemma) + if len(selected_lemmas) >= 1: + return selected_lemmas + else: + return possible_lemmas + else: + return [] + + def lemmatize_verb( + self, word: str, features: List[str], rule: Optional[str], index: List[str] + ) -> List[str]: + """ + Lemmatize a verb. + + word (str): The word to lemmatize. + features (List[str]): The morphological features as a list of Feat=Val + pairs. + index (List[str]): The POS-specific lookup list. + + RETURNS (List[str]): The list of lemmas. + """ + # Exceptions for verb+pronoun(s) + if "PronType=Prs" in features: + return self.lemmatize_verb_pron(word, features, rule, index) + + # Initialize empty lists for the generated lemmas + possible_lemmas = [] + selected_lemmas = [] + + # Apply lemmatization rules + rule = str(rule or "") + for old, new in self.lookups.get_table("lemma_rules").get(rule, []): + possible_lemma = re.sub(old + "$", new, word) + if possible_lemma != word: + possible_lemmas.append(possible_lemma) + + for lemma in possible_lemmas: + if lemma in index: + selected_lemmas.append(lemma) + if len(selected_lemmas) == 0: + # If none of the possible lemmas are in the lookup list, + # apply vocalic alternation rules and search in the lookup list + # again + for lemma in possible_lemmas: + for old, new in self.lookups.get_table("lemma_rules").get( + "voc_alt_1", [] + ): + if old in lemma: + for i, char in enumerate(lemma): + if char == old: + voc_alt_lemma = lemma[:i] + new + lemma[i + 1 :] + if voc_alt_lemma in index: + selected_lemmas.append(voc_alt_lemma) + for old, new in self.lookups.get_table("lemma_rules").get( + "voc_alt_2", [] + ): + if old in lemma: + voc_alt_lemma = lemma.replace(old, new, 1) + if voc_alt_lemma in index: + selected_lemmas.append(voc_alt_lemma) + # Additional rule for verbs that lose the accent mark when lemmatized, + # e.g., amplían -> ampliar + additional_lemmas = [] + for possible_lemma in possible_lemmas: + for old, new in self.lookups.get_table("lemma_rules").get("accents", []): + additional_lemmas.append(re.sub(old, new, possible_lemma)) + possible_lemmas.extend(additional_lemmas) + + # If one or more of the created possible lemmas are in the lookup list, + # return all of them + if len(selected_lemmas) > 0: + return selected_lemmas + elif len(possible_lemmas) > 0: + return possible_lemmas + else: + return [word] + + def lemmatize_verb_pron( + self, word: str, features: List[str], rule: Optional[str], index: List[str] + ) -> List[str]: + # Strip and collect pronouns + pron_patt = "^(.*?)([mts]e|l[aeo]s?|n?os)$" + prons: List[str] = [] + verb = word + m = re.search(pron_patt, verb) + while m is not None and len(prons) <= 3: + verb = re.sub(m.group(2) + "$", "", verb) + prons = [m.group(2)] + prons + m = re.search(pron_patt, verb) + # Strip accents from verb form + for old, new in self.lookups.get_table("lemma_rules").get("accents", []): + verb = re.sub(old, new, verb) + # Lemmatize the verb and pronouns, checking for exceptions + exc = self.lookups.get_table("lemma_exc").get("verb", {}).get(verb) + if exc is not None: + verb_lemma = exc[0] + else: + rule = self.select_rule("verb", features) + verb_lemma = self.lemmatize_verb( + verb, + features - {"PronType=Prs"}, # type: ignore[operator] + rule, + index, # type: ignore[operator] + )[0] + pron_lemmas = [] + for pron in prons: + exc = self.lookups.get_table("lemma_exc").get("pron", {}).get(pron) + if exc is not None: + pron_lemmas.append(exc[0]) + else: + rule = self.select_rule("pron", features) + pron_lemmas.append(self.lemmatize_pron(pron, features, rule, index)[0]) + return [verb_lemma + " " + " ".join(pron_lemmas)] diff --git a/spacy/lang/es/lex_attrs.py b/spacy/lang/es/lex_attrs.py new file mode 100644 index 0000000..4c477ea --- /dev/null +++ b/spacy/lang/es/lex_attrs.py @@ -0,0 +1,103 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "cero", + "uno", + "dos", + "tres", + "cuatro", + "cinco", + "seis", + "siete", + "ocho", + "nueve", + "diez", + "once", + "doce", + "trece", + "catorce", + "quince", + "dieciséis", + "diecisiete", + "dieciocho", + "diecinueve", + "veinte", + "veintiuno", + "veintidós", + "veintitrés", + "veinticuatro", + "veinticinco", + "veintiséis", + "veintisiete", + "veintiocho", + "veintinueve", + "treinta", + "cuarenta", + "cincuenta", + "sesenta", + "setenta", + "ochenta", + "noventa", + "cien", + "mil", + "millón", + "billón", + "trillón", +] + + +_ordinal_words = [ + "primero", + "segundo", + "tercero", + "cuarto", + "quinto", + "sexto", + "séptimo", + "octavo", + "noveno", + "décimo", + "undécimo", + "duodécimo", + "decimotercero", + "decimocuarto", + "decimoquinto", + "decimosexto", + "decimoséptimo", + "decimoctavo", + "decimonoveno", + "vigésimo", + "trigésimo", + "cuadragésimo", + "quincuagésimo", + "sexagésimo", + "septuagésimo", + "octogésima", + "nonagésima", + "centésima", + "milésima", + "millonésima", + "billonésima", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/es/punctuation.py b/spacy/lang/es/punctuation.py new file mode 100644 index 0000000..3d20518 --- /dev/null +++ b/spacy/lang/es/punctuation.py @@ -0,0 +1,53 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + LIST_UNITS, + PUNCT, + merge_chars, +) + +_list_units = [u for u in LIST_UNITS if u != "%"] +_units = merge_chars(" ".join(_list_units)) +_concat_quotes = CONCAT_QUOTES + "—–" + + +_suffixes = ( + ["—", "–"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=_units), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=_concat_quotes, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=_concat_quotes + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/es/stop_words.py b/spacy/lang/es/stop_words.py new file mode 100644 index 0000000..6d28854 --- /dev/null +++ b/spacy/lang/es/stop_words.py @@ -0,0 +1,80 @@ +STOP_WORDS = set( + """ +a acuerdo adelante ademas además afirmó agregó ahi ahora ahí al algo alguna +algunas alguno algunos algún alli allí alrededor ambos ante anterior antes +apenas aproximadamente aquel aquella aquellas aquello aquellos aqui aquél +aquélla aquéllas aquéllos aquí arriba aseguró asi así atras aun aunque añadió +aún + +bajo bastante bien breve buen buena buenas bueno buenos + +cada casi cierta ciertas cierto ciertos cinco claro comentó como con conmigo +conocer conseguimos conseguir considera consideró consigo consigue consiguen +consigues contigo contra creo cual cuales cualquier cuando cuanta cuantas +cuanto cuantos cuatro cuenta cuál cuáles cuándo cuánta cuántas cuánto cuántos +cómo + +da dado dan dar de debajo debe deben debido decir dejó del delante demasiado +demás dentro deprisa desde despacio despues después detras detrás dia dias dice +dicen dicho dieron diez diferente diferentes dijeron dijo dio doce donde dos +durante día días dónde + +e el ella ellas ello ellos embargo en encima encuentra enfrente enseguida +entonces entre era eramos eran eras eres es esa esas ese eso esos esta estaba +estaban estado estados estais estamos estan estar estará estas este esto estos +estoy estuvo está están excepto existe existen explicó expresó él ésa ésas ése +ésos ésta éstas éste éstos + +fin final fue fuera fueron fui fuimos + +gran grande grandes + +ha haber habia habla hablan habrá había habían hace haceis hacemos hacen hacer +hacerlo haces hacia haciendo hago han hasta hay haya he hecho hemos hicieron +hizo hoy hubo + +igual incluso indicó informo informó ir + +junto + +la lado largo las le les llegó lleva llevar lo los luego + +mal manera manifestó mas mayor me mediante medio mejor mencionó menos menudo mi +mia mias mientras mio mios mis misma mismas mismo mismos modo mucha muchas +mucho muchos muy más mí mía mías mío míos + +nada nadie ni ninguna ningunas ninguno ningunos ningún no nos nosotras nosotros +nuestra nuestras nuestro nuestros nueva nuevas nueve nuevo nuevos nunca + +o ocho once os otra otras otro otros + +para parece parte partir pasada pasado paìs peor pero pesar poca pocas poco +pocos podeis podemos poder podria podriais podriamos podrian podrias podrá +podrán podría podrían poner por porque posible primer primera primero primeros +pronto propia propias propio propios proximo próximo próximos pudo pueda puede +pueden puedo pues + +qeu que quedó queremos quien quienes quiere quiza quizas quizá quizás quién +quiénes qué + +realizado realizar realizó repente respecto + +sabe sabeis sabemos saben saber sabes salvo se sea sean segun segunda segundo +según seis ser sera será serán sería señaló si sido siempre siendo siete sigue +siguiente sin sino sobre sois sola solamente solas solo solos somos son soy su +supuesto sus suya suyas suyo suyos sé sí sólo + +tal tambien también tampoco tan tanto tarde te temprano tendrá tendrán teneis +tenemos tener tenga tengo tenido tenía tercera tercero ti tiene tienen toda +todas todavia todavía todo todos total tras trata través tres tu tus tuvo tuya +tuyas tuyo tuyos tú + +u ultimo un una unas uno unos usa usais usamos usan usar usas uso usted ustedes +última últimas último últimos + +va vais vamos van varias varios vaya veces ver verdad verdadera verdadero vez +vosotras vosotros voy vuestra vuestras vuestro vuestros + +y ya yo +""".split() +) diff --git a/spacy/lang/es/syntax_iterators.py b/spacy/lang/es/syntax_iterators.py new file mode 100644 index 0000000..96df444 --- /dev/null +++ b/spacy/lang/es/syntax_iterators.py @@ -0,0 +1,76 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "nsubj", + "nsubj:pass", + "obj", + "obl", + "nmod", + "pcomp", + "appos", + "ROOT", + ] + post_modifiers = ["flat", "fixed", "compound"] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = {doc.vocab.strings.add(label) for label in labels} + np_modifs = {doc.vocab.strings.add(modifier) for modifier in post_modifiers} + np_label = doc.vocab.strings.add("NP") + adj_label = doc.vocab.strings.add("amod") + adp_label = doc.vocab.strings.add("ADP") + conj = doc.vocab.strings.add("conj") + conj_pos = doc.vocab.strings.add("CCONJ") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + right_childs = list(word.rights) + right_child = right_childs[0] if right_childs else None + + if right_child: + if right_child.dep == adj_label: + right_end = right_child.right_edge + elif right_child.dep in np_modifs: # Check if we can expand to right + right_end = word.right_edge + else: + right_end = word + else: + right_end = word + prev_end = right_end.i + + left_index = word.left_edge.i + left_index = ( + left_index + 1 if word.left_edge.pos == adp_label else left_index + ) # Eliminate left attached de, del + + yield left_index, right_end.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + + left_index = word.left_edge.i # eliminate left attached conjunction + left_index = ( + left_index + 1 if word.left_edge.pos == conj_pos else left_index + ) + yield left_index, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/es/tokenizer_exceptions.py b/spacy/lang/es/tokenizer_exceptions.py new file mode 100644 index 0000000..2ea0ed8 --- /dev/null +++ b/spacy/lang/es/tokenizer_exceptions.py @@ -0,0 +1,82 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = { + "pal": [{ORTH: "pa"}, {ORTH: "l", NORM: "el"}], +} + + +for exc_data in [ + {ORTH: "n°"}, + {ORTH: "°C"}, + {ORTH: "aprox."}, + {ORTH: "dna."}, + {ORTH: "dpto."}, + {ORTH: "ej."}, + {ORTH: "esq."}, + {ORTH: "pág."}, + {ORTH: "p.ej."}, + {ORTH: "Ud.", NORM: "usted"}, + {ORTH: "Vd.", NORM: "usted"}, + {ORTH: "Uds.", NORM: "ustedes"}, + {ORTH: "Vds.", NORM: "ustedes"}, + {ORTH: "vol.", NORM: "volúmen"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Times + +_exc["12m."] = [{ORTH: "12"}, {ORTH: "m."}] + + +for h in range(1, 12 + 1): + for period in ["a.m.", "am"]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}"}, {ORTH: period}] + for period in ["p.m.", "pm"]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}"}, {ORTH: period}] + + +for orth in [ + "a.C.", + "a.J.C.", + "d.C.", + "d.J.C.", + "apdo.", + "Av.", + "Avda.", + "Cía.", + "Dr.", + "Dra.", + "EE.UU.", + "Ee.Uu.", + "EE. UU.", + "Ee. Uu.", + "etc.", + "fig.", + "Gob.", + "Gral.", + "Ing.", + "J.C.", + "km/h", + "Lic.", + "m.n.", + "núm.", + "P.D.", + "Prof.", + "Profa.", + "q.e.p.d.", + "Q.E.P.D.", + "S.A.", + "S.L.", + "S.R.L.", + "s.s.s.", + "Sr.", + "Sra.", + "Srta.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/et/__init__.py b/spacy/lang/et/__init__.py new file mode 100644 index 0000000..9ec7e60 --- /dev/null +++ b/spacy/lang/et/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class EstonianDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Estonian(Language): + lang = "et" + Defaults = EstonianDefaults + + +__all__ = ["Estonian"] diff --git a/spacy/lang/et/stop_words.py b/spacy/lang/et/stop_words.py new file mode 100644 index 0000000..e1da1f1 --- /dev/null +++ b/spacy/lang/et/stop_words.py @@ -0,0 +1,41 @@ +# Source: https://github.com/stopwords-iso/stopwords-et + +STOP_WORDS = set( + """ +aga +ei +et +ja +jah +kas +kui +kõik +ma +me +mida +midagi +mind +minu +mis +mu +mul +mulle +nad +nii +oled +olen +oli +oma +on +pole +sa +seda +see +selle +siin +siis +ta +te +ära +""".split() +) diff --git a/spacy/lang/eu/__init__.py b/spacy/lang/eu/__init__.py new file mode 100644 index 0000000..81f9c4a --- /dev/null +++ b/spacy/lang/eu/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS + + +class BasqueDefaults(BaseDefaults): + suffixes = TOKENIZER_SUFFIXES + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Basque(Language): + lang = "eu" + Defaults = BasqueDefaults + + +__all__ = ["Basque"] diff --git a/spacy/lang/eu/examples.py b/spacy/lang/eu/examples.py new file mode 100644 index 0000000..3b9ef71 --- /dev/null +++ b/spacy/lang/eu/examples.py @@ -0,0 +1,11 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.eu.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "bilbon ko castinga egin da eta nik jakin ez zuetako inork egin al du edota parte hartu duen ezagunik ba al du", + "gaur telebistan entzunda denok martetik gatoz hortaz martzianoak gara beno nire ustez batzuk beste batzuk baino martzianoagoak dira", +] diff --git a/spacy/lang/eu/lex_attrs.py b/spacy/lang/eu/lex_attrs.py new file mode 100644 index 0000000..a3ab018 --- /dev/null +++ b/spacy/lang/eu/lex_attrs.py @@ -0,0 +1,76 @@ +from ...attrs import LIKE_NUM + +# Source http://mylanguages.org/basque_numbers.php + + +_num_words = """ +bat +bi +hiru +lau +bost +sei +zazpi +zortzi +bederatzi +hamar +hamaika +hamabi +hamahiru +hamalau +hamabost +hamasei +hamazazpi +Hemezortzi +hemeretzi +hogei +ehun +mila +milioi +""".split() + +# source https://www.google.com/intl/ur/inputtools/try/ + +_ordinal_words = """ +lehen +bigarren +hirugarren +laugarren +bosgarren +seigarren +zazpigarren +zortzigarren +bederatzigarren +hamargarren +hamaikagarren +hamabigarren +hamahirugarren +hamalaugarren +hamabosgarren +hamaseigarren +hamazazpigarren +hamazortzigarren +hemeretzigarren +hogeigarren +behin +""".split() + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/eu/punctuation.py b/spacy/lang/eu/punctuation.py new file mode 100644 index 0000000..382bfc7 --- /dev/null +++ b/spacy/lang/eu/punctuation.py @@ -0,0 +1,3 @@ +from ..punctuation import TOKENIZER_SUFFIXES + +_suffixes = TOKENIZER_SUFFIXES diff --git a/spacy/lang/eu/stop_words.py b/spacy/lang/eu/stop_words.py new file mode 100644 index 0000000..d213b5b --- /dev/null +++ b/spacy/lang/eu/stop_words.py @@ -0,0 +1,105 @@ +# Source: https://github.com/stopwords-iso/stopwords-eu +# https://www.ranks.nl/stopwords/basque +# https://www.mustgo.com/worldlanguages/basque/ +STOP_WORDS = set( + """ +al +anitz +arabera +asko +baina +bat +batean +batek +bati +batzuei +batzuek +batzuetan +batzuk +bera +beraiek +berau +berauek +bere +berori +beroriek +beste +bezala +da +dago +dira +ditu +du +dute +edo +egin +ere +eta +eurak +ez +gainera +gu +gutxi +guzti +haiei +haiek +haietan +hainbeste +hala +han +handik +hango +hara +hari +hark +hartan +hau +hauei +hauek +hauetan +hemen +hemendik +hemengo +hi +hona +honek +honela +honetan +honi +hor +hori +horiei +horiek +horietan +horko +horra +horrek +horrela +horretan +horri +hortik +hura +izan +ni +noiz +nola +non +nondik +nongo +nor +nora +ze +zein +zen +zenbait +zenbat +zer +zergatik +ziren +zituen +zu +zuek +zuen +zuten +""".split() +) diff --git a/spacy/lang/fa/__init__.py b/spacy/lang/fa/__init__.py new file mode 100644 index 0000000..e5baa8b --- /dev/null +++ b/spacy/lang/fa/__init__.py @@ -0,0 +1,52 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ...pipeline import Lemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class PersianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} + + +class Persian(Language): + lang = "fa" + Defaults = PersianDefaults + + +@Persian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return Lemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Persian"] diff --git a/spacy/lang/fa/examples.py b/spacy/lang/fa/examples.py new file mode 100644 index 0000000..6810e48 --- /dev/null +++ b/spacy/lang/fa/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.fa.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "این یک جمله نمونه می باشد.", + "قرار ما، امروز ساعت ۲:۳۰ بعدازظهر هست!", + "دیروز علی به من ۲۰۰۰.۱﷼ پول نقد داد.", + "چطور می‌توان از تهران به کاشان رفت؟", + "حدود ۸۰٪ هوا از نیتروژن تشکیل شده است.", +] diff --git a/spacy/lang/fa/generate_verbs_exc.py b/spacy/lang/fa/generate_verbs_exc.py new file mode 100644 index 0000000..7ef82c3 --- /dev/null +++ b/spacy/lang/fa/generate_verbs_exc.py @@ -0,0 +1,651 @@ +verb_roots = """ +#هست +آخت#آهنج +آراست#آرا +آراماند#آرامان +آرامید#آرام +آرمید#آرام +آزرد#آزار +آزمود#آزما +آسود#آسا +آشامید#آشام +آشفت#آشوب +آشوبید#آشوب +آغازید#آغاز +آغشت#آمیز +آفرید#آفرین +آلود#آلا +آمد#آ +آمرزید#آمرز +آموخت#آموز +آموزاند#آموزان +آمیخت#آمیز +آورد#آر +آورد#آور +آویخت#آویز +آکند#آکن +آگاهانید#آگاهان +ارزید#ارز +افتاد#افت +افراخت#افراز +افراشت#افراز +افروخت#افروز +افروزید#افروز +افزود#افزا +افسرد#افسر +افشاند#افشان +افکند#افکن +افگند#افگن +انباشت#انبار +انجامید#انجام +انداخت#انداز +اندوخت#اندوز +اندود#اندا +اندیشید#اندیش +انگاشت#انگار +انگیخت#انگیز +انگیزاند#انگیزان +ایستاد#ایست +ایستاند#ایستان +باخت#باز +باراند#باران +بارگذاشت#بارگذار +بارید#بار +باز#بازخواه +بازآفرید#بازآفرین +بازآمد#بازآ +بازآموخت#بازآموز +بازآورد#بازآور +بازایستاد#بازایست +بازتابید#بازتاب +بازجست#بازجو +بازخواند#بازخوان +بازخوراند#بازخوران +بازداد#بازده +بازداشت#بازدار +بازرساند#بازرسان +بازرسانید#بازرسان +باززد#باززن +بازستاند#بازستان +بازشمارد#بازشمار +بازشمرد#بازشمار +بازشمرد#بازشمر +بازشناخت#بازشناس +بازشناساند#بازشناسان +بازفرستاد#بازفرست +بازماند#بازمان +بازنشست#بازنشین +بازنمایاند#بازنمایان +بازنهاد#بازنه +بازنگریست#بازنگر +بازپرسید#بازپرس +بازگذارد#بازگذار +بازگذاشت#بازگذار +بازگرداند#بازگردان +بازگردانید#بازگردان +بازگردید#بازگرد +بازگرفت#بازگیر +بازگشت#بازگرد +بازگشود#بازگشا +بازگفت#بازگو +بازیافت#بازیاب +بافت#باف +بالید#بال +باوراند#باوران +بایست#باید +بخشود#بخش +بخشود#بخشا +بخشید#بخش +بر#برخواه +برآشفت#برآشوب +برآمد#برآ +برآورد#برآور +برازید#براز +برافتاد#برافت +برافراخت#برافراز +برافراشت#برافراز +برافروخت#برافروز +برافشاند#برافشان +برافکند#برافکن +براند#بران +برانداخت#برانداز +برانگیخت#برانگیز +بربست#بربند +برتاباند#برتابان +برتابید#برتاب +برتافت#برتاب +برتنید#برتن +برجهید#برجه +برخاست#برخیز +برخورد#برخور +برد#بر +برداشت#بردار +بردمید#بردم +برزد#برزن +برشد#برشو +برشمارد#برشمار +برشمرد#برشمار +برشمرد#برشمر +برنشاند#برنشان +برنشانید#برنشان +برنشست#برنشین +برنهاد#برنه +برچید#برچین +برکرد#برکن +برکشید#برکش +برکند#برکن +برگذشت#برگذر +برگرداند#برگردان +برگردانید#برگردان +برگردید#برگرد +برگرفت#برگیر +برگزید#برگزین +برگشت#برگرد +برگشود#برگشا +برگمارد#برگمار +برگمارید#برگمار +برگماشت#برگمار +برید#بر +بست#بند +بلعید#بلع +بود#باش +بوسید#بوس +بویید#بو +بیخت#بیز +بیخت#بوز +تاباند#تابان +تابید#تاب +تاخت#تاز +تاراند#تاران +تازاند#تازان +تازید#تاز +تافت#تاب +ترادیسید#ترادیس +تراشاند#تراشان +تراشید#تراش +تراوید#تراو +ترساند#ترسان +ترسید#ترس +ترشاند#ترشان +ترشید#ترش +ترکاند#ترکان +ترکید#ترک +تفتید#تفت +تمرگید#تمرگ +تنید#تن +توانست#توان +توفید#توف +تپاند#تپان +تپید#تپ +تکاند#تکان +تکانید#تکان +جست#جه +جست#جو +جنباند#جنبان +جنبید#جنب +جنگید#جنگ +جهاند#جهان +جهید#جه +جوشاند#جوشان +جوشانید#جوشان +جوشید#جوش +جويد#جو +جوید#جو +خاراند#خاران +خارید#خار +خاست#خیز +خایید#خا +خراشاند#خراشان +خراشید#خراش +خرامید#خرام +خروشید#خروش +خرید#خر +خزید#خز +خسبید#خسب +خشکاند#خشکان +خشکید#خشک +خفت#خواب +خلید#خل +خماند#خمان +خمید#خم +خنداند#خندان +خندانید#خندان +خندید#خند +خواباند#خوابان +خوابانید#خوابان +خوابید#خواب +خواست#خواه +خواست#خیز +خواند#خوان +خوراند#خوران +خورد#خور +خیزاند#خیزان +خیساند#خیسان +داد#ده +داشت#دار +دانست#دان +در#درخواه +درآمد#درآ +درآمیخت#درآمیز +درآورد#درآور +درآویخت#درآویز +درافتاد#درافت +درافکند#درافکن +درانداخت#درانداز +درانید#دران +دربرد#دربر +دربرگرفت#دربرگیر +درخشاند#درخشان +درخشانید#درخشان +درخشید#درخش +درداد#درده +دررفت#دررو +درماند#درمان +درنمود#درنما +درنوردید#درنورد +درود#درو +دروید#درو +درکرد#درکن +درکشید#درکش +درگذشت#درگذر +درگرفت#درگیر +دریافت#دریاب +درید#در +دزدید#دزد +دمید#دم +دواند#دوان +دوخت#دوز +دوشید#دوش +دوید#دو +دید#بین +راند#ران +ربود#ربا +ربود#روب +رخشید#رخش +رساند#رسان +رسانید#رسان +رست#ره +رست#رو +رسید#رس +رشت#ریس +رفت#رو +رفت#روب +رقصاند#رقصان +رقصید#رقص +رماند#رمان +رمانید#رمان +رمید#رم +رنجاند#رنجان +رنجانید#رنجان +رنجید#رنج +رندید#رند +رهاند#رهان +رهانید#رهان +رهید#ره +روبید#روب +روفت#روب +رویاند#رویان +رویانید#رویان +رویید#رو +رویید#روی +ریخت#ریز +رید#رین +ریدن#رین +ریسید#ریس +زاد#زا +زارید#زار +زایاند#زایان +زایید#زا +زد#زن +زدود#زدا +زیست#زی +ساباند#سابان +سابید#ساب +ساخت#ساز +سایید#سا +ستاد#ستان +ستاند#ستان +سترد#ستر +ستود#ستا +ستیزید#ستیز +سراند#سران +سرایید#سرا +سرشت#سرش +سرود#سرا +سرکشید#سرکش +سرگرفت#سرگیر +سرید#سر +سزید#سز +سفت#سنب +سنجید#سنج +سوخت#سوز +سود#سا +سوزاند#سوزان +سپارد#سپار +سپرد#سپار +سپرد#سپر +سپوخت#سپوز +سگالید#سگال +شاشید#شاش +شایست# +شایست#شاید +شتاباند#شتابان +شتابید#شتاب +شتافت#شتاب +شد#شو +شست#شو +شست#شوی +شلید#شل +شمار#شمر +شمارد#شمار +شمرد#شمار +شمرد#شمر +شناخت#شناس +شناساند#شناسان +شنفت#شنو +شنید#شنو +شوتید#شوت +شوراند#شوران +شورید#شور +شکافت#شکاف +شکاند#شکان +شکاند#شکن +شکست#شکن +شکفت#شکف +طلبید#طلب +طپید#طپ +غراند#غران +غرید#غر +غلتاند#غلتان +غلتانید#غلتان +غلتید#غلت +غلطاند#غلطان +غلطانید#غلطان +غلطید#غلط +فرا#فراخواه +فراخواند#فراخوان +فراداشت#فرادار +فرارسید#فرارس +فرانمود#فرانما +فراگرفت#فراگیر +فرستاد#فرست +فرسود#فرسا +فرمود#فرما +فرهیخت#فرهیز +فرو#فروخواه +فروآمد#فروآ +فروآورد#فروآور +فروافتاد#فروافت +فروافکند#فروافکن +فروبرد#فروبر +فروبست#فروبند +فروخت#فروش +فروخفت#فروخواب +فروخورد#فروخور +فروداد#فروده +فرودوخت#فرودوز +فرورفت#فرورو +فروریخت#فروریز +فروشکست#فروشکن +فروفرستاد#فروفرست +فروماند#فرومان +فرونشاند#فرونشان +فرونشانید#فرونشان +فرونشست#فرونشین +فرونمود#فرونما +فرونهاد#فرونه +فروپاشاند#فروپاشان +فروپاشید#فروپاش +فروچکید#فروچک +فروکرد#فروکن +فروکشید#فروکش +فروکوبید#فروکوب +فروکوفت#فروکوب +فروگذارد#فروگذار +فروگذاشت#فروگذار +فروگرفت#فروگیر +فریفت#فریب +فشاند#فشان +فشرد#فشار +فشرد#فشر +فلسفید#فلسف +فهماند#فهمان +فهمید#فهم +قاپید#قاپ +قبولاند#قبول +قبولاند#قبولان +لاسید#لاس +لرزاند#لرزان +لرزید#لرز +لغزاند#لغزان +لغزید#لغز +لمباند#لمبان +لمید#لم +لنگید#لنگ +لولید#لول +لیسید#لیس +ماسید#ماس +مالاند#مالان +مالید#مال +ماند#مان +مانست#مان +مرد#میر +مویید#مو +مکید#مک +نازید#ناز +نالاند#نالان +نالید#نال +نامید#نام +نشاند#نشان +نشست#نشین +نمایاند#نما +نمایاند#نمایان +نمود#نما +نهاد#نه +نهفت#نهنب +نواخت#نواز +نوازید#نواز +نوردید#نورد +نوشاند#نوشان +نوشانید#نوشان +نوشت#نویس +نوشید#نوش +نکوهید#نکوه +نگاشت#نگار +نگرید# +نگریست#نگر +هراساند#هراسان +هراسانید#هراسان +هراسید#هراس +هشت#هل +وا#واخواه +واداشت#وادار +وارفت#وارو +وارهاند#وارهان +واماند#وامان +وانهاد#وانه +واکرد#واکن +واگذارد#واگذار +واگذاشت#واگذار +ور#ورخواه +ورآمد#ورآ +ورافتاد#ورافت +وررفت#وررو +ورزید#ورز +وزاند#وزان +وزید#وز +ویراست#ویرا +پاشاند#پاشان +پاشید#پاش +پالود#پالا +پایید#پا +پخت#پز +پذیراند#پذیران +پذیرفت#پذیر +پراند#پران +پراکند#پراکن +پرداخت#پرداز +پرستید#پرست +پرسید#پرس +پرهیخت#پرهیز +پرهیزید#پرهیز +پروراند#پروران +پرورد#پرور +پرید#پر +پسندید#پسند +پلاساند#پلاسان +پلاسید#پلاس +پلکید#پلک +پناهاند#پناهان +پناهید#پناه +پنداشت#پندار +پوساند#پوسان +پوسید#پوس +پوشاند#پوشان +پوشید#پوش +پویید#پو +پژمرد#پژمر +پژوهید#پژوه +پکید#پک +پیراست#پیرا +پیمود#پیما +پیوست#پیوند +پیچاند#پیچان +پیچانید#پیچان +پیچید#پیچ +چاپید#چاپ +چایید#چا +چراند#چران +چرانید#چران +چرباند#چربان +چربید#چرب +چرخاند#چرخان +چرخانید#چرخان +چرخید#چرخ +چروکید#چروک +چرید#چر +چزاند#چزان +چسباند#چسبان +چسبید#چسب +چسید#چس +چشاند#چشان +چشید#چش +چلاند#چلان +چلانید#چلان +چپاند#چپان +چپید#چپ +چکاند#چکان +چکید#چک +چید#چین +کاست#کاه +کاشت#کار +کاوید#کاو +کرد#کن +کشاند#کشان +کشانید#کشان +کشت#کار +کشت#کش +کشید#کش +کند#کن +کوباند#کوبان +کوبید#کوب +کوشید#کوش +کوفت#کوب +کوچانید#کوچان +کوچید#کوچ +گایید#گا +گداخت#گداز +گذارد#گذار +گذاشت#گذار +گذراند#گذران +گذشت#گذر +گرازید#گراز +گرانید#گران +گرایید#گرا +گرداند#گردان +گردانید#گردان +گردید#گرد +گرفت#گیر +گروید#گرو +گریاند#گریان +گریخت#گریز +گریزاند#گریزان +گریست#گر +گریست#گری +گزارد#گزار +گزاشت#گزار +گزید#گزین +گسارد#گسار +گستراند#گستران +گسترانید#گستران +گسترد#گستر +گسست#گسل +گسلاند#گسل +گسیخت#گسل +گشاد#گشا +گشت#گرد +گشود#گشا +گفت#گو +گمارد#گمار +گماشت#گمار +گنجاند#گنجان +گنجانید#گنجان +گنجید#گنج +گنداند#گندان +گندید#گند +گوارید#گوار +گوزید#گوز +گیراند#گیران +یازید#یاز +یافت#یاب +یونید#یون +""".strip().split() + + +# Below code is a modified version of HAZM package's verb conjugator, +# with soem extra verbs (Anything in hazm and not in here? compare needed!) +VERBS_EXC = {} +with_nots = lambda items: items + ["ن" + item for item in items] +simple_ends = ["م", "ی", "", "یم", "ید", "ند"] +narrative_ends = ["ه‌ام", "ه‌ای", "ه", "ه‌ایم", "ه‌اید", "ه‌اند"] +present_ends = ["م", "ی", "د", "یم", "ید", "ند"] + +# special case of '#هست': +VERBS_EXC.update(dict.fromkeys(["هست" + end for end in simple_ends], "هست")) +VERBS_EXC.update(dict.fromkeys(["نیست" + end for end in simple_ends], "هست")) + +for verb_root in verb_roots: + conjugations = [] + if "#" not in verb_root: + continue + past, present = verb_root.split("#") + + if past: + past_simples = [past + end for end in simple_ends] + past_imperfects = ["می‌" + item for item in past_simples] + past_narratives = [past + end for end in narrative_ends] + conjugations = with_nots(past_simples + past_imperfects + past_narratives) + if present: + imperatives = ["ب" + present, "ن" + present] + if present.endswith("ا") or present in ("آ", "گو"): + present = present + "ی" + present_simples = [present + end for end in present_ends] + present_imperfects = ["می‌" + present + end for end in present_ends] + present_subjunctives = ["ب" + present + end for end in present_ends] + conjugations += ( + with_nots(present_simples + present_imperfects) + + present_subjunctives + + imperatives + ) + + if past.startswith("آ"): + conjugations = list( + set( + map( + lambda item: item.replace("بآ", "بیا").replace("نآ", "نیا"), + conjugations, + ) + ) + ) + + VERBS_EXC.update(dict.fromkeys(conjugations, (past,) if past else present)) diff --git a/spacy/lang/fa/lex_attrs.py b/spacy/lang/fa/lex_attrs.py new file mode 100644 index 0000000..065e81b --- /dev/null +++ b/spacy/lang/fa/lex_attrs.py @@ -0,0 +1,101 @@ +from ...attrs import LIKE_NUM + +MIM = "م" +ZWNJ_O_MIM = "‌ام" +YE_NUN = "ین" + + +_num_words = set( + """ +صفر +یک +دو +سه +چهار +پنج +شش +شیش +هفت +هشت +نه +ده +یازده +دوازده +سیزده +چهارده +پانزده +پونزده +شانزده +شونزده +هفده +هجده +هیجده +نوزده +بیست +سی +چهل +پنجاه +شصت +هفتاد +هشتاد +نود +صد +یکصد +یک‌صد +دویست +سیصد +چهارصد +پانصد +پونصد +ششصد +شیشصد +هفتصد +هفصد +هشتصد +نهصد +هزار +میلیون +میلیارد +بیلیون +بیلیارد +تریلیون +تریلیارد +کوادریلیون +کادریلیارد +کوینتیلیون +""".split() +) + +_ordinal_words = set( + """ +اول +سوم +سی‌ام""".split() +) + +_ordinal_words.update({num + MIM for num in _num_words}) +_ordinal_words.update({num + ZWNJ_O_MIM for num in _num_words}) +_ordinal_words.update({num + YE_NUN for num in _ordinal_words}) + + +def like_num(text): + """ + check if text resembles a number + """ + text = ( + text.replace(",", "") + .replace(".", "") + .replace("،", "") + .replace("٫", "") + .replace("/", "") + ) + if text.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/fa/punctuation.py b/spacy/lang/fa/punctuation.py new file mode 100644 index 0000000..c1ee570 --- /dev/null +++ b/spacy/lang/fa/punctuation.py @@ -0,0 +1,24 @@ +from ..char_classes import ( + ALPHA_UPPER, + CURRENCY, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + [ + r"(?<=[0-9])\+", + r"(?<=[0-9])%", # 4% -> ["4", "%"] + # Persian is written from Right-To-Left + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/fa/stop_words.py b/spacy/lang/fa/stop_words.py new file mode 100644 index 0000000..f462f2e --- /dev/null +++ b/spacy/lang/fa/stop_words.py @@ -0,0 +1,393 @@ +# Stop words from HAZM package +STOP_WORDS = set( + """ +و +در +به +از +که +این +را +با +است +برای +آن +یک +خود +تا +کرد +بر +هم +نیز +گفت +می‌شود +وی +شد +دارد +ما +اما +یا +شده +باید +هر +آنها +بود +او +دیگر +دو +مورد +می‌کند +شود +کند +وجود +بین +پیش +شده‌است +پس +نظر +اگر +همه +یکی +حال +هستند +من +کنند +نیست +باشد +چه +بی +می +بخش +می‌کنند +همین +افزود +هایی +دارند +راه +همچنین +روی +داد +بیشتر +بسیار +سه +داشت +چند +سوی +تنها +هیچ +میان +اینکه +شدن +بعد +جدید +ولی +حتی +کردن +برخی +کردند +می‌دهد +اول +نه +کرده‌است +نسبت +بیش +شما +چنین +طور +افراد +تمام +درباره +بار +بسیاری +می‌تواند +کرده +چون +ندارد +دوم +بزرگ +طی +حدود +همان +بدون +البته +آنان +می‌گوید +دیگری +خواهد‌شد +کنیم +قابل +یعنی +رشد +می‌توان +وارد +کل +ویژه +قبل +براساس +نیاز +گذاری +هنوز +لازم +سازی +بوده‌است +چرا +می‌شوند +وقتی +گرفت +کم +جای +حالی +تغییر +پیدا +اکنون +تحت +باعث +مدت +فقط +زیادی +تعداد +آیا +بیان +رو +شدند +عدم +کرده‌اند +بودن +نوع +بلکه +جاری +دهد +برابر +مهم +بوده +اخیر +مربوط +امر +زیر +گیری +شاید +خصوص +آقای +اثر +کننده +بودند +فکر +کنار +اولین +سوم +سایر +کنید +ضمن +مانند +باز +می‌گیرد +ممکن +حل +دارای +پی +مثل +می‌رسد +اجرا +دور +منظور +کسی +موجب +طول +امکان +آنچه +تعیین +گفته +شوند +جمع +خیلی +علاوه +گونه +تاکنون +رسید +ساله +گرفته +شده‌اند +علت +چهار +داشته‌باشد +خواهد‌بود +طرف +تهیه +تبدیل +مناسب +زیرا +مشخص +می‌توانند +نزدیک +جریان +روند +بنابراین +می‌دهند +یافت +نخستین +بالا +پنج +ریزی +عالی +چیزی +نخست +بیشتری +ترتیب +شده‌بود +خاص +خوبی +خوب +شروع +فرد +کامل +غیر +می‌رود +دهند +آخرین +دادن +جدی +بهترین +شامل +گیرد +بخشی +باشند +تمامی +بهتر +داده‌است +حد +نبود +کسانی +می‌کرد +داریم +علیه +می‌باشد +دانست +ناشی +داشتند +دهه +می‌شد +ایشان +آنجا +گرفته‌است +دچار +می‌آید +لحاظ +آنکه +داده +بعضی +هستیم +اند +برداری +نباید +می‌کنیم +نشست +سهم +همیشه +آمد +اش +وگو +می‌کنم +حداقل +طبق +جا +خواهد‌کرد +نوعی +چگونه +رفت +هنگام +فوق +روش +ندارند +سعی +بندی +شمار +کلی +کافی +مواجه +همچنان +زیاد +سمت +کوچک +داشته‌است +چیز +پشت +آورد +حالا +روبه +سال‌های +دادند +می‌کردند +عهده +نیمه +جایی +دیگران +سی +بروز +یکدیگر +آمده‌است +جز +کنم +سپس +کنندگان +خودش +همواره +یافته +شان +صرف +نمی‌شود +رسیدن +چهارم +یابد +متر +ساز +داشته +کرده‌بود +باره +نحوه +کردم +تو +شخصی +داشته‌باشند +محسوب +پخش +کمی +متفاوت +سراسر +کاملا +داشتن +نظیر +آمده +گروهی +فردی +ع +همچون +خطر +خویش +کدام +دسته +سبب +عین +آوری +متاسفانه +بیرون +دار +ابتدا +شش +افرادی +می‌گویند +سالهای +درون +نیستند +یافته‌است +پر +خاطرنشان +گاه +جمعی +اغلب +دوباره +می‌یابد +لذا +زاده +گردد +اینجا""".split() +) diff --git a/spacy/lang/fa/syntax_iterators.py b/spacy/lang/fa/syntax_iterators.py new file mode 100644 index 0000000..3052369 --- /dev/null +++ b/spacy/lang/fa/syntax_iterators.py @@ -0,0 +1,51 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "nsubj", + "dobj", + "nsubjpass", + "pcomp", + "pobj", + "dative", + "appos", + "attr", + "ROOT", + ] + doc = doclike.doc # Ensure works on both Doc and Span. + + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + np_deps = [doc.vocab.strings.add(label) for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + yield word.left_edge.i, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/fa/tokenizer_exceptions.py b/spacy/lang/fa/tokenizer_exceptions.py new file mode 100644 index 0000000..3b31b7f --- /dev/null +++ b/spacy/lang/fa/tokenizer_exceptions.py @@ -0,0 +1,746 @@ +from ...symbols import NORM, ORTH + +TOKENIZER_EXCEPTIONS = { + ".ق ": [{ORTH: ".ق "}], + ".م": [{ORTH: ".م"}], + ".هـ": [{ORTH: ".هـ"}], + "ب.م": [{ORTH: "ب.م"}], + "ق.م": [{ORTH: "ق.م"}], + "آبرویت": [{ORTH: "آبروی", NORM: "آبروی"}, {ORTH: "ت", NORM: "ت"}], + "آب‌نباتش": [{ORTH: "آب‌نبات", NORM: "آب‌نبات"}, {ORTH: "ش", NORM: "ش"}], + "آثارش": [{ORTH: "آثار", NORM: "آثار"}, {ORTH: "ش", NORM: "ش"}], + "آخرش": [{ORTH: "آخر", NORM: "آخر"}, {ORTH: "ش", NORM: "ش"}], + "آدمهاست": [{ORTH: "آدمها", NORM: "آدمها"}, {ORTH: "ست", NORM: "ست"}], + "آرزومندیم": [{ORTH: "آرزومند", NORM: "آرزومند"}, {ORTH: "یم", NORM: "یم"}], + "آزادند": [{ORTH: "آزاد", NORM: "آزاد"}, {ORTH: "ند", NORM: "ند"}], + "آسیب‌پذیرند": [{ORTH: "آسیب‌پذیر", NORM: "آسیب‌پذیر"}, {ORTH: "ند", NORM: "ند"}], + "آفریده‌اند": [{ORTH: "آفریده‌", NORM: "آفریده‌"}, {ORTH: "اند", NORM: "اند"}], + "آمدنش": [{ORTH: "آمدن", NORM: "آمدن"}, {ORTH: "ش", NORM: "ش"}], + "آمریکاست": [{ORTH: "آمریکا", NORM: "آمریکا"}, {ORTH: "ست", NORM: "ست"}], + "آنجاست": [{ORTH: "آنجا", NORM: "آنجا"}, {ORTH: "ست", NORM: "ست"}], + "آنست": [{ORTH: "آن", NORM: "آن"}, {ORTH: "ست", NORM: "ست"}], + "آنند": [{ORTH: "آن", NORM: "آن"}, {ORTH: "ند", NORM: "ند"}], + "آن‌هاست": [{ORTH: "آن‌ها", NORM: "آن‌ها"}, {ORTH: "ست", NORM: "ست"}], + "آپاداناست": [{ORTH: "آپادانا", NORM: "آپادانا"}, {ORTH: "ست", NORM: "ست"}], + "اجتماعی‌مان": [{ORTH: "اجتماعی‌", NORM: "اجتماعی‌"}, {ORTH: "مان", NORM: "مان"}], + "اجدادت": [{ORTH: "اجداد", NORM: "اجداد"}, {ORTH: "ت", NORM: "ت"}], + "اجدادش": [{ORTH: "اجداد", NORM: "اجداد"}, {ORTH: "ش", NORM: "ش"}], + "اجدادی‌شان": [{ORTH: "اجدادی‌", NORM: "اجدادی‌"}, {ORTH: "شان", NORM: "شان"}], + "اجراست": [{ORTH: "اجرا", NORM: "اجرا"}, {ORTH: "ست", NORM: "ست"}], + "اختیارش": [{ORTH: "اختیار", NORM: "اختیار"}, {ORTH: "ش", NORM: "ش"}], + "اخلاقشان": [{ORTH: "اخلاق", NORM: "اخلاق"}, {ORTH: "شان", NORM: "شان"}], + "ادعایمان": [{ORTH: "ادعای", NORM: "ادعای"}, {ORTH: "مان", NORM: "مان"}], + "اذیتش": [{ORTH: "اذیت", NORM: "اذیت"}, {ORTH: "ش", NORM: "ش"}], + "اراده‌اش": [{ORTH: "اراده‌", NORM: "اراده‌"}, {ORTH: "اش", NORM: "اش"}], + "ارتباطش": [{ORTH: "ارتباط", NORM: "ارتباط"}, {ORTH: "ش", NORM: "ش"}], + "ارتباطمان": [{ORTH: "ارتباط", NORM: "ارتباط"}, {ORTH: "مان", NORM: "مان"}], + "ارزشهاست": [{ORTH: "ارزشها", NORM: "ارزشها"}, {ORTH: "ست", NORM: "ست"}], + "ارزی‌اش": [{ORTH: "ارزی‌", NORM: "ارزی‌"}, {ORTH: "اش", NORM: "اش"}], + "اره‌اش": [{ORTH: "اره‌", NORM: "اره‌"}, {ORTH: "اش", NORM: "اش"}], + "ازش": [{ORTH: "از", NORM: "از"}, {ORTH: "ش", NORM: "ش"}], + "ازین": [{ORTH: "از", NORM: "از"}, {ORTH: "ین", NORM: "ین"}], + "ازین‌هاست": [ + {ORTH: "از", NORM: "از"}, + {ORTH: "ین‌ها", NORM: "ین‌ها"}, + {ORTH: "ست", NORM: "ست"}, + ], + "استخوانند": [{ORTH: "استخوان", NORM: "استخوان"}, {ORTH: "ند", NORM: "ند"}], + "اسلامند": [{ORTH: "اسلام", NORM: "اسلام"}, {ORTH: "ند", NORM: "ند"}], + "اسلامی‌اند": [{ORTH: "اسلامی‌", NORM: "اسلامی‌"}, {ORTH: "اند", NORM: "اند"}], + "اسلحه‌هایشان": [ + {ORTH: "اسلحه‌های", NORM: "اسلحه‌های"}, + {ORTH: "شان", NORM: "شان"}, + ], + "اسمت": [{ORTH: "اسم", NORM: "اسم"}, {ORTH: "ت", NORM: "ت"}], + "اسمش": [{ORTH: "اسم", NORM: "اسم"}, {ORTH: "ش", NORM: "ش"}], + "اشتباهند": [{ORTH: "اشتباه", NORM: "اشتباه"}, {ORTH: "ند", NORM: "ند"}], + "اصلش": [{ORTH: "اصل", NORM: "اصل"}, {ORTH: "ش", NORM: "ش"}], + "اطاقش": [{ORTH: "اطاق", NORM: "اطاق"}, {ORTH: "ش", NORM: "ش"}], + "اعتقادند": [{ORTH: "اعتقاد", NORM: "اعتقاد"}, {ORTH: "ند", NORM: "ند"}], + "اعلایش": [{ORTH: "اعلای", NORM: "اعلای"}, {ORTH: "ش", NORM: "ش"}], + "افتراست": [{ORTH: "افترا", NORM: "افترا"}, {ORTH: "ست", NORM: "ست"}], + "افطارت": [{ORTH: "افطار", NORM: "افطار"}, {ORTH: "ت", NORM: "ت"}], + "اقوامش": [{ORTH: "اقوام", NORM: "اقوام"}, {ORTH: "ش", NORM: "ش"}], + "امروزیش": [{ORTH: "امروزی", NORM: "امروزی"}, {ORTH: "ش", NORM: "ش"}], + "اموالش": [{ORTH: "اموال", NORM: "اموال"}, {ORTH: "ش", NORM: "ش"}], + "امیدوارند": [{ORTH: "امیدوار", NORM: "امیدوار"}, {ORTH: "ند", NORM: "ند"}], + "امیدواریم": [{ORTH: "امیدوار", NORM: "امیدوار"}, {ORTH: "یم", NORM: "یم"}], + "انتخابهایم": [{ORTH: "انتخابها", NORM: "انتخابها"}, {ORTH: "یم", NORM: "یم"}], + "انتظارم": [{ORTH: "انتظار", NORM: "انتظار"}, {ORTH: "م", NORM: "م"}], + "انجمنم": [{ORTH: "انجمن", NORM: "انجمن"}, {ORTH: "م", NORM: "م"}], + "اندرش": [{ORTH: "اندر", NORM: "اندر"}, {ORTH: "ش", NORM: "ش"}], + "انشایش": [{ORTH: "انشای", NORM: "انشای"}, {ORTH: "ش", NORM: "ش"}], + "انگشتشان": [{ORTH: "انگشت", NORM: "انگشت"}, {ORTH: "شان", NORM: "شان"}], + "انگشتهایش": [{ORTH: "انگشتهای", NORM: "انگشتهای"}, {ORTH: "ش", NORM: "ش"}], + "اهمیتشان": [{ORTH: "اهمیت", NORM: "اهمیت"}, {ORTH: "شان", NORM: "شان"}], + "اهمیتند": [{ORTH: "اهمیت", NORM: "اهمیت"}, {ORTH: "ند", NORM: "ند"}], + "اوایلش": [{ORTH: "اوایل", NORM: "اوایل"}, {ORTH: "ش", NORM: "ش"}], + "اوست": [{ORTH: "او", NORM: "او"}, {ORTH: "ست", NORM: "ست"}], + "اولش": [{ORTH: "اول", NORM: "اول"}, {ORTH: "ش", NORM: "ش"}], + "اولشان": [{ORTH: "اول", NORM: "اول"}, {ORTH: "شان", NORM: "شان"}], + "اولم": [{ORTH: "اول", NORM: "اول"}, {ORTH: "م", NORM: "م"}], + "اکثرشان": [{ORTH: "اکثر", NORM: "اکثر"}, {ORTH: "شان", NORM: "شان"}], + "ایتالیاست": [{ORTH: "ایتالیا", NORM: "ایتالیا"}, {ORTH: "ست", NORM: "ست"}], + "ایرانی‌اش": [{ORTH: "ایرانی‌", NORM: "ایرانی‌"}, {ORTH: "اش", NORM: "اش"}], + "اینجاست": [{ORTH: "اینجا", NORM: "اینجا"}, {ORTH: "ست", NORM: "ست"}], + "این‌هاست": [{ORTH: "این‌ها", NORM: "این‌ها"}, {ORTH: "ست", NORM: "ست"}], + "بابات": [{ORTH: "بابا", NORM: "بابا"}, {ORTH: "ت", NORM: "ت"}], + "بارش": [{ORTH: "بار", NORM: "بار"}, {ORTH: "ش", NORM: "ش"}], + "بازیگرانش": [{ORTH: "بازیگران", NORM: "بازیگران"}, {ORTH: "ش", NORM: "ش"}], + "بازیگرمان": [{ORTH: "بازیگر", NORM: "بازیگر"}, {ORTH: "مان", NORM: "مان"}], + "بازیگرهایم": [{ORTH: "بازیگرها", NORM: "بازیگرها"}, {ORTH: "یم", NORM: "یم"}], + "بازی‌اش": [{ORTH: "بازی‌", NORM: "بازی‌"}, {ORTH: "اش", NORM: "اش"}], + "بالاست": [{ORTH: "بالا", NORM: "بالا"}, {ORTH: "ست", NORM: "ست"}], + "باورند": [{ORTH: "باور", NORM: "باور"}, {ORTH: "ند", NORM: "ند"}], + "بجاست": [{ORTH: "بجا", NORM: "بجا"}, {ORTH: "ست", NORM: "ست"}], + "بدان": [{ORTH: "ب", NORM: "ب"}, {ORTH: "دان", NORM: "دان"}], + "بدش": [{ORTH: "بد", NORM: "بد"}, {ORTH: "ش", NORM: "ش"}], + "بدشان": [{ORTH: "بد", NORM: "بد"}, {ORTH: "شان", NORM: "شان"}], + "بدنم": [{ORTH: "بدن", NORM: "بدن"}, {ORTH: "م", NORM: "م"}], + "بدهی‌ات": [{ORTH: "بدهی‌", NORM: "بدهی‌"}, {ORTH: "ات", NORM: "ات"}], + "بدین": [{ORTH: "ب", NORM: "ب"}, {ORTH: "دین", NORM: "دین"}], + "برابرش": [{ORTH: "برابر", NORM: "برابر"}, {ORTH: "ش", NORM: "ش"}], + "برادرت": [{ORTH: "برادر", NORM: "برادر"}, {ORTH: "ت", NORM: "ت"}], + "برادرش": [{ORTH: "برادر", NORM: "برادر"}, {ORTH: "ش", NORM: "ش"}], + "برایت": [{ORTH: "برای", NORM: "برای"}, {ORTH: "ت", NORM: "ت"}], + "برایتان": [{ORTH: "برای", NORM: "برای"}, {ORTH: "تان", NORM: "تان"}], + "برایش": [{ORTH: "برای", NORM: "برای"}, {ORTH: "ش", NORM: "ش"}], + "برایشان": [{ORTH: "برای", NORM: "برای"}, {ORTH: "شان", NORM: "شان"}], + "برایم": [{ORTH: "برای", NORM: "برای"}, {ORTH: "م", NORM: "م"}], + "برایمان": [{ORTH: "برای", NORM: "برای"}, {ORTH: "مان", NORM: "مان"}], + "برخوردارند": [{ORTH: "برخوردار", NORM: "برخوردار"}, {ORTH: "ند", NORM: "ند"}], + "برنامه‌سازهاست": [ + {ORTH: "برنامه‌سازها", NORM: "برنامه‌سازها"}, + {ORTH: "ست", NORM: "ست"}, + ], + "برهمش": [{ORTH: "برهم", NORM: "برهم"}, {ORTH: "ش", NORM: "ش"}], + "برهنه‌اش": [{ORTH: "برهنه‌", NORM: "برهنه‌"}, {ORTH: "اش", NORM: "اش"}], + "برگهایش": [{ORTH: "برگها", NORM: "برگها"}, {ORTH: "یش", NORM: "یش"}], + "برین": [{ORTH: "بر", NORM: "بر"}, {ORTH: "ین", NORM: "ین"}], + "بزرگش": [{ORTH: "بزرگ", NORM: "بزرگ"}, {ORTH: "ش", NORM: "ش"}], + "بزرگ‌تری": [{ORTH: "بزرگ‌تر", NORM: "بزرگ‌تر"}, {ORTH: "ی", NORM: "ی"}], + "بساطش": [{ORTH: "بساط", NORM: "بساط"}, {ORTH: "ش", NORM: "ش"}], + "بعدش": [{ORTH: "بعد", NORM: "بعد"}, {ORTH: "ش", NORM: "ش"}], + "بعضیهایشان": [{ORTH: "بعضیهای", NORM: "بعضیهای"}, {ORTH: "شان", NORM: "شان"}], + "بعضی‌شان": [{ORTH: "بعضی", NORM: "بعضی"}, {ORTH: "‌شان", NORM: "شان"}], + "بقیه‌اش": [{ORTH: "بقیه‌", NORM: "بقیه‌"}, {ORTH: "اش", NORM: "اش"}], + "بلندش": [{ORTH: "بلند", NORM: "بلند"}, {ORTH: "ش", NORM: "ش"}], + "بناگوشش": [{ORTH: "بناگوش", NORM: "بناگوش"}, {ORTH: "ش", NORM: "ش"}], + "بنظرم": [ + {ORTH: "ب", NORM: "ب"}, + {ORTH: "نظر", NORM: "نظر"}, + {ORTH: "م", NORM: "م"}, + ], + "بهت": [{ORTH: "به", NORM: "به"}, {ORTH: "ت", NORM: "ت"}], + "بهترش": [{ORTH: "بهتر", NORM: "بهتر"}, {ORTH: "ش", NORM: "ش"}], + "بهترم": [{ORTH: "بهتر", NORM: "بهتر"}, {ORTH: "م", NORM: "م"}], + "بهتری": [{ORTH: "بهتر", NORM: "بهتر"}, {ORTH: "ی", NORM: "ی"}], + "بهش": [{ORTH: "به", NORM: "به"}, {ORTH: "ش", NORM: "ش"}], + "به‌شان": [{ORTH: "به‌", NORM: "به‌"}, {ORTH: "شان", NORM: "شان"}], + "بودمش": [{ORTH: "بودم", NORM: "بودم"}, {ORTH: "ش", NORM: "ش"}], + "بودنش": [{ORTH: "بودن", NORM: "بودن"}, {ORTH: "ش", NORM: "ش"}], + "بودن‌شان": [{ORTH: "بودن‌", NORM: "بودن‌"}, {ORTH: "شان", NORM: "شان"}], + "بوستانش": [{ORTH: "بوستان", NORM: "بوستان"}, {ORTH: "ش", NORM: "ش"}], + "بویش": [{ORTH: "بو", NORM: "بو"}, {ORTH: "یش", NORM: "یش"}], + "بچه‌اش": [{ORTH: "بچه‌", NORM: "بچه‌"}, {ORTH: "اش", NORM: "اش"}], + "بچه‌م": [{ORTH: "بچه‌", NORM: "بچه‌"}, {ORTH: "م", NORM: "م"}], + "بچه‌هایش": [{ORTH: "بچه‌های", NORM: "بچه‌های"}, {ORTH: "ش", NORM: "ش"}], + "بیانیه‌شان": [{ORTH: "بیانیه‌", NORM: "بیانیه‌"}, {ORTH: "شان", NORM: "شان"}], + "بیدارم": [{ORTH: "بیدار", NORM: "بیدار"}, {ORTH: "م", NORM: "م"}], + "بیناتری": [{ORTH: "بیناتر", NORM: "بیناتر"}, {ORTH: "ی", NORM: "ی"}], + "بی‌اطلاعند": [{ORTH: "بی‌اطلاع", NORM: "بی‌اطلاع"}, {ORTH: "ند", NORM: "ند"}], + "بی‌اطلاعید": [{ORTH: "بی‌اطلاع", NORM: "بی‌اطلاع"}, {ORTH: "ید", NORM: "ید"}], + "بی‌بهره‌اند": [{ORTH: "بی‌بهره‌", NORM: "بی‌بهره‌"}, {ORTH: "اند", NORM: "اند"}], + "بی‌تفاوتند": [{ORTH: "بی‌تفاوت", NORM: "بی‌تفاوت"}, {ORTH: "ند", NORM: "ند"}], + "بی‌حسابش": [{ORTH: "بی‌حساب", NORM: "بی‌حساب"}, {ORTH: "ش", NORM: "ش"}], + "بی‌نیش": [{ORTH: "بی‌نی", NORM: "بی‌نی"}, {ORTH: "ش", NORM: "ش"}], + "تجربه‌هایم": [{ORTH: "تجربه‌ها", NORM: "تجربه‌ها"}, {ORTH: "یم", NORM: "یم"}], + "تحریم‌هاست": [{ORTH: "تحریم‌ها", NORM: "تحریم‌ها"}, {ORTH: "ست", NORM: "ست"}], + "تحولند": [{ORTH: "تحول", NORM: "تحول"}, {ORTH: "ند", NORM: "ند"}], + "تخیلی‌اش": [{ORTH: "تخیلی‌", NORM: "تخیلی‌"}, {ORTH: "اش", NORM: "اش"}], + "ترا": [{ORTH: "ت", NORM: "ت"}, {ORTH: "را", NORM: "را"}], + "ترسشان": [{ORTH: "ترس", NORM: "ترس"}, {ORTH: "شان", NORM: "شان"}], + "ترکش": [{ORTH: "ترک", NORM: "ترک"}, {ORTH: "ش", NORM: "ش"}], + "تشنه‌ت": [{ORTH: "تشنه‌", NORM: "تشنه‌"}, {ORTH: "ت", NORM: "ت"}], + "تشکیلاتی‌اش": [{ORTH: "تشکیلاتی‌", NORM: "تشکیلاتی‌"}, {ORTH: "اش", NORM: "اش"}], + "تعلقش": [{ORTH: "تعلق", NORM: "تعلق"}, {ORTH: "ش", NORM: "ش"}], + "تلاششان": [{ORTH: "تلاش", NORM: "تلاش"}, {ORTH: "شان", NORM: "شان"}], + "تلاشمان": [{ORTH: "تلاش", NORM: "تلاش"}, {ORTH: "مان", NORM: "مان"}], + "تماشاگرش": [{ORTH: "تماشاگر", NORM: "تماشاگر"}, {ORTH: "ش", NORM: "ش"}], + "تمامشان": [{ORTH: "تمام", NORM: "تمام"}, {ORTH: "شان", NORM: "شان"}], + "تنش": [{ORTH: "تن", NORM: "تن"}, {ORTH: "ش", NORM: "ش"}], + "تنمان": [{ORTH: "تن", NORM: "تن"}, {ORTH: "مان", NORM: "مان"}], + "تنهایی‌اش": [{ORTH: "تنهایی‌", NORM: "تنهایی‌"}, {ORTH: "اش", NORM: "اش"}], + "توانایی‌اش": [{ORTH: "توانایی‌", NORM: "توانایی‌"}, {ORTH: "اش", NORM: "اش"}], + "توجهش": [{ORTH: "توجه", NORM: "توجه"}, {ORTH: "ش", NORM: "ش"}], + "توست": [{ORTH: "تو", NORM: "تو"}, {ORTH: "ست", NORM: "ست"}], + "توصیه‌اش": [{ORTH: "توصیه‌", NORM: "توصیه‌"}, {ORTH: "اش", NORM: "اش"}], + "تیغه‌اش": [{ORTH: "تیغه‌", NORM: "تیغه‌"}, {ORTH: "اش", NORM: "اش"}], + "جاست": [{ORTH: "جا", NORM: "جا"}, {ORTH: "ست", NORM: "ست"}], + "جامعه‌اند": [{ORTH: "جامعه‌", NORM: "جامعه‌"}, {ORTH: "اند", NORM: "اند"}], + "جانم": [{ORTH: "جان", NORM: "جان"}, {ORTH: "م", NORM: "م"}], + "جایش": [{ORTH: "جای", NORM: "جای"}, {ORTH: "ش", NORM: "ش"}], + "جایشان": [{ORTH: "جای", NORM: "جای"}, {ORTH: "شان", NORM: "شان"}], + "جدیدش": [{ORTH: "جدید", NORM: "جدید"}, {ORTH: "ش", NORM: "ش"}], + "جرمزاست": [{ORTH: "جرمزا", NORM: "جرمزا"}, {ORTH: "ست", NORM: "ست"}], + "جلوست": [{ORTH: "جلو", NORM: "جلو"}, {ORTH: "ست", NORM: "ست"}], + "جلویش": [{ORTH: "جلوی", NORM: "جلوی"}, {ORTH: "ش", NORM: "ش"}], + "جمهوریست": [{ORTH: "جمهوری", NORM: "جمهوری"}, {ORTH: "ست", NORM: "ست"}], + "جنسش": [{ORTH: "جنس", NORM: "جنس"}, {ORTH: "ش", NORM: "ش"}], + "جنس‌اند": [{ORTH: "جنس‌", NORM: "جنس‌"}, {ORTH: "اند", NORM: "اند"}], + "جوانانش": [{ORTH: "جوانان", NORM: "جوانان"}, {ORTH: "ش", NORM: "ش"}], + "جویش": [{ORTH: "جوی", NORM: "جوی"}, {ORTH: "ش", NORM: "ش"}], + "جگرش": [{ORTH: "جگر", NORM: "جگر"}, {ORTH: "ش", NORM: "ش"}], + "حاضرم": [{ORTH: "حاضر", NORM: "حاضر"}, {ORTH: "م", NORM: "م"}], + "حالتهایشان": [{ORTH: "حالتهای", NORM: "حالتهای"}, {ORTH: "شان", NORM: "شان"}], + "حالیست": [{ORTH: "حالی", NORM: "حالی"}, {ORTH: "ست", NORM: "ست"}], + "حالی‌مان": [{ORTH: "حالی‌", NORM: "حالی‌"}, {ORTH: "مان", NORM: "مان"}], + "حاکیست": [{ORTH: "حاکی", NORM: "حاکی"}, {ORTH: "ست", NORM: "ست"}], + "حرامزادگی‌اش": [ + {ORTH: "حرامزادگی‌", NORM: "حرامزادگی‌"}, + {ORTH: "اش", NORM: "اش"}, + ], + "حرفتان": [{ORTH: "حرف", NORM: "حرف"}, {ORTH: "تان", NORM: "تان"}], + "حرفش": [{ORTH: "حرف", NORM: "حرف"}, {ORTH: "ش", NORM: "ش"}], + "حرفشان": [{ORTH: "حرف", NORM: "حرف"}, {ORTH: "شان", NORM: "شان"}], + "حرفم": [{ORTH: "حرف", NORM: "حرف"}, {ORTH: "م", NORM: "م"}], + "حرف‌های‌شان": [{ORTH: "حرف‌های‌", NORM: "حرف‌های‌"}, {ORTH: "شان", NORM: "شان"}], + "حرکتمان": [{ORTH: "حرکت", NORM: "حرکت"}, {ORTH: "مان", NORM: "مان"}], + "حریفانشان": [{ORTH: "حریفان", NORM: "حریفان"}, {ORTH: "شان", NORM: "شان"}], + "حضورشان": [{ORTH: "حضور", NORM: "حضور"}, {ORTH: "شان", NORM: "شان"}], + "حمایتش": [{ORTH: "حمایت", NORM: "حمایت"}, {ORTH: "ش", NORM: "ش"}], + "حواسش": [{ORTH: "حواس", NORM: "حواس"}, {ORTH: "ش", NORM: "ش"}], + "حواسشان": [{ORTH: "حواس", NORM: "حواس"}, {ORTH: "شان", NORM: "شان"}], + "حوصله‌مان": [{ORTH: "حوصله‌", NORM: "حوصله‌"}, {ORTH: "مان", NORM: "مان"}], + "حکومتش": [{ORTH: "حکومت", NORM: "حکومت"}, {ORTH: "ش", NORM: "ش"}], + "حکومتشان": [{ORTH: "حکومت", NORM: "حکومت"}, {ORTH: "شان", NORM: "شان"}], + "حیفم": [{ORTH: "حیف", NORM: "حیف"}, {ORTH: "م", NORM: "م"}], + "خاندانش": [{ORTH: "خاندان", NORM: "خاندان"}, {ORTH: "ش", NORM: "ش"}], + "خانه‌اش": [{ORTH: "خانه‌", NORM: "خانه‌"}, {ORTH: "اش", NORM: "اش"}], + "خانه‌شان": [{ORTH: "خانه‌", NORM: "خانه‌"}, {ORTH: "شان", NORM: "شان"}], + "خانه‌مان": [{ORTH: "خانه‌", NORM: "خانه‌"}, {ORTH: "مان", NORM: "مان"}], + "خانه‌هایشان": [{ORTH: "خانه‌های", NORM: "خانه‌های"}, {ORTH: "شان", NORM: "شان"}], + "خانواده‌ات": [{ORTH: "خانواده", NORM: "خانواده"}, {ORTH: "‌ات", NORM: "ات"}], + "خانواده‌اش": [{ORTH: "خانواده‌", NORM: "خانواده‌"}, {ORTH: "اش", NORM: "اش"}], + "خانواده‌ام": [{ORTH: "خانواده‌", NORM: "خانواده‌"}, {ORTH: "ام", NORM: "ام"}], + "خانواده‌شان": [{ORTH: "خانواده‌", NORM: "خانواده‌"}, {ORTH: "شان", NORM: "شان"}], + "خداست": [{ORTH: "خدا", NORM: "خدا"}, {ORTH: "ست", NORM: "ست"}], + "خدایش": [{ORTH: "خدا", NORM: "خدا"}, {ORTH: "یش", NORM: "یش"}], + "خدایشان": [{ORTH: "خدای", NORM: "خدای"}, {ORTH: "شان", NORM: "شان"}], + "خردسالش": [{ORTH: "خردسال", NORM: "خردسال"}, {ORTH: "ش", NORM: "ش"}], + "خروپفشان": [{ORTH: "خروپف", NORM: "خروپف"}, {ORTH: "شان", NORM: "شان"}], + "خسته‌ای": [{ORTH: "خسته‌", NORM: "خسته‌"}, {ORTH: "ای", NORM: "ای"}], + "خطت": [{ORTH: "خط", NORM: "خط"}, {ORTH: "ت", NORM: "ت"}], + "خوابمان": [{ORTH: "خواب", NORM: "خواب"}, {ORTH: "مان", NORM: "مان"}], + "خواندنش": [{ORTH: "خواندن", NORM: "خواندن"}, {ORTH: "ش", NORM: "ش"}], + "خواهرش": [{ORTH: "خواهر", NORM: "خواهر"}, {ORTH: "ش", NORM: "ش"}], + "خوبش": [{ORTH: "خوب", NORM: "خوب"}, {ORTH: "ش", NORM: "ش"}], + "خودت": [{ORTH: "خود", NORM: "خود"}, {ORTH: "ت", NORM: "ت"}], + "خودتان": [{ORTH: "خود", NORM: "خود"}, {ORTH: "تان", NORM: "تان"}], + "خودش": [{ORTH: "خود", NORM: "خود"}, {ORTH: "ش", NORM: "ش"}], + "خودشان": [{ORTH: "خود", NORM: "خود"}, {ORTH: "شان", NORM: "شان"}], + "خودمان": [{ORTH: "خود", NORM: "خود"}, {ORTH: "مان", NORM: "مان"}], + "خوردمان": [{ORTH: "خورد", NORM: "خورد"}, {ORTH: "مان", NORM: "مان"}], + "خوردنشان": [{ORTH: "خوردن", NORM: "خوردن"}, {ORTH: "شان", NORM: "شان"}], + "خوشش": [{ORTH: "خوش", NORM: "خوش"}, {ORTH: "ش", NORM: "ش"}], + "خوشوقتم": [{ORTH: "خوشوقت", NORM: "خوشوقت"}, {ORTH: "م", NORM: "م"}], + "خونشان": [{ORTH: "خون", NORM: "خون"}, {ORTH: "شان", NORM: "شان"}], + "خویش": [{ORTH: "خوی", NORM: "خوی"}, {ORTH: "ش", NORM: "ش"}], + "خویشتنم": [{ORTH: "خویشتن", NORM: "خویشتن"}, {ORTH: "م", NORM: "م"}], + "خیالش": [{ORTH: "خیال", NORM: "خیال"}, {ORTH: "ش", NORM: "ش"}], + "خیسش": [{ORTH: "خیس", NORM: "خیس"}, {ORTH: "ش", NORM: "ش"}], + "داراست": [{ORTH: "دارا", NORM: "دارا"}, {ORTH: "ست", NORM: "ست"}], + "داستانهایش": [{ORTH: "داستانهای", NORM: "داستانهای"}, {ORTH: "ش", NORM: "ش"}], + "دخترمان": [{ORTH: "دختر", NORM: "دختر"}, {ORTH: "مان", NORM: "مان"}], + "دخیلند": [{ORTH: "دخیل", NORM: "دخیل"}, {ORTH: "ند", NORM: "ند"}], + "درباره‌ات": [{ORTH: "درباره", NORM: "درباره"}, {ORTH: "‌ات", NORM: "ات"}], + "درباره‌اش": [{ORTH: "درباره‌", NORM: "درباره‌"}, {ORTH: "اش", NORM: "اش"}], + "دردش": [{ORTH: "درد", NORM: "درد"}, {ORTH: "ش", NORM: "ش"}], + "دردشان": [{ORTH: "درد", NORM: "درد"}, {ORTH: "شان", NORM: "شان"}], + "درسته": [{ORTH: "درست", NORM: "درست"}, {ORTH: "ه", NORM: "ه"}], + "درش": [{ORTH: "در", NORM: "در"}, {ORTH: "ش", NORM: "ش"}], + "درون‌شان": [{ORTH: "درون‌", NORM: "درون‌"}, {ORTH: "شان", NORM: "شان"}], + "درین": [{ORTH: "در", NORM: "در"}, {ORTH: "ین", NORM: "ین"}], + "دریچه‌هایش": [{ORTH: "دریچه‌های", NORM: "دریچه‌های"}, {ORTH: "ش", NORM: "ش"}], + "دزدانش": [{ORTH: "دزدان", NORM: "دزدان"}, {ORTH: "ش", NORM: "ش"}], + "دستت": [{ORTH: "دست", NORM: "دست"}, {ORTH: "ت", NORM: "ت"}], + "دستش": [{ORTH: "دست", NORM: "دست"}, {ORTH: "ش", NORM: "ش"}], + "دستمان": [{ORTH: "دست", NORM: "دست"}, {ORTH: "مان", NORM: "مان"}], + "دستهایشان": [{ORTH: "دستهای", NORM: "دستهای"}, {ORTH: "شان", NORM: "شان"}], + "دست‌یافتنی‌ست": [ + {ORTH: "دست‌یافتنی‌", NORM: "دست‌یافتنی‌"}, + {ORTH: "ست", NORM: "ست"}, + ], + "دشمنند": [{ORTH: "دشمن", NORM: "دشمن"}, {ORTH: "ند", NORM: "ند"}], + "دشمنیشان": [{ORTH: "دشمنی", NORM: "دشمنی"}, {ORTH: "شان", NORM: "شان"}], + "دشمنیم": [{ORTH: "دشمن", NORM: "دشمن"}, {ORTH: "یم", NORM: "یم"}], + "دفترش": [{ORTH: "دفتر", NORM: "دفتر"}, {ORTH: "ش", NORM: "ش"}], + "دفنشان": [{ORTH: "دفن", NORM: "دفن"}, {ORTH: "شان", NORM: "شان"}], + "دلت": [{ORTH: "دل", NORM: "دل"}, {ORTH: "ت", NORM: "ت"}], + "دلش": [{ORTH: "دل", NORM: "دل"}, {ORTH: "ش", NORM: "ش"}], + "دلشان": [{ORTH: "دل", NORM: "دل"}, {ORTH: "شان", NORM: "شان"}], + "دلم": [{ORTH: "دل", NORM: "دل"}, {ORTH: "م", NORM: "م"}], + "دلیلش": [{ORTH: "دلیل", NORM: "دلیل"}, {ORTH: "ش", NORM: "ش"}], + "دنبالش": [{ORTH: "دنبال", NORM: "دنبال"}, {ORTH: "ش", NORM: "ش"}], + "دنباله‌اش": [{ORTH: "دنباله‌", NORM: "دنباله‌"}, {ORTH: "اش", NORM: "اش"}], + "دهاتی‌هایش": [{ORTH: "دهاتی‌های", NORM: "دهاتی‌های"}, {ORTH: "ش", NORM: "ش"}], + "دهانت": [{ORTH: "دهان", NORM: "دهان"}, {ORTH: "ت", NORM: "ت"}], + "دهنش": [{ORTH: "دهن", NORM: "دهن"}, {ORTH: "ش", NORM: "ش"}], + "دورش": [{ORTH: "دور", NORM: "دور"}, {ORTH: "ش", NORM: "ش"}], + "دوروبریهاشان": [ + {ORTH: "دوروبریها", NORM: "دوروبریها"}, + {ORTH: "شان", NORM: "شان"}, + ], + "دوستانش": [{ORTH: "دوستان", NORM: "دوستان"}, {ORTH: "ش", NORM: "ش"}], + "دوستانشان": [{ORTH: "دوستان", NORM: "دوستان"}, {ORTH: "شان", NORM: "شان"}], + "دوستت": [{ORTH: "دوست", NORM: "دوست"}, {ORTH: "ت", NORM: "ت"}], + "دوستش": [{ORTH: "دوست", NORM: "دوست"}, {ORTH: "ش", NORM: "ش"}], + "دومش": [{ORTH: "دوم", NORM: "دوم"}, {ORTH: "ش", NORM: "ش"}], + "دویدنش": [{ORTH: "دویدن", NORM: "دویدن"}, {ORTH: "ش", NORM: "ش"}], + "دکورهایمان": [{ORTH: "دکورهای", NORM: "دکورهای"}, {ORTH: "مان", NORM: "مان"}], + "دیدگاهش": [{ORTH: "دیدگاه", NORM: "دیدگاه"}, {ORTH: "ش", NORM: "ش"}], + "دیرت": [{ORTH: "دیر", NORM: "دیر"}, {ORTH: "ت", NORM: "ت"}], + "دیرم": [{ORTH: "دیر", NORM: "دیر"}, {ORTH: "م", NORM: "م"}], + "دینت": [{ORTH: "دین", NORM: "دین"}, {ORTH: "ت", NORM: "ت"}], + "دینش": [{ORTH: "دین", NORM: "دین"}, {ORTH: "ش", NORM: "ش"}], + "دین‌شان": [{ORTH: "دین‌", NORM: "دین‌"}, {ORTH: "شان", NORM: "شان"}], + "دیواره‌هایش": [{ORTH: "دیواره‌های", NORM: "دیواره‌های"}, {ORTH: "ش", NORM: "ش"}], + "دیوانه‌ای": [{ORTH: "دیوانه‌", NORM: "دیوانه‌"}, {ORTH: "ای", NORM: "ای"}], + "دیوی": [{ORTH: "دیو", NORM: "دیو"}, {ORTH: "ی", NORM: "ی"}], + "دیگرم": [{ORTH: "دیگر", NORM: "دیگر"}, {ORTH: "م", NORM: "م"}], + "دیگرمان": [{ORTH: "دیگر", NORM: "دیگر"}, {ORTH: "مان", NORM: "مان"}], + "ذهنش": [{ORTH: "ذهن", NORM: "ذهن"}, {ORTH: "ش", NORM: "ش"}], + "ذهنشان": [{ORTH: "ذهن", NORM: "ذهن"}, {ORTH: "شان", NORM: "شان"}], + "ذهنم": [{ORTH: "ذهن", NORM: "ذهن"}, {ORTH: "م", NORM: "م"}], + "رئوسش": [{ORTH: "رئوس", NORM: "رئوس"}, {ORTH: "ش", NORM: "ش"}], + "راهشان": [{ORTH: "راه", NORM: "راه"}, {ORTH: "شان", NORM: "شان"}], + "راهگشاست": [{ORTH: "راهگشا", NORM: "راهگشا"}, {ORTH: "ست", NORM: "ست"}], + "رایانه‌هایشان": [ + {ORTH: "رایانه‌های", NORM: "رایانه‌های"}, + {ORTH: "شان", NORM: "شان"}, + ], + "رعایتشان": [{ORTH: "رعایت", NORM: "رعایت"}, {ORTH: "شان", NORM: "شان"}], + "رفتارش": [{ORTH: "رفتار", NORM: "رفتار"}, {ORTH: "ش", NORM: "ش"}], + "رفتارشان": [{ORTH: "رفتار", NORM: "رفتار"}, {ORTH: "شان", NORM: "شان"}], + "رفتارمان": [{ORTH: "رفتار", NORM: "رفتار"}, {ORTH: "مان", NORM: "مان"}], + "رفتارهاست": [{ORTH: "رفتارها", NORM: "رفتارها"}, {ORTH: "ست", NORM: "ست"}], + "رفتارهایشان": [{ORTH: "رفتارهای", NORM: "رفتارهای"}, {ORTH: "شان", NORM: "شان"}], + "رفقایم": [{ORTH: "رفقا", NORM: "رفقا"}, {ORTH: "یم", NORM: "یم"}], + "رقیق‌ترش": [{ORTH: "رقیق‌تر", NORM: "رقیق‌تر"}, {ORTH: "ش", NORM: "ش"}], + "رنجند": [{ORTH: "رنج", NORM: "رنج"}, {ORTH: "ند", NORM: "ند"}], + "رهگشاست": [{ORTH: "رهگشا", NORM: "رهگشا"}, {ORTH: "ست", NORM: "ست"}], + "رواست": [{ORTH: "روا", NORM: "روا"}, {ORTH: "ست", NORM: "ست"}], + "روبروست": [{ORTH: "روبرو", NORM: "روبرو"}, {ORTH: "ست", NORM: "ست"}], + "روحی‌اش": [{ORTH: "روحی‌", NORM: "روحی‌"}, {ORTH: "اش", NORM: "اش"}], + "روزنامه‌اش": [{ORTH: "روزنامه‌", NORM: "روزنامه‌"}, {ORTH: "اش", NORM: "اش"}], + "روزه‌ست": [{ORTH: "روزه‌", NORM: "روزه‌"}, {ORTH: "ست", NORM: "ست"}], + "روسری‌اش": [{ORTH: "روسری‌", NORM: "روسری‌"}, {ORTH: "اش", NORM: "اش"}], + "روشتان": [{ORTH: "روش", NORM: "روش"}, {ORTH: "تان", NORM: "تان"}], + "رویش": [{ORTH: "روی", NORM: "روی"}, {ORTH: "ش", NORM: "ش"}], + "زبانش": [{ORTH: "زبان", NORM: "زبان"}, {ORTH: "ش", NORM: "ش"}], + "زحماتشان": [{ORTH: "زحمات", NORM: "زحمات"}, {ORTH: "شان", NORM: "شان"}], + "زدنهایشان": [{ORTH: "زدنهای", NORM: "زدنهای"}, {ORTH: "شان", NORM: "شان"}], + "زرنگشان": [{ORTH: "زرنگ", NORM: "زرنگ"}, {ORTH: "شان", NORM: "شان"}], + "زشتش": [{ORTH: "زشت", NORM: "زشت"}, {ORTH: "ش", NORM: "ش"}], + "زشتکارانند": [{ORTH: "زشتکاران", NORM: "زشتکاران"}, {ORTH: "ند", NORM: "ند"}], + "زلفش": [{ORTH: "زلف", NORM: "زلف"}, {ORTH: "ش", NORM: "ش"}], + "زمن": [{ORTH: "ز", NORM: "ز"}, {ORTH: "من", NORM: "من"}], + "زنبوری‌اش": [{ORTH: "زنبوری‌", NORM: "زنبوری‌"}, {ORTH: "اش", NORM: "اش"}], + "زندانم": [{ORTH: "زندان", NORM: "زندان"}, {ORTH: "م", NORM: "م"}], + "زنده‌ام": [{ORTH: "زنده‌", NORM: "زنده‌"}, {ORTH: "ام", NORM: "ام"}], + "زندگانی‌اش": [{ORTH: "زندگانی‌", NORM: "زندگانی‌"}, {ORTH: "اش", NORM: "اش"}], + "زندگی‌اش": [{ORTH: "زندگی‌", NORM: "زندگی‌"}, {ORTH: "اش", NORM: "اش"}], + "زندگی‌ام": [{ORTH: "زندگی‌", NORM: "زندگی‌"}, {ORTH: "ام", NORM: "ام"}], + "زندگی‌شان": [{ORTH: "زندگی‌", NORM: "زندگی‌"}, {ORTH: "شان", NORM: "شان"}], + "زنش": [{ORTH: "زن", NORM: "زن"}, {ORTH: "ش", NORM: "ش"}], + "زنند": [{ORTH: "زن", NORM: "زن"}, {ORTH: "ند", NORM: "ند"}], + "زو": [{ORTH: "ز", NORM: "ز"}, {ORTH: "و", NORM: "و"}], + "زیاده": [{ORTH: "زیاد", NORM: "زیاد"}, {ORTH: "ه", NORM: "ه"}], + "زیباست": [{ORTH: "زیبا", NORM: "زیبا"}, {ORTH: "ست", NORM: "ست"}], + "زیبایش": [{ORTH: "زیبای", NORM: "زیبای"}, {ORTH: "ش", NORM: "ش"}], + "زیبایی": [{ORTH: "زیبای", NORM: "زیبای"}, {ORTH: "ی", NORM: "ی"}], + "زیربناست": [{ORTH: "زیربنا", NORM: "زیربنا"}, {ORTH: "ست", NORM: "ست"}], + "زیرک‌اند": [{ORTH: "زیرک‌", NORM: "زیرک‌"}, {ORTH: "اند", NORM: "اند"}], + "سؤالتان": [{ORTH: "سؤال", NORM: "سؤال"}, {ORTH: "تان", NORM: "تان"}], + "سؤالم": [{ORTH: "سؤال", NORM: "سؤال"}, {ORTH: "م", NORM: "م"}], + "سابقه‌اش": [{ORTH: "سابقه‌", NORM: "سابقه‌"}, {ORTH: "اش", NORM: "اش"}], + "ساختنم": [{ORTH: "ساختن", NORM: "ساختن"}, {ORTH: "م", NORM: "م"}], + "ساده‌اش": [{ORTH: "ساده‌", NORM: "ساده‌"}, {ORTH: "اش", NORM: "اش"}], + "ساده‌اند": [{ORTH: "ساده‌", NORM: "ساده‌"}, {ORTH: "اند", NORM: "اند"}], + "سازمانش": [{ORTH: "سازمان", NORM: "سازمان"}, {ORTH: "ش", NORM: "ش"}], + "ساعتم": [{ORTH: "ساعت", NORM: "ساعت"}, {ORTH: "م", NORM: "م"}], + "سالته": [ + {ORTH: "سال", NORM: "سال"}, + {ORTH: "ت", NORM: "ت"}, + {ORTH: "ه", NORM: "ه"}, + ], + "سالش": [{ORTH: "سال", NORM: "سال"}, {ORTH: "ش", NORM: "ش"}], + "سالهاست": [{ORTH: "سالها", NORM: "سالها"}, {ORTH: "ست", NORM: "ست"}], + "ساله‌اش": [{ORTH: "ساله‌", NORM: "ساله‌"}, {ORTH: "اش", NORM: "اش"}], + "ساکتند": [{ORTH: "ساکت", NORM: "ساکت"}, {ORTH: "ند", NORM: "ند"}], + "ساکنند": [{ORTH: "ساکن", NORM: "ساکن"}, {ORTH: "ند", NORM: "ند"}], + "سبزشان": [{ORTH: "سبز", NORM: "سبز"}, {ORTH: "شان", NORM: "شان"}], + "سبیل‌مان": [{ORTH: "سبیل‌", NORM: "سبیل‌"}, {ORTH: "مان", NORM: "مان"}], + "ستم‌هایش": [{ORTH: "ستم‌های", NORM: "ستم‌های"}, {ORTH: "ش", NORM: "ش"}], + "سخنانش": [{ORTH: "سخنان", NORM: "سخنان"}, {ORTH: "ش", NORM: "ش"}], + "سخنانشان": [{ORTH: "سخنان", NORM: "سخنان"}, {ORTH: "شان", NORM: "شان"}], + "سخنتان": [{ORTH: "سخن", NORM: "سخن"}, {ORTH: "تان", NORM: "تان"}], + "سخنش": [{ORTH: "سخن", NORM: "سخن"}, {ORTH: "ش", NORM: "ش"}], + "سخنم": [{ORTH: "سخن", NORM: "سخن"}, {ORTH: "م", NORM: "م"}], + "سردش": [{ORTH: "سرد", NORM: "سرد"}, {ORTH: "ش", NORM: "ش"}], + "سرزمینشان": [{ORTH: "سرزمین", NORM: "سرزمین"}, {ORTH: "شان", NORM: "شان"}], + "سرش": [{ORTH: "سر", NORM: "سر"}, {ORTH: "ش", NORM: "ش"}], + "سرمایه‌دارهاست": [ + {ORTH: "سرمایه‌دارها", NORM: "سرمایه‌دارها"}, + {ORTH: "ست", NORM: "ست"}, + ], + "سرنوشتش": [{ORTH: "سرنوشت", NORM: "سرنوشت"}, {ORTH: "ش", NORM: "ش"}], + "سرنوشتشان": [{ORTH: "سرنوشت", NORM: "سرنوشت"}, {ORTH: "شان", NORM: "شان"}], + "سروتهش": [{ORTH: "سروته", NORM: "سروته"}, {ORTH: "ش", NORM: "ش"}], + "سرچشمه‌اش": [{ORTH: "سرچشمه‌", NORM: "سرچشمه‌"}, {ORTH: "اش", NORM: "اش"}], + "سقمش": [{ORTH: "سقم", NORM: "سقم"}, {ORTH: "ش", NORM: "ش"}], + "سنش": [{ORTH: "سن", NORM: "سن"}, {ORTH: "ش", NORM: "ش"}], + "سپاهش": [{ORTH: "سپاه", NORM: "سپاه"}, {ORTH: "ش", NORM: "ش"}], + "سیاسیشان": [{ORTH: "سیاسی", NORM: "سیاسی"}, {ORTH: "شان", NORM: "شان"}], + "سیاه‌چاله‌هاست": [ + {ORTH: "سیاه‌چاله‌ها", NORM: "سیاه‌چاله‌ها"}, + {ORTH: "ست", NORM: "ست"}, + ], + "شاخه‌هایشان": [{ORTH: "شاخه‌های", NORM: "شاخه‌های"}, {ORTH: "شان", NORM: "شان"}], + "شالوده‌اش": [{ORTH: "شالوده‌", NORM: "شالوده‌"}, {ORTH: "اش", NORM: "اش"}], + "شانه‌هایش": [{ORTH: "شانه‌های", NORM: "شانه‌های"}, {ORTH: "ش", NORM: "ش"}], + "شاهدیم": [{ORTH: "شاهد", NORM: "شاهد"}, {ORTH: "یم", NORM: "یم"}], + "شاهکارهایش": [{ORTH: "شاهکارهای", NORM: "شاهکارهای"}, {ORTH: "ش", NORM: "ش"}], + "شخصیتش": [{ORTH: "شخصیت", NORM: "شخصیت"}, {ORTH: "ش", NORM: "ش"}], + "شدنشان": [{ORTH: "شدن", NORM: "شدن"}, {ORTH: "شان", NORM: "شان"}], + "شرکتیست": [{ORTH: "شرکتی", NORM: "شرکتی"}, {ORTH: "ست", NORM: "ست"}], + "شعارهاشان": [{ORTH: "شعارها", NORM: "شعارها"}, {ORTH: "شان", NORM: "شان"}], + "شعورش": [{ORTH: "شعور", NORM: "شعور"}, {ORTH: "ش", NORM: "ش"}], + "شغلش": [{ORTH: "شغل", NORM: "شغل"}, {ORTH: "ش", NORM: "ش"}], + "شماست": [{ORTH: "شما", NORM: "شما"}, {ORTH: "ست", NORM: "ست"}], + "شمشیرش": [{ORTH: "شمشیر", NORM: "شمشیر"}, {ORTH: "ش", NORM: "ش"}], + "شنیدنش": [{ORTH: "شنیدن", NORM: "شنیدن"}, {ORTH: "ش", NORM: "ش"}], + "شوراست": [{ORTH: "شورا", NORM: "شورا"}, {ORTH: "ست", NORM: "ست"}], + "شومت": [{ORTH: "شوم", NORM: "شوم"}, {ORTH: "ت", NORM: "ت"}], + "شیرینترش": [{ORTH: "شیرینتر", NORM: "شیرینتر"}, {ORTH: "ش", NORM: "ش"}], + "شیطان‌اند": [{ORTH: "شیطان‌", NORM: "شیطان‌"}, {ORTH: "اند", NORM: "اند"}], + "شیوه‌هاست": [{ORTH: "شیوه‌ها", NORM: "شیوه‌ها"}, {ORTH: "ست", NORM: "ست"}], + "صاحبش": [{ORTH: "صاحب", NORM: "صاحب"}, {ORTH: "ش", NORM: "ش"}], + "صحنه‌اش": [{ORTH: "صحنه‌", NORM: "صحنه‌"}, {ORTH: "اش", NORM: "اش"}], + "صدایش": [{ORTH: "صدای", NORM: "صدای"}, {ORTH: "ش", NORM: "ش"}], + "صددند": [{ORTH: "صدد", NORM: "صدد"}, {ORTH: "ند", NORM: "ند"}], + "صندوق‌هاست": [{ORTH: "صندوق‌ها", NORM: "صندوق‌ها"}, {ORTH: "ست", NORM: "ست"}], + "صندوق‌هایش": [{ORTH: "صندوق‌های", NORM: "صندوق‌های"}, {ORTH: "ش", NORM: "ش"}], + "صورتش": [{ORTH: "صورت", NORM: "صورت"}, {ORTH: "ش", NORM: "ش"}], + "ضروری‌اند": [{ORTH: "ضروری‌", NORM: "ضروری‌"}, {ORTH: "اند", NORM: "اند"}], + "ضمیرش": [{ORTH: "ضمیر", NORM: "ضمیر"}, {ORTH: "ش", NORM: "ش"}], + "طرفش": [{ORTH: "طرف", NORM: "طرف"}, {ORTH: "ش", NORM: "ش"}], + "طلسمش": [{ORTH: "طلسم", NORM: "طلسم"}, {ORTH: "ش", NORM: "ش"}], + "طوره": [{ORTH: "طور", NORM: "طور"}, {ORTH: "ه", NORM: "ه"}], + "عاشوراست": [{ORTH: "عاشورا", NORM: "عاشورا"}, {ORTH: "ست", NORM: "ست"}], + "عبارتند": [{ORTH: "عبارت", NORM: "عبارت"}, {ORTH: "ند", NORM: "ند"}], + "عزیزانتان": [{ORTH: "عزیزان", NORM: "عزیزان"}, {ORTH: "تان", NORM: "تان"}], + "عزیزانش": [{ORTH: "عزیزان", NORM: "عزیزان"}, {ORTH: "ش", NORM: "ش"}], + "عزیزش": [{ORTH: "عزیز", NORM: "عزیز"}, {ORTH: "ش", NORM: "ش"}], + "عشرت‌طلبی‌اش": [ + {ORTH: "عشرت‌طلبی‌", NORM: "عشرت‌طلبی‌"}, + {ORTH: "اش", NORM: "اش"}, + ], + "عقبیم": [{ORTH: "عقب", NORM: "عقب"}, {ORTH: "یم", NORM: "یم"}], + "علاقه‌اش": [{ORTH: "علاقه‌", NORM: "علاقه‌"}, {ORTH: "اش", NORM: "اش"}], + "علمیمان": [{ORTH: "علمی", NORM: "علمی"}, {ORTH: "مان", NORM: "مان"}], + "عمرش": [{ORTH: "عمر", NORM: "عمر"}, {ORTH: "ش", NORM: "ش"}], + "عمرشان": [{ORTH: "عمر", NORM: "عمر"}, {ORTH: "شان", NORM: "شان"}], + "عملش": [{ORTH: "عمل", NORM: "عمل"}, {ORTH: "ش", NORM: "ش"}], + "عملی‌اند": [{ORTH: "عملی‌", NORM: "عملی‌"}, {ORTH: "اند", NORM: "اند"}], + "عمویت": [{ORTH: "عموی", NORM: "عموی"}, {ORTH: "ت", NORM: "ت"}], + "عمویش": [{ORTH: "عموی", NORM: "عموی"}, {ORTH: "ش", NORM: "ش"}], + "عمیقش": [{ORTH: "عمیق", NORM: "عمیق"}, {ORTH: "ش", NORM: "ش"}], + "عواملش": [{ORTH: "عوامل", NORM: "عوامل"}, {ORTH: "ش", NORM: "ش"}], + "عوضشان": [{ORTH: "عوض", NORM: "عوض"}, {ORTH: "شان", NORM: "شان"}], + "غذایی‌شان": [{ORTH: "غذایی‌", NORM: "غذایی‌"}, {ORTH: "شان", NORM: "شان"}], + "غریبه‌اند": [{ORTH: "غریبه‌", NORM: "غریبه‌"}, {ORTH: "اند", NORM: "اند"}], + "غلامانش": [{ORTH: "غلامان", NORM: "غلامان"}, {ORTH: "ش", NORM: "ش"}], + "غلطهاست": [{ORTH: "غلطها", NORM: "غلطها"}, {ORTH: "ست", NORM: "ست"}], + "فراموشتان": [{ORTH: "فراموش", NORM: "فراموش"}, {ORTH: "تان", NORM: "تان"}], + "فردی‌اند": [{ORTH: "فردی‌", NORM: "فردی‌"}, {ORTH: "اند", NORM: "اند"}], + "فرزندانش": [{ORTH: "فرزندان", NORM: "فرزندان"}, {ORTH: "ش", NORM: "ش"}], + "فرزندش": [{ORTH: "فرزند", NORM: "فرزند"}, {ORTH: "ش", NORM: "ش"}], + "فرم‌هایش": [{ORTH: "فرم‌های", NORM: "فرم‌های"}, {ORTH: "ش", NORM: "ش"}], + "فرهنگی‌مان": [{ORTH: "فرهنگی‌", NORM: "فرهنگی‌"}, {ORTH: "مان", NORM: "مان"}], + "فریادشان": [{ORTH: "فریاد", NORM: "فریاد"}, {ORTH: "شان", NORM: "شان"}], + "فضایی‌شان": [{ORTH: "فضایی‌", NORM: "فضایی‌"}, {ORTH: "شان", NORM: "شان"}], + "فقیرشان": [{ORTH: "فقیر", NORM: "فقیر"}, {ORTH: "شان", NORM: "شان"}], + "فوری‌شان": [{ORTH: "فوری‌", NORM: "فوری‌"}, {ORTH: "شان", NORM: "شان"}], + "قائلند": [{ORTH: "قائل", NORM: "قائل"}, {ORTH: "ند", NORM: "ند"}], + "قائلیم": [{ORTH: "قائل", NORM: "قائل"}, {ORTH: "یم", NORM: "یم"}], + "قادرند": [{ORTH: "قادر", NORM: "قادر"}, {ORTH: "ند", NORM: "ند"}], + "قانونمندش": [{ORTH: "قانونمند", NORM: "قانونمند"}, {ORTH: "ش", NORM: "ش"}], + "قبلند": [{ORTH: "قبل", NORM: "قبل"}, {ORTH: "ند", NORM: "ند"}], + "قبلی‌اش": [{ORTH: "قبلی‌", NORM: "قبلی‌"}, {ORTH: "اش", NORM: "اش"}], + "قبلی‌مان": [{ORTH: "قبلی‌", NORM: "قبلی‌"}, {ORTH: "مان", NORM: "مان"}], + "قدریست": [{ORTH: "قدری", NORM: "قدری"}, {ORTH: "ست", NORM: "ست"}], + "قدمش": [{ORTH: "قدم", NORM: "قدم"}, {ORTH: "ش", NORM: "ش"}], + "قسمتش": [{ORTH: "قسمت", NORM: "قسمت"}, {ORTH: "ش", NORM: "ش"}], + "قضایاست": [{ORTH: "قضایا", NORM: "قضایا"}, {ORTH: "ست", NORM: "ست"}], + "قضیه‌شان": [{ORTH: "قضیه‌", NORM: "قضیه‌"}, {ORTH: "شان", NORM: "شان"}], + "قهرمانهایشان": [ + {ORTH: "قهرمانهای", NORM: "قهرمانهای"}, + {ORTH: "شان", NORM: "شان"}, + ], + "قهرمانیش": [{ORTH: "قهرمانی", NORM: "قهرمانی"}, {ORTH: "ش", NORM: "ش"}], + "قومت": [{ORTH: "قوم", NORM: "قوم"}, {ORTH: "ت", NORM: "ت"}], + "لازمه‌اش": [{ORTH: "لازمه‌", NORM: "لازمه‌"}, {ORTH: "اش", NORM: "اش"}], + "مأموریتش": [{ORTH: "مأموریت", NORM: "مأموریت"}, {ORTH: "ش", NORM: "ش"}], + "مأموریتم": [{ORTH: "مأموریت", NORM: "مأموریت"}, {ORTH: "م", NORM: "م"}], + "مأموریت‌اند": [{ORTH: "مأموریت‌", NORM: "مأموریت‌"}, {ORTH: "اند", NORM: "اند"}], + "مادرانشان": [{ORTH: "مادران", NORM: "مادران"}, {ORTH: "شان", NORM: "شان"}], + "مادرت": [{ORTH: "مادر", NORM: "مادر"}, {ORTH: "ت", NORM: "ت"}], + "مادرش": [{ORTH: "مادر", NORM: "مادر"}, {ORTH: "ش", NORM: "ش"}], + "مادرم": [{ORTH: "مادر", NORM: "مادر"}, {ORTH: "م", NORM: "م"}], + "ماست": [{ORTH: "ما", NORM: "ما"}, {ORTH: "ست", NORM: "ست"}], + "مالی‌اش": [{ORTH: "مالی‌", NORM: "مالی‌"}, {ORTH: "اش", NORM: "اش"}], + "ماهیتش": [{ORTH: "ماهیت", NORM: "ماهیت"}, {ORTH: "ش", NORM: "ش"}], + "مایی": [{ORTH: "ما", NORM: "ما"}, {ORTH: "یی", NORM: "یی"}], + "مجازاتش": [{ORTH: "مجازات", NORM: "مجازات"}, {ORTH: "ش", NORM: "ش"}], + "مجبورند": [{ORTH: "مجبور", NORM: "مجبور"}, {ORTH: "ند", NORM: "ند"}], + "محتاجند": [{ORTH: "محتاج", NORM: "محتاج"}, {ORTH: "ند", NORM: "ند"}], + "محرمم": [{ORTH: "محرم", NORM: "محرم"}, {ORTH: "م", NORM: "م"}], + "محلش": [{ORTH: "محل", NORM: "محل"}, {ORTH: "ش", NORM: "ش"}], + "مخالفند": [{ORTH: "مخالف", NORM: "مخالف"}, {ORTH: "ند", NORM: "ند"}], + "مخدرش": [{ORTH: "مخدر", NORM: "مخدر"}, {ORTH: "ش", NORM: "ش"}], + "مدتهاست": [{ORTH: "مدتها", NORM: "مدتها"}, {ORTH: "ست", NORM: "ست"}], + "مدرسه‌ات": [{ORTH: "مدرسه", NORM: "مدرسه"}, {ORTH: "‌ات", NORM: "ات"}], + "مدرکم": [{ORTH: "مدرک", NORM: "مدرک"}, {ORTH: "م", NORM: "م"}], + "مدیرانش": [{ORTH: "مدیران", NORM: "مدیران"}, {ORTH: "ش", NORM: "ش"}], + "مدیونم": [{ORTH: "مدیون", NORM: "مدیون"}, {ORTH: "م", NORM: "م"}], + "مذهبی‌اند": [{ORTH: "مذهبی‌", NORM: "مذهبی‌"}, {ORTH: "اند", NORM: "اند"}], + "مرا": [{ORTH: "م", NORM: "م"}, {ORTH: "را", NORM: "را"}], + "مرادت": [{ORTH: "مراد", NORM: "مراد"}, {ORTH: "ت", NORM: "ت"}], + "مردمشان": [{ORTH: "مردم", NORM: "مردم"}, {ORTH: "شان", NORM: "شان"}], + "مردمند": [{ORTH: "مردم", NORM: "مردم"}, {ORTH: "ند", NORM: "ند"}], + "مردم‌اند": [{ORTH: "مردم‌", NORM: "مردم‌"}, {ORTH: "اند", NORM: "اند"}], + "مرزشان": [{ORTH: "مرز", NORM: "مرز"}, {ORTH: "شان", NORM: "شان"}], + "مرزهاشان": [{ORTH: "مرزها", NORM: "مرزها"}, {ORTH: "شان", NORM: "شان"}], + "مزدورش": [{ORTH: "مزدور", NORM: "مزدور"}, {ORTH: "ش", NORM: "ش"}], + "مسئولیتش": [{ORTH: "مسئولیت", NORM: "مسئولیت"}, {ORTH: "ش", NORM: "ش"}], + "مسائلش": [{ORTH: "مسائل", NORM: "مسائل"}, {ORTH: "ش", NORM: "ش"}], + "مستحضرید": [{ORTH: "مستحضر", NORM: "مستحضر"}, {ORTH: "ید", NORM: "ید"}], + "مسلمانم": [{ORTH: "مسلمان", NORM: "مسلمان"}, {ORTH: "م", NORM: "م"}], + "مسلمانند": [{ORTH: "مسلمان", NORM: "مسلمان"}, {ORTH: "ند", NORM: "ند"}], + "مشتریانش": [{ORTH: "مشتریان", NORM: "مشتریان"}, {ORTH: "ش", NORM: "ش"}], + "مشتهایمان": [{ORTH: "مشتهای", NORM: "مشتهای"}, {ORTH: "مان", NORM: "مان"}], + "مشخصند": [{ORTH: "مشخص", NORM: "مشخص"}, {ORTH: "ند", NORM: "ند"}], + "مشغولند": [{ORTH: "مشغول", NORM: "مشغول"}, {ORTH: "ند", NORM: "ند"}], + "مشغولیم": [{ORTH: "مشغول", NORM: "مشغول"}, {ORTH: "یم", NORM: "یم"}], + "مشهورش": [{ORTH: "مشهور", NORM: "مشهور"}, {ORTH: "ش", NORM: "ش"}], + "مشکلاتشان": [{ORTH: "مشکلات", NORM: "مشکلات"}, {ORTH: "شان", NORM: "شان"}], + "مشکلم": [{ORTH: "مشکل", NORM: "مشکل"}, {ORTH: "م", NORM: "م"}], + "مطمئنم": [{ORTH: "مطمئن", NORM: "مطمئن"}, {ORTH: "م", NORM: "م"}], + "معامله‌مان": [{ORTH: "معامله‌", NORM: "معامله‌"}, {ORTH: "مان", NORM: "مان"}], + "معتقدم": [{ORTH: "معتقد", NORM: "معتقد"}, {ORTH: "م", NORM: "م"}], + "معتقدند": [{ORTH: "معتقد", NORM: "معتقد"}, {ORTH: "ند", NORM: "ند"}], + "معتقدیم": [{ORTH: "معتقد", NORM: "معتقد"}, {ORTH: "یم", NORM: "یم"}], + "معرفی‌اش": [{ORTH: "معرفی‌", NORM: "معرفی‌"}, {ORTH: "اش", NORM: "اش"}], + "معروفش": [{ORTH: "معروف", NORM: "معروف"}, {ORTH: "ش", NORM: "ش"}], + "معضلاتمان": [{ORTH: "معضلات", NORM: "معضلات"}, {ORTH: "مان", NORM: "مان"}], + "معلمش": [{ORTH: "معلم", NORM: "معلم"}, {ORTH: "ش", NORM: "ش"}], + "معنایش": [{ORTH: "معنای", NORM: "معنای"}, {ORTH: "ش", NORM: "ش"}], + "مغزشان": [{ORTH: "مغز", NORM: "مغز"}, {ORTH: "شان", NORM: "شان"}], + "مفیدند": [{ORTH: "مفید", NORM: "مفید"}, {ORTH: "ند", NORM: "ند"}], + "مقابلش": [{ORTH: "مقابل", NORM: "مقابل"}, {ORTH: "ش", NORM: "ش"}], + "مقاله‌اش": [{ORTH: "مقاله‌", NORM: "مقاله‌"}, {ORTH: "اش", NORM: "اش"}], + "مقدمش": [{ORTH: "مقدم", NORM: "مقدم"}, {ORTH: "ش", NORM: "ش"}], + "مقرش": [{ORTH: "مقر", NORM: "مقر"}, {ORTH: "ش", NORM: "ش"}], + "مقصدشان": [{ORTH: "مقصد", NORM: "مقصد"}, {ORTH: "شان", NORM: "شان"}], + "مقصرند": [{ORTH: "مقصر", NORM: "مقصر"}, {ORTH: "ند", NORM: "ند"}], + "مقصودتان": [{ORTH: "مقصود", NORM: "مقصود"}, {ORTH: "تان", NORM: "تان"}], + "ملاقاتهایش": [{ORTH: "ملاقاتهای", NORM: "ملاقاتهای"}, {ORTH: "ش", NORM: "ش"}], + "ممکنشان": [{ORTH: "ممکن", NORM: "ممکن"}, {ORTH: "شان", NORM: "شان"}], + "ممیزیهاست": [{ORTH: "ممیزیها", NORM: "ممیزیها"}, {ORTH: "ست", NORM: "ست"}], + "منظورم": [{ORTH: "منظور", NORM: "منظور"}, {ORTH: "م", NORM: "م"}], + "منی": [{ORTH: "من", NORM: "من"}, {ORTH: "ی", NORM: "ی"}], + "منید": [{ORTH: "من", NORM: "من"}, {ORTH: "ید", NORM: "ید"}], + "مهربانش": [{ORTH: "مهربان", NORM: "مهربان"}, {ORTH: "ش", NORM: "ش"}], + "مهم‌اند": [{ORTH: "مهم‌", NORM: "مهم‌"}, {ORTH: "اند", NORM: "اند"}], + "مواجهند": [{ORTH: "مواجه", NORM: "مواجه"}, {ORTH: "ند", NORM: "ند"}], + "مواجه‌اند": [{ORTH: "مواجه‌", NORM: "مواجه‌"}, {ORTH: "اند", NORM: "اند"}], + "مواخذه‌ات": [{ORTH: "مواخذه", NORM: "مواخذه"}, {ORTH: "‌ات", NORM: "ات"}], + "مواضعشان": [{ORTH: "مواضع", NORM: "مواضع"}, {ORTH: "شان", NORM: "شان"}], + "مواضعمان": [{ORTH: "مواضع", NORM: "مواضع"}, {ORTH: "مان", NORM: "مان"}], + "موافقند": [{ORTH: "موافق", NORM: "موافق"}, {ORTH: "ند", NORM: "ند"}], + "موجوداتش": [{ORTH: "موجودات", NORM: "موجودات"}, {ORTH: "ش", NORM: "ش"}], + "موجودند": [{ORTH: "موجود", NORM: "موجود"}, {ORTH: "ند", NORM: "ند"}], + "موردش": [{ORTH: "مورد", NORM: "مورد"}, {ORTH: "ش", NORM: "ش"}], + "موضعشان": [{ORTH: "موضع", NORM: "موضع"}, {ORTH: "شان", NORM: "شان"}], + "موظفند": [{ORTH: "موظف", NORM: "موظف"}, {ORTH: "ند", NORM: "ند"}], + "موهایش": [{ORTH: "موهای", NORM: "موهای"}, {ORTH: "ش", NORM: "ش"}], + "موهایمان": [{ORTH: "موهای", NORM: "موهای"}, {ORTH: "مان", NORM: "مان"}], + "مویم": [{ORTH: "مو", NORM: "مو"}, {ORTH: "یم", NORM: "یم"}], + "ناخرسندند": [{ORTH: "ناخرسند", NORM: "ناخرسند"}, {ORTH: "ند", NORM: "ند"}], + "ناراحتیش": [{ORTH: "ناراحتی", NORM: "ناراحتی"}, {ORTH: "ش", NORM: "ش"}], + "ناراضی‌اند": [{ORTH: "ناراضی‌", NORM: "ناراضی‌"}, {ORTH: "اند", NORM: "اند"}], + "نارواست": [{ORTH: "ناروا", NORM: "ناروا"}, {ORTH: "ست", NORM: "ست"}], + "نازش": [{ORTH: "ناز", NORM: "ناز"}, {ORTH: "ش", NORM: "ش"}], + "نامش": [{ORTH: "نام", NORM: "نام"}, {ORTH: "ش", NORM: "ش"}], + "نامشان": [{ORTH: "نام", NORM: "نام"}, {ORTH: "شان", NORM: "شان"}], + "نامم": [{ORTH: "نام", NORM: "نام"}, {ORTH: "م", NORM: "م"}], + "نامه‌ات": [{ORTH: "نامه", NORM: "نامه"}, {ORTH: "‌ات", NORM: "ات"}], + "نامه‌ام": [{ORTH: "نامه‌", NORM: "نامه‌"}, {ORTH: "ام", NORM: "ام"}], + "ناچارم": [{ORTH: "ناچار", NORM: "ناچار"}, {ORTH: "م", NORM: "م"}], + "نخست‌وزیری‌اش": [ + {ORTH: "نخست‌وزیری‌", NORM: "نخست‌وزیری‌"}, + {ORTH: "اش", NORM: "اش"}, + ], + "نزدش": [{ORTH: "نزد", NORM: "نزد"}, {ORTH: "ش", NORM: "ش"}], + "نشانم": [{ORTH: "نشان", NORM: "نشان"}, {ORTH: "م", NORM: "م"}], + "نظرات‌شان": [{ORTH: "نظرات‌", NORM: "نظرات‌"}, {ORTH: "شان", NORM: "شان"}], + "نظرتان": [{ORTH: "نظر", NORM: "نظر"}, {ORTH: "تان", NORM: "تان"}], + "نظرش": [{ORTH: "نظر", NORM: "نظر"}, {ORTH: "ش", NORM: "ش"}], + "نظرشان": [{ORTH: "نظر", NORM: "نظر"}, {ORTH: "شان", NORM: "شان"}], + "نظرم": [{ORTH: "نظر", NORM: "نظر"}, {ORTH: "م", NORM: "م"}], + "نظرهایشان": [{ORTH: "نظرهای", NORM: "نظرهای"}, {ORTH: "شان", NORM: "شان"}], + "نفاقش": [{ORTH: "نفاق", NORM: "نفاق"}, {ORTH: "ش", NORM: "ش"}], + "نفرند": [{ORTH: "نفر", NORM: "نفر"}, {ORTH: "ند", NORM: "ند"}], + "نفوذیند": [{ORTH: "نفوذی", NORM: "نفوذی"}, {ORTH: "ند", NORM: "ند"}], + "نقطه‌نظراتتان": [ + {ORTH: "نقطه‌نظرات", NORM: "نقطه‌نظرات"}, + {ORTH: "تان", NORM: "تان"}, + ], + "نمایشی‌مان": [{ORTH: "نمایشی‌", NORM: "نمایشی‌"}, {ORTH: "مان", NORM: "مان"}], + "نمایندگی‌شان": [ + {ORTH: "نمایندگی‌", NORM: "نمایندگی‌"}, + {ORTH: "شان", NORM: "شان"}, + ], + "نمونه‌اش": [{ORTH: "نمونه‌", NORM: "نمونه‌"}, {ORTH: "اش", NORM: "اش"}], + "نمی‌پذیرندش": [{ORTH: "نمی‌پذیرند", NORM: "نمی‌پذیرند"}, {ORTH: "ش", NORM: "ش"}], + "نوآوری‌اش": [{ORTH: "نوآوری‌", NORM: "نوآوری‌"}, {ORTH: "اش", NORM: "اش"}], + "نوشته‌هایشان": [ + {ORTH: "نوشته‌های", NORM: "نوشته‌های"}, + {ORTH: "شان", NORM: "شان"}, + ], + "نوشته‌هایم": [{ORTH: "نوشته‌ها", NORM: "نوشته‌ها"}, {ORTH: "یم", NORM: "یم"}], + "نکردنشان": [{ORTH: "نکردن", NORM: "نکردن"}, {ORTH: "شان", NORM: "شان"}], + "نگاهداری‌شان": [ + {ORTH: "نگاهداری‌", NORM: "نگاهداری‌"}, + {ORTH: "شان", NORM: "شان"}, + ], + "نگاهش": [{ORTH: "نگاه", NORM: "نگاه"}, {ORTH: "ش", NORM: "ش"}], + "نگرانم": [{ORTH: "نگران", NORM: "نگران"}, {ORTH: "م", NORM: "م"}], + "نگرشهایشان": [{ORTH: "نگرشهای", NORM: "نگرشهای"}, {ORTH: "شان", NORM: "شان"}], + "نیازمندند": [{ORTH: "نیازمند", NORM: "نیازمند"}, {ORTH: "ند", NORM: "ند"}], + "هدفش": [{ORTH: "هدف", NORM: "هدف"}, {ORTH: "ش", NORM: "ش"}], + "همانست": [{ORTH: "همان", NORM: "همان"}, {ORTH: "ست", NORM: "ست"}], + "همراهش": [{ORTH: "همراه", NORM: "همراه"}, {ORTH: "ش", NORM: "ش"}], + "همسرتان": [{ORTH: "همسر", NORM: "همسر"}, {ORTH: "تان", NORM: "تان"}], + "همسرش": [{ORTH: "همسر", NORM: "همسر"}, {ORTH: "ش", NORM: "ش"}], + "همسرم": [{ORTH: "همسر", NORM: "همسر"}, {ORTH: "م", NORM: "م"}], + "همفکرانش": [{ORTH: "همفکران", NORM: "همفکران"}, {ORTH: "ش", NORM: "ش"}], + "همه‌اش": [{ORTH: "همه‌", NORM: "همه‌"}, {ORTH: "اش", NORM: "اش"}], + "همه‌شان": [{ORTH: "همه‌", NORM: "همه‌"}, {ORTH: "شان", NORM: "شان"}], + "همکارانش": [{ORTH: "همکاران", NORM: "همکاران"}, {ORTH: "ش", NORM: "ش"}], + "هم‌نظریم": [{ORTH: "هم‌نظر", NORM: "هم‌نظر"}, {ORTH: "یم", NORM: "یم"}], + "هنرش": [{ORTH: "هنر", NORM: "هنر"}, {ORTH: "ش", NORM: "ش"}], + "هواست": [{ORTH: "هوا", NORM: "هوا"}, {ORTH: "ست", NORM: "ست"}], + "هویتش": [{ORTH: "هویت", NORM: "هویت"}, {ORTH: "ش", NORM: "ش"}], + "وابسته‌اند": [{ORTH: "وابسته‌", NORM: "وابسته‌"}, {ORTH: "اند", NORM: "اند"}], + "واقفند": [{ORTH: "واقف", NORM: "واقف"}, {ORTH: "ند", NORM: "ند"}], + "والدینشان": [{ORTH: "والدین", NORM: "والدین"}, {ORTH: "شان", NORM: "شان"}], + "وجدان‌تان": [{ORTH: "وجدان‌", NORM: "وجدان‌"}, {ORTH: "تان", NORM: "تان"}], + "وجودشان": [{ORTH: "وجود", NORM: "وجود"}, {ORTH: "شان", NORM: "شان"}], + "وطنم": [{ORTH: "وطن", NORM: "وطن"}, {ORTH: "م", NORM: "م"}], + "وعده‌اش": [{ORTH: "وعده‌", NORM: "وعده‌"}, {ORTH: "اش", NORM: "اش"}], + "وقتمان": [{ORTH: "وقت", NORM: "وقت"}, {ORTH: "مان", NORM: "مان"}], + "ولادتش": [{ORTH: "ولادت", NORM: "ولادت"}, {ORTH: "ش", NORM: "ش"}], + "پایانش": [{ORTH: "پایان", NORM: "پایان"}, {ORTH: "ش", NORM: "ش"}], + "پایش": [{ORTH: "پای", NORM: "پای"}, {ORTH: "ش", NORM: "ش"}], + "پایین‌ترند": [{ORTH: "پایین‌تر", NORM: "پایین‌تر"}, {ORTH: "ند", NORM: "ند"}], + "پدرت": [{ORTH: "پدر", NORM: "پدر"}, {ORTH: "ت", NORM: "ت"}], + "پدرش": [{ORTH: "پدر", NORM: "پدر"}, {ORTH: "ش", NORM: "ش"}], + "پدرشان": [{ORTH: "پدر", NORM: "پدر"}, {ORTH: "شان", NORM: "شان"}], + "پدرم": [{ORTH: "پدر", NORM: "پدر"}, {ORTH: "م", NORM: "م"}], + "پربارش": [{ORTH: "پربار", NORM: "پربار"}, {ORTH: "ش", NORM: "ش"}], + "پروردگارت": [{ORTH: "پروردگار", NORM: "پروردگار"}, {ORTH: "ت", NORM: "ت"}], + "پسرتان": [{ORTH: "پسر", NORM: "پسر"}, {ORTH: "تان", NORM: "تان"}], + "پسرش": [{ORTH: "پسر", NORM: "پسر"}, {ORTH: "ش", NORM: "ش"}], + "پسرعمویش": [{ORTH: "پسرعموی", NORM: "پسرعموی"}, {ORTH: "ش", NORM: "ش"}], + "پسر‌عمویت": [{ORTH: "پسر‌عموی", NORM: "پسر‌عموی"}, {ORTH: "ت", NORM: "ت"}], + "پشتش": [{ORTH: "پشت", NORM: "پشت"}, {ORTH: "ش", NORM: "ش"}], + "پشیمونی": [{ORTH: "پشیمون", NORM: "پشیمون"}, {ORTH: "ی", NORM: "ی"}], + "پولش": [{ORTH: "پول", NORM: "پول"}, {ORTH: "ش", NORM: "ش"}], + "پژوهش‌هایش": [{ORTH: "پژوهش‌های", NORM: "پژوهش‌های"}, {ORTH: "ش", NORM: "ش"}], + "پیامبرش": [{ORTH: "پیامبر", NORM: "پیامبر"}, {ORTH: "ش", NORM: "ش"}], + "پیامبری": [{ORTH: "پیامبر", NORM: "پیامبر"}, {ORTH: "ی", NORM: "ی"}], + "پیامش": [{ORTH: "پیام", NORM: "پیام"}, {ORTH: "ش", NORM: "ش"}], + "پیداست": [{ORTH: "پیدا", NORM: "پیدا"}, {ORTH: "ست", NORM: "ست"}], + "پیراهنش": [{ORTH: "پیراهن", NORM: "پیراهن"}, {ORTH: "ش", NORM: "ش"}], + "پیروانش": [{ORTH: "پیروان", NORM: "پیروان"}, {ORTH: "ش", NORM: "ش"}], + "پیشانی‌اش": [{ORTH: "پیشانی‌", NORM: "پیشانی‌"}, {ORTH: "اش", NORM: "اش"}], + "پیمانت": [{ORTH: "پیمان", NORM: "پیمان"}, {ORTH: "ت", NORM: "ت"}], + "پیوندشان": [{ORTH: "پیوند", NORM: "پیوند"}, {ORTH: "شان", NORM: "شان"}], + "چاپش": [{ORTH: "چاپ", NORM: "چاپ"}, {ORTH: "ش", NORM: "ش"}], + "چت": [{ORTH: "چ", NORM: "چ"}, {ORTH: "ت", NORM: "ت"}], + "چته": [{ORTH: "چ", NORM: "چ"}, {ORTH: "ت", NORM: "ت"}, {ORTH: "ه", NORM: "ه"}], + "چرخ‌هایش": [{ORTH: "چرخ‌های", NORM: "چرخ‌های"}, {ORTH: "ش", NORM: "ش"}], + "چشمم": [{ORTH: "چشم", NORM: "چشم"}, {ORTH: "م", NORM: "م"}], + "چشمهایش": [{ORTH: "چشمهای", NORM: "چشمهای"}, {ORTH: "ش", NORM: "ش"}], + "چشمهایشان": [{ORTH: "چشمهای", NORM: "چشمهای"}, {ORTH: "شان", NORM: "شان"}], + "چمنم": [{ORTH: "چمن", NORM: "چمن"}, {ORTH: "م", NORM: "م"}], + "چهره‌اش": [{ORTH: "چهره‌", NORM: "چهره‌"}, {ORTH: "اش", NORM: "اش"}], + "چکاره‌اند": [{ORTH: "چکاره‌", NORM: "چکاره‌"}, {ORTH: "اند", NORM: "اند"}], + "چیزهاست": [{ORTH: "چیزها", NORM: "چیزها"}, {ORTH: "ست", NORM: "ست"}], + "چیزهایش": [{ORTH: "چیزهای", NORM: "چیزهای"}, {ORTH: "ش", NORM: "ش"}], + "چیزیست": [{ORTH: "چیزی", NORM: "چیزی"}, {ORTH: "ست", NORM: "ست"}], + "چیست": [{ORTH: "چی", NORM: "چی"}, {ORTH: "ست", NORM: "ست"}], + "کارش": [{ORTH: "کار", NORM: "کار"}, {ORTH: "ش", NORM: "ش"}], + "کارشان": [{ORTH: "کار", NORM: "کار"}, {ORTH: "شان", NORM: "شان"}], + "کارم": [{ORTH: "کار", NORM: "کار"}, {ORTH: "م", NORM: "م"}], + "کارند": [{ORTH: "کار", NORM: "کار"}, {ORTH: "ند", NORM: "ند"}], + "کارهایم": [{ORTH: "کارها", NORM: "کارها"}, {ORTH: "یم", NORM: "یم"}], + "کافیست": [{ORTH: "کافی", NORM: "کافی"}, {ORTH: "ست", NORM: "ست"}], + "کتابخانه‌اش": [{ORTH: "کتابخانه‌", NORM: "کتابخانه‌"}, {ORTH: "اش", NORM: "اش"}], + "کتابش": [{ORTH: "کتاب", NORM: "کتاب"}, {ORTH: "ش", NORM: "ش"}], + "کتابهاشان": [{ORTH: "کتابها", NORM: "کتابها"}, {ORTH: "شان", NORM: "شان"}], + "کجاست": [{ORTH: "کجا", NORM: "کجا"}, {ORTH: "ست", NORM: "ست"}], + "کدورتهایشان": [{ORTH: "کدورتهای", NORM: "کدورتهای"}, {ORTH: "شان", NORM: "شان"}], + "کردنش": [{ORTH: "کردن", NORM: "کردن"}, {ORTH: "ش", NORM: "ش"}], + "کرم‌خورده‌اش": [ + {ORTH: "کرم‌خورده‌", NORM: "کرم‌خورده‌"}, + {ORTH: "اش", NORM: "اش"}, + ], + "کشش": [{ORTH: "کش", NORM: "کش"}, {ORTH: "ش", NORM: "ش"}], + "کشورش": [{ORTH: "کشور", NORM: "کشور"}, {ORTH: "ش", NORM: "ش"}], + "کشورشان": [{ORTH: "کشور", NORM: "کشور"}, {ORTH: "شان", NORM: "شان"}], + "کشورمان": [{ORTH: "کشور", NORM: "کشور"}, {ORTH: "مان", NORM: "مان"}], + "کشورهاست": [{ORTH: "کشورها", NORM: "کشورها"}, {ORTH: "ست", NORM: "ست"}], + "کلیشه‌هاست": [{ORTH: "کلیشه‌ها", NORM: "کلیشه‌ها"}, {ORTH: "ست", NORM: "ست"}], + "کمبودهاست": [{ORTH: "کمبودها", NORM: "کمبودها"}, {ORTH: "ست", NORM: "ست"}], + "کمتره": [{ORTH: "کمتر", NORM: "کمتر"}, {ORTH: "ه", NORM: "ه"}], + "کمکم": [{ORTH: "کمک", NORM: "کمک"}, {ORTH: "م", NORM: "م"}], + "کنارش": [{ORTH: "کنار", NORM: "کنار"}, {ORTH: "ش", NORM: "ش"}], + "کودکانشان": [{ORTH: "کودکان", NORM: "کودکان"}, {ORTH: "شان", NORM: "شان"}], + "کوچکش": [{ORTH: "کوچک", NORM: "کوچک"}, {ORTH: "ش", NORM: "ش"}], + "کیست": [{ORTH: "کی", NORM: "کی"}, {ORTH: "ست", NORM: "ست"}], + "کیفش": [{ORTH: "کیف", NORM: "کیف"}, {ORTH: "ش", NORM: "ش"}], + "گذشته‌اند": [{ORTH: "گذشته‌", NORM: "گذشته‌"}, {ORTH: "اند", NORM: "اند"}], + "گرانقدرش": [{ORTH: "گرانقدر", NORM: "گرانقدر"}, {ORTH: "ش", NORM: "ش"}], + "گرانقدرشان": [{ORTH: "گرانقدر", NORM: "گرانقدر"}, {ORTH: "شان", NORM: "شان"}], + "گردنتان": [{ORTH: "گردن", NORM: "گردن"}, {ORTH: "تان", NORM: "تان"}], + "گردنش": [{ORTH: "گردن", NORM: "گردن"}, {ORTH: "ش", NORM: "ش"}], + "گرفتارند": [{ORTH: "گرفتار", NORM: "گرفتار"}, {ORTH: "ند", NORM: "ند"}], + "گرفتنت": [{ORTH: "گرفتن", NORM: "گرفتن"}, {ORTH: "ت", NORM: "ت"}], + "گروهند": [{ORTH: "گروه", NORM: "گروه"}, {ORTH: "ند", NORM: "ند"}], + "گروگانهایش": [{ORTH: "گروگانهای", NORM: "گروگانهای"}, {ORTH: "ش", NORM: "ش"}], + "گریمش": [{ORTH: "گریم", NORM: "گریم"}, {ORTH: "ش", NORM: "ش"}], + "گفتارمان": [{ORTH: "گفتار", NORM: "گفتار"}, {ORTH: "مان", NORM: "مان"}], + "گلهایش": [{ORTH: "گلهای", NORM: "گلهای"}, {ORTH: "ش", NORM: "ش"}], + "گلویش": [{ORTH: "گلوی", NORM: "گلوی"}, {ORTH: "ش", NORM: "ش"}], + "گناهت": [{ORTH: "گناه", NORM: "گناه"}, {ORTH: "ت", NORM: "ت"}], + "گوشش": [{ORTH: "گوش", NORM: "گوش"}, {ORTH: "ش", NORM: "ش"}], + "گوشم": [{ORTH: "گوش", NORM: "گوش"}, {ORTH: "م", NORM: "م"}], + "گولش": [{ORTH: "گول", NORM: "گول"}, {ORTH: "ش", NORM: "ش"}], + "یادتان": [{ORTH: "یاد", NORM: "یاد"}, {ORTH: "تان", NORM: "تان"}], + "یادم": [{ORTH: "یاد", NORM: "یاد"}, {ORTH: "م", NORM: "م"}], + "یادمان": [{ORTH: "یاد", NORM: "یاد"}, {ORTH: "مان", NORM: "مان"}], + "یارانش": [{ORTH: "یاران", NORM: "یاران"}, {ORTH: "ش", NORM: "ش"}], +} diff --git a/spacy/lang/fi/__init__.py b/spacy/lang/fi/__init__.py new file mode 100644 index 0000000..3e371b9 --- /dev/null +++ b/spacy/lang/fi/__init__.py @@ -0,0 +1,23 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class FinnishDefaults(BaseDefaults): + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + syntax_iterators = SYNTAX_ITERATORS + + +class Finnish(Language): + lang = "fi" + Defaults = FinnishDefaults + + +__all__ = ["Finnish"] diff --git a/spacy/lang/fi/examples.py b/spacy/lang/fi/examples.py new file mode 100644 index 0000000..930fac2 --- /dev/null +++ b/spacy/lang/fi/examples.py @@ -0,0 +1,15 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.fi.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Itseajavat autot siirtävät vakuutusvastuun autojen valmistajille", + "San Francisco harkitsee toimitusrobottien liikkumisen kieltämistä jalkakäytävillä", + "Lontoo on suuri kaupunki Yhdistyneessä Kuningaskunnassa.", + "Missä sinä olet?", + "Mikä on Yhdysvaltojen pääkaupunki?", + "Kuka on Suomen presidentti?", + "Milloin Sauli Niinistö on syntynyt?", +] diff --git a/spacy/lang/fi/lex_attrs.py b/spacy/lang/fi/lex_attrs.py new file mode 100644 index 0000000..9eec41b --- /dev/null +++ b/spacy/lang/fi/lex_attrs.py @@ -0,0 +1,55 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nolla", + "yksi", + "kaksi", + "kolme", + "neljä", + "viisi", + "kuusi", + "seitsemän", + "kahdeksan", + "yhdeksän", + "kymmenen", + "yksitoista", + "kaksitoista", + "kolmetoista", + "neljätoista", + "viisitoista", + "kuusitoista", + "seitsemäntoista", + "kahdeksantoista", + "yhdeksäntoista", + "kaksikymmentä", + "kolmekymmentä", + "neljäkymmentä", + "viisikymmentä", + "kuusikymmentä", + "seitsemänkymmentä", + "kahdeksankymmentä", + "yhdeksänkymmentä", + "sata", + "tuhat", + "miljoona", + "miljardi", + "triljoona", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(".", "").replace(",", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/fi/punctuation.py b/spacy/lang/fi/punctuation.py new file mode 100644 index 0000000..29ddc31 --- /dev/null +++ b/spacy/lang/fi/punctuation.py @@ -0,0 +1,35 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + LIST_ELLIPSES, + LIST_HYPHENS, + LIST_ICONS, +) +from ..punctuation import TOKENIZER_SUFFIXES + +_quotes = CONCAT_QUOTES.replace("'", "") +DASHES = "|".join(x for x in LIST_HYPHENS if x != "-") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])(?:{d})(?=[{a}])".format(a=ALPHA, d=DASHES), + r"(?<=[{a}0-9])[<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = [ + suffix + for suffix in TOKENIZER_SUFFIXES + if suffix not in ["'s", "'S", "’s", "’S", r"\'"] +] + + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/fi/stop_words.py b/spacy/lang/fi/stop_words.py new file mode 100644 index 0000000..8e8dcfa --- /dev/null +++ b/spacy/lang/fi/stop_words.py @@ -0,0 +1,110 @@ +# Source https://github.com/stopwords-iso/stopwords-fi/blob/master/stopwords-fi.txt +# Reformatted with some minor corrections +STOP_WORDS = set( + """ +aiemmin aika aikaa aikaan aikaisemmin aikaisin aikana aikoina aikoo aikovat +aina ainakaan ainakin ainoa ainoat aiomme aion aiotte aivan ajan alas alemmas +alkuisin alkuun alla alle aloitamme aloitan aloitat aloitatte aloitattivat +aloitettava aloitettavaksi aloitettu aloitimme aloitin aloitit aloititte +aloittaa aloittamatta aloitti aloittivat alta aluksi alussa alusta annettavaksi +annettava annettu ansiosta antaa antamatta antoi apu asia asiaa asian asiasta +asiat asioiden asioihin asioita asti avuksi avulla avun avutta + +edelle edelleen edellä edeltä edemmäs edes edessä edestä ehkä ei eikä eilen +eivät eli ellei elleivät ellemme ellen ellet ellette emme en enemmän eniten +ennen ensi ensimmäinen ensimmäiseksi ensimmäisen ensimmäisenä ensimmäiset +ensimmäisiksi ensimmäisinä ensimmäisiä ensimmäistä ensin entinen entisen +entisiä entisten entistä enää eri erittäin erityisesti eräiden eräs eräät esi +esiin esillä esimerkiksi et eteen etenkin ette ettei että + +halua haluaa haluamatta haluamme haluan haluat haluatte haluavat halunnut +halusi halusimme halusin halusit halusitte halusivat halutessa haluton he hei +heidän heidät heihin heille heillä heiltä heissä heistä heitä helposti heti +hetkellä hieman hitaasti huolimatta huomenna hyvien hyviin hyviksi hyville +hyviltä hyvin hyvinä hyvissä hyvistä hyviä hyvä hyvät hyvää hän häneen hänelle +hänellä häneltä hänen hänessä hänestä hänet häntä + +ihan ilman ilmeisesti itse itsensä itseään + +ja jo johon joiden joihin joiksi joilla joille joilta joina joissa joista joita +joka jokainen jokin joko joksi joku jolla jolle jolloin jolta jompikumpi jona +jonka jonkin jonne joo jopa jos joskus jossa josta jota jotain joten jotenkin +jotenkuten jotka jotta jouduimme jouduin jouduit jouduitte joudumme joudun +joudutte joukkoon joukossa joukosta joutua joutui joutuivat joutumaan joutuu +joutuvat juuri jälkeen jälleen jää + +kahdeksan kahdeksannen kahdella kahdelle kahdelta kahden kahdessa kahdesta +kahta kahteen kai kaiken kaikille kaikilta kaikkea kaikki kaikkia kaikkiaan +kaikkialla kaikkialle kaikkialta kaikkien kaikkiin kaksi kannalta kannattaa +kanssa kanssaan kanssamme kanssani kanssanne kanssasi kauan kauemmas kaukana +kautta kehen keiden keihin keiksi keille keillä keiltä keinä keissä keistä +keitten keittä keitä keneen keneksi kenelle kenellä keneltä kenen kenenä +kenessä kenestä kenet kenettä kenties kerran kerta kertaa keskellä kesken +keskimäärin ketkä ketä kiitos kohti koko kokonaan kolmas kolme kolmen kolmesti +koska koskaan kovin kuin kuinka kuinkaan kuitenkaan kuitenkin kuka kukaan kukin +kumpainen kumpainenkaan kumpi kumpikaan kumpikin kun kuten kuuden kuusi kuutta +kylliksi kyllä kymmenen kyse + +liian liki lisäksi lisää lla luo luona lähekkäin lähelle lähellä läheltä +lähemmäs lähes lähinnä lähtien läpi + +mahdollisimman mahdollista me meidän meidät meihin meille meillä meiltä meissä +meistä meitä melkein melko menee menemme menen menet menette menevät meni +menimme menin menit menivät mennessä mennyt menossa mihin miksi mikä mikäli +mikään mille milloin milloinkan millä miltä minkä minne minua minulla minulle +minulta minun minussa minusta minut minuun minä missä mistä miten mitkä mitä +mitään moi molemmat mones monesti monet moni moniaalla moniaalle moniaalta +monta muassa muiden muita muka mukaan mukaansa mukana mutta muu muualla muualle +muualta muuanne muulloin muun muut muuta muutama muutaman muuten myöhemmin myös +myöskin myöskään myötä + +ne neljä neljän neljää niiden niihin niiksi niille niillä niiltä niin niinä +niissä niistä niitä noiden noihin noiksi noilla noille noilta noin noina noissa +noista noita nopeammin nopeasti nopeiten nro nuo nyt näiden näihin näiksi +näille näillä näiltä näin näinä näissä näistä näitä nämä + +ohi oikea oikealla oikein ole olemme olen olet olette oleva olevan olevat oli +olimme olin olisi olisimme olisin olisit olisitte olisivat olit olitte olivat +olla olleet ollut oma omaa omaan omaksi omalle omalta oman omassa omat omia +omien omiin omiksi omille omilta omissa omista on onkin onko ovat + +paikoittain paitsi pakosti paljon paremmin parempi parhaillaan parhaiten +perusteella peräti pian pieneen pieneksi pienelle pienellä pieneltä pienempi +pienestä pieni pienin poikki puolesta puolestaan päälle + +runsaasti + +saakka sama samaa samaan samalla saman samat samoin satojen se +seitsemän sekä sen seuraavat siellä sieltä siihen siinä siis siitä sijaan siksi +sille silloin sillä silti siltä sinne sinua sinulla sinulle sinulta sinun +sinussa sinusta sinut sinuun sinä sisäkkäin sisällä siten sitten sitä ssa sta +suoraan suuntaan suuren suuret suuri suuria suurin suurten + +taa taas taemmas tahansa tai takaa takaisin takana takia tallä tapauksessa +tarpeeksi tavalla tavoitteena te teidän teidät teihin teille teillä teiltä +teissä teistä teitä tietysti todella toinen toisaalla toisaalle toisaalta +toiseen toiseksi toisella toiselle toiselta toisemme toisen toisensa toisessa +toisesta toista toistaiseksi toki tosin tule tulee tulemme tulen +tulet tulette tulevat tulimme tulin tulisi tulisimme tulisin tulisit tulisitte +tulisivat tulit tulitte tulivat tulla tulleet tullut tuntuu tuo tuohon tuoksi +tuolla tuolle tuolloin tuolta tuon tuona tuonne tuossa tuosta tuota tuskin tykö +tähän täksi tälle tällä tällöin tältä tämä tämän tänne tänä tänään tässä tästä +täten tätä täysin täytyvät täytyy täällä täältä + +ulkopuolella usea useasti useimmiten usein useita uudeksi uudelleen uuden uudet +uusi uusia uusien uusinta uuteen uutta + +vaan vai vaiheessa vaikea vaikean vaikeat vaikeilla vaikeille vaikeilta +vaikeissa vaikeista vaikka vain varmasti varsin varsinkin varten vasen +vasemmalla vasta vastaan vastakkain vastan verran vielä vierekkäin vieressä +vieri viiden viime viimeinen viimeisen viimeksi viisi voi voidaan voimme voin +voisi voit voitte voivat vuoden vuoksi vuosi vuosien vuosina vuotta vähemmän +vähintään vähiten vähän välillä + +yhdeksän yhden yhdessä yhteen yhteensä yhteydessä yhteyteen yhtä yhtäälle +yhtäällä yhtäältä yhtään yhä yksi yksin yksittäin yleensä ylemmäs yli ylös +ympäri + +älköön älä +""".split() +) diff --git a/spacy/lang/fi/syntax_iterators.py b/spacy/lang/fi/syntax_iterators.py new file mode 100644 index 0000000..6e22167 --- /dev/null +++ b/spacy/lang/fi/syntax_iterators.py @@ -0,0 +1,80 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on both Doc and Span.""" + labels = [ + "appos", + "nsubj", + "nsubj:cop", + "obj", + "obl", + "ROOT", + ] + extend_labels = [ + "amod", + "compound", + "compound:nn", + "flat:name", + "nmod", + "nmod:gobj", + "nmod:gsubj", + "nmod:poss", + "nummod", + ] + + def potential_np_head(word): + return word.pos in (NOUN, PROPN) and ( + word.dep in np_deps or word.head.pos == PRON + ) + + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + np_deps = [doc.vocab.strings[label] for label in labels] + extend_deps = [doc.vocab.strings[label] for label in extend_labels] + np_label = doc.vocab.strings.add("NP") + conj_label = doc.vocab.strings.add("conj") + + rbracket = 0 + prev_end = -1 + for i, word in enumerate(doclike): + if i < rbracket: + continue + + # Is this a potential independent NP head or coordinated with + # a NOUN that is itself an independent NP head? + # + # e.g. "Terveyden ja hyvinvoinnin laitos" + if potential_np_head(word) or ( + word.dep == conj_label and potential_np_head(word.head) + ): + # Try to extend to the left to include adjective/num + # modifiers, compound words etc. + lbracket = word.i + for ldep in word.lefts: + if ldep.dep in extend_deps: + lbracket = ldep.left_edge.i + break + + # Prevent nested chunks from being produced + if lbracket <= prev_end: + continue + + rbracket = word.i + # Try to extend the span to the right to capture + # appositions and noun modifiers + for rdep in word.rights: + if rdep.dep in extend_deps: + rbracket = rdep.i + prev_end = rbracket + + yield lbracket, rbracket + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/fi/tokenizer_exceptions.py b/spacy/lang/fi/tokenizer_exceptions.py new file mode 100644 index 0000000..0bbd7bd --- /dev/null +++ b/spacy/lang/fi/tokenizer_exceptions.py @@ -0,0 +1,111 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +# Source https://www.cs.tut.fi/~jkorpela/kielenopas/5.5.html +for exc_data in [ + {ORTH: "aik."}, + {ORTH: "alk."}, + {ORTH: "alv."}, + {ORTH: "ark."}, + {ORTH: "as."}, + {ORTH: "eaa."}, + {ORTH: "ed."}, + {ORTH: "esim."}, + {ORTH: "huom."}, + {ORTH: "jne."}, + {ORTH: "joht."}, + {ORTH: "k."}, + {ORTH: "ks."}, + {ORTH: "lk."}, + {ORTH: "lkm."}, + {ORTH: "lyh."}, + {ORTH: "läh."}, + {ORTH: "miel."}, + {ORTH: "milj."}, + {ORTH: "Mm."}, + {ORTH: "mm."}, + {ORTH: "myöh."}, + {ORTH: "n."}, + {ORTH: "nimim."}, + {ORTH: "n:o"}, + {ORTH: "N:o"}, + {ORTH: "nro"}, + {ORTH: "ns."}, + {ORTH: "nyk."}, + {ORTH: "oik."}, + {ORTH: "os."}, + {ORTH: "p."}, + {ORTH: "par."}, + {ORTH: "per."}, + {ORTH: "pj."}, + {ORTH: "puh.joht."}, + {ORTH: "prof."}, + {ORTH: "puh."}, + {ORTH: "pvm."}, + {ORTH: "rak."}, + {ORTH: "ry."}, + {ORTH: "s."}, + {ORTH: "siht."}, + {ORTH: "synt."}, + {ORTH: "t."}, + {ORTH: "tark."}, + {ORTH: "til."}, + {ORTH: "tms."}, + {ORTH: "toim."}, + {ORTH: "v."}, + {ORTH: "vas."}, + {ORTH: "vast."}, + {ORTH: "vrt."}, + {ORTH: "yht."}, + {ORTH: "yl."}, + {ORTH: "ym."}, + {ORTH: "yms."}, + {ORTH: "yo."}, + {ORTH: "yliopp."}, + {ORTH: "ao."}, + {ORTH: "em."}, + {ORTH: "ko."}, + {ORTH: "ml."}, + {ORTH: "po."}, + {ORTH: "so."}, + {ORTH: "ts."}, + {ORTH: "vm."}, + {ORTH: "srk."}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +# Source: https://kaino.kotus.fi/visk/sisallys.php?p=141 +conj_contraction_bases = [ + ("ett", "että"), + ("jott", "jotta"), + ("kosk", "koska"), + ("mutt", "mutta"), + ("vaikk", "vaikka"), + ("ehk", "ehkä"), + ("miks", "miksi"), + ("siks", "siksi"), + ("joll", "jos"), + ("ell", "jos"), +] +conj_contraction_negations = [ + ("en", "en"), + ("et", "et"), + ("ei", "ei"), + ("emme", "emme"), + ("ette", "ette"), + ("eivat", "eivät"), + ("eivät", "eivät"), +] +for base_lower, base_norm in conj_contraction_bases: + for base in [base_lower, base_lower.title()]: + for suffix, suffix_norm in conj_contraction_negations: + _exc[base + suffix] = [ + {ORTH: base, NORM: base_norm}, + {ORTH: suffix, NORM: suffix_norm}, + ] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/fo/__init__.py b/spacy/lang/fo/__init__.py new file mode 100644 index 0000000..db18f1a --- /dev/null +++ b/spacy/lang/fo/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class FaroeseDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + prefixes = TOKENIZER_PREFIXES + + +class Faroese(Language): + lang = "fo" + Defaults = FaroeseDefaults + + +__all__ = ["Faroese"] diff --git a/spacy/lang/fo/tokenizer_exceptions.py b/spacy/lang/fo/tokenizer_exceptions.py new file mode 100644 index 0000000..856b722 --- /dev/null +++ b/spacy/lang/fo/tokenizer_exceptions.py @@ -0,0 +1,90 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for orth in [ + "apr.", + "aug.", + "avgr.", + "árg.", + "ávís.", + "beinl.", + "blkv.", + "blaðkv.", + "blm.", + "blaðm.", + "bls.", + "blstj.", + "blaðstj.", + "des.", + "eint.", + "febr.", + "fyrrv.", + "góðk.", + "h.m.", + "innt.", + "jan.", + "kl.", + "m.a.", + "mðr.", + "mió.", + "nr.", + "nto.", + "nov.", + "nút.", + "o.a.", + "o.a.m.", + "o.a.tíl.", + "o.fl.", + "ff.", + "o.m.a.", + "o.o.", + "o.s.fr.", + "o.tíl.", + "o.ø.", + "okt.", + "omf.", + "pst.", + "ritstj.", + "sbr.", + "sms.", + "smst.", + "smb.", + "sb.", + "sbrt.", + "sp.", + "sept.", + "spf.", + "spsk.", + "t.e.", + "t.s.", + "t.s.s.", + "tlf.", + "tel.", + "tsk.", + "t.o.v.", + "t.d.", + "uml.", + "ums.", + "uppl.", + "upprfr.", + "uppr.", + "útg.", + "útl.", + "útr.", + "vanl.", + "v.", + "v.h.", + "v.ø.o.", + "viðm.", + "viðv.", + "vm.", + "v.m.", +]: + _exc[orth] = [{ORTH: orth}] + capitalized = orth.capitalize() + _exc[capitalized] = [{ORTH: capitalized}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/fr/__init__.py b/spacy/lang/fr/__init__.py new file mode 100644 index 0000000..a8bc7f5 --- /dev/null +++ b/spacy/lang/fr/__init__.py @@ -0,0 +1,54 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import FrenchLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKEN_MATCH, TOKENIZER_EXCEPTIONS + + +class FrenchDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + token_match = TOKEN_MATCH + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class French(Language): + lang = "fr" + Defaults = FrenchDefaults + + +@French.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return FrenchLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["French"] diff --git a/spacy/lang/fr/_tokenizer_exceptions_list.py b/spacy/lang/fr/_tokenizer_exceptions_list.py new file mode 100644 index 0000000..50f4395 --- /dev/null +++ b/spacy/lang/fr/_tokenizer_exceptions_list.py @@ -0,0 +1,15630 @@ +FR_BASE_EXCEPTIONS = [ + "(+)-amphétamine", + "(5R,6S)-7,8-didehydro-4,5-époxy-3-méthoxy-N-méthylmorphinan-6-ol", + "(R)-amphétamine", + "(S)-amphétamine", + "(−)-amphétamine", + "0-day", + "0-days", + "1,1-diméthylhydrazine", + "1,2,3-tris-nitrooxy-propane", + "1,2-diazine", + "1,2-dichloropropane", + "1,3-diazine", + "1,3-dichloropropène", + "1,4-diazine", + "1-DDOL", + "1-TDOL", + "1-alpha,2-alpha,3-bêta,4-alpha,5-alpha,6-bêta-hexachlorocyclohexane", + "1-dodécanol", + "1-méthyl-2,4,6-trinitrobenzène", + "1-tétradécanol", + "1000Base-T", + "100Base-T", + "100Base-T4", + "100Base-TX", + "10BASE-F", + "10Base-T", + "11-Septembre", + "11-septembre", + "120-cellules", + "14-18", + "16-cellules", + "1T-SRAM", + "2,2'-iminodi(éthylamine)", + "2,3,6-TBA", + "2,4,5-T", + "2,4,5-TP", + "2,4,6-trinitrophénol", + "2,4-D", + "2,4-DB", + "2,4-DP", + "2,4-MCPA", + "2,4-MCPB", + "2-désoxyribose", + "2-méthylpropane", + "2-méthylpropanes", + "22-dihydroergocalciférol", + "24-cellules", + "2′-O-méthyla", + "2′-O-méthylai", + "2′-O-méthylaient", + "2′-O-méthylais", + "2′-O-méthylait", + "2′-O-méthylant", + "2′-O-méthylas", + "2′-O-méthylasse", + "2′-O-méthylassent", + "2′-O-méthylasses", + "2′-O-méthylassiez", + "2′-O-méthylassions", + "2′-O-méthyle", + "2′-O-méthylent", + "2′-O-méthyler", + "2′-O-méthylera", + "2′-O-méthylerai", + "2′-O-méthyleraient", + "2′-O-méthylerais", + "2′-O-méthylerait", + "2′-O-méthyleras", + "2′-O-méthylerez", + "2′-O-méthyleriez", + "2′-O-méthylerions", + "2′-O-méthylerons", + "2′-O-méthyleront", + "2′-O-méthyles", + "2′-O-méthylez", + "2′-O-méthyliez", + "2′-O-méthylions", + "2′-O-méthylons", + "2′-O-méthylâmes", + "2′-O-méthylât", + "2′-O-méthylâtes", + "2′-O-méthylèrent", + "2′-O-méthylé", + "2′-O-méthylée", + "2′-O-méthylées", + "2′-O-méthylés", + "3,4-DCPA", + "3,6-DCP", + "3-hydroxyflavone", + "3-méthylmorphine", + "33-tours", + "39-45", + "4-3-3", + "4-5-1", + "4-CPA", + "4-acétylaminophénol", + "5-4-1", + "5-HPETE", + "5-cellules", + "6-benzyladénine", + "600-cellules", + "8-hydroxyquinoléine", + "9-2", + "9-3", + "A-EF", + "A-OF", + "A-ÉF", + "A.-Vict.", + "AAAA-MM-JJ", + "Aarle-Rixtel", + "Abanto-Zierbena", + "Abaucourt-Hautecourt", + "Abbans-Dessous", + "Abbans-Dessus", + "Abcoude-Baambrugge", + "Abcoude-Proostdij", + "Abel-François", + "Abergement-Clémenciat", + "Abergement-Sainte-Colombe", + "Abitibi-Témiscamien", + "Abitibi-Témiscamingue", + "Ablaincourt-Pressoir", + "Aboncourt-Gesincourt", + "Achard-Bourgeois", + "Achard-Bourgeoise", + "Achard-Bourgeoises", + "Achter-Drempt", + "Achter-Lindt", + "Achter-Thesinge", + "Acquae-Sextien", + "Acquae-Sextienne", + "Acquae-Sextiennes", + "Acquae-Sextiens", + "Acquin-Westbécourt", + "Acy-Romance", + "Ad-Dawr", + "Addis-Abeba", + "Addis-Abebien", + "Addis-Abébien", + "Adelans-et-le-Val-de-Bithaine", + "Adervielle-Pouchergues", + "Admannshagen-Bargeshagen", + "Africain-Américain", + "Africaine-Américaine", + "Africaines-Américaines", + "Africains-Américains", + "Agen-d'Aveyron", + "Agnam-Goly", + "Agon-Coutainville", + "Agos-Vidalos", + "Ahaxe-Alciette-Bascassan", + "Ahlefeld-Bistensee", + "Ahrenshagen-Daskow", + "Aibar-Oibar", + "Aichach-Friedberg", + "Aigrefeuille-d'Aunis", + "Aillières-Beauvoir", + "Ailly-le-Haut-Clocher", + "Ainhice-Mongelos", + "Ainval-Septoutre", + "Aire-sur-la-Lys", + "Airon-Notre-Dame", + "Aiseau-Presles", + "Aiseau-Preslois", + "Aiseau-Presloise", + "Aix-Noulette", + "Aix-Villemaur-Pâlis", + "Aizy-Jouy", + "Ajoupa-Bouillon", + "Al-Andalus", + "Al-Dour", + "Al-Khwarizmi", + "Alb-Danube", + "Albaret-Sainte-Marie", + "Albefeuille-Lagarde", + "Albepierre-Bredons", + "Albergaria-a-Velha", + "Albiez-Montrond", + "Albon-d'Ardèche", + "Aldridge-Brownhills", + "Alegría-Dulantzi", + "Alise-Sainte-Reine", + "Allaines-Mervilliers", + "Alland'Huy", + "Alland'Huy-et-Sausseuil", + "Allas-Bocage", + "Allas-Champagne", + "Allemanche-Launay-et-Soyer", + "Alligny-Cosne", + "Allouville-Bellefosse", + "Alos-Sibas-Abense", + "Aloxe-Corton", + "Alpes-Maritimes", + "Alpes-de-Haute-Provence", + "Alphen-Boshoven", + "Alphen-Chaam", + "Alphen-Oosterwijk", + "Alphen-sur-le-Rhin", + "Alsace-Champagne-Ardenne-Lorraine", + "Alsace-Lorraine", + "Alsbach-Hähnlein", + "Altmark-Salzwedel", + "Alzey-Worms", + "Alçay-Alçabéhéty-Sunharette", + "Amagne-Lucquy", + "Amareins-Francheleins-Cesseins", + "Amathay-Vésigneux", + "Amberg-Sulzbach", + "Ambillou-Château", + "Ambly-Fleury", + "Amendeuix-Oneix", + "Amfreville-la-Mi-Voie", + "Amfreville-sous-les-Monts", + "Amigny-Rouy", + "Amlikon-Bissegg", + "Amorebieta-Etxano", + "Amorots-Succos", + "Amélie-les-Bains-Palalda", + "Ance-Féas", + "Ancy-Dornot", + "Andelot-Blancheville", + "Andelot-Morval", + "Andouillé-Neuville", + "Andrézien-Bouthéonnais", + "Andrézienne-Bouthéonnaise", + "Andréziennes-Bouthéonnaises", + "Andréziens-Bouthéonnais", + "Andrézieux-Bouthéon", + "Ange-Gardienois", + "Ange-Gardienoise", + "Angeac-Champagne", + "Angeac-Charente", + "Angerville-Bailleul", + "Angerville-l'Orcher", + "Anglards-de-Saint-Flour", + "Anglars-Juillac", + "Anglars-Nozac", + "Anglesqueville-l'Esneval", + "Anglesqueville-la-Bras-Long", + "Angoustrine-Villeneuve-des-Escaldes", + "Anhalt-Bitterfeld", + "Annaberg-Buchholz", + "Anne-Charlotte", + "Anne-Laure", + "Anne-Marie", + "Anne-Sophie", + "Anneville-Ambourville", + "Annevoie-Rouillon", + "Annoisin-Chatelans", + "Annouville-Vilmesnil", + "Anse-Bertrand", + "Antheuil-Portes", + "Anticostien-Minganien", + "Antiguais-Barbudien", + "Antiguais-Barbudiens", + "Antiguaise-Barbudienne", + "Antiguaises-Barbudiennes", + "Antoine-Labellois", + "Any-Martin-Rieux", + "Anéran-Camors", + "Apenburg-Winterfeld", + "Appelterre-Eichem", + "Appeville-Annebault", + "Aquae-Sextien", + "Aquae-Sextienne", + "Aquae-Sextiennes", + "Aquae-Sextiens", + "Aquitaine-Limousin-Poitou-Charentes", + "Arandon-Passins", + "Arbedo-Castione", + "Arbouet-Sussaute", + "Arbérats-Sillègue", + "Arces-Dilo", + "Arcizac-Adour", + "Arcizac-ez-Angles", + "Arcizans-Avant", + "Arcizans-Dessus", + "Arcy-Sainte-Restitue", + "Ardeuil-Montfauxelles", + "Arfeuille-Châtain", + "Argelès-Bagnères", + "Argelès-Gazost", + "Argens-Minervois", + "Argenton-Château", + "Argenton-Notre-Dame", + "Argenton-l'Eglise", + "Argenton-l'Église", + "Argiusta-Moriccio", + "Argut-Dessous", + "Argut-Dessus", + "Ariaco-Dompierrois", + "Ariaco-Dompierroise", + "Ariaco-Dompierroises", + "Aries-Espénan", + "Arivey-Lingeois", + "Arivey-Lingeoise", + "Arivey-Lingeoises", + "Armançon-Martinois", + "Armançon-Martinoise", + "Armançon-Martinoises", + "Armbouts-Cappel", + "Armbouts-Cappellois", + "Armbouts-Cappelloise", + "Armbouts-Cappelloises", + "Arnac-Pompadour", + "Arnaud-Guilhem", + "Arnaud-Guilhémois", + "Arnaud-Guilhémoise", + "Arnaud-Guilhémoises", + "Aroue-Ithorots-Olhaïby", + "Arraia-Maeztu", + "Arrast-Larrebieu", + "Arratzua-Ubarrundia", + "Arraute-Charritte", + "Arrayou-Lahitte", + "Arrens-Marsous", + "Arricau-Bordes", + "Arrodets-ez-Angles", + "Arros-d'Oloron", + "Ars-Laquenexois", + "Ars-Laquenexoise", + "Ars-Laquenexoises", + "Ars-Laquenexy", + "Arsure-Arsurette", + "Artalens-Souin", + "Arthaz-Pont-Notre-Dame", + "Arthez-d'Armagnac", + "Arthez-d'Asson", + "Arzacq-Arraziguet", + "Arzenc-d'Apcher", + "Arzillières-Neuville", + "Asasp-Arros", + "Asbach-Bäumenheim", + "Asbach-Sickenberg", + "Asnans-Beauvoisin", + "Aspach-Michelbach", + "Aspin-Aure", + "Aspret-Sarrat", + "Assyro-Chaldéen", + "Aste-Béon", + "Aston-Jonction", + "Athesans-Etroitefontaine", + "Athesans-Étroitefontaine", + "Athis-Mons", + "Athis-Val de Rouvre", + "Athos-Aspis", + "Attenrode-Wever", + "Attignat-Oncin", + "Aubepierre-Ozouer-le-Repos", + "Aubermesnil-Beaumais", + "Aubert-Gallionnais", + "Aubigny-Les Clouzeaux", + "Aubigné-Briand", + "Aubigné-Racan", + "Auboncourt-Vauzelles", + "Aubusson-d'Auvergne", + "Auchy-lez-Orchies", + "Aude-Line", + "Audenhove-Sainte-Marie", + "Auffreville-Brasseuil", + "Aujan-Mournède", + "Aulhat-Flat", + "Aulnay-l'Aître", + "Aulnoy-lez-Valenciennes", + "Aulnoye-Aymeries", + "Aumeville-Lestre", + "Aumont-Aubrac", + "Auneau-Bleury-Saint-Symphorien", + "Aurelle-Verlac", + "Auriac-Lagast", + "Auriac-l'Eglise", + "Auriac-l'Église", + "Aurions-Idernes", + "Aussac-Vadalle", + "Australie-Méridionale", + "Australie-Occidentale", + "Austro-Hongrois", + "Austro-Hongroise", + "Austro-Hongroises", + "Autechaux-Roide", + "Autevielle-Saint-Martin-Bideren", + "Autheuil-Authouillet", + "Authieux-Ratiéville", + "Authon-Ebéon", + "Authon-Ébéon", + "Autrans-Méaudre en Vercors", + "Autre-Église", + "Autre-Églisois", + "Autre-Églisoise", + "Autreville-sur-la-Renne", + "Autriche-Hongrie", + "Autry-Issards", + "Auvergne-Rhône-Alpes", + "Auvet-et-la-Chapelotte", + "Aux-Aussat", + "Auxelles-Bas", + "Auxelles-Haut", + "Auxey-Duresses", + "Auxon-Dessous", + "Auxon-Dessus", + "Auzeville-Tolosane", + "Auzouville-Auberbosc", + "Auzouville-l'Esneval", + "Availles-Limouzine", + "Availles-Thouarsais", + "Avanne-Aveney", + "Avenay-Val-d'Or", + "Avesnes-Chaussoy", + "Avezac-Prat-Lahitte", + "Avignon-lès-Saint-Claude", + "Avignonet-Lauragais", + "Avillers-Sainte-Croix", + "Avirey-Lingey", + "Avrigney-Virey", + "Avéron-Bergelle", + "Awala-Yalimapo", + "Axo-Missien", + "Axo-Missienne", + "Axo-Missiennes", + "Axo-Missiens", + "Ayala-Aiara", + "Ayer's-Cliffois", + "Ayguatébia-Talau", + "Ayros-Arbouix", + "Ayzac-Ost", + "Azanuy-Alins", + "Azat-Châtenet", + "Azilone-Ampaza", + "Aínsa-Sobrarbe", + "Aïcirits-Camou-Suhast", + "Aïd-el-Kébir", + "Aïd-el-Séghir", + "Aÿ-Champagne", + "B-52", + "B-frame", + "B-spline", + "B-splines", + "BVD-MD", + "Baaks-Sweijer", + "Baar-Ebenhausen", + "Baarle-Nassau", + "Baarle-Nassau-Grens", + "Babeau-Bouldoux", + "Bachos-Binos", + "Bachte-Maria-Leerne", + "Bade-Wurtemberg", + "Badefols-d'Ans", + "Baden-Baden", + "Badonvilliers-Gérauvilliers", + "Baerle-Duc", + "Bagnoles de l'Orne Normandie", + "Baguer-Morvan", + "Baguer-Pican", + "Bahus-Soubiran", + "Bahá'u'lláh", + "Baie-Catherinois", + "Baie-Comelien", + "Baie-Comellien", + "Baie-Comien", + "Baie-Comois", + "Baie-Jolien", + "Baie-Mahault", + "Baie-Mahaultien", + "Baie-Mahaultienne", + "Baie-Mahaultiennes", + "Baie-Mahaultiens", + "Baie-Trinitois", + "Baignes-Sainte-Radegonde", + "Bailleau-Armenonville", + "Bailleau-l'Evêque", + "Bailleau-l'Évêque", + "Bailleul-Neuville", + "Bailleul-Sir-Berthoult", + "Bailly-Carrois", + "Bailly-Romainvilliers", + "Bairon-Mont-Dieu", + "Bairon-le-Mont-Dieu", + "Baisy-Thy", + "Bakkum-Noord", + "Balaguier-d'Olt", + "Baliracq-Maumusson", + "Ballan-Miré", + "Ballon-Saint Mars", + "Ballrechten-Dottingen", + "Ban-sur-Meurthe-Clefcy", + "Banassac-Canilhac", + "Bannost-Villegagnon", + "Banogne-Recouvrance", + "Banyuls-dels-Aspres", + "Barbazan-Debat", + "Barbazan-Dessus", + "Barbe-Bleue", + "Barbey-Seroux", + "Barbonne-Fayel", + "Barenton-Bugny", + "Barenton-Cel", + "Barger-Compascuum", + "Barger-Erfscheidenveen", + "Barger-Oosterveen", + "Barger-Oosterveld", + "Bargfeld-Stegen", + "Barletta-Andria-Trani", + "Barneville-Carteret", + "Barrais-Bussolles", + "Barraute-Camu", + "Barrow-in-Furness", + "Barry-d'Islemade", + "Bartenshagen-Parkentin", + "Barvaux-Condroz", + "Basadingen-Schlattingen", + "Bassillon-Vauzé", + "Bassoles-Aulers", + "Batignolles-Monceaux", + "Battenans-Varin", + "Baussery-Montain", + "Baussery-Montaine", + "Baussery-Montaines", + "Baussery-Montains", + "Bayerfeld-Steckweiler", + "Bazoches-sur-le-Betz", + "Bazouges-sur-le-Loir", + "Bazus-Aure", + "Bazus-Neste", + "Beaucamps-Ligny", + "Beaufort-Blavincourt", + "Beaufour-Druval", + "Beaujeu-Saint-Vallier-Pierrejux-et-Quitteur", + "Beaulieu-sous-la-Roche", + "Beaumont-Hague", + "Beaumont-Hamel", + "Beaumont-Louestault", + "Beaumont-Monteux", + "Beaumont-Pied-de-Bœuf", + "Beaumont-Sardolles", + "Beaumont-Village", + "Beaumotte-Aubertans", + "Beaumotte-lès-Montbozon-et-Aubertans", + "Beaune-d'Allier", + "Beauregard-Baret", + "Beauregard-Vendon", + "Beauregard-l'Evêque", + "Beauregard-l'Évêque", + "Beaussais-Vitré", + "Beauvoir-Rivière", + "Beauvoir-Wavans", + "Beaux-Arts", + "Beaux-Rivageois", + "Bech-Kleinmacher", + "Bedburg-Hau", + "Beemte-Broekland", + "Beffu-et-le-Morthomme", + "Behren-Lübchin", + "Beiersdorf-Freudenberg", + "Beine-Nauroy", + "Beintza-Labaien", + "Belbèze-Escoulis", + "Belhomert-Guéhouville", + "Belin-Béliet", + "Bellecombe-Tarendol", + "Bellegarde-Marsal", + "Bellegarde-Poussieu", + "Bellegarde-Sainte-Marie", + "Bellevaux-Ligneuville", + "Belleville-et-Châtillon-sur-Bar", + "Belmont-Bretenoux", + "Belmont-Luthézieu", + "Belmont-Sainte-Foi", + "Belmont-Tramonet", + "Belmont-d'Azergues", + "Belmont-de-la-Loire", + "Belt-Schutsloot", + "Belval-Bois-des-Dames", + "Belvédère-Campomoro", + "Ben-Ahin", + "Ben-Ahinois", + "Ben-Ahinoise", + "Beneden-Haastrecht", + "Beneden-Leeuwen", + "Beni-Khiran", + "Benque-Dessous-et-Dessus", + "Benqué-Molère", + "Bentayou-Sérée", + "Berbérust-Lias", + "Berchem-Sainte-Agathe", + "Berd'huis", + "Berendrecht-Zandvliet-Lillo", + "Berg-op-Zoom", + "Bergouey-Viellenave", + "Berkel-Enschot", + "Berkholz-Meyenburg", + "Berles-Monchel", + "Berlin-Est", + "Berlin-Ouest", + "Bernac-Debat", + "Bernac-Dessus", + "Bernadets-Debat", + "Bernadets-Dessus", + "Bernay-Vilbert", + "Berne-Mittelland", + "Bernico-Montois", + "Bernico-Montoise", + "Bernico-Montoises", + "Bernières-d'Ailly", + "Bernkastel-Kues", + "Bernkastel-Wittlich", + "Bernos-Beaulac", + "Bernuy-Zapardiel", + "Berny-Rivière", + "Berre-l'Etang", + "Berre-l'Étang", + "Berrogain-Laruns", + "Berry-Bouy", + "Bersillies-l'Abbaye", + "Bertaucourt-Epourdon", + "Bertric-Burée", + "Bertsdorf-Hörnitz", + "Besse-et-Saint-Anastaise", + "Betbezer-d'Armagnac", + "Betcave-Aguin", + "Beton-Bazoches", + "Betpouey-Barèges", + "Bettencourt-Rivière", + "Betting-lès-Saint-Avold", + "Betton-Bettonet", + "Beura-Cardezza", + "Beurey-Bauguay", + "Beuvry-Nord", + "Beuzec-Cap-Sizun", + "Beuzec-Conq", + "Beveland-Nord", + "Bexhill-on-Sea", + "Beyne-Heusay", + "Beyrède-Jumet", + "Bezins-Garraux", + "Bichelsee-Balterswil", + "Bidania-Goiatz", + "Biel-Benken", + "Bienne-lez-Happart", + "Bierry-les-Belles-Fontaines", + "Biest-Houtakker", + "Bietigheim-Bissingen", + "Big-bang", + "Billens-Hennens", + "Billigheim-Ingenheim", + "Billy-Berclau", + "Billy-Chevannes", + "Billy-Montigny", + "Billy-sous-les-Côtes", + "Binic-Étables-sur-Mer", + "Binnen-Moerdijk", + "Bioley-Magnoux", + "Bioley-Orjulaz", + "Birken-Honigsessen", + "Bissao-Guinéen", + "Bissau-Guinéen", + "Bissau-Guinéenne", + "Bissau-Guinéennes", + "Bithaine-et-le-Val", + "Bitterfeld-Wolfen", + "Biurrun-Olcoz", + "Bize-Minervois", + "Biéville-Beuville", + "Biéville-Quétiéville", + "Black-Lakien", + "Blainville-Crevon", + "Blaison-Gohier", + "Blaisy-Bas", + "Blaisy-Haut", + "Blanche-Eglise", + "Blanche-Neige", + "Blanche-Église", + "Blangerval-Blangermont", + "Blangy-Tronville", + "Blankenfelde-Mahlow", + "Blanzac-Porcheresse", + "Blaydon-on-Tyne", + "Blaye-et-Sainte-Luce", + "Bleyen-Genschmar", + "Blies-Ebersing", + "Blies-Ebersingeois", + "Blies-Ebersingeoise", + "Blies-Ebersingeoises", + "Blies-Guersviller", + "Blies-Ébersing", + "Blies-Ébersingeois", + "Blies-Ébersingeoise", + "Blies-Ébersingeoises", + "Blosseville-Bonsecours", + "Blot-l'Eglise", + "Blot-l'Église", + "Blousson-Sérian", + "Blénod-lès-Pont-à-Mousson", + "Bobenheim-Roxheim", + "Bobo-Dioulasso", + "Bodegraven-Reeuwijk", + "Bodenrode-Westhausen", + "Bodman-Ludwigshafen", + "Boeil-Bezing", + "Bogis-Bossey", + "Bohas-Meyriat-Rignat", + "Boiry-Becquerelle", + "Boiry-Notre-Dame", + "Boiry-Sainte-Rictrude", + "Boisné-La Tude", + "Boisse-Penchot", + "Boissy-Fresnoy", + "Boissy-Lamberville", + "Boissy-Maugien", + "Boissy-Maugis", + "Boissy-Mauvoisin", + "Boissy-l'Aillerie", + "Boissy-sans-Avoir", + "Boissy-sous-Saint-Yon", + "Boisville-la-Saint-Père", + "Bokholt-Hanredder", + "Bollendorf-Pont", + "Bonac-Irazein", + "Bonneuil-Matours", + "Bonnevent-Velloreille", + "Bonnevent-et-Velloreille-lès-Bonnevent", + "Bonneville-Aptot", + "Bonneville-et-Saint-Avit-de-Fumadières", + "Bonrepos-Riquet", + "Bonvillers-Mont", + "Bootle-cum-Linacre", + "Bora-Bora", + "Bordel's", + "Bordes-Uchentein", + "Bordères-Louron", + "Borgdorf-Seedorf", + "Borger-Odoorn", + "Bors (Canton de Baignes-Sainte-Radegonde)", + "Bors (Canton de Montmoreau-Saint-Cybard)", + "Borstel-Hohenraden", + "Bort-l'Etang", + "Bort-l'Étang", + "Bosc-Bordel", + "Bosc-Bénard-Commin", + "Bosc-Bénard-Crescy", + "Bosc-Bérenger", + "Bosc-Edeline", + "Bosc-Guérard-Saint-Adrien", + "Bosc-Guérardais", + "Bosc-Guérardaise", + "Bosc-Guérardaises", + "Bosc-Hyons", + "Bosc-Mesnil", + "Bosc-Renoult-en-Ouche", + "Bosc-Renoult-en-Roumois", + "Bosc-Renoulthien", + "Bosc-Renoulthienne", + "Bosc-Renoulthiennes", + "Bosc-Renoulthiens", + "Bosc-Roger-sur-Buchy", + "Bosc-Édeline", + "Bosmie-l'Aiguille", + "Bosnie-Herzégovine", + "Bossut-Gottechain", + "Bouchavesnes-Bergen", + "Bouconville-Vauclair", + "Bouconville-Vauclerc", + "Boueilh-Boueilho-Lasque", + "Bougy-Villars", + "Bougy-lez-Neuville", + "Bougé-Chambalud", + "Bouilh-Devant", + "Bouilh-Péreuilh", + "Bouillé-Courdault", + "Bouillé-Loretz", + "Bouillé-Ménard", + "Bouin-Plumoison", + "Boulay-Morinois", + "Boulay-Morinoise", + "Boulay-Morinoises", + "Boulay-Moselle", + "Boule-d'Amont", + "Boulogne-Billancourt", + "Bourbon-Lancy", + "Bourbon-Vendée", + "Bourbon-l'Archambault", + "Bourbourg-Campagne", + "Bourgneuf-Val-d'Or", + "Bourgogne-Franche-Comté", + "Bourgogne-Fresne", + "Bourgoin-Jallieu", + "Bourgtheroulde-Infreville", + "Bourguignon-lès-la-Charité", + "Bournainville-Faverolles", + "Bourneville-Sainte-Croix", + "Bouroum-Bouroum", + "Bourriot-Bergonce", + "Bourron-Marlotte", + "Bourseigne-Neuve", + "Bourseigne-Vieille", + "Boussac-Bourg", + "Boussu-lez-Walcourt", + "Bout-du-Pont-de-Larn", + "Bout-du-Pont-de-l'Arnais", + "Bout-du-Pont-de-l'Arnaise", + "Bout-du-Pont-de-l'Arnaises", + "Boutenac-Touvent", + "Boutigny-Prouais", + "Bouvesse-Quirieu", + "Bouvigny-Boyeffles", + "Bouy-Luxembourg", + "Bouzon-Gellenave", + "Boven-Haastrecht", + "Boven-Hardinxveld", + "Boven-Leeuwen", + "Bovisio-Masciago", + "Boô-Silhen", + "Brabant-Septentrional", + "Brabant-Wallon", + "Bragelogne-Beauvoir", + "Brailly-Cornehotte", + "Braine-l'Alleud", + "Brains-sur-les-Marches", + "Brande-Hörnerkirchen", + "Brandebourg-sur-la-Havel", + "Branville-Hague", + "Bras-Panon", + "Bras-d'Asse", + "Bras-d'Assien", + "Bras-d'Assienne", + "Bras-d'Assiennes", + "Bras-d'Assiens", + "Braud-et-Saint-Louis", + "Braunau-am-Inn", + "Braux-Sainte-Cohière", + "Bray-Dunes", + "Bray-Dunois", + "Bray-Dunoise", + "Bray-Dunoises", + "Brazza-Congolais", + "Breitenbach-Haut-Rhin", + "Bretagne-d'Armagnac", + "Bretigney-Notre-Dame", + "Bretnig-Hauswalde", + "Bretteville-du-Grand-Caux", + "Bretteville-l'Orgueilleuse", + "Breuil-Barret", + "Breuil-Bernardin", + "Breuil-Bernardine", + "Breuil-Bernardines", + "Breuil-Bernardins", + "Breuil-Bois-Robert", + "Breuil-Chaussée", + "Breuil-Magné", + "Breukelen-Nijenrode", + "Breukelen-Sint-Pieters", + "Breux-Jouy", + "Brey-et-Maison-du-Bois", + "Brie-Comte-Robert", + "Brieskow-Finkenheerd", + "Brig-Glis", + "Brignano-Frascata", + "Brignogan-Plages", + "Brigue-Glis", + "Brillet-Pontin", + "Brillet-Pontine", + "Brillet-Pontines", + "Brillet-Pontins", + "Briquemesnil-Floxicourt", + "Brisgau-Haute-Forêt-Noire", + "Brissac-Quincé", + "Brissago-Valtravaglia", + "Brissay-Choigny", + "Brissy-Hamégicourt", + "Britanno-Colombien", + "Britanno-Colombienne", + "Britanno-Colombiennes", + "Britanno-Colombiens", + "Brives-Charensac", + "Brohl-Lützing", + "Bromont-Lamothe", + "Brot-Dessous", + "Brot-Plamboz", + "Broussey-Raulecourt", + "Brovello-Carpugnino", + "Broye-Aubigney-Montseugny", + "Broye-Vully", + "Broye-les-Loups-et-Verfontaine", + "Broye-lès-Pesmes-Aubigney-Montseugny", + "Broût-Vernet", + "Broût-Vernetois", + "Broût-Vernetoise", + "Broût-Vernetoises", + "Bruchhausen-Vilsen", + "Bruchmühlbach-Miesau", + "Bruchweiler-Bärenbach", + "Brue-Auriac", + "Bruges-Capbis-Mifaget", + "Brugny-Vaudancourt", + "Bruille-lez-Marchiennes", + "Brunstatt-Didenheim", + "Bruttig-Fankel", + "Bruxelles-ville", + "Bruère-Allichamps", + "Brécy-Brières", + "Brécy-Brièrois", + "Brécy-Brièroise", + "Brécy-Brièroises", + "Brégnier-Cordon", + "Bréhan-Loudéac", + "Brémontier-Merval", + "Bréxent-Enocq", + "Bréxent-Énocq", + "Brücken-Hackpfüffel", + "Budel-Dorplein", + "Budel-Schoot", + "Buenos-Airien", + "Buenos-Ayres", + "Buhl-Lorraine", + "Buigny-l'Abbé", + "Buire-Courcelles", + "Buis-Prévenchais", + "Buis-Prévenchaise", + "Buis-Prévenchaises", + "Bulat-Pestivien", + "Buno-Bonnevaux", + "Burg-Reuland", + "Burg-Reulandais", + "Burg-Reulandaise", + "Burkina-Faso", + "Burkina-be", + "Burkina-bes", + "Burkina-fassien", + "Burosse-Mendousse", + "Bussac-Forêt", + "Bussière-Badil", + "Bussière-Boffy", + "Bussière-Dunoise", + "Bussière-Galant", + "Bussière-Nouvelle", + "Bussière-Poitevine", + "Bussunarits-Sarrasquette", + "Bussus-Bussuel", + "Bussy-Albieux", + "Bussy-Chardonney", + "Bussy-Lettrée", + "Bustince-Iriberry", + "Butot-Vénesville", + "Butte-Montmartre", + "Buttes-Chaumont", + "Buxières-d'Aillac", + "Buxières-sous-les-Côtes", + "Buzy-Darmont", + "Byhleguhre-Byhlen", + "Bœurs-en-Othe", + "Bâle-Campagne", + "Bâle-Ville", + "Béard-Géovreissiat", + "Bécordel-Bécourt", + "Bédouès-Cocurès", + "Béhasque-Lapiste", + "Bénesse-Maremne", + "Béning-lès-Saint-Avold", + "Bénivay-Ollon", + "Bény-Bocain", + "Bény-Bocaine", + "Bény-Bocaines", + "Bény-Bocains", + "Bénévent-l'Abbaye", + "Bérig-Vintrange", + "Bézues-Bajon", + "Böhl-Iggelheim", + "Börde-Hakel", + "Börgerende-Rethwisch", + "Bösleben-Wüllersleben", + "C-4", + "C-blanc", + "C-blancs", + "C.-Antip.", + "CD-R", + "CD-ROM", + "CD-RW", + "CD-WORM", + "Cabanac-Cazaux", + "Cabanac-Séguenville", + "Cabas-Loumassès", + "Cabrières-d'Aigues", + "Cabrières-d'Avignon", + "Cadegliano-Viconago", + "Cadeilhan-Trachère", + "Cadzand-Bad", + "Caillouet-Orgeville", + "Caillouël-Crépigny", + "Calasca-Castiglione", + "Calatafimi-Segesta", + "Calleville-les-Deux-Eglises", + "Calleville-les-Deux-Églises", + "Calmels-et-le-Viala", + "Calo-Moulinotin", + "Calo-Moulinotine", + "Calo-Moulinotines", + "Calo-Moulinotins", + "Calonne-Ricouart", + "Calonne-sur-la-Lys", + "Calp.-Flac.", + "Calumet-Pointois", + "Calumet-Pontois", + "Camblain-Châtelain", + "Camblain-l'Abbé", + "Cambounet-sur-le-Sor", + "Cambron-Casteau", + "Camiac-et-Saint-Denis", + "Camou-Cihigue", + "Camou-Mixe-Suhast", + "Camp-Auriol", + "Camp-Dumy", + "Camp-Mégier", + "Camp-Méjan", + "Camp-Public", + "Camp-Réal", + "Campagne-d'Armagnac", + "Campandré-Valcongrain", + "Campezo-Kanpezu", + "Campiglione-Fenile", + "Campillos-Paravientos", + "Campillos-Sierra", + "Camping-Gaz", + "Camplong-d'Aude", + "Campo-Haltien", + "Campo-Laïcien", + "Campo-Laïcienne", + "Campo-Laïciennes", + "Campo-Laïciens", + "Camps-Saint-Mathurin-Léobazel", + "Canada-Uni", + "Canale-di-Verde", + "Candau-Casteidois", + "Candau-Casteidoise", + "Candau-Casteidoises", + "Cannes-Ecluse", + "Cannes-Écluse", + "Cante-Greil", + "Cante-Grel", + "Cante-Grillet", + "Cante-Perdris", + "Cantenay-Epinard", + "Cantenay-Épinard", + "Canville-les-Deux-Eglises", + "Canville-les-Deux-Églises", + "Cany-Barville", + "Caouënnec-Lanvézéac", + "Capaccio-Paestum", + "Capdenac-Gare", + "Capelle-Fermont", + "Capelle-Filismontin", + "Capelle-Filismontine", + "Capelle-Filismontines", + "Capelle-Filismontins", + "Capesterre-Belle-Eau", + "Capesterre-de-Marie-Galante", + "Capitale-Nationale", + "Cappelle-Brouck", + "Capélo-Hugonais", + "Capélo-Hugonaise", + "Capélo-Hugonaises", + "Carbon-Blanc", + "Carbonia-Iglesias", + "Carcarès-Sainte-Croix", + "Carcen-Ponson", + "Carcheto-Brustico", + "Cardo-Torgia", + "Carhaix-Plouguer", + "Carla-Bayle", + "Carmzow-Wallmow", + "Carnac-Rouffiac", + "Caro-Percyais", + "Caro-Percyaise", + "Caro-Percyaises", + "Carresse-Cassaber", + "Carsac-Aillac", + "Cartigny-l'Epinay", + "Cartigny-l'Épinay", + "Carville-Pot-de-Fer", + "Case-Pilote", + "Cassagnabère-Tournas", + "Cassagnes-Bégonhès", + "Castaignos-Souslens", + "Castanet-Tolosan", + "Casteide-Cami", + "Casteide-Candau", + "Casteide-Doat", + "Castelbello-Ciardes", + "Castell'Alfero", + "Castell'Arquato", + "Castell'Azzara", + "Castell'Umberto", + "Castellare-di-Casinca", + "Castellare-di-Mercurio", + "Castello-di-Rostino", + "Castelmoron-d'Albret", + "Castelnau d'Auzan Labarrère", + "Castelnau-Barbarens", + "Castelnau-Chalosse", + "Castelnau-Durban", + "Castelnau-Durbannais", + "Castelnau-Durbannaise", + "Castelnau-Durbannaises", + "Castelnau-Magnoac", + "Castelnau-Montratier", + "Castelnau-Montratier-Sainte-Alauzie", + "Castelnau-Picampeau", + "Castelnau-Pégayrols", + "Castelnau-Rivière-Basse", + "Castelnau-Tursan", + "Castelnau-Valence", + "Castelnau-d'Anglès", + "Castelnau-d'Arbieu", + "Castelnau-d'Aude", + "Castelnau-d'Auzan", + "Castelnau-d'Estrétefonds", + "Castet-Arrouy", + "Castet-Arrouyais", + "Castet-Arrouyaise", + "Castet-Arrouyaises", + "Castetnau-Camblong", + "Castex-d'Armagnac", + "Casti-Wergenstein", + "Casties-Labrande", + "Castillejo-Sierra", + "Castillo-Albaráñez", + "Castillon (Canton d'Arthez-de-Béarn)", + "Castillon-Debats", + "Castillon-Massas", + "Castillon-Savès", + "Castillon-de-Saint-Martory", + "Castres-Gironde", + "Castrillo-Tejeriego", + "Castro-Urdiales", + "Castrop-Rauxel", + "Castéra-Bouzet", + "Castéra-Lanusse", + "Castéra-Lectourois", + "Castéra-Lou", + "Castéra-Loubix", + "Castéra-Verduzan", + "Castéra-Vignoles", + "Catillon-Fumechon", + "Caubios-Loos", + "Cauchy-à-la-Tour", + "Caumont-l'Eventé", + "Caumont-l'Éventé", + "Caunes-Minervois", + "Caupenne-d'Armagnac", + "Caussade-Rivière", + "Causse-Bégon", + "Causse-de-la-Selle", + "Cavaglio-Spoccia", + "Cavallino-Treporti", + "Cayre-four", + "Cazaril-Laspènes", + "Cazaril-Tambourès", + "Cazarilh-Laspènes", + "Cazaux-Debat", + "Cazaux-Fréchet-Anéran-Camors", + "Cazaux-Layrisse", + "Cazaux-Savès", + "Cazaux-Villecomtal", + "Cazaux-d'Anglès", + "Cazenave-Serres-et-Allens", + "Cazeneuve-Montaut", + "Cazes-Mondenard", + "Cazouls-d'Hérault", + "Celle-Lévescault", + "Cenne-Monestiés", + "Centre-Est", + "Centre-Mauricien", + "Centre-Nord", + "Centre-Ouest", + "Centre-Sud", + "Centro-Américain", + "Centro-Américaine", + "Centro-Américains", + "Cerexhe-Heuseux", + "Cerisy-Belle-Etoile", + "Cerisy-Belle-Étoile", + "Cerisy-Buleux", + "Cerisy-Gailly", + "Cernay-l'Eglise", + "Cernay-l'Église", + "Cesny-Bois-Halbout", + "Cesson-Sévigné", + "Cette-Eygun", + "Chaillé-sous-les-Ormeaux", + "Chaintrix-Bierges", + "Chaise-Dieu-du-Theil", + "Chalain-d'Uzore", + "Chalandry-Elaire", + "Chalivoy-Milon", + "Chalmazel-Jeansagnière", + "Chalonnes-sous-le-Lude", + "Chalou-Moulineux", + "Chamarandes-Choignes", + "Chambley-Bussières", + "Chambolle-Musigny", + "Chambon-Sainte-Croix", + "Chambost-Allières", + "Chambost-Longessaigne", + "Chamonix-Mont-Blanc", + "Champagne-Ardenne", + "Champagne-Mouton", + "Champagne-Vigny", + "Champagne-au-Mont-d'Or", + "Champdor-Corcelles", + "Champeaux-et-la-Chapelle-Pommier", + "Champigneul-Champagne", + "Champignol-lez-Mondeville", + "Champs-Romain", + "Champs-sur-Tarentaine-Marchal", + "Champs-zé", + "Champs-Élysées", + "Chang-Haï", + "Chanos-Curson", + "Chanos-Cursonnais", + "Chanos-Cursonnaise", + "Chanos-Cursonnaises", + "Chanoz-Châtenay", + "Chante-Clair", + "Chantemerle-sur-la-Soie", + "Chantenay-Villedieu", + "Chapdes-Beaufort", + "Chapellois-Fortinien", + "Chapellois-Fortiniens", + "Chapelloise-Fortinienne", + "Chapelloises-Fortiniennes", + "Chapon-Seraing", + "Chapon-Sérésien", + "Chapon-Sérésienne", + "Charcé-Saint-Ellier-sur-Aubance", + "Chareil-Cintrat", + "Charency-Vezin", + "Charente-Inférieure", + "Charente-Maritime", + "Charette-Varennes", + "Charles-Quint", + "Charleville-Mézières", + "Charly-Oradour", + "Charmois-devant-Bruyères", + "Charmois-l'Orgueilleux", + "Charmontois-l'Abbé", + "Chartrier-Ferrière", + "Charvieu-Chavagneux", + "Chassagne-Montrachet", + "Chassey-Beaupré", + "Chastel-Arnaud", + "Chastel-Nouvel", + "Chastre-Villeroux-Blanmont", + "Chatel-Chéhéry", + "Chatenay-Mâcheron", + "Chatenay-Vaudin", + "Chatonrupt-Sommermont", + "Chauconin-Neufmontiers", + "Chaudière-Appalaches", + "Chaudon-Norante", + "Chaufour-Notre-Dame", + "Chaumont-Gistoux", + "Chaumont-Porcien", + "Chaumont-d'Anjou", + "Chaumont-devant-Damvillers", + "Chaumoux-Marcilly", + "Chaussoy-Epagny", + "Chaussée-Notre-Dame-Louvignies", + "Chauvac-Laux-Montaux", + "Chauvincourt-Provemont", + "Chaux-Champagny", + "Chaux-Neuve", + "Chavagneux-Montbertand", + "Chavaniac-Lafayette", + "Chavigny-Bailleul", + "Chavot-Courcourt", + "Chazay-d'Azergues", + "Chazey-Bons", + "Chazé-Henry", + "Chef-Boutonnais", + "Chef-Boutonnaise", + "Chef-Boutonnaises", + "Chef-Boutonne", + "Chef-Haut", + "Cheffreville-Tonnencourt", + "Chein-Dessus", + "Chelle-Debat", + "Chelle-Spou", + "Chemillé-Melay", + "Chemin-d'Aisey", + "Chenac-Saint-Seurin-d'Uzet", + "Chenailler-Mascheix", + "Chenecey-Buillon", + "Chenillé-Champteussé", + "Chenillé-Changé", + "Cherbourg-Octeville", + "Chermizy-Ailles", + "Cherveix-Cubas", + "Cherves-Châtelars", + "Cherves-Richemont", + "Cheseaux-Noréaz", + "Chesne-Arnoul", + "Chesne-Carré", + "Chesne-Dolley", + "Chesnois-Auboncourt", + "Cheval-Blanc", + "Chevannes-Changy", + "Chevilly-Larue", + "Chevresis-Monceau", + "Chevry-Cossigny", + "Chevry-sous-le-Bignon", + "Cheylard-l'Evêque", + "Cheylard-l'Évêque", + "Chezal-Benoît", + "Chibougamo-Chapien", + "Chilly-Mazarin", + "Chiopris-Viscone", + "Chirac-Bellevue", + "Chirat-l'Eglise", + "Chirat-l'Église", + "Chiry-Ourscamp", + "Chiry-Ourscamps", + "Chivres-Val", + "Choilley-Dardenay", + "Choloy-Ménillot", + "Chonas-l'Amballan", + "Chonville-Malaumont", + "Chuffilly-Roche", + "Châlon's", + "Château-Chinon (Campagne)", + "Château-Chinon (Ville)", + "Châtel-Censoir", + "Châtel-Guyon", + "Châtel-Gérard", + "Châtel-Montagne", + "Châtel-Moron", + "Châtelaillon-Plage", + "Châtelus-Malvaleix", + "Châtenay-Malabry", + "Châtillon-Coligny", + "Châtillon-Guyotte", + "Châtillon-sous-les-Côtes", + "Chémery-Chéhéry", + "Chéry-Chartreuve", + "Chéry-Chartreuvois", + "Chéry-Chartreuvoise", + "Chéry-Chartreuvoises", + "Chézery-Forens", + "Chêne-Arnoult", + "Chêne-Bernard", + "Chêne-Bougeries", + "Chêne-Bourg", + "Chêne-Carré", + "Chêne-Chenu", + "Chêne-Dolley", + "Chêne-Pâquier", + "Chêne-Sec", + "Chênehutte-Trèves-Cunault", + "Cierp-Gaud", + "Cinq-Mars-la-Pile", + "Cinq-Marsien", + "Cinq-Marsienne", + "Cinq-Marsiennes", + "Cinq-Marsiens", + "Ciry-Salsogne", + "Ciré-d'Aunis", + "Cissac-Médoc", + "Civrieux-d'Azergues", + "Clacton-on-Sea", + "Clairvaux-d'Aveyron", + "Clairy-Saulchoix", + "Clarafond-Arcine", + "Clausthal-Zellerfeld", + "Clavans-en-Haut-Oisans", + "Claville-Motteville", + "Clavy-Warby", + "Claye-Souilly", + "Clef Vallée d'Eure", + "Clermont-Créans", + "Clermont-Dessous", + "Clermont-Dessus", + "Clermont-Ferrand", + "Clermont-Pouyguillès", + "Clermont-Savès", + "Clermont-Soubiran", + "Clermont-d'Excideuil", + "Clermont-l'Hérault", + "Clohars-Carnoët", + "Clohars-Fouesnant", + "Clos-Fontaine", + "Clos-Fontainois", + "Clos-Fontainoise", + "Clos-Fontainoises", + "Cloyes-les-Trois-Rivières", + "Cloyes-sur-le-Loir", + "Cluj-Napoca", + "Clun's", + "Clux-Villeneuve", + "Cléden-Cap-Sizun", + "Cléden-Poher", + "Clémence-d'Ambel", + "Cléon-d'Andran", + "Cléry-Grand", + "Cléry-Petit", + "Coat-Méal", + "Coat-Méalien", + "Coat-Méalienne", + "Coat-Méaliennes", + "Coat-Méaliens", + "Cochem-Zell", + "Cocquio-Trevisago", + "Cognat-Lyonne", + "Cognocoli-Monticchi", + "Coise-Saint-Jean-Pied-Gauthier", + "Coizard-Joches", + "Collandres-Quincarnon", + "Colleville-Montgomery", + "Collex-Bossy", + "Colligis-Crandelain", + "Colligny-Maizery", + "Colline-Beaumont", + "Collombey-Muraz", + "Collonge-Bellerive", + "Collonges-au-Mont-d'Or", + "Colmar-Berg", + "Colmesnil-Manneville", + "Colombey-les-Deux-Eglises", + "Colombey-les-Deux-Églises", + "Colombie-Anglaise", + "Colombie-Britannique", + "Colombier-Châtelot", + "Colombier-Fontaine", + "Colombier-Saugnieu", + "Colomby-Anguerny", + "Colonard-Corubert", + "Colpach-Bas", + "Colpach-Haut", + "Comblain-Fairon", + "Combres-sous-les-Côtes", + "Comezzano-Cizzago", + "Comin-Yanga", + "Comines-Warneton", + "Commelle-Vernay", + "Comps-la-Grand-Ville", + "Condeixa-a-Nova", + "Condom-d'Aubrac", + "Condé-Folie", + "Condé-Northen", + "Condé-Sainte-Libiaire", + "Conflans-Sainte-Honorine", + "Confolent-Port-Dieu", + "Confort-Meilars", + "Congerville-Thionville", + "Congo-Brazzaville", + "Congo-Kinshasa", + "Congo-Léo", + "Congo-Léopoldville", + "Conie-Molitard", + "Conilhac-Corbières", + "Conilhac-de-la-Montagne", + "Connantray-Vaurefroy", + "Conne-de-la-Barde", + "Cons-Sainte-Colombe", + "Consolation-Maisonnettes", + "Contamine-Sarzin", + "Corbeil-Cerf", + "Corbeil-Essonnes", + "Corbeil-Essonnois", + "Corbeil-Essonnoise", + "Corbeil-Essonnoises", + "Corbère-Abères", + "Corcelle-Mieslot", + "Corcelles-Cormondrèche", + "Corcelles-Ferrières", + "Cordes-Tolosannes", + "Corme-Ecluse", + "Corme-Royal", + "Corme-Écluse", + "Cormot-Vauchignon", + "Corneilla-del-Vercol", + "Cornillon-Confoux", + "Corny-Machéroménil", + "Corpataux-Magnedens", + "Corps-Nuds", + "Corral-Rubio", + "Cortil-Noirmont", + "Cortil-Noirmontois", + "Cortil-Noirmontoise", + "Cortil-Wodon", + "Corvol-d'Embernard", + "Corvol-l'Orgueilleux", + "Coslédaà-Lube-Boast", + "Cosne-Cours-sur-Loire", + "Cosne-d'Allier", + "Cossé-d'Anjou", + "Costa-Rica", + "Costa-Ricain", + "Costa-Ricien", + "Costa-Ricienne", + "Costa-Riciennes", + "Costa-Riciens", + "Coteau-Landais", + "Coteau-Libre", + "Coti-Chiavari", + "Coucy-le-Château-Auffrique", + "Coudekerque-Branche", + "Coudekerque-Village", + "Coudekerque-sur-le-Rhin", + "Coudray-Rabut", + "Couesmes-Vaucé", + "Couilly-Pont-aux-Dames", + "Coulonges-Cohan", + "Coulonges-Thouarsais", + "Couloumé-Mondebat", + "Coulounieix-Chamiers", + "Coulouvray-Boisbenâtre", + "Coupelle-Neuve", + "Coupelle-Vieille", + "Cour-Cheverny", + "Cour-Maugis sur Huisne", + "Cour-l'Evêque", + "Cour-l'Évêque", + "Courcelles-Chaussy", + "Courcelles-Epayelles", + "Courcelles-Frémoy", + "Courcelles-Sapicourt", + "Courcelles-Val-d'Esnoms", + "Cournon-d'Auvergne", + "Courson-Monteloup", + "Courtemont-Varennes", + "Courtonne-les-Deux-Eglises", + "Courtonne-les-Deux-Églises", + "Courtrai-Dutsel", + "Coussac-Bonneval", + "Couture-d'Argenson", + "Coux et Bigaroque-Mouzens", + "Couze-et-Saint-Front", + "Couzon-au-Mont-d'Or", + "Cran-Gevrien", + "Cran-Gevrienne", + "Cran-Gevriennes", + "Cran-Gevriens", + "Cran-Gevrier", + "Cranves-Sales", + "Cranves-Salien", + "Cranves-Saliens", + "Cranves-Saliène", + "Cranves-Saliènes", + "Cras-Avernas", + "Crempigny-Bonneguête", + "Cressin-Rochefort", + "Cressy-Omencourt", + "Crest-Voland", + "Crest-Volantain", + "Crest-Volantaine", + "Crest-Volantaines", + "Crest-Volantains", + "Crevans-et-la-Chapelle-lès-Granges", + "Crevant-Laveine", + "Creys-Mépieu", + "Criquetot-l'Esneval", + "Cro-Magnon", + "Cro-Magnons", + "Croissy-Beaubourg", + "Croix-Caluois", + "Croix-Caluoise", + "Croix-Caluoises", + "Croix-Caluyau", + "Croix-Chapeau", + "Croix-Fonsomme", + "Croix-Fonsommes", + "Croix-Mare", + "Croix-Moligneaux", + "Croix-Rousse", + "Croix-Valmer", + "Croix-lez-Rouveroy", + "Crouzet-Migette", + "Crozes-Hermitage", + "Crucey-Villages", + "Cruci-Capétien", + "Cruci-Capétienne", + "Cruci-Capétiennes", + "Cruci-Capétiens", + "Cruci-Falgardien", + "Cruci-Falgardienne", + "Cruci-Falgardiennes", + "Cruci-Falgardiens", + "Cruquius-Oost", + "Cruviers-Lascours", + "Crèvecœur-en-Auge", + "Crèvecœur-en-Brie", + "Crèvecœur-le-Grand", + "Crèvecœur-le-Petit", + "Crèvecœur-sur-l'Escaut", + "Crécy-Couvé", + "Créon-d'Armagnac", + "Cubjac-Auvézère-Val-d'Ans", + "Cugliate-Fabiasco", + "Cuiry-Housse", + "Cuiry-Houssien", + "Cuiry-Houssienne", + "Cuiry-Houssiennes", + "Cuiry-Houssiens", + "Cujavie-Poméranie", + "Cumières-le-Mort-Homme", + "Cuq-Toulza", + "Curciat-Dongalon", + "Cureghem-lez-Bruxelles", + "Curis-au-Mont-d'Or", + "Cursolo-Orasso", + "Curti-Marignacais", + "Curti-Marignacaise", + "Curti-Marignacaises", + "Curtil-Vergy", + "Cussac-Fort-Médoc", + "Cuttoli-Corticchiato", + "Cuxac-Cabardès", + "Cuxac-d'Aude", + "Cuyk-Sainte-Agathe", + "Cœuvres-et-Valsery", + "Céaux-d'Allègre", + "Céleste-Empire", + "Cénac-et-Saint-Julien", + "Cérans-Foulletourte", + "Céroux-Mousty", + "Césarville-Dossainville", + "Côtes-d'Armor", + "D'Huison-Longueville", + "D-Day", + "D-glucuronate", + "D-glucuronates", + "D-glycéraldéhyde", + "D-sucre", + "D-sucres", + "DIN-31635", + "DMTA-P", + "DOM-ROM", + "DOM-TOM", + "DVD-RAM", + "DVD-ROM", + "DVD-RW", + "Dagny-Lambercy", + "Dahme-Forêt-de-Spree", + "Dainville-Bertheléville", + "Dalberg-Wendelstorf", + "Dallgow-Döberitz", + "Dame-Marie", + "Dame-Marie-les-Bois", + "Dammartin-Marpain", + "Dampierre-sur-le-Doubs", + "Dancourt-Popincourt", + "Danne-et-Quatre-Vents", + "Dannstadt-Schauernheim", + "Danube-Ries", + "Darmstadt-Dieburg", + "Daubeuf-Serville", + "Delap-Uliga-Darrit", + "Delley-Portalban", + "Delouze-Rosières", + "Demitz-Thumitz", + "Dennweiler-Frohnbach", + "Dessau-Rosslau", + "Deux-Acren", + "Deux-Chaises", + "Deux-Chaisois", + "Deux-Chaisoise", + "Deux-Chaisoises", + "Deux-Evailles", + "Deux-Jumeaux", + "Deux-Montagnais", + "Deux-Ponts", + "Deux-Rivières", + "Deux-Sèvres", + "Deux-Verges", + "Deux-Évailles", + "Deuxnouds-devant-Beauzée", + "Devrai-Gondragnier", + "Devrai-Gondragniers", + "Devrai-Gondragnière", + "Devrai-Gondragnières", + "Dhuys et Morin-en-Brie", + "Diane-Capelle", + "Dieffenbach-lès-Wœrth", + "Diekhusen-Fahrstedt", + "Diennes-Aubigny", + "Diensdorf-Radlow", + "Diera-Zehren", + "Dietzenrode-Vatterode", + "Dilsen-Stokkem", + "Dingolfing-Landau", + "Dion-Valmont", + "Diony-Sapinois", + "Diony-Sapinoise", + "Diony-Sapinoises", + "Dissen-Striesow", + "Dissé-sous-le-Lude", + "Dittelsheim-Heßloch", + "Divitz-Spoldershagen", + "Doati-Casteidois", + "Doati-Casteidoise", + "Doati-Casteidoises", + "Dobbin-Linstow", + "Doberlug-Kirchhain", + "Doberschau-Gaußig", + "Dohm-Lammersdorf", + "Dolus-d'Oléron", + "Domart-sur-la-Luce", + "Dombasle-devant-Darney", + "Domburg-Binnen", + "Domburg-Buiten", + "Domecy-sur-le-Vault", + "Domezain-Berraute", + "Domléger-Longvillers", + "Dommarie-Eulmont", + "Dommartin-Dampierre", + "Dommartin-Lettrée", + "Dommartin-Varimont", + "Dommartin-le-Saint-Père", + "Dommary-Baroncourt", + "Dompierre-Becquincourt", + "Domremy-Landéville", + "Dongen-Vaart", + "Donnemarie-Dontilly", + "Dore-l'Eglise", + "Dore-l'Église", + "Dorn-Dürkheim", + "Dornburg-Camburg", + "Dossenheim-Kochersberg", + "Doubet-Talibautier", + "Doubet-Talibautiers", + "Doubet-Talibautière", + "Doubet-Talibautières", + "Douchy-Montcorbon", + "Doulaincourt-Saucourt", + "Doumely-Bégny", + "Dourd'Hal", + "Drachenbronn-Birlenbach", + "Dragey-Ronthon", + "Dreis-Brück", + "Dreuil-Hamel", + "Driebergen-Rijsenburg", + "Droupt-Sainte-Marie", + "Drumettaz-Clarafond", + "Druy-Parigny", + "Druyes-les-Belles-Fontaines", + "Drémil-Lafage", + "Ducey-Les Chéris", + "Ducy-Sainte-Marguerite", + "Duhort-Bachen", + "Duino-Aurisina", + "Dupont-Lajoie", + "Durban-Corbières", + "Durdat-Larequille", + "Durfort-Lacapelette", + "Durfort-et-Saint-Martin-de-Sossenac", + "Dœuil-sur-le-Mignon", + "Dão-Lafões", + "Débats-Rivière-d'Orpra", + "Décines-Charpieu", + "Dégrad-Edmond", + "Dégrad-Samson", + "Dénezé-sous-le-Lude", + "Dörfles-Esbach", + "Dürrröhrsdorf-Dittersbach", + "Dœuil-sur-le-Mignon", + "E7,Z9-12:Ac", + "E7-Z9-dodécadiénylacétate", + "E8,E10-dodécadiène-1-ol", + "EE-8,10-DDDOL", + "Eaux-Bonnais", + "Eaux-Bonnaise", + "Eaux-Bonnaises", + "Eaux-Bonnes", + "Eaux-Puiseaux", + "Eberbach-Seltz", + "Eberbach-Wœrth", + "Ebersbach-Musbach", + "Ebnat-Kappel", + "Ecalles-Alix", + "Eccica-Suarella", + "Echarri-Aranaz", + "Echenans-sous-Mont-Vaudois", + "Echt-Susteren", + "Eclans-Nenon", + "Eclaron-Braucourt-Sainte-Livière", + "Eclose-Badinières", + "Eclusier-Vaux", + "Ecole-Valentin", + "Ecotay-l'Olme", + "Ectot-l'Auber", + "Edam-Volendam", + "Ediger-Eller", + "Edingen-Neckarhausen", + "Eelde-Paterswolde", + "Effelder-Rauenstein", + "Efringen-Kirchen", + "Eggenstein-Leopoldshafen", + "Eglise-Neuve-d'Issac", + "Eglise-Neuve-de-Vergt", + "Egliseneuve-d'Entraigues", + "Egmond-Binnen", + "Eguzon-Chantôme", + "Egée-Méridionale", + "Egée-Septentrionale", + "Ehra-Lessien", + "Eifel-Bitburg-Prüm", + "Eijsden-Margraten", + "Elbe-Elster", + "Elbe-Parey", + "Elburgo-Burgelu", + "Elchesheim-Illingen", + "Eleu-dit-Leauwette", + "Elincourt-Sainte-Marguerite", + "Elisabeth-Sophien-Koog", + "Elise-Daucourt", + "Ellenz-Poltersdorf", + "Ellignies-Sainte-Anne", + "Elsdorf-Westermühlen", + "Elvillar-Bilar", + "Emmelsbüll-Horsbüll", + "Emmer-Compascuum", + "Emmer-Erfscheidenveen", + "Emmingen-Liptingen", + "Enclave-de-la-Martinière", + "Enencourt-Léage", + "Enge-Sande", + "Enkenbach-Alsenborn", + "Ennepe-Ruhr", + "Enquin-lez-Guinegatte", + "Entraigues-sur-la-Sorgue", + "Epagne-Epagnette", + "Epagny Metz-Tessy", + "Epaux-Bézu", + "Epi-Contois", + "Epi-Contoise", + "Epi-Contoises", + "Epiais-Rhus", + "Epinay-Champlâtreux", + "Eppe-Sauvage", + "Eppenberg-Wöschnau", + "Epreville-près-le-Neubourg", + "Equennes-Eramecourt", + "Equeurdreville-Hainneville", + "Equihen-Plage", + "Erbes-Büdesheim", + "Ergué-Armel", + "Ergué-Gabéric", + "Erlangen-Höchstadt", + "Erlbach-Kirchberg", + "Ernemont-Boutavent", + "Ernolsheim-Bruche", + "Ernolsheim-Saverne", + "Erpe-Mere", + "Erps-Kwerps", + "Erquinghem-Lys", + "Esboz-Brest", + "Eschêne-Autrage", + "Esclassan-Labastide", + "Esclavolles-Lurey", + "Escolives-Sainte-Camille", + "Escombres-et-le-Chesnois", + "Escoubès-Pouts", + "Escueillens-et-Saint-Just-de-Bélengard", + "Eslourenties-Daban", + "Esmery-Hallon", + "Espagnac-Sainte-Eulalie", + "Espierres-Helchin", + "Espinasse-Vozelle", + "Esplantas-Vazeilles", + "Esprit-Saint", + "Espès-Undurein", + "Esquay-Notre-Dame", + "Esquièze-Sère", + "Essert-Pittet", + "Essert-Romanais", + "Essert-Romanaise", + "Essert-Romanaises", + "Essert-Romand", + "Esserts-Blay", + "Esserts-Salève", + "Esserval-Combe", + "Esserval-Tartre", + "Estouteville-Ecalles", + "Estouteville-Écalles", + "Estrée-Blanche", + "Estrée-Blanchois", + "Estrée-Blanchoise", + "Estrée-Blanchoises", + "Estrée-Cauchois", + "Estrée-Cauchoise", + "Estrée-Cauchoises", + "Estrée-Cauchy", + "Estrée-Wamin", + "Estrée-Waminois", + "Estrée-Waminoise", + "Estrée-Waminoises", + "Estrées-Deniécourt", + "Estrées-Mons", + "Etang-Bertrand", + "Etang-Salé", + "Etang-Saléen", + "Etang-Saléenne", + "Etang-Saléennes", + "Etang-Saléens", + "Etang-Vergy", + "Etival-Clairefontaine", + "Etival-lès-le-Mans", + "Etrelles-et-la-Montbleuse", + "Etricourt-Manancourt", + "Etricourt-Manancourtois", + "Etricourt-Manancourtoise", + "Etricourt-Manancourtoises", + "Etten-Leur", + "Etxarri-Aranatz", + "Euro-Afrique", + "Eurville-Bienville", + "Evette-Salbert", + "Evin-Malmaison", + "Evry-Grégy-sur-Yerre", + "Eygluy-Escoulin", + "Eyres-Moncube", + "Eyzin-Pinet", + "Ezkio-Itsaso", + "Eßleben-Teutleben", + "Faches-Thumesnil", + "Faches-Thumesnilois", + "Faches-Thumesniloise", + "Faches-Thumesniloises", + "Faget-Abbatial", + "Fains-Véel", + "Far-West", + "Farcy-Pontain", + "Farcy-Pontaine", + "Farcy-Pontaines", + "Farcy-Pontains", + "Fargau-Pratjau", + "Farges-Allichamps", + "Fatouville-Grestain", + "Fatu-Hiva", + "Faucogney-et-la-Mer", + "Faveraye-Mâchelles", + "Faverges-Seythenex", + "Faverges-de-la-Tour", + "Faye-d'Anjou", + "Faye-l'Abbesse", + "Fayet-Ronaye", + "Fayl-Billot", + "Fayl-Billotin", + "Fayl-Billotine", + "Fayl-Billotines", + "Fayl-Billotins", + "Fayt-lez-Manage", + "Febvin-Palfart", + "Fehl-Ritzhausen", + "Felben-Wellhausen", + "Feldkirchen-Westerham", + "Ferme-Neuvien", + "Ferney-Voltaire", + "Fernán-Núñez", + "Ferreux-Quincey", + "Ferrière-Larçon", + "Ferrières-Haut-Clocher", + "Ferrières-Poussarou", + "Fessanvilliers-Mattanvilliers", + "Fessey-Dessous-et-Dessus", + "Festes-et-Saint-André", + "Feuguerolles-Bully", + "Fexhe-Slins", + "Fexhe-le-Haut-Clocher", + "Fichous-Riumayou", + "Fieffes-Montrelet", + "Fierville-Bray", + "Figaró-Montmany", + "Finkenbach-Gersweiler", + "Fiquefleur-Equainville", + "Fiquefleur-Équainville", + "Fischbach-Göslikon", + "Fischbach-Oberraden", + "Fitz-James", + "Fitz-Jamois", + "Fitz-Jamoise", + "Fitz-Jamoises", + "Fize-Fontaine", + "Flagey-Echézeaux", + "Flagey-Rigney", + "Flagey-Échézeaux", + "Flaignes-Havys", + "Flamets-Frétils", + "Flancourt-Catelon", + "Flancourt-Crescy-en-Roumois", + "Flandre-Occidentale", + "Flandre-Orientale", + "Flaujac-Gare", + "Flaujac-Poujols", + "Flaumont-Waudrechies", + "Flavigny-le-Grand-et-Beaurain", + "Flers-lez-Lille", + "Fleurey-lès-Saint-Loup", + "Fleury-Montmarin", + "Fleury-Mérogis", + "Fleury-Vallée-d'Aillant", + "Fleury-devant-Douaumont", + "Flez-Cuzy", + "Flieth-Stegelitz", + "Flines-lez-Raches", + "Flins-Neuve-Eglise", + "Flins-Neuve-Église", + "Floh-Seligenthal", + "Florimont-Gaumier", + "Flottemanville-Hague", + "Fluorn-Winzeln", + "Flémalle-Grande", + "Flémalle-Haute", + "Fléville-Lixières", + "Fléville-devant-Nancy", + "Flörsheim-Dalsheim", + "Foameix-Ornel", + "Fohren-Linden", + "Follainville-Dennemont", + "Fonches-Fonchette", + "Fondachelli-Fantina", + "Font-Romeu-Odeillo-Via", + "Font-rubí", + "Fontaines-d'Ozillac", + "Fontanil-Cornillon", + "Fontenay-Mauvoisin", + "Fontenay-Torcy", + "Fontenay-Trésigny", + "Fontenelle-Montby", + "Fontenille-Saint-Martin-d'Entraigues", + "Fontenilles-d'Aigueparse", + "Fontevraud-l'Abbaye", + "Fontiers-Cabardès", + "Fontiès-d'Aude", + "Forest-Montiers", + "Forest-l'Abbaye", + "Forge-Philippe", + "Forlì-Cesena", + "Forst-Längenbühl", + "Forêt-Noire", + "Forêt-Noire-Baar", + "Fosbury-flop", + "Foucaucourt-Hors-Nesle", + "Foufflin-Ricametz", + "Fourcatier-et-Maison-Neuve", + "Fournes-Cabardès", + "Fournet-Blancheroche", + "Fournets-Luisans", + "Foussais-Payré", + "Fouta-Diallon", + "Fouta-Djalon", + "Fox-Amphoux", + "Foy-Notre-Dame", + "Foy-Notre-Damien", + "Foy-Notre-Damienne", + "Foz-Calanda", + "Fragnes-La Loyère", + "Fraisse-Cabardès", + "Framerville-Rainecourt", + "Francfort-sur-le-Main", + "Franche-Comté", + "Franches-Montagnes", + "Francilly-Selency", + "Frasnay-Reugny", + "Frasnes-lez-Anvaing", + "Frasnes-lez-Buissenal", + "Frasnes-lez-Couvin", + "Frasnes-lez-Gosselies", + "Fredersdorf-Vogelsdorf", + "Frei-Laubersheim", + "Freienstein-Teufen", + "Freix-Anglards", + "Fresnay-l'Evêque", + "Fresnay-l'Évêque", + "Fresne-Cauverville", + "Fresne-Léguillon", + "Fresne-l'Archevêque", + "Fresneaux-Montchevreuil", + "Fresnes-Mazancourt", + "Fresnes-Tilloloy", + "Fresnoy-Andainville", + "Fresnoy-Folny", + "Freyming-Merlebach", + "Freyming-Merlebachois", + "Freyming-Merlebachoise", + "Freyming-Merlebachoises", + "Freyung-Grafenau", + "Friedrich-Wilhelm-Lübke-Koog", + "Frise-Occidentale", + "Friville-Escarbotin", + "Frières-Faillouël", + "Froidmont-Cohartille", + "Frontenay-Rohan-Rohan", + "Frontignan-Savès", + "Frutigen-Bas-Simmental", + "Fränkisch-Crumbach", + "Fréchet-Aure", + "Fréchou-Fréchet", + "Frédéric-Fontaine", + "Frémeréville-sous-les-Côtes", + "Frévin-Capelle", + "Fuente-Olmedo", + "Fuente-Tójar", + "Fuente-Álamo", + "Full-Reuenthal", + "Fère-Champenoise", + "Félines-Minervois", + "Félines-Termenès", + "Férolles-Attilly", + "Fêche-l'Eglise", + "Fêche-l'Église", + "Fête-Dieu", + "Föhrden-Barl", + "Gaag-Maasland", + "Gaag-Schipluiden", + "Gaasterlân-Sleat", + "Gabbioneta-Binanuova", + "Gadz'Arette", + "Gadz'Arettes", + "Gadz'Arts", + "Gagne-monopanglotte", + "Gaillac-Toulza", + "Gaillac-d'Aveyron", + "Gaillardbois-Cressenville", + "Gallin-Kuppentin", + "Gamiz-Fika", + "Gampel-Bratsch", + "Garcelles-Secqueville", + "Gargilesse-Dampierre", + "Garlède-Mondebat", + "Garmisch-Partenkirchen", + "Garrigues-Sainte-Eulalie", + "Garzau-Garzin", + "Gaspé-Nordien", + "Gaspésie-Îles-de-la-Madeleine", + "Gasville-Oisème", + "Gau-Algesheim", + "Gau-Bickelheim", + "Gau-Bischofsheim", + "Gau-Heppenheim", + "Gau-Odernheim", + "Gau-Weinheim", + "Gauchin-Légal", + "Gauchin-Verloingt", + "Gaurain-Ramecroix", + "Gavarnie-Gèdre", + "Geest-Gérompont", + "Geest-Gérompont-Petit-Rosière", + "Geldrop-Mierlo", + "Gemert-Bakel", + "Genac-Bignac", + "Gennes-Ivergny", + "Gennes-Val de Loire", + "Gentioux-Pigerolles", + "Georges-Fontaine", + "Germano-Iranien", + "Germigny-Pend-la-Pie", + "Germigny-l'Evêque", + "Germigny-l'Exempt", + "Germigny-l'Évêque", + "Germo-Roburien", + "Germo-Roburienne", + "Germo-Roburiennes", + "Germo-Roburiens", + "Germond-Rouvre", + "Gernika-Lumo", + "Gevrey-Chambertin", + "Gez-ez-Angles", + "Geüs-d'Oloron", + "Giardini-Naxos", + "Giel-Courteilles", + "Giessen-Nieuwkerk", + "Giessen-Oudekerk", + "Giffaumont-Champaubert", + "Gigny-Bussy", + "Ginsheim-Gustavsburg", + "Gipf-Oberfrick", + "Girmont-Val-d'Ajol", + "Gironville-sous-les-Côtes", + "Givenchy-lès-la-Bassée", + "Givet-Notre-Dame", + "Glabbeek-Zuurbemde", + "Glaine-Montaigut", + "Glaire-Latour", + "Glan-Münchweiler", + "Glane-Beekhoek", + "Glaude-Arbourois", + "Gleiszellen-Gleishorbach", + "Goeree-Overflakkee", + "Gognies-Chaussée", + "Goldbach-Altenbach", + "Gommenec'h", + "Gomzé-Andoumont", + "Gond-Pontouvre", + "Gondenans-Montby", + "Gondenans-Moulins", + "Gondrecourt-Aix", + "Gonfreville-Caillot", + "Gonfreville-l'Orcher", + "Gonneville-Le Theil", + "Gorden-Staupitz", + "Gorges-du-Tarn-Causses", + "Gornate-Olona", + "Gorom-Gorom", + "Gors-Opleeuw", + "Gossersweiler-Stein", + "Gotein-Libarrenx", + "Gourdan-Polignan", + "Gourdan-Polignanais", + "Gourdan-Polignanaise", + "Gourdan-Polignanaises", + "Gourdon-Murat", + "Gouris-Taitien", + "Gouris-Taitienne", + "Gouris-Taitiennes", + "Gouris-Taitiens", + "Gournay-Loizé", + "Gout-Rossignol", + "Gouy-Servins", + "Gouy-l'Hôpital", + "Gouy-lez-Piéton", + "Graal-Müritz", + "Graben-Neudorf", + "Grabow-Below", + "Gracieux-Berluron", + "Gracieux-Berluronne", + "Graffigny-Chemin", + "Graignes-Mesnil-Angot", + "Grainville-Langannerie", + "Grainville-Ymauville", + "Grancey-le-Château-Neuvelle", + "Grandcamp-Maisy", + "Grandpuits-Bailly-Carrois", + "Grandvelle-et-le-Perrenot", + "Grandville-Gaudreville", + "Granges-Aumontzey", + "Granges-Maillot", + "Granges-Narboz", + "Granges-Paccot", + "Granges-Sainte-Marie", + "Granges-d'Ans", + "Granzay-Gript", + "Graveron-Sémerville", + "Grebs-Niendorf", + "Gremersdorf-Buchholz", + "Grenier-Montgon", + "Grenville-sur-la-Rouge", + "Grenzach-Wyhlen", + "Gretz-Armainvilliers", + "Grez-Doiceau", + "Grez-Neuville", + "Grez-Neuvillois", + "Grez-Neuvilloise", + "Grez-Neuvilloises", + "Grindorff-Bizing", + "Grisy-Suisnes", + "Grivy-Loisy", + "Groot-Abeele", + "Groot-Ammers", + "Groot-Gelmen", + "Groot-Loo", + "Groot-Loon", + "Groot-Valkenisse", + "Groot-Wetsinge", + "Gross-Gerau", + "Grosse-Islois", + "Grosseto-Prugna", + "Grote-Brogel", + "Grote-Spouwen", + "Grouches-Luchuel", + "Groß-Bieberau", + "Groß-Gerau", + "Groß-Rohrheim", + "Groß-Umstadt", + "Groß-Zimmern", + "Großtreben-Zwethau", + "Grugé-l'Hôpital", + "Grun-Bordas", + "Grunow-Dammendorf", + "Grâce-Berleur", + "Grâce-Hollogne", + "Grâce-Uzel", + "Grâcieux-Hollognois", + "Grâcieux-Hollognoise", + "Grèce-Centrale", + "Grèce-Occidentale", + "Grébault-Mesnil", + "Grésigny-Sainte-Reine", + "Gréville-Hague", + "Grézet-Cavagnan", + "Grünhain-Beierfeld", + "Gudmont-Villiers", + "Guessling-Hémering", + "Guilherand-Granges", + "Guilherandais-Grangeois", + "Guilherandaise-Grangeoise", + "Guilherandaises-Grangeoises", + "Guilligomarc'h", + "Guinarthe-Parenties", + "Guinée-Bissao", + "Guinée-Bissau", + "Guinée-Conakry", + "Guipry-Messac", + "Guitalens-L'Albarède", + "Gujan-Mestras", + "Gulpen-Wittem", + "Gumbrechtshoffen-Oberbronn", + "Gusow-Platkow", + "Gutenzell-Hürbel", + "Gutierre-Muñoz", + "Guttet-Feschel", + "Guyans-Durnes", + "Guyans-Vennes", + "Guyencourt-Saulcourt", + "Gué-d'Allérien", + "Gué-d'Allérienne", + "Gué-d'Allériennes", + "Gué-d'Allériens", + "Gué-d'Hossus", + "Guémené-Penfao", + "Gy-l'Evêque", + "Gy-l'Évêque", + "Gère-Bélesten", + "Gère-Bélestinois", + "Gère-Bélestinoise", + "Gère-Bélestinoises", + "Gée-Rivière", + "Géfosse-Fontenay", + "Géus-d'Arzacq", + "Göhren-Döhlen", + "Göhren-Lebbin", + "Gülitz-Reetz", + "Gülzow-Prüzen", + "Günthersleben-Wechmar", + "Gœgnies-Chaussée", + "Ha'ava", + "Habère-Lullin", + "Habère-Poche", + "Hadancourt-le-Haut-Clocher", + "Halenbeck-Rohlsdorf", + "Halle-Booienhoven", + "Halle-Heide", + "Halle-Nijman", + "Hallennes-lez-Haubourdin", + "Halles-sous-les-Côtes", + "Halling-lès-Boulay-Moselle", + "Ham-Nalinnois", + "Ham-Nalinnoise", + "Ham-Nordois", + "Ham-sans-Culottes", + "Ham-sur-Heure-Nalinnes", + "Hamelin-Pyrmont", + "Hames-Boucres", + "Hames-Boucrois", + "Hames-Boucroise", + "Hames-Boucroises", + "Hamme-Mille", + "Hamme-Millois", + "Hamme-Milloise", + "Hamont-Achel", + "Han-devant-Pierrepont", + "Hanerau-Hademarschen", + "Hangen-Weisheim", + "Hannonville-Suzémont", + "Hannonville-sous-les-Côtes", + "Hantes-Wihéries", + "Hardencourt-Cocherel", + "Hardinxveld-Giessendam", + "Harth-Pöllnitz", + "Hartmannsdorf-Reichenau", + "Hastière-Lavaux", + "Hastière-par-delà", + "Haucourt-Moulaine", + "Hauenstein-Ifenthal", + "Hauptwil-Gottshaus", + "Hautecourt-Romanèche", + "Hautem-Sainte-Marguerite", + "Hauteville-Gondon", + "Hauteville-Lompnes", + "Hauteville-Lompnés", + "Hautot-l'Auvray", + "Hautteville-Bocage", + "Hautvillers-Ouville", + "Hazerswoude-Dorp", + "Hazerswoude-Rijndijk", + "Hechtel-Eksel", + "Heckelberg-Brunow", + "Heeswijk-Dinther", + "Heeze-Leende", + "Heiltz-l'Evêque", + "Heiltz-l'Évêque", + "Heist-op-den-Berg", + "Heist-sur-la-Montagne", + "Helen-Bos", + "Hellemmes-Lille", + "Hellenhahn-Schellenberg", + "Hellschen-Heringsand-Unterschaar", + "Helmstadt-Bargen", + "Hem-Hardinval", + "Hem-Lenglet", + "Hem-Monacu", + "Hendrik-Ido-Ambacht", + "Henri-Chapelle", + "Henstedt-Ulzburg", + "Herdwangen-Schönach", + "Hermitage-Lorge", + "Hernán-Pérez", + "Herpy-l'Arlésienne", + "Herren-Sulzbach", + "Herschweiler-Pettersheim", + "Hersfeld-Rotenburg", + "Hersin-Coupigny", + "Herzebrock-Clarholz", + "Hesdin-l'Abbé", + "Hettange-Grande", + "Heubécourt-Haricourt", + "Heuchelheim-Klingen", + "Heucourt-Croquoison", + "Heudicourt-sous-les-Côtes", + "Heuilley-Cotton", + "Heume-l'Eglise", + "Heume-l'Église", + "Heusden-Zolder", + "Hiers-Brouage", + "Higuères-Souye", + "Hilgertshausen-Tandern", + "Hinzert-Pölert", + "Hirz-Maulsbach", + "Hiva-Oa", + "Hochdorf-Assenheim", + "Hochstetten-Dhaun", + "Hodenc-l'Evêque", + "Hodenc-l'Évêque", + "Hodeng-Hodenger", + "Hofstetten-Flüh", + "Hohen-Sülzen", + "Hohenberg-Krusemark", + "Hohenfels-Essingen", + "Hohenstein-Ernstthal", + "Hollande-Méridionale", + "Hollande-Septentrionale", + "Hollern-Twielenfleth", + "Hombourg-Budange", + "Hombourg-Haut", + "Hon-Hergeois", + "Hon-Hergeoise", + "Hon-Hergeoises", + "Hon-Hergies", + "Hong-Kong", + "Honguemare-Guenouville", + "Hoog-Baarlo", + "Hoog-Caestert", + "Hoog-Geldrop", + "Hoog-Keppel", + "Hoogezand-Sappemeer", + "Hoorebeke-Sainte-Marie", + "Hoppstädten-Weiersbach", + "Horbourg-Wihr", + "Horion-Hozémont", + "Horndon-on-the-Hill", + "Hornow-Wadelsdorf", + "Horrenbach-Buchen", + "Hoste-Haut", + "Hostello-Flavien", + "Hostello-Flavienne", + "Hostello-Flaviennes", + "Hostello-Flaviens", + "Houdain-lez-Bavay", + "Houdeng-Aimeries", + "Houdeng-Goegnies", + "Houdeng-Gœgnies", + "Houlbec-Cocherel", + "Houlbec-près-le-Gros-Theil", + "Houphouët-Boigny", + "Houplin-Ancoisne", + "Hout-Blerick", + "Houtain-l'Évêque", + "Houthalen-Helchteren", + "Houvin-Houvigneul", + "Hoya-Gonzalo", + "Huanne-Montmartin", + "Hubert-Folie", + "Huison-Longueville", + "Humes-Jorquenay", + "Hurons-Wendat", + "Husseren-Wesserling", + "Hussigny-Godbrange", + "Huércal-Overa", + "Huétor-Tájar", + "Hyèvre-Magny", + "Hyèvre-Paroisse", + "Häg-Ehrsberg", + "Hédé-Bazouges", + "Hénin-Beaumont", + "Hérinnes-lez-Enghien", + "Hô-Chi-Minh-Ville", + "Hôme-Chamondot", + "Hôpital-Camfrout", + "Hôpital-d'Orion", + "Höhenkirchen-Siegertsbrunn", + "Höhr-Grenzhausen", + "Hörselberg-Hainich", + "I-frame", + "II-VI", + "III-V", + "IS-IS", + "Iamalo-Nénètsie", + "Idanha-a-Nova", + "Idar-Oberstein", + "Idaux-Mendy", + "Idrac-Respaillès", + "Igny-Comblizy", + "Igny-Marin", + "Igny-Marine", + "Igny-Marines", + "Igny-Marins", + "Illiers-Combray", + "Illiers-l'Evêque", + "Illiers-l'Évêque", + "Illkirch-Graffenstaden", + "Illnau-Effretikon", + "Ilo-Dionysien", + "Ilo-Dionysienne", + "Ilo-Dionysiennes", + "Ilo-Dionysiens", + "Iléos-Meldois", + "Iléos-Meldoise", + "Iléos-Meldoises", + "Ingrandes-Le Fresne sur Loire", + "Injoux-Génissiat", + "Interlaken-Oberhasli", + "Inval-Boiron", + "Inzinzac-Lochrist", + "Iruraiz-Gauna", + "Isle-Adam", + "Isle-Arné", + "Isle-Aubigny", + "Isle-Aumont", + "Isle-Bouzon", + "Isle-Jourdain", + "Isle-Vertois", + "Isle-d'Abeau", + "Isle-d'Espagnac", + "Isle-sur-la-Sorgue", + "Isle-sur-le-Doubs", + "Isolaccio-di-Fiumorbo", + "Issoudun-Létrieix", + "Issy-l'Evêque", + "Issy-l'Évêque", + "Ithorots-Olhaïby", + "Ivano-Fracena", + "Ivoz-Ramet", + "J-pop", + "J-rock", + "JAX-RPC", + "JAX-RS", + "Jacob-Bellecombette", + "Jandrain-Jandrenouille", + "Jarnac-Champagne", + "Jarzé-Villages", + "Jassans-Riottier", + "Jau-Dignac-et-Loirac", + "Jaunay-Clan", + "Jaunay-Clanais", + "Jaunay-Clanaise", + "Jaunay-Clanaises", + "Jaunay-Marigny", + "Javerlhac-et-la-Chapelle-Saint-Robert", + "Jeannois-Mitissien", + "Jettingen-Scheppach", + "Jeu-Maloches", + "Jeu-Malochois", + "Jeu-Malochoise", + "Jeu-Malochoises", + "Ji-hu", + "Ji-hun", + "Jodoigne-Souveraine", + "John-Bull", + "Jollain-Merlin", + "Jonzier-Epagny", + "Jonzier-Épagny", + "Jorat-Menthue", + "Jouars-Pontchartrain", + "Jouxtens-Mézery", + "Jouy-Mauvoisin", + "Joué-l'Abbé", + "Joué-Étiau", + "Juaye-Mondaye", + "Jubbega-Schurega", + "Jugeals-Nazareth", + "Jugon-les-Lacs - Commune nouvelle", + "Juif-Errant", + "Juifs-Errants", + "Juigné-Béné", + "Jully-Sarçois", + "Jully-Sarçoise", + "Jully-Sarçoises", + "Jussecourt-Minecourt", + "Jussy-Champagne", + "Justine-Herbigny", + "Juvigny Val d'Andaine", + "Juvigny-Val-d'Andaine", + "Juzet-d'Izaut", + "Jésus-Christ", + "Jû-Belloc", + "K-POP", + "K-Pop", + "K-bis", + "K-pop", + "Kaala-Gomen", + "Kabardino-Balkarie", + "Kaiser-Wilhelm-Koog", + "Kalenborn-Scheuern", + "Kamerik-Houtdijken", + "Kamerik-Mijzijde", + "Kamp-Bornhofen", + "Kamp-Lintfort", + "Kamperzeedijk-Oost", + "Kamperzeedijk-West", + "Kani-Kéli", + "Kapel-Avezaath", + "Kapelle-op-den-Bos", + "Kapellen-Drusweiler", + "Kappel-Grafenhausen", + "Kara-Koum", + "Karangasso-Sambla", + "Karangasso-Vigué", + "Karatchaïévo-Tcherkassie", + "Karbow-Vietlübbe", + "Karlsdorf-Neuthard", + "Karlstadt-sur-le-Main", + "Kasbach-Ohlenberg", + "Kasel-Golzig", + "Kastel-Staadt", + "Katlenburg-Lindau", + "Kaysersberg-Vignoble", + "Kelpen-Oler", + "Kenz-Küstrow", + "Kerckom-lez-Saint-Trond", + "Kergrist-Moëlou", + "Kerk-Avezaath", + "Kerkom-bij-Sint-Truiden", + "Kermaria-Sulard", + "Kermoroc'h", + "Kerry-Blue-terrier", + "Kersaint-Plabennec", + "Kersbeek-Miskom", + "Kessel-Eik", + "Kessel-Lo", + "Kiel-Windeweer", + "Kingston-upon-Hull", + "Kingston-upon-Thames", + "Kino-Congolais", + "Kirkby-in-Ashfield", + "Kirrwiller-Bosselshausen", + "Kizil-Arvat", + "Klazienaveen-Noord", + "Klein-Amsterdam", + "Klein-Bedaf", + "Klein-Brabant", + "Klein-Delfgauw", + "Klein-Doenrade", + "Klein-Dongen", + "Klein-Overleek", + "Klein-Ulsda", + "Klein-Valkenisse", + "Klein-Wetsinge", + "Klein-Winternheim", + "Klein-Zundert", + "Kleine-Brogel", + "Kleine-Spouwen", + "Kleßen-Görne", + "Klooster-Lidlum", + "Klosters-Serneus", + "Knokke-Heist", + "Knopp-Labach", + "Kobern-Gondorf", + "Kohren-Sahlis", + "Komki-Ipala", + "Korbeek-Dijle", + "Korbeek-Lo", + "Korntal-Münchingen", + "Kottweiler-Schwanden", + "Kradolf-Schönenberg", + "Kreba-Neudorf", + "Kreimbach-Kaulbach", + "Kröppelshagen-Fahrendorf", + "Kuhlen-Wendorf", + "KwaZulu-Natal", + "Kyzyl-Arvat", + "Kœur-la-Grande", + "Kœur-la-Petite", + "Kölln-Reisiek", + "Königsbach-Stein", + "Königshain-Wiederau", + "L'Abergement-Clémenciat", + "L'Abergement-Sainte-Colombe", + "L'Abergement-de-Cuisery", + "L'Abergement-de-Varey", + "L'Absie", + "L'Aigle", + "L'Aiguillon", + "L'Aiguillon-sur-Mer", + "L'Aiguillon-sur-Vie", + "L'Ajoupa-Bouillon", + "L'Albenc", + "L'Albère", + "L'Arbresle", + "L'Argentière-la-Bessée", + "L'Aubépin", + "L'Escale", + "L'Escarène", + "L'Estréchure", + "L'Habit", + "L'Haÿ-les-Roses", + "L'Herbergement", + "L'Herm", + "L'Hermenault", + "L'Hermitage", + "L'Honor-de-Cos", + "L'Horme", + "L'Hosmes", + "L'Hospitalet", + "L'Hospitalet-du-Larzac", + "L'Hospitalet-près-l'Andorre", + "L'Houmeau", + "L'Huisserie", + "L'Hôme-Chamondot", + "L'Hôpital", + "L'Hôpital-Saint-Blaise", + "L'Hôpital-Saint-Lieffroy", + "L'Hôpital-d'Orion", + "L'Hôpital-du-Grosbois", + "L'Hôpital-le-Grand", + "L'Hôpital-le-Mercier", + "L'Hôpital-sous-Rochefort", + "L'Hôtellerie", + "L'Hôtellerie-de-Flée", + "L'Isle-Adam", + "L'Isle-Arné", + "L'Isle-Bouzon", + "L'Isle-Jourdain", + "L'Isle-d'Abeau", + "L'Isle-d'Espagnac", + "L'Isle-de-Noé", + "L'Isle-en-Dodon", + "L'Isle-sur-Serein", + "L'Isle-sur-la-Sorgue", + "L'Isle-sur-le-Doubs", + "L'Orbrie", + "L'Oudon", + "L'Union", + "L'Écaille", + "L'Échelle", + "L'Échelle-Saint-Aurin", + "L'Écouvotte", + "L'Église-aux-Bois", + "L'Éguille", + "L'Épine", + "L'Épine-aux-Bois", + "L'Étang-Bertrand", + "L'Étang-Salé", + "L'Étang-Vergy", + "L'Étang-la-Ville", + "L'Étoile", + "L'Étrat", + "L'Île-Bouchard", + "L'Île-Rousse", + "L'Île-Saint-Denis", + "L'Île-d'Elle", + "L'Île-d'Olonne", + "L'Île-d'Yeu", + "L-aminoacide", + "L-aminoacides", + "L-flampropisopropyl", + "L-glycéraldéhyde", + "LGBTI-friendly", + "LGBTI-phobie", + "LGBTI-phobies", + "La Balme-d'Épy", + "La Balme-de-Sillingy", + "La Balme-de-Thuy", + "La Balme-les-Grottes", + "La Barre-de-Monts", + "La Barre-de-Semilly", + "La Barthe-de-Neste", + "La Basse-Vaivre", + "La Bastide-Clairence", + "La Bastide-Pradines", + "La Bastide-Puylaurent", + "La Bastide-Solages", + "La Bastide-d'Engras", + "La Bastide-de-Besplas", + "La Bastide-de-Bousignac", + "La Bastide-de-Lordat", + "La Bastide-de-Sérou", + "La Bastide-des-Jourdans", + "La Bastide-du-Salat", + "La Bastide-sur-l'Hers", + "La Baule-Escoublac", + "La Baume-Cornillane", + "La Baume-d'Hostun", + "La Baume-de-Transit", + "La Bazoche-Gouet", + "La Bazoge-Montpinçon", + "La Bazouge-de-Chemeré", + "La Bazouge-des-Alleux", + "La Bazouge-du-Désert", + "La Bernerie-en-Retz", + "La Besseyre-Saint-Mary", + "La Boissière-d'Ans", + "La Boissière-de-Montaigu", + "La Boissière-des-Landes", + "La Boissière-du-Doré", + "La Boissière-en-Gâtine", + "La Boissière-École", + "La Bollène-Vésubie", + "La Bonneville-sur-Iton", + "La Bosse-de-Bretagne", + "La Bourdinière-Saint-Loup", + "La Breille-les-Pins", + "La Bretonnière-la-Claye", + "La Brosse-Montceaux", + "La Bruère-sur-Loir", + "La Brée-les-Bains", + "La Bussière-sur-Ouche", + "La Bâtie-Montgascon", + "La Bâtie-Montsaléon", + "La Bâtie-Neuve", + "La Bâtie-Rolland", + "La Bâtie-Vieille", + "La Bâtie-des-Fonds", + "La Bégude-de-Mazenc", + "La Bénisson-Dieu", + "La Cadière-d'Azur", + "La Cadière-et-Cambo", + "La Caillère-Saint-Hilaire", + "La Capelle-Balaguier", + "La Capelle-Bleys", + "La Capelle-Bonance", + "La Capelle-et-Masmolène", + "La Capelle-lès-Boulogne", + "La Celle-Condé", + "La Celle-Dunoise", + "La Celle-Guenand", + "La Celle-Saint-Avant", + "La Celle-Saint-Cloud", + "La Celle-Saint-Cyr", + "La Celle-en-Morvan", + "La Celle-les-Bordes", + "La Celle-sous-Chantemerle", + "La Celle-sous-Gouzon", + "La Celle-sur-Loire", + "La Celle-sur-Morin", + "La Celle-sur-Nièvre", + "La Chaise-Baudouin", + "La Chaise-Dieu", + "La Chaize-Giraud", + "La Chaize-le-Vicomte", + "La Charité-sur-Loire", + "La Chartre-sur-le-Loir", + "La Chaussée-Saint-Victor", + "La Chaussée-Tirancourt", + "La Chaussée-d'Ivry", + "La Chaussée-sur-Marne", + "La Chaux-du-Dombief", + "La Chaux-en-Bresse", + "La Chaze-de-Peyre", + "La Châtre-Langlin", + "La Cluse-et-Mijoux", + "La Colle-sur-Loup", + "La Combe-de-Lancey", + "La Condamine-Châtelard", + "La Couarde-sur-Mer", + "La Cour-Marigny", + "La Couture-Boussey", + "La Croisille-sur-Briance", + "La Croix-Avranchin", + "La Croix-Blanche", + "La Croix-Comtesse", + "La Croix-Helléan", + "La Croix-Valmer", + "La Croix-aux-Bois", + "La Croix-aux-Mines", + "La Croix-de-la-Rochette", + "La Croix-du-Perche", + "La Croix-en-Brie", + "La Croix-en-Champagne", + "La Croix-en-Touraine", + "La Croix-sur-Gartempe", + "La Croix-sur-Ourcq", + "La Croix-sur-Roudoule", + "La Côte-Saint-André", + "La Côte-d'Arbroz", + "La Côte-en-Couzan", + "La Digne-d'Amont", + "La Digne-d'Aval", + "La Fage-Montivernoux", + "La Fage-Saint-Julien", + "La Fare-en-Champsaur", + "La Fare-les-Oliviers", + "La Faute-sur-Mer", + "La Ferrière-Airoux", + "La Ferrière-Bochard", + "La Ferrière-Béchet", + "La Ferrière-au-Doyen", + "La Ferrière-aux-Étangs", + "La Ferrière-de-Flée", + "La Ferrière-en-Parthenay", + "La Ferrière-sur-Risle", + "La Ferté-Alais", + "La Ferté-Beauharnais", + "La Ferté-Bernard", + "La Ferté-Chevresis", + "La Ferté-Gaucher", + "La Ferté-Hauterive", + "La Ferté-Imbault", + "La Ferté-Loupière", + "La Ferté-Macé", + "La Ferté-Milon", + "La Ferté-Saint-Aubin", + "La Ferté-Saint-Cyr", + "La Ferté-Saint-Samson", + "La Ferté-Vidame", + "La Ferté-Villeneuil", + "La Ferté-en-Ouche", + "La Ferté-sous-Jouarre", + "La Ferté-sur-Chiers", + "La Folletière-Abenon", + "La Fontaine-Saint-Martin", + "La Forest-Landerneau", + "La Forêt-Fouesnant", + "La Forêt-Sainte-Croix", + "La Forêt-de-Tessé", + "La Forêt-du-Parc", + "La Forêt-du-Temple", + "La Forêt-le-Roi", + "La Forêt-sur-Sèvre", + "La Fosse-Corduan", + "La Foye-Monjault", + "La Fresnaie-Fayel", + "La Frette-sur-Seine", + "La Garde-Adhémar", + "La Garde-Freinet", + "La Garenne-Colombes", + "La Gonterie-Boulouneix", + "La Grand-Combe", + "La Grand-Croix", + "La Grande-Fosse", + "La Grande-Motte", + "La Grande-Paroisse", + "La Grande-Résie", + "La Grande-Verrière", + "La Gripperie-Saint-Symphorien", + "La Grève-sur-Mignon", + "La Grée-Saint-Laurent", + "La Guerche-de-Bretagne", + "La Guerche-sur-l'Aubois", + "La Haie-Fouassière", + "La Haie-Traversaine", + "La Haute-Beaume", + "La Haute-Maison", + "La Haye-Aubrée", + "La Haye-Bellefond", + "La Haye-Malherbe", + "La Haye-Pesnel", + "La Haye-Saint-Sylvestre", + "La Haye-d'Ectot", + "La Haye-de-Calleville", + "La Haye-de-Routot", + "La Haye-du-Theil", + "La Haye-le-Comte", + "La Houssaye-Béranger", + "La Houssaye-en-Brie", + "La Jaille-Yvon", + "La Jarrie-Audouin", + "La Jonchère-Saint-Maurice", + "La Lande-Chasles", + "La Lande-Patry", + "La Lande-Saint-Léger", + "La Lande-Saint-Siméon", + "La Lande-d'Airou", + "La Lande-de-Fronsac", + "La Lande-de-Goult", + "La Lande-de-Lougé", + "La Lande-sur-Drôme", + "La Lanterne-et-les-Armonts", + "La Loge-Pomblin", + "La Loge-aux-Chèvres", + "La Londe-les-Maures", + "La Louptière-Thénard", + "La Louvière-Lauragais", + "La Lucerne-d'Outremer", + "La Madelaine-sous-Montreuil", + "La Madeleine-Bouvet", + "La Madeleine-Villefrouin", + "La Madeleine-de-Nonancourt", + "La Madeleine-sur-Loing", + "La Magdelaine-sur-Tarn", + "La Maison-Dieu", + "La Marolle-en-Sologne", + "La Mazière-aux-Bons-Hommes", + "La Meilleraie-Tillay", + "La Meilleraye-de-Bretagne", + "La Membrolle-sur-Choisille", + "La Monnerie-le-Montel", + "La Mothe-Achard", + "La Mothe-Saint-Héray", + "La Motte-Chalancon", + "La Motte-Fanjas", + "La Motte-Feuilly", + "La Motte-Fouquet", + "La Motte-Saint-Jean", + "La Motte-Saint-Martin", + "La Motte-Servolex", + "La Motte-Ternant", + "La Motte-Tilly", + "La Motte-d'Aigues", + "La Motte-d'Aveillans", + "La Motte-de-Galaure", + "La Motte-du-Caire", + "La Motte-en-Bauges", + "La Motte-en-Champsaur", + "La Mure-Argens", + "La Neuve-Grange", + "La Neuve-Lyre", + "La Neuvelle-lès-Lure", + "La Neuvelle-lès-Scey", + "La Neuveville-devant-Lépanges", + "La Neuveville-sous-Châtenois", + "La Neuveville-sous-Montfort", + "La Neuville-Bosmont", + "La Neuville-Chant-d'Oisel", + "La Neuville-Garnier", + "La Neuville-Housset", + "La Neuville-Roy", + "La Neuville-Saint-Pierre", + "La Neuville-Sire-Bernard", + "La Neuville-Vault", + "La Neuville-au-Pont", + "La Neuville-aux-Bois", + "La Neuville-aux-Joûtes", + "La Neuville-aux-Larris", + "La Neuville-d'Aumont", + "La Neuville-du-Bosc", + "La Neuville-en-Beine", + "La Neuville-en-Hez", + "La Neuville-en-Tourne-à-Fuy", + "La Neuville-lès-Bray", + "La Neuville-lès-Dorengt", + "La Neuville-lès-Wasigny", + "La Neuville-sur-Essonne", + "La Neuville-sur-Oudeuil", + "La Neuville-sur-Ressons", + "La Neuville-à-Maire", + "La Nocle-Maulaix", + "La Noë-Blanche", + "La Noë-Poulain", + "La Palud-sur-Verdon", + "La Penne-sur-Huveaune", + "La Penne-sur-l'Ouvèze", + "La Petite-Boissière", + "La Petite-Fosse", + "La Petite-Marche", + "La Petite-Pierre", + "La Petite-Raon", + "La Petite-Verrière", + "La Plaine-des-Palmistes", + "La Plaine-sur-Mer", + "La Poterie-Cap-d'Antifer", + "La Poterie-Mathieu", + "La Proiselière-et-Langle", + "La Queue-en-Brie", + "La Queue-les-Yvelines", + "La Rivière-Drugeon", + "La Rivière-Enverse", + "La Rivière-Saint-Sauveur", + "La Rivière-de-Corps", + "La Robine-sur-Galabre", + "La Roche-Bernard", + "La Roche-Blanche", + "La Roche-Canillac", + "La Roche-Chalais", + "La Roche-Clermault", + "La Roche-Derrien", + "La Roche-Guyon", + "La Roche-Mabile", + "La Roche-Maurice", + "La Roche-Morey", + "La Roche-Noire", + "La Roche-Posay", + "La Roche-Rigault", + "La Roche-Vanneau", + "La Roche-Vineuse", + "La Roche-de-Glun", + "La Roche-de-Rame", + "La Roche-des-Arnauds", + "La Roche-en-Brenil", + "La Roche-l'Abeille", + "La Roche-sur-Foron", + "La Roche-sur-Grane", + "La Roche-sur-Yon", + "La Roche-sur-le-Buis", + "La Rochebeaucourt-et-Argentine", + "La Rochette-du-Buis", + "La Ronde-Haye", + "La Roque-Alric", + "La Roque-Baignard", + "La Roque-Esclapon", + "La Roque-Gageac", + "La Roque-Sainte-Marguerite", + "La Roque-d'Anthéron", + "La Roque-en-Provence", + "La Roque-sur-Cèze", + "La Roque-sur-Pernes", + "La Roquette-sur-Siagne", + "La Roquette-sur-Var", + "La Rue-Saint-Pierre", + "La Répara-Auriples", + "La Résie-Saint-Martin", + "La Salette-Fallavaux", + "La Salle-en-Beaumont", + "La Salle-les-Alpes", + "La Salvetat-Belmontet", + "La Salvetat-Lauragais", + "La Salvetat-Peyralès", + "La Salvetat-Saint-Gilles", + "La Salvetat-sur-Agout", + "La Sauvetat-de-Savères", + "La Sauvetat-du-Dropt", + "La Sauvetat-sur-Lède", + "La Sauzière-Saint-Jean", + "La Selle-Craonnaise", + "La Selle-Guerchaise", + "La Selle-en-Coglès", + "La Selle-en-Hermoy", + "La Selle-en-Luitré", + "La Selle-la-Forge", + "La Selle-sur-le-Bied", + "La Serre-Bussière-Vieille", + "La Seyne-sur-Mer", + "La Suze-sur-Sarthe", + "La Séauve-sur-Semène", + "La Terrasse-sur-Dorlay", + "La Teste-de-Buch", + "La Tour-Blanche", + "La Tour-Saint-Gelin", + "La Tour-d'Aigues", + "La Tour-d'Auvergne", + "La Tour-de-Salvagny", + "La Tour-de-Sçay", + "La Tour-du-Crieu", + "La Tour-du-Meix", + "La Tour-du-Pin", + "La Tour-en-Jarez", + "La Tour-sur-Orb", + "La Tourette-Cabardès", + "La Tranche-sur-Mer", + "La Trinité-Porhoët", + "La Trinité-Surzur", + "La Trinité-de-Réville", + "La Trinité-de-Thouberville", + "La Trinité-des-Laitiers", + "La Trinité-du-Mont", + "La Trinité-sur-Mer", + "La Vacheresse-et-la-Rouillie", + "La Vacquerie-et-Saint-Martin-de-Castries", + "La Valette-du-Var", + "La Valla-en-Gier", + "La Valla-sur-Rochefort", + "La Vallée-Mulâtre", + "La Vallée-au-Blé", + "La Vendue-Mignot", + "La Vespière-Friardel", + "La Vicomté-sur-Rance", + "La Vieille-Loye", + "La Vieille-Lyre", + "La Vieux-Rue", + "La Ville-Dieu-du-Temple", + "La Ville-aux-Bois", + "La Ville-aux-Bois-lès-Dizy", + "La Ville-aux-Bois-lès-Pontavert", + "La Ville-aux-Clercs", + "La Ville-aux-Dames", + "La Ville-du-Bois", + "La Ville-sous-Orbais", + "La Ville-ès-Nonais", + "La Villedieu-du-Clain", + "La Villedieu-en-Fontenette", + "La Villeneuve-Bellenoye-et-la-Maize", + "La Villeneuve-au-Châtelot", + "La Villeneuve-au-Chêne", + "La Villeneuve-en-Chevrie", + "La Villeneuve-les-Convers", + "La Villeneuve-lès-Charleville", + "La Villeneuve-sous-Thury", + "La Voulte-sur-Rhône", + "La Vraie-Croix", + "La-Fertois", + "La-Fertoise", + "La-Fertoises", + "Laag-Caestert", + "Laag-Keppel", + "Laag-Nieuwkoop", + "Laag-Soeren", + "Labarthe-Bleys", + "Labarthe-Inard", + "Labarthe-Rivière", + "Labastide-Beauvoir", + "Labastide-Castel-Amouroux", + "Labastide-Chalosse", + "Labastide-Clairence", + "Labastide-Clermont", + "Labastide-Cézéracq", + "Labastide-Dénat", + "Labastide-Esparbairenque", + "Labastide-Gabausse", + "Labastide-Marnhac", + "Labastide-Monréjeau", + "Labastide-Murat", + "Labastide-Paumès", + "Labastide-Rouairoux", + "Labastide-Savès", + "Labastide-Villefranche", + "Labastide-d'Anjou", + "Labastide-d'Armagnac", + "Labastide-du-Haut-Mont", + "Labatie-d'Andaure", + "Labatut-Rivière", + "Labergement-Foigney", + "Labergement-Sainte-Marie", + "Labessière-Candeil", + "Labets-Biscay", + "Labruyère-Dorsa", + "Labécède-Lauragais", + "Lac-Beauportois", + "Lac-Bouchettien", + "Lac-Carréen", + "Lac-Etcheminois", + "Lac-Humquien", + "Lac-Poulinois", + "Lac-Saguayen", + "Lac-des-Rouges-Truites", + "Lac-ou-Villers", + "Lac-Édouard", + "Lacam-d'Ourcet", + "Lacapelle-Barrès", + "Lacapelle-Biron", + "Lacapelle-Cabanac", + "Lacapelle-Livron", + "Lacapelle-Marival", + "Lacapelle-Pinet", + "Lacapelle-Ségalar", + "Lacapelle-Viescamp", + "Lacapelle-del-Fraisse", + "Lacarry-Arhan-Charritte-de-Haut", + "Lachamp-Raphaël", + "Lachapelle-Auzac", + "Lachapelle-Graillouse", + "Lachaussée-du-Bois-d'Ecu", + "Lachaussée-du-Bois-d'Écu", + "Lacougotte-Cadoul", + "Lacour-d'Arcenay", + "Lacroix-Barrez", + "Lacroix-Falgarde", + "Ladevèze-Rivière", + "Ladevèze-Ville", + "Ladoix-Serrigny", + "Laethem-Sainte-Marie", + "Laffite-Toupière", + "Lafitte-Vigordane", + "Lagarde-Enval", + "Lagarde-Hachan", + "Lagarde-Paréol", + "Lagarde-d'Apt", + "Lagarde-sur-le-Né", + "Lagnicourt-Marcel", + "Lagrâce-Dieu", + "Laguian-Mazous", + "Laguinge-Restoue", + "Lahitte-Toupière", + "Lahn-Dill", + "Laissac-Sévérac l'Église", + "Laize-Clinchamps", + "Lalanne-Arqué", + "Lalanne-Trie", + "Lalevade-d'Ardèche", + "Lalouret-Laffiteau", + "Lamadeleine-Val-des-Anges", + "Lamarque-Pontacq", + "Lamarque-Rustaing", + "Lamazière-Basse", + "Lamazière-Haute", + "Lambres-lez-Aire", + "Lambres-lez-Douai", + "Lamonzie-Montastruc", + "Lamothe-Capdeville", + "Lamothe-Cassel", + "Lamothe-Cumont", + "Lamothe-Fénelon", + "Lamothe-Goas", + "Lamothe-Landerron", + "Lamothe-Montravel", + "Lamotte-Beuvron", + "Lamotte-Brebière", + "Lamotte-Buleux", + "Lamotte-Warfusée", + "Lampaul-Guimiliau", + "Lampaul-Plouarzel", + "Lampaul-Ploudalmézeau", + "Lanciego-Lantziego", + "Lancken-Granitz", + "Landes-Vieilles-et-Neuves", + "Landrecourt-Lempire", + "Landres-et-Saint-Georges", + "Landsberg-am-Lech", + "Laneuveville-derrière-Foug", + "Laneuveville-devant-Bayon", + "Laneuveville-devant-Nancy", + "Langemark-Poelkapelle", + "Langenleuba-Niederhain", + "Languedoc-Roussillon", + "Languedoc-Roussillon-Midi-Pyrénées", + "Languevoisin-Quiquery", + "Lanitz-Hassel-Tal", + "Lanne-Soubiran", + "Lanne-Soubiranais", + "Lanne-Soubiranaise", + "Lanne-Soubiranaises", + "Lannoy-Cuillère", + "Lans-l'Hermitage", + "Lansen-Schönau", + "Lanslebourg-Mont-Cenis", + "Lantenne-Vertière", + "Lançon-Provence", + "Lapeyrouse-Fossat", + "Lapeyrouse-Mornay", + "Laragne-Montéglin", + "Larceveau-Arros-Cibits", + "Largillay-Marsonnay", + "Larivière-Arnoncourt", + "Larmor-Baden", + "Larmor-Plage", + "Laroque-Timbaut", + "Laroque-d'Olmes", + "Larribar-Sorhapuru", + "Larroque-Engalin", + "Larroque-Toirac", + "Lasarte-Oria", + "Lascellas-Ponzano", + "Lasne-Chapelle-Saint-Lambert", + "Lasseube-Propre", + "Latouille-Lentillac", + "Latour-Bas-Elne", + "Latrecey-Ormoy-sur-Aube", + "Lau-Balagnas", + "Lau-Balutin", + "Lau-Balutine", + "Lau-Balutines", + "Lau-Balutins", + "Lauda-Königshofen", + "Laudio-Llodio", + "Laudun-l'Ardoise", + "Laufen-Uhwiesen", + "Launay-Villersois", + "Launay-Villersoise", + "Launay-Villersoises", + "Launay-Villiers", + "Laure-Minervois", + "Lauwin-Planque", + "Laux-Montaux", + "Laval-Atger", + "Laval-Morency", + "Laval-Pradel", + "Laval-Roquecezière", + "Laval-d'Aix", + "Laval-d'Aurelle", + "Lavancia-Epercy", + "Lavans-Quingeois", + "Lavans-Quingeoise", + "Lavans-Quingeoises", + "Lavans-Quingey", + "Lavans-Vuillafans", + "Lavans-lès-Saint-Claude", + "Lavault-Sainte-Anne", + "Lavaux-Oron", + "Lavaux-Sainte-Anne", + "Laveline-devant-Bruyères", + "Laveno-Mombello", + "Lavernose-Lacasse", + "Lavey-Morcles", + "Lavoûte-Chilhac", + "Lawarde-Mauger-l'Hortoy", + "Lay-Lamidou", + "Lays-sur-le-Doubs", + "Laà-Mondrans", + "Le Ban-Saint-Martin", + "Le Bar-sur-Loup", + "Le Bec-Hellouin", + "Le Bec-Thomas", + "Le Bellay-en-Vexin", + "Le Bignon-Mirabeau", + "Le Bignon-du-Maine", + "Le Blanc-Mesnil", + "Le Bois-Hellain", + "Le Bois-Plage-en-Ré", + "Le Bois-Robert", + "Le Bois-d'Oingt", + "Le Bosc-Renoult", + "Le Bosc-Roger-en-Roumois", + "Le Bouchet-Mont-Charvin", + "Le Bouchet-Saint-Nicolas", + "Le Bouchon-sur-Saulx", + "Le Boulay-Morin", + "Le Boullay-Mivoye", + "Le Boullay-Thierry", + "Le Boullay-les-Deux-Églises", + "Le Bourg-Dun", + "Le Bourg-Saint-Léonard", + "Le Bourg-d'Hem", + "Le Bourg-d'Iré", + "Le Bourg-d'Oisans", + "Le Bourget-du-Lac", + "Le Bourgneuf-la-Forêt", + "Le Bousquet-d'Orb", + "Le Breil-sur-Mérize", + "Le Breuil-Bernard", + "Le Breuil-en-Auge", + "Le Breuil-en-Bessin", + "Le Breuil-sur-Couze", + "Le Brouilh-Monbert", + "Le Buisson-de-Cadouin", + "Le Bû-sur-Rouvres", + "Le Cannet-des-Maures", + "Le Castellard-Mélan", + "Le Cateau-Cambrésis", + "Le Caule-Sainte-Beuve", + "Le Chaffaut-Saint-Jurson", + "Le Chambon-Feugerolles", + "Le Chambon-sur-Lignon", + "Le Champ-Saint-Père", + "Le Champ-de-la-Pierre", + "Le Champ-près-Froges", + "Le Château-d'Almenêches", + "Le Château-d'Oléron", + "Le Châtelet-en-Brie", + "Le Châtelet-sur-Meuse", + "Le Châtelet-sur-Retourne", + "Le Châtelet-sur-Sormonne", + "Le Châtenet-en-Dognon", + "Le Cloître-Pleyben", + "Le Cloître-Saint-Thégonnec", + "Le Collet-de-Dèze", + "Le Coudray-Macouard", + "Le Coudray-Montceaux", + "Le Coudray-Saint-Germer", + "Le Coudray-sur-Thelle", + "Le Fay-Saint-Quentin", + "Le Freney-d'Oisans", + "Le Fresne-Camilly", + "Le Fresne-Poret", + "Le Frestoy-Vaux", + "Le Gault-Perche", + "Le Gault-Saint-Denis", + "Le Gault-Soigny", + "Le Genest-Saint-Isle", + "Le Grand-Bornand", + "Le Grand-Bourg", + "Le Grand-Celland", + "Le Grand-Lemps", + "Le Grand-Lucé", + "Le Grand-Madieu", + "Le Grand-Pressigny", + "Le Grand-Quevilly", + "Le Grand-Serre", + "Le Grand-Village-Plage", + "Le Grau-du-Roi", + "Le Gué-d'Alleré", + "Le Gué-de-Longroi", + "Le Gué-de-Velluire", + "Le Gué-de-la-Chaîne", + "Le Haut-Corlay", + "Le Hommet-d'Arthenay", + "Le Housseau-Brétignolles", + "Le Hérie-la-Viéville", + "Le Kremlin-Bicêtre", + "Le Lac-d'Issarlès", + "Le Lardin-Saint-Lazare", + "Le Lauzet-Ubaye", + "Le Lion-d'Angers", + "Le Loroux-Bottereau", + "Le Louroux-Béconnais", + "Le Malzieu-Forain", + "Le Malzieu-Ville", + "Le Marais-la-Chapelle", + "Le Mas-d'Agenais", + "Le Mas-d'Artige", + "Le Mas-d'Azil", + "Le Mas-de-Tence", + "Le Masnau-Massuguiès", + "Le May-sur-Èvre", + "Le Mayet-d'École", + "Le Mayet-de-Montagne", + "Le Meix-Saint-Epoing", + "Le Meix-Tiercelin", + "Le Minihic-sur-Rance", + "Le Molay-Littry", + "Le Monastier-sur-Gazeille", + "Le Monestier-du-Percy", + "Le Mont-Dieu", + "Le Mont-Saint-Adrien", + "Le Mont-Saint-Michel", + "Le Monteil-au-Vicomte", + "Le Monêtier-les-Bains", + "Le Morne-Rouge", + "Le Morne-Vert", + "Le Moulinet-sur-Solin", + "Le Mée-sur-Seine", + "Le Ménil-Broût", + "Le Ménil-Bérard", + "Le Ménil-Ciboult", + "Le Ménil-Guyon", + "Le Ménil-Scelleur", + "Le Ménil-Vicomte", + "Le Ménil-de-Briouze", + "Le Mêle-sur-Sarthe", + "Le Nouvion-en-Thiérache", + "Le Noyer-en-Ouche", + "Le Palais-sur-Vienne", + "Le Pas-Saint-l'Homer", + "Le Pavillon-Sainte-Julie", + "Le Perray-en-Yvelines", + "Le Perreux-sur-Marne", + "Le Petit-Bornand-les-Glières", + "Le Petit-Celland", + "Le Petit-Fougeray", + "Le Petit-Mercey", + "Le Petit-Pressigny", + "Le Petit-Quevilly", + "Le Pian-Médoc", + "Le Pian-sur-Garonne", + "Le Pin-Murelet", + "Le Pin-au-Haras", + "Le Pin-la-Garenne", + "Le Plan-de-la-Tour", + "Le Plessier-Huleu", + "Le Plessier-Rozainvillers", + "Le Plessier-sur-Bulles", + "Le Plessier-sur-Saint-Just", + "Le Plessis-Belleville", + "Le Plessis-Bouchard", + "Le Plessis-Brion", + "Le Plessis-Dorin", + "Le Plessis-Feu-Aussoux", + "Le Plessis-Gassot", + "Le Plessis-Grammoire", + "Le Plessis-Grimoult", + "Le Plessis-Grohan", + "Le Plessis-Hébert", + "Le Plessis-Lastelle", + "Le Plessis-Luzarches", + "Le Plessis-Patte-d'Oie", + "Le Plessis-Placy", + "Le Plessis-Pâté", + "Le Plessis-Robinson", + "Le Plessis-Sainte-Opportune", + "Le Plessis-Trévise", + "Le Plessis-aux-Bois", + "Le Plessis-l'Échelle", + "Le Plessis-l'Évêque", + "Le Poiré-sur-Velluire", + "Le Poiré-sur-Vie", + "Le Poizat-Lalleyriat", + "Le Pont-Chrétien-Chabenet", + "Le Pont-de-Beauvoisin", + "Le Pont-de-Claix", + "Le Port-Marly", + "Le Poujol-sur-Orb", + "Le Poët-Célard", + "Le Poët-Laval", + "Le Poët-Sigillat", + "Le Poët-en-Percip", + "Le Pré-Saint-Gervais", + "Le Pré-d'Auge", + "Le Puy-Notre-Dame", + "Le Puy-Sainte-Réparade", + "Le Puy-en-Velay", + "Le Péage-de-Roussillon", + "Le Quesnel-Aubry", + "Le Quesnoy-en-Artois", + "Le Relecq-Kerhuon", + "Le Revest-les-Eaux", + "Le Rouget-Pers", + "Le Rousset-Marizy", + "Le Sap-André", + "Le Sappey-en-Chartreuse", + "Le Sauze-du-Lac", + "Le Sel-de-Bretagne", + "Le Taillan-Médoc", + "Le Tartre-Gaudran", + "Le Temple-de-Bretagne", + "Le Temple-sur-Lot", + "Le Tertre-Saint-Denis", + "Le Theil-Nolent", + "Le Theil-de-Bretagne", + "Le Theil-en-Auge", + "Le Thil-Riberpré", + "Le Thoult-Trosnay", + "Le Thuit de l'Oison", + "Le Tilleul-Lambert", + "Le Tilleul-Othon", + "Le Torp-Mesnil", + "Le Touquet-Paris-Plage", + "Le Tour-du-Parc", + "Le Tremblay-Omonville", + "Le Tremblay-sur-Mauldre", + "Le Val d'Hazey", + "Le Val d'Ocre", + "Le Val-David", + "Le Val-Saint-Germain", + "Le Val-Saint-Père", + "Le Val-Saint-Éloi", + "Le Val-d'Ajol", + "Le Val-d'Esnoms", + "Le Val-de-Gouhenans", + "Le Val-de-Guéblange", + "Le Vanneau-Irleau", + "Le Verdon-sur-Mer", + "Le Vernet-Sainte-Marguerite", + "Le Vieil-Dampierre", + "Le Vieil-Évreux", + "Le Vieux-Bourg", + "Le Vieux-Cérier", + "Le Vieux-Marché", + "Le Vivier-sur-Mer", + "Leers-Nord", + "Leidschendam-Voorburg", + "Leinefelde-Worbis", + "Leinfelden-Echterdingen", + "Leintz-Gatzaga", + "Lelin-Lapujolle", + "Leménil-Mitry", + "Lens-Lestang", + "Lentillac-Lauzès", + "Leo-Stichting", + "Lepuix-Neuf", + "Les Adrets-de-l'Estérel", + "Les Aix-d'Angillon", + "Les Alluets-le-Roi", + "Les Ancizes-Comps", + "Les Angles-sur-Corrèze", + "Les Anses-d'Arlet", + "Les Artigues-de-Lussac", + "Les Autels-Villevillon", + "Les Authieux-Papion", + "Les Authieux-du-Puits", + "Les Authieux-sur-Calonne", + "Les Authieux-sur-le-Port-Saint-Ouen", + "Les Avanchers-Valmorel", + "Les Avenières Veyrins-Thuellin", + "Les Baux-Sainte-Croix", + "Les Baux-de-Breteuil", + "Les Baux-de-Provence", + "Les Bois d'Anjou", + "Les Bordes-Aumont", + "Les Bordes-sur-Arize", + "Les Bordes-sur-Lez", + "Les Cent-Acres", + "Les Champs-Géraux", + "Les Champs-de-Losque", + "Les Chapelles-Bourbon", + "Les Chavannes-en-Maurienne", + "Les Châtelliers-Notre-Dame", + "Les Clayes-sous-Bois", + "Les Contamines-Montjoie", + "Les Corvées-les-Yys", + "Les Costes-Gozon", + "Les Côtes-d'Arey", + "Les Côtes-de-Corps", + "Les Deux-Fays", + "Les Deux-Villes", + "Les Essards-Taignevaux", + "Les Essarts-le-Roi", + "Les Essarts-le-Vicomte", + "Les Essarts-lès-Sézanne", + "Les Eyzies-de-Tayac-Sireuil", + "Les Grandes-Armoises", + "Les Grandes-Chapelles", + "Les Grandes-Loges", + "Les Grandes-Ventes", + "Les Grands-Chézeaux", + "Les Granges-Gontardes", + "Les Granges-le-Roi", + "Les Hautes-Rivières", + "Les Hauts-de-Chée", + "Les Hôpitaux-Neufs", + "Les Hôpitaux-Vieux", + "Les Isles-Bardel", + "Les Istres-et-Bury", + "Les Landes-Genusson", + "Les Loges-Marchis", + "Les Loges-Margueron", + "Les Loges-Saulces", + "Les Loges-en-Josas", + "Les Loges-sur-Brécey", + "Les Lucs-sur-Boulogne", + "Les Lèves-et-Thoumeyragues", + "Les Magnils-Reigniers", + "Les Martres-d'Artière", + "Les Martres-de-Veyre", + "Les Moitiers-d'Allonne", + "Les Moitiers-en-Bauptois", + "Les Monts d'Andaine", + "Les Monts-Verts", + "Les Moutiers-en-Auge", + "Les Moutiers-en-Cinglais", + "Les Moutiers-en-Retz", + "Les Noës-près-Troyes", + "Les Ollières-sur-Eyrieux", + "Les Ormes-sur-Voulzie", + "Les Pavillons-sous-Bois", + "Les Pennes-Mirabeau", + "Les Petites-Armoises", + "Les Petites-Loges", + "Les Plains-et-Grands-Essarts", + "Les Planches-en-Montagne", + "Les Planches-près-Arbois", + "Les Ponts-de-Cé", + "Les Portes-en-Ré", + "Les Quatre-Routes-du-Lot", + "Les Rivières-Henruel", + "Les Roches-de-Condrieu", + "Les Roches-l'Évêque", + "Les Rosiers-sur-Loire", + "Les Rouges-Eaux", + "Les Rues-des-Vignes", + "Les Sables-d'Olonne", + "Les Salles-Lavauguyon", + "Les Salles-de-Castillon", + "Les Salles-du-Gardon", + "Les Salles-sur-Verdon", + "Les Souhesmes-Rampont", + "Les Terres-de-Chaux", + "Les Thilliers-en-Vexin", + "Les Touches-de-Périgny", + "Les Trois-Bassins", + "Les Trois-Domaines", + "Les Trois-Moutiers", + "Les Trois-Pierres", + "Les Trois-Îlets", + "Les Ventes-de-Bourse", + "Les Verchers-sur-Layon", + "Les Villards-sur-Thônes", + "Les Églises-d'Argenteuil", + "Les Églisottes-et-Chalaures", + "Leschères-sur-le-Blaiseron", + "Lescouët-Gouarec", + "Lescouët-Jugon", + "Lescure-Jaoul", + "Lescure-d'Albigeois", + "Lesparre-Médoc", + "Lespielle-Germenaud-Lannegrasse", + "Lessard-et-le-Chêne", + "Lestelle-Bétharram", + "Lestelle-de-Saint-Martory", + "Leuben-Schleinitz", + "Leulinghen-Bernes", + "Leusden-Zuid", + "Leval-Chaudeville", + "Leval-Trahegnies", + "Levallois-Perret", + "Leyritz-Moncassin", + "Lez-Fontaine", + "Li-Fi", + "Liancourt-Fosse", + "Lias-d'Armagnac", + "Libramont-Chevigny", + "Libre-Ville", + "Lichans-Sunhar", + "Lichterfeld-Schacksdorf", + "Licq-Athérey", + "Licy-Clignon", + "Lierde-Sainte-Marie", + "Liesse-Notre-Dame", + "Lieuran-Cabrières", + "Lignières-Châtelain", + "Lignières-Orgères", + "Lignières-Sonneville", + "Ligny-Haucourt", + "Ligny-Thilloy", + "Lillois-Witterzée", + "Limbach-Oberfrohna", + "Limburg-Weilburg", + "Limeil-Brévannes", + "Limetz-Villez", + "Limey-Remenauville", + "Limoges-Fourches", + "Limont-Fontaine", + "Lincheux-Hallivillers", + "Lindre-Basse", + "Lindre-Haute", + "Linières-Bouton", + "Linkenheim-Hochstetten", + "Liny-devant-Dun", + "Lion-devant-Dun", + "Lippersdorf-Erdmannsdorf", + "Lissay-Lochy", + "Listrac-Médoc", + "Livarot-Pays-d'Auge", + "Livers-Cazelles", + "Livry-Gargan", + "Livry-Louvercy", + "Lixing-lès-Saint-Avold", + "Lo-Reninge", + "Loc-Brévalaire", + "Loc-Eguiner", + "Loc-Eguiner-Saint-Thégonnec", + "Loc-Envel", + "Loc-Éguiner", + "Loc-Éguiner-Saint-Thégonnec", + "Locmaria-Berrien", + "Locmaria-Grand-Champ", + "Locmaria-Plouzané", + "Locoal-Mendon", + "Locoalo-Mendonnais", + "Locoalo-Mendonnaise", + "Locoalo-Mendonnaises", + "Loenen-Kronenburg", + "Loge-Fougereuse", + "Logny-Bogny", + "Logonna-Daoulas", + "Logonna-Quimerch", + "Logrian-Florian", + "Logrian-et-Comiac-de-Florian", + "Loguivy-Plougras", + "Lohe-Föhrden", + "Lohe-Rickelshof", + "Lohitzun-Oyhercq", + "Lohn-Ammannsegg", + "Loire-Atlantique", + "Loire-Authion", + "Loire-Inférieure", + "Loiron-Ruillé", + "Loisey-Culey", + "Loitsche-Heinrichsberg", + "Lombeek-Notre-Dame", + "Lona-Lases", + "Longeau-Percey", + "Longeville-lès-Saint-Avold", + "Longeville-sur-la-Laines", + "Longevilles-Mont-d'Or", + "Longpré-les-Corps-Saints", + "Longue-Rivois", + "Longueil-Annel", + "Longueil-Sainte-Marie", + "Longueval-Barbonval", + "Longué-Jumelles", + "Longwy-sur-le-Doubs", + "Longwé-l'Abbaye", + "Lonlay-l'Abbaye", + "Loon-Plage", + "Loreto-di-Casinca", + "Loreto-di-Tallano", + "Lorp-Sentaraille", + "Lorrez-le-Bocage-Préaux", + "Lorry-Mardigny", + "Louan-Villegruis-Fontaine", + "Loubens-Lauragais", + "Loubès-Bernac", + "Louchy-Montfand", + "Loulans-Verchamp", + "Loup-Maëlle", + "Loupiac-de-la-Réole", + "Lourdios-Ichère", + "Loures-Barousse", + "Louresse-Rochemenier", + "Louroux-Bourbonnais", + "Louroux-Hodement", + "Lourouzien-Bourbonnais", + "Lourouzienne-Bourbonnaise", + "Lourouziennes-Bourbonnaises", + "Lourouziens-Bourbonnais", + "Lourties-Monbrun", + "Loussous-Débat", + "Louvemont-Côte-du-Poivre", + "Louvie-Juzon", + "Louvie-Soubiron", + "Louvie-Soubironnais", + "Louvie-Soubironnaise", + "Louvie-Soubironnaises", + "Louvignies-Bavay", + "Louvignies-Quesnoy", + "Lozoyuela-Navas-Sieteiglesias", + "Loèche-Ville", + "Luby-Betmont", + "Luc-Armau", + "Lucenay-l'Evêque", + "Lucenay-l'Évêque", + "Luché-Pringé", + "Luché-Thouarsais", + "Ludon-Médoc", + "Lugaut-Retjons", + "Lugny-Bourbonnais", + "Lugny-Champagne", + "Lugo-di-Nazza", + "Lugon-et-l'Île-du-Carnay", + "Luhe-Wildenau", + "Lumigny-Nesles-Ormeaux", + "Lunel-Viel", + "Lunow-Stolzenhagen", + "Lupiñén-Ortilla", + "Luppé-Violles", + "Lurcy-Lévis", + "Lurcy-Lévy", + "Lus-la-Croix-Haute", + "Lusignan-Grand", + "Lusignan-Petit", + "Lussagnet-Lusson", + "Lussan-Adeilhac", + "Lussery-Villars", + "Luthenay-Uxeloup", + "Luxe-Sumberraute", + "Ly-Fontaine", + "Lys-Haut-Layon", + "Lys-lez-Lannoy", + "Lâ-Todin", + "Lège-Cap-Ferret", + "Léa-Lisa", + "Lées-Athas", + "Légion-d'Honneur", + "Lésignac-Durand", + "Lézignan-Corbières", + "Lüchow-Dannenberg", + "Lüterkofen-Ichertswil", + "Lüterswil-Gächliwil", + "Lüttow-Valluhn", + "M'Tsangamouji", + "Maarke-Kerkem", + "Macédoine-Centrale", + "Macédoine-Occidentale", + "Macédoine-Orientale-et-Thrace", + "Madlitz-Wilmersdorf", + "Mae-West", + "Mae-Wests", + "Maen-Roch", + "Magnac-Bourg", + "Magnac-Laval", + "Magnac-Lavalette-Villars", + "Magnat-l'Etrange", + "Magnat-l'Étrange", + "Magneux-Haute-Rive", + "Magny-Châtelard", + "Magny-Cours", + "Magny-Danigon", + "Magny-Fouchard", + "Magny-Jobert", + "Magny-Lambert", + "Magny-Lormes", + "Magny-Montarlot", + "Magny-Vernois", + "Maignaut-Tauzia", + "Maignelay-Montigny", + "Mailleroncourt-Charette", + "Mailly-Castellois", + "Mailly-Castelloise", + "Mailly-Castelloises", + "Mailly-Champagne", + "Mailly-Maillet", + "Mailly-Raineval", + "Main-Kinzig", + "Main-Spessart", + "Main-Tauber", + "Main-Taunus", + "Mairy-Mainville", + "Mairé-Levescault", + "Maisières-Notre-Dame", + "Maison-Blanche", + "Maison-Feyne", + "Maison-Maugis", + "Maison-Ponthieu", + "Maison-Roland", + "Maison-Rouge", + "Maisoncelle-Tuilerie", + "Maisoncelles-Pelvey", + "Maisons-Alfort", + "Maisons-Laffitte", + "Maisons-du-Bois-Lièvremont", + "Maizières-la-Grande-Paroisse", + "Malarce-sur-la-Thines", + "Malayo-Polynésiens", + "Malborghetto-Valbruna", + "Mallefougasse-Augès", + "Malleret-Boussac", + "Mallersdorf-Pfaffenberg", + "Malleville-sur-le-Bec", + "Mals-Peignées", + "Mals-Peignés", + "Malsburg-Marzell", + "Malèves-Sainte-Marie-Wastines", + "Manas-Bastanous", + "Mancenans-Lizerne", + "Mandres-aux-Quatre-Tours", + "Manent-Montané", + "Mango-Rosa", + "Maninghen-Henne", + "Manneken-pis", + "Manneville-ès-Plains", + "Mannweiler-Cölln", + "Mansfeld-Harz-du-Sud", + "Mantenay-Montlin", + "Mantes-Gassicourt", + "Marais-Vernier", + "Marange-Silvange", + "Marange-Zondrange", + "Marbourg-Biedenkopf", + "Marcellaz-Albanais", + "Marchais-Beton", + "Marchais-Béton", + "Marche-lez-Écaussinnes", + "Marchiennes-Campagne", + "Marché-Allouarde", + "Marcillac-Lanville", + "Marcillac-Vallon", + "Marcilly-Ogny", + "Marcilly-d'Azergues", + "Marcillé-Raoul", + "Marcillé-Robert", + "Marcq-en-Barœul", + "Marcy-l'Etoile", + "Marcy-l'Étoile", + "Mareil-Marly", + "Maren-Kessel", + "Maresquel-Ecquemicourt", + "Marest-Dampcourt", + "Mareuil-Caubert", + "Mareuil-sur-Lay-Dissais", + "Margaux-Cantenac", + "Margerie-Chantagret", + "Margerie-Hancourt", + "Margouët-Meymes", + "Maria-Hoop", + "Marignac-Lasclares", + "Marignac-Laspeyres", + "Marigny-Brizay", + "Marigny-Chemereau", + "Marigny-Marmande", + "Marigny-l'Eglise", + "Marigny-l'Église", + "Marigné-Laillé", + "Marigné-Peuton", + "Maring-Noviand", + "Marizy-Sainte-Geneviève", + "Markina-Xemein", + "Marly-Gomont", + "Marmont-Pachas", + "Marolles-lès-Saint-Calais", + "Marquette-lez-Lille", + "Marsais-Sainte-Radégonde", + "Martainville-Epreville", + "Martainville-Épreville", + "Martigny-Combe", + "Martigny-Courpierre", + "Martigné-Briand", + "Martigné-Ferchaud", + "Martin-Eglise", + "Martin-Église", + "Martouzin-Neuville", + "Martres-Tolosane", + "Martres-d'Artières", + "Maruri-Jatabe", + "Marvaux-Vieux", + "Marville-Moutiers-Brûlé", + "Mas-Blanc", + "Mas-Blanc-des-Alpilles", + "Mas-Cabardès", + "Mas-Chélyen", + "Mas-Chélyenne", + "Mas-Chélyennes", + "Mas-Chélyens", + "Mas-Grenier", + "Mas-Saintes-Puelles", + "Mas-Tençois", + "Mas-Tençoise", + "Mas-Tençoises", + "Mas-d'Auvignon", + "Mas-d'Orcières", + "Masbaraud-Mérignat", + "Mascaraàs-Haron", + "Masevaux-Niederbruck", + "Maspie-Lalonquère-Juillacq", + "Massa-Carrara", + "Massac-Séran", + "Massen-Niederlausitz", + "Massillargues-Attuech", + "Matafelon-Granges", + "Matignicourt-Goncourt", + "Matzlow-Garwitz", + "Maubert-Fontaine", + "Mauléon-Barousse", + "Mauléon-Licharre", + "Mauléon-d'Armagnac", + "Maulévrier-Sainte-Gertrude", + "Maumusson-Laguian", + "Maureillas-las-Illas", + "Maurens-Scopont", + "Mauvezin-d'Armagnac", + "Mauvezin-de-Sainte-Croix", + "Mauzac-et-Grand-Castang", + "Mauzé-Thouarsais", + "Mauzé-sur-le-Mignon", + "Mavilly-Mandelot", + "Mawashi-geri", + "Maxhütte-Haidhof", + "Mayen-Coblence", + "Mayence-Bingen", + "Mayres-Savel", + "Mayrinhac-Lentour", + "Mazan-l'Abbaye", + "Mazerat-Aurouze", + "Mazeyrat-Aurouze", + "Mazeyrat-d'Allier", + "Mazières-Naresse", + "Mazères-Lezons", + "Mazé-Milon", + "Maël-Carhaisien", + "Maël-Carhaisienne", + "Maël-Carhaisiennes", + "Maël-Carhaisiens", + "Maël-Carhaix", + "Maël-Pestivien", + "Mbanza-Ngungu", + "McDonald's", + "Meaulne-Vitray", + "Mechelen-aan-de-Maas", + "Mecklembourg-Poméranie-Occidentale", + "Mecklembourg-Strelitz", + "Mecklembourg-du-Nord-Ouest", + "Medina-Sidonia", + "Meensel-Kiezegem", + "Meerlo-Wanssum", + "Meeuwen-Gruitrode", + "Meillier-Fontaine", + "Meix-devant-Virton", + "Mellenbach-Glasbach", + "Melun-Sénart", + "Menetou-Couture", + "Menetou-Râtel", + "Menetou-Salon", + "Mengersgereuth-Hämmern", + "Mentque-Nortbécourt", + "Merbes-Sainte-Marie", + "Mercurol-Veaunes", + "Mercury-Gémilly", + "Mercus-Garrabet", + "Merkers-Kieselbach", + "Merkwiller-Pechelbronn", + "Merle-Leignec", + "Meroux-Moval", + "Merry-Sec", + "Merville-Franceville-Plage", + "Merzig-Wadern", + "Mesbrecourt-Richecourt", + "Meslin-l'Évêque", + "Metz-Campagne", + "Metz-Robert", + "Metz-Tesseran", + "Metz-Tesseranne", + "Metz-Tesserannes", + "Metz-Tesserans", + "Metz-Tessy", + "Metz-Ville", + "Metzerlen-Mariastein", + "Meunet-Planches", + "Meuselbach-Schwarzmühle", + "Meyrieux-Trouet", + "Meyrignac-l'Eglise", + "Meyrignac-l'Église", + "Mezzovico-Vira", + "Michel-Ange", + "Midden-Delfland", + "Midden-Drenthe", + "Midden-Eierland", + "Midi-Pyrénéen", + "Midi-Pyrénéens", + "Midi-Pyrénées", + "Midsland-Noord", + "Mielen-boven-Aalst", + "Mierlo-Hout", + "Mignaloux-Beauvoir", + "Migné-Auxances", + "Milhac-d'Auberoche", + "Milizac-Guipronvel", + "Mille-Islois", + "Millienhagen-Oebelitz", + "Milly-Lamartine", + "Min-jun", + "Min-seo", + "Minaucourt-le-Mesnil-lès-Hurlus", + "Minden-Lübbecke", + "Minho-Lima", + "Miniac-Morvan", + "Minihy-Tréguier", + "Minn'Gotain", + "Minn'Gotaine", + "Minn'Gotaines", + "Minn'Gotains", + "Miossens-Lanusse", + "Miquelon-Langlade", + "Miramont-Latour", + "Miramont-Sensacq", + "Miramont-d'Astarac", + "Mirandol-Bourgnounac", + "Miraval-Cabardes", + "Mireval-Lauragais", + "Miribel-Lanchâtre", + "Miserey-Salines", + "Misery-Courtion", + "Missen-Wilhams", + "Mitry-Mory", + "Mittainvilliers-Vérigny", + "Moca-Croce", + "Moidieu-Détourbe", + "Moirey-Flabas-Crépion", + "Moissac-Bellevue", + "Moissac-Vallée-Française", + "Moissy-Cramayel", + "Moissy-Moulinot", + "Molenbeek-Wersbeek", + "Molières-Cavaillac", + "Molières-Glandaz", + "Molliens-Dreuil", + "Monacia-d'Aullène", + "Monacia-d'Orezza", + "Monassut-Audiracq", + "Moncayolle-Larrory-Mendibieu", + "Monceau-Imbrechies", + "Monceau-le-Neuf-et-Faucouzy", + "Monceaux-l'Abbaye", + "Moncetz-Longevas", + "Moncetz-l'Abbaye", + "Monchaux-Soreng", + "Monchy-Breton", + "Monchy-Cayeux", + "Monchy-Humières", + "Monchy-Lagache", + "Moncorneil-Grazan", + "Mondariz-Balneario", + "Mondement-Montgivroux", + "Monestier-Merlines", + "Monestier-Port-Dieu", + "Monestier-d'Ambel", + "Monferran-Plavès", + "Monferran-Savès", + "Monflorite-Lascasas", + "Mongolie-intérieure", + "Monguelfo-Tesido", + "Monistrol-d'Allier", + "Monlaur-Bernet", + "Monlezun-d'Armagnac", + "Monléon-Magnoac", + "Monnetier-Mornex", + "Mons-en-Barœul", + "Monsempron-Libos", + "Monsteroux-Milieu", + "Montacher-Villegardin", + "Montagnac-Montpezat", + "Montagnac-d'Auberoche", + "Montagne-Fayel", + "Montagney-Servigney", + "Montagny-Sainte-Félicité", + "Montaigu-Zichem", + "Montalba-d'Amélie", + "Montalieu-Vercieu", + "Montambert-Tannay", + "Montaren-et-Saint-Médiers", + "Montastruc-Savès", + "Montbrun-Bocage", + "Montbrun-Lauragais", + "Montceaux-Ragny", + "Montceaux-l'Etoile", + "Montceaux-l'Étoile", + "Montclar-Lauragais", + "Montcourt-Fromonville", + "Montcuq-en-Quercy-Blanc", + "Montcy-Notre-Dame", + "Monte-Carlo", + "Montemor-o-Novo", + "Montemor-o-Velho", + "Montereau-Fault-Yonne", + "Montereau-faut-Yonne", + "Montereau-sur-le-Jard", + "Montescourt-Lizerolles", + "Montesquieu-Avantès", + "Montesquieu-Guittaut", + "Montesquieu-Lauragais", + "Montesquieu-Volvestre", + "Montfaucon-Montigné", + "Montfaucon-d'Argonne", + "Montfort-l'Amaury", + "Montgaillard-Lauragais", + "Monti-Corcellois", + "Monti-Corcelloise", + "Monti-Corcelloises", + "Monties-Aussos", + "Montignac-Charente", + "Montignac-Toupinerie", + "Montignies-lez-Lens", + "Montigny-Lencoup", + "Montigny-Lengrain", + "Montigny-Montfort", + "Montigny-Mornay-Villeneuve-sur-Vingeanne", + "Montigny-devant-Sassey", + "Montigny-l'Allier", + "Montis-Fagussin", + "Montis-Fagussine", + "Montis-Fagussines", + "Montis-Fagussins", + "Montmorency-Beaufort", + "Montmort-Lucy", + "Montoire-sur-le-Loir", + "Montoy-Flanville", + "Montpon-Ménestérol", + "Montredon-Labessonnié", + "Montreuil-Bellay", + "Montreuil-Bonnin", + "Montreuil-Juigné", + "Montreuil-Poulay", + "Montreuil-l'Argillé", + "Montreux-Château", + "Montreux-Jeune", + "Montreux-Vieux", + "Montrichard-Val-de-Cher", + "Montricher-Albanne", + "Montrol-Sénard", + "Montsecret-Clairefougère", + "Montsinéry-Tonnegrande", + "Montégut-Arros", + "Montégut-Bourjac", + "Montégut-Lauragais", + "Montégut-Plantaurel", + "Montégut-Savès", + "Monêtier-Allemont", + "Moor-Rolofshagen", + "Moorea-Maiao", + "Morainville-Jouveaux", + "Morgenröthe-Rautenkranz", + "Morigny-Champigny", + "Morlanwelz-Mariemont", + "Mornay-Berry", + "Mortain-Bocage", + "Morteaux-Couliboeuf", + "Morteaux-Coulibœuf", + "Morteaux-Coulibœuf", + "Mortes-Frontières", + "Mory-Montcrux", + "Motey-Besuche", + "Mouans-Sartoux", + "Mougon-Thorigné", + "Moulin-Mage", + "Moulin-Mageois", + "Moulin-Mageoise", + "Moulin-Mageoises", + "Moulin-Neuf", + "Moulin-l'Évêque", + "Moulins-Engilbert", + "Moult-Chicheboville", + "Mounes-Prohencoux", + "Mourioux-Vieilleville", + "Mournans-Charbonny", + "Mourvilles-Basses", + "Mourvilles-Hautes", + "Mousseaux-Neuville", + "Moussy-Verneuil", + "Moustier-Ventadour", + "Moustiers-Sainte-Marie", + "Moustoir-Ac", + "Moustoir-Remungol", + "Moutaine-Aresches", + "Mouterre-Silly", + "Mouthier-Haute-Pierre", + "Moutier-Malcard", + "Moutier-Rozeille", + "Moutier-d'Ahun", + "Moutiers-sur-le-Lay", + "Mouzieys-Panens", + "Mouzieys-Teulet", + "Moyenne-Franconie", + "Moyeuvre-Grande", + "Moyeuvre-Petite", + "Muad-Dib", + "Muille-Villette", + "Muncq-Nieurlet", + "Murtin-Bogny", + "Murtin-et-le-Châtelet", + "Mœurs-Verdey", + "Ménestérol-Montignac", + "Ménil'muche", + "Ménil-Annelles", + "Ménil-Annellois", + "Ménil-Annelloise", + "Ménil-Annelloises", + "Ménil-Erreux", + "Ménil-Froger", + "Ménil-Gondouin", + "Ménil-Gondoyen", + "Ménil-Gondoyenne", + "Ménil-Gondoyennes", + "Ménil-Gondoyens", + "Ménil-Hermei", + "Ménil-Hubert-en-Exmes", + "Ménil-Hubert-sur-Orne", + "Ménil-Jean", + "Ménil-Lépinois", + "Ménil-Vin", + "Méolans-Revel", + "Mérey-Vieilley", + "Méricourt-l'Abbé", + "Méry-Bissières-en-Auge", + "Méry-Corbon", + "Méry-Prémecy", + "Méry-ès-Bois", + "Méso-Amérique", + "Mévergnies-lez-Lens", + "Mézidon-Canon", + "Mézières-lez-Cléry", + "Mézy-Moulins", + "Mönchpfiffel-Nikolausrieth", + "Mörel-Filet", + "Mörfelden-Walldorf", + "Möriken-Wildegg", + "Mûrs-Erigné", + "Mûrs-Érigné", + "Mühlhausen-Ehingen", + "Mülheim-Kärlich", + "Mümliswil-Ramiswil", + "Münster-Geschinen", + "Münster-Sarmsheim", + "Mœurs-Verdey", + "N'Djamena", + "N'Djaména", + "N'Tcham", + "N'dorola", + "N,N-dinitronitramide", + "N-(4-hydroxyphényl)éthanamide", + "N-acétylcystéine", + "N-déméthyla", + "N-déméthylai", + "N-déméthylaient", + "N-déméthylais", + "N-déméthylait", + "N-déméthylant", + "N-déméthylas", + "N-déméthylasse", + "N-déméthylassent", + "N-déméthylasses", + "N-déméthylassiez", + "N-déméthylassions", + "N-déméthyle", + "N-déméthylent", + "N-déméthyler", + "N-déméthylera", + "N-déméthylerai", + "N-déméthyleraient", + "N-déméthylerais", + "N-déméthylerait", + "N-déméthyleras", + "N-déméthylerez", + "N-déméthyleriez", + "N-déméthylerions", + "N-déméthylerons", + "N-déméthyleront", + "N-déméthyles", + "N-déméthylez", + "N-déméthyliez", + "N-déméthylions", + "N-déméthylons", + "N-déméthylâmes", + "N-déméthylât", + "N-déméthylâtes", + "N-déméthylèrent", + "N-déméthylé", + "N-déméthylée", + "N-déméthylées", + "N-déméthylés", + "N-méthyla", + "N-méthylai", + "N-méthylaient", + "N-méthylais", + "N-méthylait", + "N-méthylant", + "N-méthylas", + "N-méthylasse", + "N-méthylassent", + "N-méthylasses", + "N-méthylassiez", + "N-méthylassions", + "N-méthyle", + "N-méthylent", + "N-méthyler", + "N-méthylera", + "N-méthylerai", + "N-méthyleraient", + "N-méthylerais", + "N-méthylerait", + "N-méthyleras", + "N-méthylerez", + "N-méthyleriez", + "N-méthylerions", + "N-méthylerons", + "N-méthyleront", + "N-méthyles", + "N-méthylez", + "N-méthyliez", + "N-méthylions", + "N-méthylons", + "N-méthylâmes", + "N-méthylât", + "N-méthylâtes", + "N-méthylèrent", + "N-méthylé", + "N-méthylée", + "N-méthylées", + "N-méthylés", + "N-éthyléthanamine", + "N.-D.", + "N.-W.", + "NDM-1", + "Nachrodt-Wiblingwerde", + "Nagel-Séez-Mesnil", + "Nago-Torbole", + "Nagorno-Karabakh", + "Nagorny-Karabagh", + "Nagorny-Karabakh", + "Nahetal-Waldau", + "Naives-Rosières", + "Namps-Maisnil", + "Nanc-lès-Saint-Amour", + "Nans-sous-Sainte-Anne", + "Nanteuil-Auriac-de-Bourzac", + "Nanteuil-Notre-Dame", + "Napoléon-Vendée", + "Naussac-Fontanes", + "Navailles-Angos", + "Nay-Bourdettes", + "Naz-Sciaves", + "Nazelles-Négron", + "Neaufles-Auvergny", + "Neckar-Odenwald", + "Neder-Betuwe", + "Neder-Hardinxveld", + "Neder-Over-Heembeek", + "Neder-over-Heembeek", + "Nederhemert-Noord", + "Nederhemert-Zuid", + "Nederweert-Eind", + "Nederzwalm-Hermelgem", + "Neige-Côtier", + "Neiße-Malxetal", + "Nemsdorf-Göhrendorf", + "Nesle-Hodeng", + "Nesle-Normandeuse", + "Nesle-l'Hôpital", + "Neu-Anspach", + "Neu-Bamberg", + "Neu-Eichenberg", + "Neu-Isenburg", + "Neu-Moresnet", + "Neu-Seeland", + "Neu-Ulm", + "Neublans-Abergement", + "Neubourg-sur-le-Danube", + "Neuburg-Schrobenhausen", + "Neuchâtel-Urtière", + "Neudorf-Bornstein", + "Neuendorf-Sachsenbande", + "Neuenkirchen-Vörden", + "Neuf-Berquin", + "Neuf-Berquinois", + "Neuf-Berquinoise", + "Neuf-Berquinoises", + "Neuf-Brisach", + "Neuf-Eglise", + "Neuf-Marché", + "Neuf-Mesnil", + "Neuf-Église", + "Neufchâtel-Hardelot", + "Neugartheim-Ittlenheim", + "Neuhaus-Schierschnitz", + "Neuilly-Plaisance", + "Neuilly-l'Evêque", + "Neuilly-l'Hôpital", + "Neuilly-l'Évêque", + "Neuillé-Pont-Pierre", + "Neukirchen-Balbini", + "Neukirchen-Vluyn", + "Neumagen-Dhron", + "Neunkirchen-Seelscheid", + "Neurey-lès-la-Demie", + "Neusalza-Spremberg", + "Neussargues-Moissac", + "Neustadt-Glewe", + "Neuve-Chapelle", + "Neuve-Chapellois", + "Neuve-Chapelloise", + "Neuve-Chapelloises", + "Neuve-Eglise", + "Neuve-Grangeais", + "Neuve-Grangeaise", + "Neuve-Grangeaises", + "Neuve-Maison", + "Neuve-Église", + "Neuvelle-lès-la-Charité", + "Neuves-Maisons", + "Neuvic-Entier", + "Neuvicq-Montguyon", + "Neuville-lès-Lœuilly", + "Neuvy-Bouin", + "Neuvy-Deux-Clochers", + "Neuvy-Grandchamp", + "Neuvy-Pailloux", + "Neuvy-Sautour", + "Neuvy-Sautourien", + "Neuvy-Sautourienne", + "Neuvy-Sautouriennes", + "Neuvy-Sautouriens", + "Nevi'im", + "New-Glasgois", + "Newcastle-under-Lyme", + "Newton-in-Makerfield", + "Ni-Skutterudites", + "Nieder-Hilbersheim", + "Nieder-Olm", + "Nieder-Wiesen", + "Niefern-Öschelbronn", + "Niel-bij-As", + "Niel-bij-Sint-Truiden", + "Nieuil-l'Espoir", + "Nieuw-Amsterdam", + "Nieuw-Annerveen", + "Nieuw-Balinge", + "Nieuw-Beerta", + "Nieuw-Beijerland", + "Nieuw-Bergen", + "Nieuw-Buinen", + "Nieuw-Dijk", + "Nieuw-Dordrecht", + "Nieuw-Ginneken", + "Nieuw-Heeten", + "Nieuw-Helvoet", + "Nieuw-Loosdrecht", + "Nieuw-Milligen", + "Nieuw-Namen", + "Nieuw-Reemst", + "Nieuw-Roden", + "Nieuw-Scheemda", + "Nieuw-Schoonebeek", + "Nieuw-Strijen", + "Nieuw-Vennep", + "Nieuw-Vossemeer", + "Nieuw-Weerdinge", + "Nieuw-Wehl", + "Nieuwe-Tonge", + "Nieuwer-Amstel", + "Nieuwkerken-Waas", + "Nieuwolda-Oost", + "Niger-Congo", + "Nijni-Taguil", + "Nil-Saint-Vincent-Saint-Martin", + "Nissan-lez-Enserune", + "Nister-Möhrendorf", + "Nistos-Haut-et-Bas", + "Nivolas-Vermelle", + "Nivollet-Montgriffon", + "Nixéville-Blercourt", + "Nizan-Gesse", + "Nlle-Calédonie", + "Nlle-Zélande", + "Nlle-Écosse", + "Nogent-l'Abbesse", + "Nogent-l'Artaud", + "Nohant-Vic", + "Noidant-Chatenoy", + "Noisy-Rudignon", + "Noisy-Rudignonais", + "Noisy-Rudignonaise", + "Noisy-Rudignonaises", + "Noncourt-sur-le-Rongeant", + "Nonette-Orsonnette", + "Nonsard-Lamarche", + "Nonvilliers-Grandhoux", + "Noord-Polsbroek", + "Noord-Scharwoude", + "Noord-Sleen", + "Noord-Spierdijk", + "Noord-Stroe", + "Noord-Waddinxveen", + "Noorder-Koggenland", + "Noordwijk-Binnen", + "Noordwolde-Zuid", + "Noron-l'Abbaye", + "Norrent-Fontes", + "Norroy-lès-Pont-à-Mousson", + "Nort-Leulinghem", + "Nort-Leulinghemois", + "Nort-Leulinghemoise", + "Nort-Leulinghemoises", + "Norwich-terrier", + "Nouaillé-Maupertuis", + "Nouveau-Brunswick", + "Nouveau-Connecticut", + "Nouveau-Continent", + "Nouveau-Cornouaille", + "Nouveau-Cornouailles", + "Nouveau-Cornwall", + "Nouveau-Hanovre", + "Nouveau-Léon", + "Nouveau-Mexique", + "Nouveau-Monde", + "Nouveau-Norfolk", + "Nouveau-Santander", + "Nouveau-Shetland", + "Nouveaux-Pays-Bas", + "Nouvel-Âge", + "Nouvelle-Albion", + "Nouvelle-Amsterdam", + "Nouvelle-Andalousie", + "Nouvelle-Angleterre", + "Nouvelle-Aquitaine", + "Nouvelle-Bretagne", + "Nouvelle-Calédonie", + "Nouvelle-Cornouaille", + "Nouvelle-Cornouailles", + "Nouvelle-Cythère", + "Nouvelle-Eglise", + "Nouvelle-Espagne", + "Nouvelle-France", + "Nouvelle-Galles", + "Nouvelle-Grenade", + "Nouvelle-Guinée", + "Nouvelle-Géorgie", + "Nouvelle-Hanovre", + "Nouvelle-Hollande", + "Nouvelle-Irlande", + "Nouvelle-Norfolk", + "Nouvelle-Néerlande", + "Nouvelle-Orléans", + "Nouvelle-Poméranie", + "Nouvelle-Sibérie", + "Nouvelle-Zamble", + "Nouvelle-Zemble", + "Nouvelle-Zélande", + "Nouvelle-Écosse", + "Nouvelle-Église", + "Nouvelles-Hébrides", + "Nouvron-Vingré", + "Novion-Porcien", + "Novy-Chevrières", + "Noyal-Châtillon-sur-Seiche", + "Noyal-Muzillac", + "Noyal-Pontivy", + "Noyant-d'Allier", + "Noyelle-Vion", + "Noyelles-Godault", + "Noyers-Auzécourt", + "Noyers-Bocage", + "Noyers-Missy", + "Noyers-Pont-Maugis", + "Noyers-Thélonne", + "Noël-Cerneux", + "Nuaillé-d'Aunis", + "Nueil-sous-les-Aubiers", + "Nuka-Hiva", + "Nuku-Hiva", + "Nuncq-Hautecôte", + "Nurieux-Volognat", + "Nuthe-Urstromtal", + "Nœux-les-Mines", + "Nœux-lès-Auxi", + "Nâves-Parmelan", + "Nézignan-l'Evêque", + "Nézignan-l'Évêque", + "Nörten-Hardenberg", + "O-desvenlafaxine", + "O-déméthyla", + "O-déméthylai", + "O-déméthylaient", + "O-déméthylais", + "O-déméthylait", + "O-déméthylant", + "O-déméthylas", + "O-déméthylasse", + "O-déméthylassent", + "O-déméthylasses", + "O-déméthylassiez", + "O-déméthylassions", + "O-déméthyle", + "O-déméthylent", + "O-déméthyler", + "O-déméthylera", + "O-déméthylerai", + "O-déméthyleraient", + "O-déméthylerais", + "O-déméthylerait", + "O-déméthyleras", + "O-déméthylerez", + "O-déméthyleriez", + "O-déméthylerions", + "O-déméthylerons", + "O-déméthyleront", + "O-déméthyles", + "O-déméthylez", + "O-déméthyliez", + "O-déméthylions", + "O-déméthylons", + "O-déméthylâmes", + "O-déméthylât", + "O-déméthylâtes", + "O-déméthylèrent", + "O-déméthylé", + "O-déméthylée", + "O-déméthylées", + "O-déméthylés", + "O-méthyla", + "O-méthylai", + "O-méthylaient", + "O-méthylais", + "O-méthylait", + "O-méthylant", + "O-méthylas", + "O-méthylasse", + "O-méthylassent", + "O-méthylasses", + "O-méthylassiez", + "O-méthylassions", + "O-méthyle", + "O-méthylent", + "O-méthyler", + "O-méthylera", + "O-méthylerai", + "O-méthyleraient", + "O-méthylerais", + "O-méthylerait", + "O-méthyleras", + "O-méthylerez", + "O-méthyleriez", + "O-méthylerions", + "O-méthylerons", + "O-méthyleront", + "O-méthyles", + "O-méthylez", + "O-méthyliez", + "O-méthylions", + "O-méthylons", + "O-méthylâmes", + "O-méthylât", + "O-méthylâtes", + "O-méthylèrent", + "O-méthylé", + "O-méthylée", + "O-méthylées", + "O-méthylés", + "Ober-Flörsheim", + "Ober-Hilbersheim", + "Ober-Mörlen", + "Ober-Olm", + "Ober-Ramstadt", + "Oberdorf-Spachbach", + "Oberehe-Stroheich", + "Oberhausen-Rheinhausen", + "Oberhonnefeld-Gierend", + "Obermaßfeld-Grimmenthal", + "Obermodern-Zutzendorf", + "Obernheim-Kirchenarnbach", + "Oberweiler-Tiefenbach", + "Oberwil-Lieli", + "Octeville-l'Avenel", + "Odeillo-Via", + "Oder-Spree", + "Oebisfelde-Weferlingen", + "Oer-Erkenschwick", + "Oestrich-Winkel", + "Offenbach-Hundheim", + "Offenbach-sur-le-Main", + "Ogenne-Camptort", + "Ogooué-Maritime", + "Ogy-Montoy-Flanville", + "Ojos-Albos", + "Olbia-Tempio", + "Olizy-Primat", + "Olmeta-di-Capocorso", + "Olmeta-di-Tuda", + "Olmi-Cappella", + "Oloron-Sainte-Marie", + "Olst-Wijhe", + "Oléac-Debat", + "Oléac-Dessus", + "Ombret-Rawsa", + "Onesse-Laharie", + "Onze-Lieve-Vrouw-Waver", + "Oost-Barendrecht", + "Oost-Cappel", + "Oost-Cappelois", + "Oost-Cappeloise", + "Oost-Cappeloises", + "Oost-Graftdijk", + "Oost-Maarland", + "Oost-Souburg", + "Oost-Vlieland", + "Ooster-Dalfsen", + "Oosterzee-Buren", + "Ophain-Bois-Seigneur-Isaac", + "Opoul-Périllos", + "Opéra-Comique", + "Or-Blanois", + "Oradour-Fanais", + "Oranienbaum-Wörlitz", + "Orbais-l'Abbaye", + "Orchamps-Vennes", + "Ordan-Larroque", + "Orgeans-Blanchefontaine", + "Orgnac-l'Aven", + "Origny-Sainte-Benoite", + "Ormont-Dessous", + "Ormont-Dessus", + "Ormoy-Villers", + "Ornolac-Ussat-les-Bains", + "Oroz-Betelu", + "Orp-Jauche", + "Orp-Jauchois", + "Orp-Jauchoise", + "Orsingen-Nenzingen", + "Orsmaal-Gussenhoven", + "Orthoux-Sérignac-Quilhan", + "Orveau-Bellesauve", + "Orvillers-Sorel", + "Orée d'Anjou", + "Orée-d'Anjou", + "Os-Marsillon", + "Osann-Monzel", + "Osly-Courtil", + "Ossas-Suhare", + "Osselle-Routelle", + "Osserain-Rivareyte", + "Ossey-les-Trois-Maisons", + "Ossun-ez-Angles", + "Ossétie-du-Nord-Alanie", + "Ostabat-Asme", + "Oster-Ohrstedt", + "Osterholz-Scharmbeck", + "Osthausen-Wülfershausen", + "Ottendorf-Okrilla", + "Ottignies-Louvain-la-Neuve", + "Oud-Aa", + "Oud-Alblas", + "Oud-Annerveen", + "Oud-Beijerland", + "Oud-Bodegraven", + "Oud-Dijk", + "Oud-Drimmelen", + "Oud-Empel", + "Oud-Gastel", + "Oud-Heverlee", + "Oud-Kamerik", + "Oud-Leusden", + "Oud-Loosdrecht", + "Oud-Maarsseveen", + "Oud-Reemst", + "Oud-Reeuwijk", + "Oud-Sabbinge", + "Oud-Turnhout", + "Oud-Valkenburg", + "Oud-Vossemeer", + "Oud-Vroenhoven", + "Oud-Wulven", + "Oud-Zuilen", + "Oude-Tonge", + "Ouder-Amstel", + "Ougney-Douvot", + "Oui-Oui", + "Oulan-Bator", + "Oulches-la-Vallée-Foulon", + "Ourcel-Maison", + "Ourdis-Cotdoussan", + "Ourgou-Manega", + "Ouroux-sous-le-Bois-Sainte-Marie", + "Ours-Mons", + "Oursel-Maison", + "Ousse-Suzan", + "Oust-Marest", + "Ouve-Wirquin", + "Ouve-Wirquinois", + "Ouve-Wirquinoise", + "Ouve-Wirquinoises", + "Ouville-l'Abbaye", + "Ouville-la-Bien-Tournée", + "Ouwster-Nijega", + "Ouzilly-Vignolles", + "Ouzoun-Ada", + "Over-Diemen", + "Oy-Mittelberg", + "Oye-Plage", + "Oyón-Oion", + "Oza-Cesuras", + "Ozenx-Montestrucq", + "P-ATA", + "P-DG", + "P-frame", + "P.-D.G.", + "PPD-T", + "Pa-O", + "Pagney-derrière-Barine", + "Pagny-la-Blanche-Côte", + "Paisy-Cosdon", + "Paizay-Naudouin-Embourie", + "Palais-Bourbon", + "Palatinat-Sud-Ouest", + "Palau-del-Vidre", + "Palau-sator", + "Palau-saverdera", + "Palmas d'Aveyron", + "Pancy-Courtecon", + "Panschwitz-Kuckau", + "Pant'ruche", + "Papouasie-Nouvelle-Guinée", + "Paray-Douaville", + "Paray-Vieille-Poste", + "Parc-d'Anxtot", + "Parc-d'Anxtotais", + "Parc-d'Anxtotaise", + "Parc-d'Anxtotaises", + "Parcoul-Chenaud", + "Pardies-Piétat", + "Parfouru-l'Éclin", + "Pargny-Filain", + "Pargny-Resson", + "Parigné-l'Evêque", + "Parigné-l'Évêque", + "Paris-Brest", + "Paris-l'Hôpital", + "Parsac-Rimondeix", + "Parva-Pétricien", + "Parva-Pétricienne", + "Parva-Pétriciennes", + "Parva-Pétriciens", + "Parçay-Meslay", + "Passy-Grigny", + "Paul-Olivier", + "Paussac-et-Saint-Vivien", + "Pautaines-Augeville", + "Payrin-Augmontel", + "Payros-Cazautets", + "Pays-Bas", + "Pays-d'Altenbourg", + "Pays-d'Enhaut", + "Peau-Rouge", + "Peau-Verte", + "Peaux-Rouges", + "Peaux-Vertes", + "Pech-Luna", + "Pedro-Rodríguez", + "Peisey-Nancroix", + "Penne-d'Agenais", + "Penta-Acquatella", + "Penta-di-Casinca", + "Pergain-Taillac", + "Perly-Certoux", + "Pernand-Vergelesses", + "Pero-Casevecchie", + "Perrancey-les-Vieux-Moulins", + "Perros-Guirec", + "Pers-Jussy", + "Pertheville-Ners", + "Pessa'h", + "Pessat-Villeneuve", + "Petegem-aan-de-Leie", + "Petegem-aan-de-Schelde", + "Peterswald-Löffelscheid", + "Petreto-Bicchisano", + "Peumerit-Quintin", + "Peypin-d'Aigues", + "Peyrelongue-Abos", + "Peyriac-Minervois", + "Peyrusse-Grande", + "Peyrusse-Massas", + "Peyrusse-Vieille", + "Peñacerrada-Urizaharra", + "Peñarroya-Pueblonuevo", + "Pfaffen-Schwabenheim", + "Pi-Ramsès", + "Pianotolli-Caldarello", + "Pianottoli-Caldarello", + "Picto-Charentais", + "Pie-d'Orezza", + "Piedicorte-di-Gaggio", + "Piennes-Onvillers", + "Pierrefitte-Nestalas", + "Pierrefitte-ès-Bois", + "Pietra-di-Verde", + "Piets-Plasence-Moustrou", + "Pijnacker-Nootdorp", + "Pila-Canale", + "Pin-Balma", + "Pin-Moriès", + "Pinel-Hauterive", + "Pino-Balméen", + "Pino-Balméenne", + "Pino-Balméennes", + "Pino-Balméens", + "Pins-Justaret", + "Pins-Justarétois", + "Pins-Justarétoise", + "Pins-Justarétoises", + "Pissy-Pôvillais", + "Pissy-Pôvillaise", + "Pissy-Pôvillaises", + "Pissy-Pôville", + "Piégut-Pluviers", + "Piñuécar-Gandullas", + "Plachy-Buyon", + "Plachy-Buyonnais", + "Plachy-Buyonnaise", + "Plachy-Buyonnaises", + "Placy-Montaigu", + "Plaimbois-Vennes", + "Plaimpied-Givaudins", + "Plaine-Haute", + "Plan-d'Aups", + "Plan-d'Aups-Sainte-Baume", + "Plan-d'Orgon", + "Plan-de-la-Tour", + "Plancher-Bas", + "Plancy-l'Abbaye", + "Plassac-Rouffiac", + "Plateau-Central", + "Pleine-Fougerais", + "Pleine-Fougeraise", + "Pleine-Fougeraises", + "Pleine-Fougères", + "Pleine-Selve", + "Pleine-Sève", + "Pleines-Œuvres", + "Pleisweiler-Oberhofen", + "Pleslin-Trigavou", + "Plessix-Balisson", + "Pleumeur-Bodou", + "Pleumeur-Gautier", + "Pleyber-Christ", + "Plobannalec-Lesconil", + "Ploeuc-L'Hermitage", + "Plonéour-Lanvern", + "Plonévez-Porzay", + "Plouezoc'h", + "Plougastel-Daoulas", + "Ploulec'h", + "Ploumanac'h", + "Plounéour-Brignogan-Plages", + "Plounéour-Ménez", + "Plounéour-Trez", + "Plounéour-Trezien", + "Plounéour-Trezienne", + "Plounéour-Treziennes", + "Plounéour-Treziens", + "Plounévez-Lochrist", + "Plounévez-Moëdec", + "Plounévez-Quintin", + "Plourac'h", + "Plouégat-Guérand", + "Plouégat-Moysan", + "Pléneuf-Val-André", + "Plénée-Jugon", + "Plœuc-L'Hermitage", + "Pobé-Mengao", + "Podio-Régalien", + "Podio-Régalienne", + "Podio-Régaliennes", + "Podio-Régaliens", + "Poey-d'Oloron", + "Poggio-Marinaccio", + "Poggio-Mezzana", + "Poggio-Mezzanais", + "Poggio-Mezzanaise", + "Poggio-Mezzanaises", + "Poggio-d'Oletta", + "Poggio-di-Nazza", + "Poggio-di-Tallano", + "Poggio-di-Venaco", + "Poilcourt-Sydney", + "Poilly-lez-Gien", + "Pointe-Claire", + "Pointe-Fortunais", + "Pointe-Fortunien", + "Pointe-Noirais", + "Pointe-Noiraise", + "Pointe-Noiraises", + "Pointe-Noire", + "Pointis-Inard", + "Poiseul-la-Ville-et-Laperrière", + "Poitou-Charentes", + "Poix-Terron", + "Poleymieux-au-Mont-d'Or", + "Poliez-Pittet", + "Pommerit-Jaudy", + "Pommiers-Moulons", + "Poméranie-Occidentale-de-l'Est", + "Poméranie-Occidentale-du-Nord", + "Ponches-Estruval", + "Poncé-sur-le-Loir", + "Ponet-et-Saint-Auban", + "Ponlat-Taillebourg", + "Ponsan-Soubiran", + "Ponson-Debat-Pouts", + "Ponson-Dessus", + "Pont de Montvert - Sud Mont Lozère", + "Pontamafrey-Montpascal", + "Pontault-Combault", + "Ponte-Avancé", + "Pontfaverger-Moronvilliers", + "Pontiacq-Viellepinte", + "Porcieu-Amblagnieu", + "Portet-d'Aspet", + "Porto-Novo", + "Porto-Ricain", + "Porto-Ricaine", + "Porto-Ricaines", + "Porto-Ricains", + "Porto-Rico", + "Porto-Vecchiais", + "Porto-Vecchiaise", + "Porto-Vecchiaises", + "Porto-Vecchio", + "Porté-Puymorens", + "Postbauer-Heng", + "Potsdam-Mittelmark", + "Pougne-Hérisson", + "Pouilley-Français", + "Poulan-Pouzols", + "Pouligney-Lusans", + "Pouligny-Notre-Dame", + "Pouques-Lormes", + "Poursay-Garnaud", + "Poursiugues-Boucoue", + "Pouy-Loubrin", + "Pouy-Roquelain", + "Pouy-Roquelaine", + "Pouy-Roquelaines", + "Pouy-Roquelains", + "Pouy-Roquelaure", + "Pouzols-Minervois", + "Pouzy-Mésangy", + "Pozo-Lorente", + "Poët-Lavalien", + "Poët-Lavalienne", + "Poët-Lavaliennes", + "Poët-Lavaliens", + "Pradelles-Cabardès", + "Prades-Salars", + "Prades-d'Aubrac", + "Prads-Haute-Bléone", + "Prat-Bonrepaux", + "Prato-di-Giovellina", + "Prats-de-Mollo-la-Preste", + "Pregny-Chambésy", + "Premeaux-Prissey", + "Premosello-Chiovenda", + "Pressagny-l'Orgueilleux", + "Pressignac-Vicq", + "Pressignaco-Vicois", + "Pressignaco-Vicoise", + "Pressignaco-Vicoises", + "Preutin-Higny", + "Preux-Romanien", + "Preux-Romanienne", + "Preux-Romaniennes", + "Preux-Romaniens", + "Preyssac-d'Excideuil", + "Prez-vers-Noréaz", + "Prignitz-de-l'Est-Ruppin", + "Prin-Deyrançon", + "Prince-Édouardien", + "Prince-Édouardienne", + "Prince-Édouardiennes", + "Prince-Édouardiens", + "Principauté-Ultérieure", + "Prinsuéjols-Malbouzon", + "Proche-Orient", + "Proença-a-Nova", + "Provence-Alpes-Côte-d'Azur", + "Provinces-Unies", + "Pruillé-l'Eguillé", + "Pruillé-l'Éguillé", + "Prunay-Belleville", + "Prunay-Cassereau", + "Prunelli-di-Casacconi", + "Prunelli-di-Fiumorbo", + "Prunet-Puigois", + "Prunet-Puigoise", + "Prunet-Puigoises", + "Prusse-Orientale", + "Préaux-Bocage", + "Préchacq-Josbaig", + "Préchacq-Navarrenx", + "Précy-Notre-Dame", + "Prétot-Sainte-Suzanne", + "Prétot-Vicquemare", + "Prévessin-Moëns", + "Puch-d'Agenais", + "Puech-Cabrier", + "Puget-Rostang", + "Puget-Théniers", + "Puget-Ville", + "Pugny-Chatenod", + "Puig-reig", + "Puiset-Doréen", + "Puiset-Doréenne", + "Puiset-Doréennes", + "Puiset-Doréens", + "Puiseux-Pontoise", + "Puligny-Montrachet", + "Putanges-Pont-Ecrepin", + "Putanges-Pont-Écrepin", + "Putot-Bessinois", + "Putot-Bessinoise", + "Putot-Bessinoises", + "Puyol-Cazalet", + "Pyrénées-Atlantiques", + "Pyrénées-Orientales", + "Pätow-Steegen", + "Pécharic-et-le-Py", + "Péret-Bel-Air", + "Périers-sur-le-Dan", + "Pérignat-ès-Allier", + "Péronnes-lez-Antoing", + "Pétrus-Colien", + "Pétrus-Colienne", + "Pétrus-Coliennes", + "Pétrus-Coliens", + "Qo'noS", + "Quemigny-Poisot", + "Quemper-Guézennec", + "Quesnay-Guesnon", + "Quincampoix-Fleuzy", + "Quincy-Basse", + "Quincy-Landzécourt", + "Quincy-Voisins", + "Quincy-sous-le-Mont", + "Quint-Fonsegrives", + "Quœux-Haut-Maînil", + "Quœux-Haut-Maînil", + "Qwa-Qwa", + "R.-V.", + "RS-232", + "Raa-Besenbek", + "Rabat-les-Trois-Seigneurs", + "Rabenkirchen-Faulück", + "Rachecourt-Suzémont", + "Ragow-Merz", + "Raguhn-Jeßnitz", + "Raillencourt-Sainte-Olle", + "Raissac-d'Aude", + "Ralbitz-Rosenthal", + "Rambluzin-et-Benoite-Vaux", + "Ramegnies-Chin", + "Ramillies-Offus", + "Ramstein-Miesenbach", + "Ransbach-Baumbach", + "Ranville-Breuillaud", + "Raon-l'Etape", + "Raon-l'Étape", + "Rapide-Danseurois", + "Rapperswil-Jona", + "Raschau-Markersbach", + "Rayol-Canadel-sur-Mer", + "Razac-d'Eymet", + "Raï'n'B", + "Rebecq-Rognon", + "Rebreuve-Ranchicourt", + "Rechenberg-Bienenmühle", + "Reckingen-Gluringen", + "Recoubeau-Jansac", + "Recoules-Prévinquières", + "Recoules-d'Aubrac", + "Regis-Breitingen", + "Regnière-Ecluse", + "Regnière-Écluse", + "Rehburg-Loccum", + "Rehlingen-Siersburg", + "Rehm-Flehde-Bargen", + "Reichenbach-Steegen", + "Reichenow-Möglin", + "Reigneville-Bocage", + "Reignier-Esery", + "Reignier-Ésery", + "Reinhardtsdorf-Schöna", + "Rembercourt-Sommaisne", + "Remda-Teichel", + "Remilly-Aillicourt", + "Remilly-Wirquin", + "Remilly-Wirquinois", + "Remilly-Wirquinoise", + "Remilly-Wirquinoises", + "Remire-Montjoly", + "Remoray-Boujeons", + "Rems-Murr", + "Rendsburg-Eckernförde", + "Ressons-l'Abbaye", + "Reuilly-Sauvigny", + "Reulle-Vergy", + "Revel-Tourdan", + "Reventin-Vaugris", + "Rheda-Wiedenbrück", + "Rheingau-Taunus", + "Rhin-Berg", + "Rhin-Erft", + "Rhin-Hunsrück", + "Rhin-Lahn", + "Rhin-Neckar", + "Rhin-Palatinat", + "Rhin-Sieg", + "Rhode-Sainte-Agathe", + "Rhénanie-Palatinat", + "Rhénanie-du-Nord-Westphalie", + "Rhêmes-Notre-Dame", + "Rhône-Alpes", + "Rhön-Grabfeld", + "Ria-Sirach", + "Ria-Sirachois", + "Ria-Sirachoise", + "Ria-Sirachoises", + "Ribnitz-Damgarten", + "Ribécourt-Dreslincourt", + "Richelieu-Yamaskois", + "Ried-Brig", + "Rielasingen-Worblingen", + "Rieschweiler-Mühlbach", + "Rietheim-Weilheim", + "Rietz-Neuendorf", + "Rietzneuendorf-Staakow", + "Rieux-Minervois", + "Rieux-Volvestre", + "Rig-Véda", + "Rigaud-Montain", + "Rigaud-Montaine", + "Rigaud-Montaines", + "Rigaud-Montains", + "Rigil-K", + "Rigny-Ussé", + "Rigny-Usséen", + "Rigny-Usséenne", + "Rigny-Usséennes", + "Rigny-Usséens", + "Rijssen-Holten", + "Rilhac-Lastours", + "Rilhac-Rancon", + "Rilhac-Treignac", + "Rilhac-Xaintrie", + "Rilland-Bath", + "Rilly-Sainte-Syre", + "Riom-Parsonz", + "Riom-ès-Montagnes", + "Rioux-Martin", + "Ris-Orangis", + "Risch-Rotkreuz", + "Risum-Lindholm", + "Rivas-Vaciamadrid", + "Rive-Nord", + "Rive-Sud", + "Rive-Sudois", + "Rivedoux-Plage", + "Rives d'Andaine", + "Rives de l'Yon", + "Riviera-Pays-d'Enhaut", + "Rivière-Devant", + "Rivière-Pilote", + "Rivière-Saas-et-Gourby", + "Rivière-Salée", + "Rizaucourt-Buchey", + "Ro-Ro", + "Ro-Ros", + "Robert-Espagne", + "Robert-Magny", + "Robert-Messin", + "Robert-Messine", + "Robert-Messines", + "Robert-Messins", + "Robiac-Rochessadoule", + "Robleda-Cervantes", + "Roc-Libre", + "Roche-Blanchais", + "Roche-Blanchaise", + "Roche-Blanchaises", + "Roche-Charles-la-Mayrand", + "Roche-Saint-Secret-Béconne", + "Roche-d'Agoux", + "Roche-lez-Beaupré", + "Roche-sur-Linotte-et-Sorans-les-Cordiers", + "Rochefort-Montagne", + "Rochefort-Samson", + "Rochefort-sur-la-Côte", + "Roches-Bettaincourt", + "Roches-Prémarie-Andillé", + "Rochy-Condé", + "Roclenge-Looz", + "Rodengo-Saiano", + "Roesbrugge-Haringe", + "Roge-Bougeron", + "Roge-Bougeronne", + "Roge-Bougeronnes", + "Roge-Bougerons", + "Rogny-les-Sept-Ecluses", + "Rogny-les-Sept-Écluses", + "Roi-Soleil", + "Rollegem-Kapelle", + "Rolleghem-Cappelle", + "Romagne-sous-les-Côtes", + "Romagny-Fontenay", + "Romainmôtier-Envy", + "Romanèche-Thorins", + "Romorantin-Lanthenay", + "Roncherolles-sur-le-Vivier", + "Ronzo-Chienis", + "Roost-Warendin", + "Roost-Warendinois", + "Roost-Warendinoise", + "Roost-Warendinoises", + "Roquebrune-Cap-Martin", + "Roquecourbe-Minervois", + "Roquestéron-Grasse", + "Rose-Marie", + "Rosenthal-Bielatal", + "Roset-Fluans", + "Rosiers-d'Egletons", + "Rosiers-d'Égletons", + "Rosières-devant-Bar", + "Rosnay-l'Hôpital", + "Rosoux-Crenwick", + "Rosti-Montois", + "Rosti-Montoise", + "Rosti-Montoises", + "Rotheux-Rimière", + "Rottach-Egern", + "Rottal-Inn", + "Rou-Marson", + "Rouessé-Fontaine", + "Rouessé-Vassé", + "Rouffiac-Tolosan", + "Rouffiac-d'Aude", + "Rouffignac-Saint-Cernin-de-Reilhac", + "Rouge-Perriers", + "Rouilly-Sacey", + "Roullet-Stéphanois", + "Roullet-Stéphanoise", + "Roullet-Stéphanoises", + "Roumazières-Loubert", + "Rousseau-esque", + "Rousseau-esques", + "Rouvray-Catillon", + "Rouvray-Sainte-Croix", + "Rouvroy-Ripont", + "Roux-Mirien", + "Roux-Mirienne", + "Roux-Miroir", + "Rouxmesnil-Bouteilles", + "Roville-devant-Bayon", + "Roy-Boissy", + "Royaume-Uni", + "Roz-Landrieux", + "Rozier-Côtes-d'Aurec", + "Rozoy-Bellevalle", + "Rudeau-Ladosse", + "Rudolfstetten-Friedlisberg", + "Rueil-Malmaison", + "Ruillé-Froid-Fonds", + "Ruppach-Goldhausen", + "Rupt-devant-Saint-Mihiel", + "Russy-Bémont", + "Ruttersdorf-Lotschen", + "Ruy-Montceau", + "Râlé-Poussé", + "Réaup-Lisse", + "Réez-Fosse-Martin", + "Régis-Borgien", + "Régis-Borgienne", + "Régis-Borgiennes", + "Régis-Borgiens", + "Régnié-Durette", + "Rémondans-Vaivre", + "Rémy-Montais", + "Rémy-Montaise", + "Rémy-Montaises", + "Rödersheim-Gronau", + "Rötsweiler-Nockenthal", + "Rüdtligen-Alchenflüh", + "S-6-verbénol", + "S-chanf", + "S-métolachlore", + "S.-E.", + "S.-W.", + "Saalburg-Ebersdorf", + "Saaldorf-Surheim", + "Saale-Holzland", + "Saale-Orla", + "Saalfeld-Rudolstadt", + "Saar-Mark", + "Saas-Almagell", + "Saas-Balen", + "Saas-Fee", + "Saas-Grund", + "Sabadel-Latronquière", + "Sabadel-Lauzès", + "Saceda-Trasierra", + "Sacré-Cœur", + "Sacré-Cœurin", + "Sacré-Cœurois", + "Saguenay-Jeannois", + "Saguenay-Lac-Saint-Jean", + "Sailly-Achâtel", + "Sailly-Flibeaucourt", + "Sailly-Labourse", + "Sailly-Laurette", + "Sailly-Saillisel", + "Sailly-lez-Cambrai", + "Sailly-lez-Lannoy", + "Sailly-sur-la-Lys", + "Sain-Bel", + "Sain-Belois", + "Sain-Beloise", + "Sain-Beloises", + "Saincaize-Meauce", + "Sains-Morainvillers", + "Sains-Richaumont", + "Saint Antoine l'Abbaye", + "Saint Aulaye-Puymangou", + "Saint Geniez d'Olt et d'Aubrac", + "Saint Martin de l'If", + "Saint-Denœux", + "Saint-Jean-de-Bœuf", + "Saint-Martin-le-Nœud", + "Saint-Michel-Tubœuf", + "Saint-Paul - Flaugnac", + "Saint-Pierre-de-Bœuf", + "Saint-Thegonnec Loc-Eguiner", + "Sainte-Alvère-Saint-Laurent Les Bâtons", + "Salignac-Eyvignes", + "Salignac-Eyvigues", + "Salins-Fontaine", + "Salle-Prunetais", + "Salle-Prunetaise", + "Salle-Prunetaises", + "Salles-Adour", + "Salles-Arbuissonnas-en-Beaujolais", + "Salles-Courbatiès", + "Salles-Curan", + "Salles-Lavalette", + "Salles-Mongiscard", + "Salles-Sourçois", + "Salles-Sourçoise", + "Salles-Sourçoises", + "Salles-d'Angles", + "Salles-d'Armagnac", + "Salles-d'Aude", + "Sallèles-Cabardès", + "Sallèles-d'Aude", + "Salm-Salm", + "Salvagnac-Cajarc", + "Salvatierra-Agurain", + "Samsons-Lion", + "Sana'a", + "Sancti-Spíritus", + "Sancto-Bénédictin", + "Sancto-Bénédictine", + "Sancto-Bénédictines", + "Sancto-Bénédictins", + "Sancto-Julianais", + "Sancto-Julianaise", + "Sancto-Julianaises", + "Sancto-Prixin", + "Sancto-Prixine", + "Sancto-Prixines", + "Sancto-Prixins", + "Sandersdorf-Brehna", + "Sanilhac-Sagriès", + "Sansac-Veinazès", + "Sanssac-l'Eglise", + "Sanssac-l'Église", + "Sant'Agapito", + "Sant'Agnello", + "Sant'Agostino", + "Sant'Alfio", + "Sant'Anastasia", + "Sant'Andréa-d'Orcino", + "Sant'Andréa-di-Bozio", + "Sant'Andréa-di-Cotone", + "Sant'Andréa-di-Tallano", + "Sant'Antimo", + "Sant'Antioco", + "Sant'Antonino", + "Sant'Antonio", + "Sant'Apollinare", + "Sant'Arcangelo", + "Sant'Arpino", + "Sant'Arsenio", + "Sant'Elena", + "Sant'Ippolito", + "Sant'Olcese", + "Sant'Omero", + "Sant'Onofrio", + "Sant'Oreste", + "Sant'Urbano", + "Santa-Lucia-di-Mercurio", + "Santa-Lucia-di-Moriani", + "Santa-Maria-Figaniella", + "Santa-Maria-Poggio", + "Santa-Maria-Siché", + "Santa-Maria-di-Lota", + "Santa-Reparata-di-Balagna", + "Santa-Reparata-di-Moriani", + "Santi-Johanien", + "Santi-Johanienne", + "Santi-Johaniennes", + "Santi-Johaniens", + "Santiago-Pontones", + "Santo-Pietro-di-Tenda", + "Santo-Pietro-di-Venaco", + "Santpoort-Noord", + "Santpoort-Zuid", + "Sapogne-Feuchères", + "Sargé-lès-le-Mans", + "Sari-Solenzara", + "Sari-d'Orcino", + "Sari-di-Porto-Vecchio", + "Sariac-Magnoac", + "Sarre-Palatinat", + "Sarre-Union", + "Sarre-Unionnais", + "Sarre-Unionnaise", + "Sarre-Unionnaises", + "Sarriac-Bigorre", + "Sarrola-Carcopino", + "Sars-Poteries", + "Sart-Bernard", + "Sart-Custinne", + "Sart-Dames-Avelines", + "Sart-Dames-Avelinois", + "Sart-Dames-Avelinoise", + "Sart-Eustache", + "Sart-Eustachois", + "Sart-Eustachoise", + "Sart-Messire-Guillaume", + "Sart-Risbart", + "Sart-Risbartois", + "Sart-Risbartoise", + "Sartilly-Baie-Bocage", + "Sassen-Trantow", + "Sathonay-Camp", + "Sathonay-Village", + "Sauchy-Cauchy", + "Sauchy-Lestrée", + "Saulces-Champenoises", + "Saulces-Monclin", + "Saulces-Vieille", + "Saulgé-l'Hôpital", + "Sault-Brénaz", + "Sault-Rethelois", + "Sault-Retheloise", + "Sault-Retheloises", + "Saulx-Marchais", + "Sauret-Besserve", + "Sauvage-Magny", + "Sauvagnat-Sainte-Marthe", + "Sauvillers-Mongival", + "Sauzé-Vaussais", + "Savas-Mépin", + "Savignac-Lédrier", + "Savignac-Mona", + "Savigny-Lévescault", + "Savigny-Poil-Fol", + "Savigny-en-Terre-Plaine", + "Savigné-l'Evêque", + "Savigné-l'Évêque", + "Savigné-sous-le-Lude", + "Savonnières-devant-Bar", + "Savy-Berlette", + "Saxe-Anhalt", + "Saxi-Bourdon", + "Saxon-Sion", + "Sceaux-d'Anjou", + "Scey-Maisières", + "Scey-sur-Saône-et-Saint-Albin", + "Schacht-Audorf", + "Scharrachbergheim-Irmstett", + "Scheibe-Alsbach", + "Schieder-Schwalenberg", + "Schinznach-Bad", + "Schiphol-Oost", + "Schiphol-Rijk", + "Schlatt-Haslen", + "Schleswig-Flensburg", + "Schleswig-Holstein", + "Schmalkalden-Meiningen", + "Schmogrow-Fehrow", + "Schmölln-Putzkau", + "Schnarup-Thumby", + "Schouwen-Duiveland", + "Schwalm-Eder", + "Schweigen-Rechtenbach", + "Schweighouse-Thann", + "Schönau-Berzdorf", + "Schönenberg-Kübelberg", + "Schönwalde-Glien", + "Scorbé-Clairvaux", + "Scy-Chazelles", + "Sealyham-terrier", + "Sedze-Maubecq", + "See-Gaster", + "Seeheim-Jugenheim", + "Seeon-Seebruck", + "Seeth-Ekholt", + "Seiches-sur-le-Loir", + "Seillons-Source-d'Argens", + "Seine-Inférieure", + "Seine-Maritime", + "Seine-Port", + "Seine-Portais", + "Seine-Portaise", + "Seine-Portaises", + "Seitingen-Oberflacht", + "Selke-Aue", + "Selon-Jean", + "Selon-Luc", + "Selon-Marc", + "Selon-Matthieu", + "Semoutiers-Montsaon", + "Senargent-Mignafans", + "Sencenac-Puy-de-Fourches", + "Sens-Beaujeu", + "Sentenac-d'Oust", + "Senven-Léhart", + "Seo-yeon", + "Sept-Forges", + "Sept-Frères", + "Sept-Frèrien", + "Sept-Frèrienne", + "Sept-Frèriennes", + "Sept-Frèriens", + "Sept-Ilien", + "Sept-Insulaire", + "Sept-Insulaires", + "Sept-Lacquois", + "Sept-Meules", + "Sept-Meulois", + "Sept-Meuloise", + "Sept-Meuloises", + "Sept-Saulx", + "Sept-Sortais", + "Sept-Sortaise", + "Sept-Sortaises", + "Sept-Sorts", + "Sept-Ventais", + "Sept-Ventaise", + "Sept-Ventaises", + "Sept-Vents", + "Sept-Îles", + "Sept-Îlien", + "Sept-Îlois", + "Sepulcro-Hilario", + "Serra-di-Ferro", + "Serra-di-Fiumorbo", + "Serra-di-Scopamène", + "Serres-Castet", + "Serres-Gaston", + "Serres-Gastonnais", + "Serres-Gastonnaise", + "Serres-Gastonnaises", + "Serres-Morlanais", + "Serres-Morlanaise", + "Serres-Morlanaises", + "Serres-Morlaàs", + "Serres-Sainte-Marie", + "Serri-Sapinois", + "Serri-Sapinoise", + "Serri-Sapinoises", + "Servance-Miellin", + "Servaville-Salmonville", + "Servigny-lès-Sainte-Barbe", + "Servon-Melzicourt", + "Serémange-Erzange", + "Seuil-d'Argonne", + "Seysses-Savès", + "Seyssinet-Pariset", + "Shai-hulud", + "Shang-Haï", + "Shoreham-by-Sea", + "Siaugues-Sainte-Marie", + "Siccieu-Saint-Julien-et-Carisieu", + "Siegen-Wittgenstein", + "Sierra-Léonais", + "Sierra-Léonaise", + "Sierra-Léonaises", + "Sierra-Léonien", + "Sieversdorf-Hohenofen", + "Signy-Avenex", + "Signy-Librecy", + "Signy-Montlibert", + "Signy-Signets", + "Signy-l'Abbaye", + "Silley-Amancey", + "Silley-Bléfond", + "Silly-Tillard", + "Simiane-Collongue", + "Siouville-Hague", + "Sittard-Geleen", + "Sivry-Ante", + "Sivry-Courtry", + "Sivry-Rance", + "Six-Fournais", + "Six-Fournaise", + "Six-Fournaises", + "Six-Fours-la-Plage", + "Six-Fours-les-Plages", + "Six-Planes", + "Sixt-Fer-à-Cheval", + "Skelton-in-Cleveland", + "Slijk-Ewijk", + "Sluis-Aardenburg", + "Smeerebbe-Vloerzegem", + "Snijders-Chaam", + "Soheit-Tinlot", + "Soing-Cubry-Charentenay", + "Soirans-Fouffrans", + "Soisy-Bouy", + "Sollières-Sardières", + "Solliès-Pont", + "Solliès-Pontois", + "Solliès-Pontoise", + "Solliès-Pontoises", + "Solliès-Toucas", + "Solliès-Villain", + "Solliès-Villaine", + "Solliès-Villaines", + "Solliès-Villains", + "Solliès-Ville", + "Soltau-Fallingbostel", + "Solutré-Pouilly", + "Someren-Eind", + "Someren-Heide", + "Somme-Bionne", + "Somme-Leuze", + "Somme-Leuzien", + "Somme-Leuzienne", + "Somme-Suippas", + "Somme-Suippase", + "Somme-Suippases", + "Somme-Suippe", + "Somme-Tourbe", + "Somme-Vesle", + "Somme-Yèvre", + "Sommepy-Tahure", + "Sommette-Eaucourt", + "Sonceboz-Sombeval", + "Sonnenberg-Winnenberg", + "Soo-hyun", + "Soorts-Hossegor", + "Sorbo-Ocagnano", + "Sorcy-Bauthémont", + "Sorde-l'Abbaye", + "Sorel-Moussel", + "Sornzig-Ablaß", + "Sotteville-sous-le-Val", + "Souain-Perthes-lès-Hurlus", + "Soudaine-Lavinadière", + "Soudé-Notre-Dame-ou-le-Petit", + "Soueix-Rogalle", + "Sougné-Remouchamps", + "Soulages-Bonneval", + "Soulaines-Dhuys", + "Soulce-Cernay", + "Souligné-Flacé", + "Soulosse-sous-Saint-Elophe", + "Soulosse-sous-Saint-Élophe", + "Soultz-Haut-Rhin", + "Source-Seine", + "Sourdeval-Vengeons", + "Southend-on-Sea", + "Souzay-Champigny", + "Spaarndam-Oost", + "Spaarndam-West", + "Spider-Man", + "Spiesen-Elversberg", + "Sprang-Capelle", + "Spree-Neisse", + "Sri-Lankais", + "Sri-Lankaise", + "Sri-Lankaises", + "St-Jean", + "Stadecken-Elsheim", + "Stafordshire-bull-terrier", + "Staines-upon-Thames", + "Starrkirch-Wil", + "Staudach-Egerndach", + "Stechow-Ferchesar", + "Steenhuize-Wijnhuize", + "Steg-Hohtenn", + "Stein-Bockenheim", + "Stein-Neukirch", + "Stein-Wingert", + "Steinbach-Hallenberg", + "Stelle-Wittenwurth", + "Stiring-Wendel", + "Stockhausen-Illfurth", + "Stockton-on-Tees", + "Stockum-Püschen", + "Stoke-on-Trent", + "Storbeck-Frankendorf", + "Stratford-on-Avon", + "Straubing-Bogen", + "Straßlach-Dingharting", + "Strombeek-Bever", + "Strépy-Bracquegnies", + "Stutzheim-Offenheim", + "Stéphano-Carladésien", + "Stéphano-Carladésienne", + "Stéphano-Carladésiennes", + "Stéphano-Carladésiens", + "Stüdenitz-Schönermark", + "Suisse-Saxonne-Monts-Métallifères-de-l'Est", + "Sukow-Levitzow", + "Sulzbach-Laufen", + "Sulzbach-Rosenberg", + "Sunbury-on-Thames", + "Sury-ès-Bois", + "Sutton-in-Ashfield", + "Sutz-Lattrigen", + "Syr-Daria", + "Sère-Lanso", + "Sère-Rustaing", + "Sèvres-Anxaumont", + "Sébazac-Concourès", + "Séez-Mesnil", + "Ségrie-Fontaine", + "Séméacq-Blachon", + "Sémézies-Cachan", + "Sénaillac-Latronquière", + "Sénaillac-Lauzès", + "Sépeaux-Saint Romain", + "Séquano-Dionysien", + "Séranvillers-Forenville", + "Sérignac-Péboudou", + "Séry-Magneval", + "Sérézin-de-la-Tour", + "Sévignacq-Meyracq", + "Sévignacq-Thèze", + "Sévigny-Waleppe", + "Sévérac d'Aveyron", + "Sévérac-d'Aveyron", + "Sévérac-l'Eglise", + "Sévérac-l'Église", + "Súdwest-Fryslân", + "T'ien-ngan-men", + "T-SQL", + "T-calculable", + "T-calculables", + "Ta-Nehisi", + "Tabaille-Usquain", + "Taben-Rodt", + "Tadousse-Ussau", + "Taglio-Isolaccio", + "Tahiti-Iti", + "Tahu-Ata", + "Taiarapu-Est", + "Taiarapu-Ouest", + "Tain-l'Hermitage", + "Taizé-Aizie", + "Talloires-Montmin", + "Tallud-Sainte-Gemme", + "Tambach-Dietharz", + "Tanghin-Dassouri", + "Taponnat-Fleurignac", + "Tarawa-Sud", + "Tardets-Sorholus", + "Taron-Sadirac-Viellenave", + "Tart-l'Abbaye", + "Tassin-la-Demi-Lune", + "Taulhac-près-le-Puy", + "Taurignan-Castet", + "Taurignan-Vieux", + "Tauxières-Mutry", + "Taxat-Senat", + "Tchéco-Slovaque", + "Tchéco-Slovaques", + "Teillet-Argenty", + "Tel-Aviv-Jaffa", + "Tella-Sin", + "Teltow-Fläming", + "Temmen-Ringenwalde", + "Temple-Laguyon", + "Tenero-Contra", + "Tensbüttel-Röst", + "Termes-d'Armagnac", + "Ternuay-Melay-et-Saint-Hilaire", + "Terny-Sorny", + "Terrasson-Lavilledieu", + "Terre-Clapier", + "Terre-Natale", + "Terre-Neuve", + "Terre-Neuve-et-Labrador", + "Terre-Neuvien", + "Terre-Neuvien-et-Labradorien", + "Terre-Neuvienne", + "Terre-Neuvienne-et-Labradorienne", + "Terre-Neuviennes", + "Terre-Neuviennes-et-Labradoriennes", + "Terre-Neuviens", + "Terre-Neuviens-et-Labradoriens", + "Tessé-Froulay", + "Test-Milon", + "Testorf-Steinfort", + "Teurthéville-Bocage", + "Teurthéville-Hague", + "Thal-Drulingen", + "Thal-Marmoutier", + "Thaleischweiler-Fröschen", + "Theil-Rabier", + "Theix-Noyalo", + "Thermes-Magnoac", + "Theuvy-Achères", + "Thiaucourt-Regniéville", + "Thieulloy-l'Abbaye", + "Thil-Manneville", + "Thimert-Gâtelles", + "Thimister-Clermont", + "Thimistérien-Clermontois", + "Thimistérien-Clermontoise", + "Thiron-Gardais", + "Thiverval-Grignon", + "Thiéblemont-Farémont", + "Thoirette-Coisia", + "Thon-Samson", + "Thon-Samsonais", + "Thon-Samsonaise", + "Thorame-Basse", + "Thorame-Haute", + "Thorens-Glières", + "Thorey-Lyautey", + "Thorigny-sur-le-Mignon", + "Thorigné-Fouillard", + "Thorigné-d'Anjou", + "Thornaby-on-Tees", + "Thornton-Cleveleys", + "Thoré-Folléen", + "Thoré-Folléenne", + "Thoré-Folléennes", + "Thoré-Folléens", + "Thouarsais-Bouildroux", + "Thoult-Tronaisien", + "Thoult-Tronaisienne", + "Thoult-Tronaisiennes", + "Thoult-Tronaisiens", + "Thoury-Ferrottes", + "Thoury-Férottes", + "Thugny-Trugny", + "Thuit-Angevin", + "Thuit-Angevine", + "Thuit-Angevines", + "Thuit-Angevins", + "Thuit-Hébert", + "Thuit-Signolais", + "Thuit-Signolaise", + "Thuit-Signolaises", + "Thuit-Simérien", + "Thuit-Simérienne", + "Thuit-Simériennes", + "Thuit-Simériens", + "Thun-Episcopien", + "Thun-Episcopienne", + "Thun-Episcopiennes", + "Thun-Episcopiens", + "Thun-l'Evêque", + "Thun-l'Évêque", + "Thun-Épiscopien", + "Thun-Épiscopienne", + "Thun-Épiscopiennes", + "Thun-Épiscopiens", + "Thury-Harcourt", + "Thuès-Entre-Valls", + "Thézy-Glimont", + "Thézy-Glimontois", + "Thézy-Glimontoise", + "Thézy-Glimontoises", + "Tian'anmen", + "Tibiran-Jaunac", + "Tielt-Winge", + "Tieste-Uragnoux", + "Tignieu-Jameyzieu", + "Tigny-Noyelle", + "Til-Châtel", + "Tilleul-Dame-Agnès", + "Tilleul-Othonnais", + "Tilleul-Othonnaise", + "Tilleul-Othonnaises", + "Tilloy-Floriville", + "Tilloy-lez-Cambrai", + "Tilloy-lez-Marchiennes", + "Tilly-Capelle", + "Tin-Akof", + "Tinchebray-Bocage", + "Tincourt-Boucly", + "Tinizong-Rona", + "Tirent-Pontéjac", + "Tite-Live", + "Titisee-Neustadt", + "Tobel-Tägerschen", + "Togny-aux-Bœufs", + "Tongre-Notre-Dame", + "Tonnay-Boutonne", + "Tonnay-Charente", + "Tonnegrande-Montsinery", + "Torcé-Viviers-en-Charnie", + "Torgelow-Holländerei", + "Torre-Cardela", + "Torre-serona", + "Torricella-Taverne", + "Torteval-Quesnay", + "Toscolano-Maderno", + "Toulx-Sainte-Croix", + "Tourcelles-Chaumont", + "Tourcelles-Chaumont-Quilly-et-Chardeny", + "Tournedos-Bois-Hubert", + "Tournehem-sur-la-Hem", + "Tournon-d'Agenais", + "Tournous-Darré", + "Tournous-Devant", + "Tourrette-Levens", + "Tourville-sur-Pont-Audemer", + "Toury-Lurcy", + "Tourzel-Ronzières", + "Toy-Viam", + "Traben-Trarbach", + "Tracy-Bocage", + "Tramont-Emy", + "Tramont-Lassus", + "Tramont-Émy", + "Tranqueville-Graux", + "Travedona-Monate", + "Treffort-Cuisiat", + "Treis-Karden", + "Treize-Septiers", + "Treize-Vents", + "Trentin-Haut-Adige", + "Trentola-Ducenta", + "Treschenu-Creyers", + "Trespoux-Rassiels", + "Treuzy-Levelay", + "Triac-Lautrait", + "Trie-Château", + "Tritteling-Redlach", + "Trizay-Coutretot-Saint-Serge", + "Trockenborn-Wolfersdorf", + "Trosly-Breuil", + "Trosly-Loire", + "Trouley-Labarthe", + "Troye-d'Ariège", + "Trucios-Turtzioz", + "Trucy-l'Orgueilleux", + "Très-Haut", + "Trèves-Cunault", + "Trèves-Sarrebourg", + "Trébons-sur-la-Grasse", + "Trédrez-Locquémeau", + "Trévou-Tréguignec", + "Tschiertschen-Praden", + "Tue-Vaques", + "Turing-calculable", + "Turing-calculables", + "Turnow-Preilack", + "Turquestein-Blancrupt", + "Téteghem-Coudekerque-Village", + "Tümlauer-Koog", + "U-turn", + "U-turns", + "UTF-8", + "Ua-Huka", + "Ua-Pou", + "Ubaye-Serre-Ponçon", + "Ubstadt-Weiher", + "Uebigau-Wahrenbrück", + "Uecker-Randow", + "Uesslingen-Buch", + "Ugao-Miraballes", + "Uggiate-Trevano", + "Ugny-l'Equipée", + "Ugny-l'Équipée", + "Uhart-Cize", + "Uhart-Mixe", + "Uharte-Arakil", + "Uhldingen-Mühlhofen", + "Uhlstädt-Kirchhasel", + "Unstrut-Hainich", + "Upgant-Schott", + "Urroz-Villa", + "Urtenen-Schönbühl", + "Urville-Bocage", + "Urville-Nacqueville", + "Usclas-d'Hérault", + "Ussel-d'Allier", + "Uvernet-Fours", + "Vabre-Tizac", + "Vabres-l'Abbaye", + "Vacognes-Neuilly", + "Vacqueriette-Erquières", + "Vahl-Ebersing", + "Vaire-Arcier", + "Val Buëch-Méouge", + "Val d'Arcomie", + "Val d'Issoire", + "Val d'Oronaye", + "Val d'Oust", + "Val d'épy", + "Vald'orien", + "Valde-Ucieza", + "Valdegovía-Gaubea", + "Valdemoro-Sierra", + "Valdeolmos-Alalpardo", + "Valdieu-Lutran", + "Valence-d'Albigeois", + "Valfin-lès-Saint-Claude", + "Valkenburg-Houthem", + "Vall-llobrega", + "Valle-d'Alesani", + "Valle-d'Orezza", + "Valle-di-Campoloro", + "Valle-di-Mezzana", + "Valle-di-Rostino", + "Vallerois-Lorioz", + "Vallon-Pont-d'Arc", + "Vallouise-Pelvoux", + "Valras-Plage", + "Vals-près-le-Puy", + "Valverde-Enrique", + "Valzin-en-Petite-Montagne", + "Vandœuvre-lès-Nancy", + "Varces-Allières-et-Risset", + "Varenne-l'Arconce", + "Varenne-sur-le-Doubs", + "Varennes-Changy", + "Varennes-Jarcy", + "Varennes-Vauzelles", + "Varmie-Mazurie", + "Varneville-Bretteville", + "Vauclerc-et-la-Vallée-Foulon", + "Vauconcourt-Nervezain", + "Vaudreuil-Lacois", + "Vaulx-Milieu", + "Vaulx-Vraucourt", + "Vazeilles-Limandre", + "Veigy-Foncenex", + "Velesmes-Echevanne", + "Velesmes-Essarts", + "Velesmes-Échevanne", + "Vellexon-Queutey-et-Vaudey", + "Vellexon-Queutrey-et-Vaudey", + "Velone-Orneto", + "Velsen-Noord", + "Velsen-Zuid", + "Veltem-Beisem", + "Velzeke-Ruddershove", + "Ven-Zelderheide", + "Vendays-Montalivet", + "Vendeuil-Caply", + "Vendredi-Saint", + "Vendresse-Beaulne", + "Ventenac-Cabardès", + "Ventenac-d'Aude", + "Verbano-Cusio-Ossola", + "Vercel-Villedieu-le-Camp", + "Verchain-Maugré", + "Verdun-sur-le-Doubs", + "Verel-Pragondran", + "Verlhac-Tescou", + "Vern-d'Anjou", + "Verneuil-Grand", + "Verneuil-Moustiers", + "Verneuil-Petit", + "Verneuil-d'Avre-et-d'Iton", + "Verneuil-l'Etang", + "Verneuil-l'Étang", + "Vernou-la-Celle-sur-Seine", + "Verrens-Arvey", + "Vers-Pont-du-Gard", + "Vert-Toulon", + "Verteuil-d'Agenais", + "Vetula-Domussien", + "Vetula-Domussienne", + "Vetula-Domussiennes", + "Vetula-Domussiens", + "Veurey-Voroize", + "Veyre-Monton", + "Veyrins-Thuellin", + "Vezels-Roussy", + "Viala-du-Pas-de-Jaux", + "Vic-Fezensac", + "Vichel-Nanteuil", + "Vicq-Exemplet", + "Vicq-d'Auribat", + "Victot-Pontfol", + "Viellenave-d'Arthez", + "Viels-Maisons", + "Vier-Bordes", + "Viereth-Trunstadt", + "Vierset-Barse", + "Viet-Nam", + "Vieu-d'Izenave", + "Vigneux-Hocquet", + "Vignola-Falesina", + "Vignoux-sous-les-Aix", + "Vigoulet-Auzil", + "Vila-real", + "Vila-rodona", + "Vila-sacra", + "Vila-sana", + "Vila-seca", + "Vildé-Guingalan", + "Villabona-Amasa", + "Village-Neuf", + "Villar-Loubière", + "Villar-d'Arêne", + "Villard-Bonnot", + "Villard-Léger", + "Villard-Notre-Dame", + "Villard-Reculas", + "Villard-Reymond", + "Villard-Sallet", + "Villard-d'Hérien", + "Villard-d'Hérienne", + "Villard-d'Hériennes", + "Villard-d'Hériens", + "Villard-d'Héry", + "Villards-d'Héria", + "Villarejo-Periesteban", + "Villarodin-Bourget", + "Villars-Brandis", + "Villars-Colmars", + "Villars-Fontaine", + "Villars-Sainte-Croix", + "Villars-Santenoge", + "Villars-Épeney", + "Villarta-Quintana", + "Villarzel-Cabardès", + "Villaverde-Mogina", + "Villebois-Lavalette", + "Villedieu-les-Poêles-Rouffigny", + "Villefranche-d'Albigeois", + "Villefranche-d'Allier", + "Villeloin-Coulangé", + "Villelongue-d'Aude", + "Villelongue-de-la-Salanque", + "Villelongue-dels-Monts", + "Villemagne-l'Argentière", + "Villenave-d'Ornon", + "Villequier-Aumont", + "Villerouge-Termenès", + "Villers-aux-Nœuds", + "Villez-sur-le-Neubourg", + "Villiers-en-Désœuvre", + "Villieu-Loyes-Mollon", + "Villingen-Schwenningen", + "Villié-Morgon", + "Villotte-devant-Louppy", + "Vilosnes-Haraumont", + "Vilters-Wangs", + "Vincent-Froideville", + "Vincy-Manœuvre", + "Vincy-Manœuvre", + "Vincy-Reuil-et-Magny", + "Vindrac-Alayrac", + "Viodos-Abense-de-Bas", + "Vireux-Molhain", + "Vireux-Wallerand", + "Virginal-Samme", + "Virginie-Occidentale", + "Viry-Châtillon", + "Viry-Noureuil", + "Visa-Bourgien", + "Visa-Bourgienne", + "Visa-Bourgiennes", + "Visa-Bourgiens", + "Vissac-Auteyrac", + "Vitoria-Gasteiz", + "Vitry-Laché", + "Vittel-menthe", + "Vivier-Danger", + "Viéville-sous-les-Côtes", + "Vlaardinger-Ambacht", + "Vlagtwedder-Barlage", + "Vlagtwedder-Veldhuis", + "Vlodrop-Station", + "Vogelsang-Warsin", + "Void-Vacon", + "Voivres-lès-le-Mans", + "Vollore-Montagne", + "Vollore-Ville", + "Voor-Drempt", + "Voroux-Goreux", + "Voroux-lez-Liers", + "Vortum-Mullem", + "Vosne-Romanée", + "Vouthon-Bas", + "Vouthon-Haut", + "Vrigne-Meuse", + "Vrigne-Meusien", + "Vrigne-Meusienne", + "Vrigne-Meusiennes", + "Vrigne-Meusiens", + "Vrijhoeve-Capelle", + "Vuisternens-devant-Romont", + "Vœlfling-lès-Bouzonville", + "Vœuil-et-Giget", + "Vélez-Blanco", + "Vélez-Málaga", + "Vélez-Rubio", + "Vélizy-Villacoublay", + "Vérizet-Fleurville", + "Vésenex-Crassy", + "Vétraz-Monthoux", + "Vézeronce-Curtin", + "Wageningen-Hoog", + "Wailly-Beaucamp", + "Wald-Michelbach", + "Waldeck-Frankenberg", + "Waldfischbach-Burgalben", + "Waldhof-Falkenstein", + "Waldshut-Tiengen", + "Walincourt-Selvigny", + "Wallendorf-Pont", + "Wallers-Trélon", + "Wallon-Cappel", + "Wallon-Cappelois", + "Wallon-Cappeloise", + "Wallon-Cappeloises", + "Walton-on-Thames", + "Wanchy-Capval", + "Wandignies-Hamage", + "Wanfercée-Baulet", + "Wangen-Brüttisellen", + "Wangenbourg-Engenthal", + "Wannegem-Lede", + "Wanzleben-Börde", + "Waret-l'Évêque", + "Warfusée-Abancourt", + "Wargemoulin-Hurlus", + "Warlencourt-Eaucourt", + "Warloy-Baillon", + "Warnant-Dreye", + "Warneton-Sud", + "Wartenberg-Rohrbach", + "Warth-Weiningen", + "Wasmes-Audemez-Briffœil", + "Waterland-Oudeman", + "Watermael-Boitsfort", + "Wath-on-Dearne", + "Wath-upon-Dearne", + "Wauthier-Braine", + "Wauthier-Brainois", + "Wauthier-Brainoise", + "Wavre-Notre-Dame", + "Wavre-Sainte-Catherine", + "Weiler-Simmerberg", + "Weilheim-Schongau", + "Weimar-Campagne", + "Weißenborn-Lüderode", + "Weißenburg-Gunzenhausen", + "Welles-Pérennes", + "Wemaers-Cappel", + "Wemaers-Cappelois", + "Wemaers-Cappeloise", + "Wemaers-Cappeloises", + "Wenningstedt-Braderup", + "Wenum-Wiesel", + "Wernberg-Köblitz", + "Werra-Meissner", + "Wervicq-Nord", + "Wervicq-Sud", + "Wesembeek-Ophem", + "West-Barendrecht", + "West-Cappel", + "West-Cappelois", + "West-Cappeloise", + "West-Cappeloises", + "West-Graftdijk", + "West-Knollendam", + "West-Souburg", + "West-Terschelling", + "Wester-Koggenland", + "Wester-Ohrstedt", + "Westerhaar-Vriezenveensewijk", + "Westhouse-Marmoutier", + "Westkapelle-Binnen", + "Wettin-Löbejün", + "Wez-Velvain", + "Wezembeek-Oppem", + "Wi-Fi", + "Wickersheim-Wilshausen", + "Wiencourt-l'Equipée", + "Wiencourt-l'Équipée", + "Wierre-Effroy", + "Wilkau-Haßlau", + "Wilp-Achterhoek", + "Wilzenberg-Hußweiler", + "Winkel-Sainte-Croix", + "Wintzenheim-Kochersberg", + "Wiège-Faty", + "Wokuhl-Dabelow", + "Wolframs-Eschenbach", + "Wolfsburg-Unkeroda", + "Wormeldange-Haut", + "Wortegem-Petegem", + "Wust-Fischbeck", + "Wutha-Farnroda", + "Wy-dit-Joli-Village", + "Wœlfling-lès-Sarreguemines", + "Wünnewil-Flamatt", + "X-SAMPA", + "X-arbre", + "X-arbres", + "X-board", + "X-boards", + "Xanton-Chassenon", + "Xivry-Circourt", + "Xonrupt-Longemer", + "Yaucourt-Bussus", + "Yernée-Fraineux", + "Yo-kai", + "Ypreville-Biville", + "Yves-Gomezée", + "Yvetot-Bocage", + "Yvré-l'Evêque", + "Yvré-l'Évêque", + "Yécora-Iekora", + "Z-grille", + "Z-grilles", + "Z/E-8-DDA", + "Z9-12:Ac", + "Z9-dodécénylacétate", + "Zahna-Elster", + "Zella-Mehlis", + "Zeltingen-Rachtig", + "Zend-avesta", + "Zernitz-Lohm", + "Zeulenroda-Triebes", + "Zevenhuizen-Moerkapelle", + "Zichen-Zussen-Bolder", + "Ziegra-Knobelsdorf", + "Zihlschlacht-Sitterdorf", + "Zillis-Reischen", + "Ziortza-Bolibar", + "Zoerle-Parwijs", + "Zoeterwoude-Dorp", + "Zoeterwoude-Rijndijk", + "Zschaitz-Ottewig", + "Zuid-Beijerland", + "Zuid-Eierland", + "Zuid-Polsbroek", + "Zuid-Scharwoude", + "Zuid-Spierdijk", + "Zuid-Waddinxveen", + "Zwaagdijk-Oost", + "Zwaagdijk-West", + "Zétrud-Lumay", + "abaisse-langue", + "abaisse-langues", + "abou-hannès", + "abou-mengel", + "abou-mengels", + "abri-vent", + "abricot-pêche", + "abricotier-pays", + "abricots-pêches", + "abris-vent", + "absorbeur-neutralisateur", + "acajou-amer", + "acajou-bois", + "acajous-amers", + "acajous-bois", + "accord-cadre", + "accords-cadres", + "accroche-coeur", + "accroche-coeurs", + "accroche-cœur", + "accroche-cœurs", + "accroche-pied", + "accroche-pieds", + "accroche-plat", + "accroche-plats", + "achard-bourgeois", + "achard-bourgeoise", + "achard-bourgeoises", + "acibenzolar-S-méthyle", + "acide-N-1-naphtyl-phtalamique", + "acide-phénol", + "acides-phénols", + "acido-alcalimétrie", + "acido-alcoolo-résistance", + "acido-alcoolo-résistances", + "acido-alcoolo-résistant", + "acido-alcoolo-résistante", + "acido-alcoolo-résistantes", + "acido-alcoolo-résistants", + "acido-basique", + "acido-résistant", + "acido-résistants", + "acqua-toffana", + "acqua-toffanas", + "acquae-sextien", + "acquae-sextienne", + "acquae-sextiennes", + "acquae-sextiens", + "acquit-patent", + "acquits-patents", + "acting-out", + "actino-uranium", + "acétyl-salicylate", + "acétyl-salicylates", + "add-on", + "adieu-mes-couilles", + "adieu-tout", + "adieu-touts", + "adieu-va", + "adieu-vas", + "adieu-vat", + "adieu-vats", + "adiposo-génital", + "adiposo-génitale", + "adiposo-génitales", + "adiposo-génitaux", + "adjudant-chef", + "adjudants-chefs", + "africain-américain", + "africaine-américaine", + "africaines-américaines", + "africains-américains", + "africano-brésilien", + "africano-brésilienne", + "africano-brésiliennes", + "africano-brésiliens", + "africano-taïwanais", + "africano-taïwanaise", + "africano-taïwanaises", + "agace-pissette", + "agar-agar", + "agasse-tambourinette", + "agatha-christien", + "agit-prop", + "agnus-castus", + "agnus-dei", + "agora-phobie", + "agora-phobies", + "ai-cham", + "aide-comptable", + "aide-mémoire", + "aide-mémoires", + "aide-soignant", + "aide-soignante", + "aide-soignantes", + "aide-soignants", + "aide-écuyer", + "aide-écuyers", + "aide-éducateur", + "aides-soignantes", + "aides-soignants", + "aigle-bar", + "aigre-douce", + "aigre-doux", + "aigre-moines", + "aigres-douces", + "aigres-doux", + "aiguise-crayon", + "aiguise-crayons", + "ainu-ken", + "airelle-myrtille", + "aiseau-preslois", + "aka-bea", + "aka-bo", + "aka-cari", + "aka-jeru", + "aka-kede", + "aka-kora", + "akar-bale", + "akhal-teke", + "akua-ba", + "al-Anbar", + "al-Anbâr", + "al-Anbār", + "al-Kachi", + "al-Qaida", + "al-Qaïda", + "albano-letton", + "alcalino-terreuse", + "alcalino-terreuses", + "alcalino-terreux", + "alcool-phénol", + "alcoolo-dépendance", + "alcoolo-dépendances", + "alcools-phénols", + "algo-carburant", + "algo-carburants", + "algéro-marocain", + "algéro-tuniso-lybien", + "algéro-tuniso-marocain", + "allanto-chorion", + "allanto-chorions", + "aller-retour", + "aller-retours", + "allers-retours", + "allez-vous-en", + "allez-y", + "alloxydime-sodium", + "allume-cigare", + "allume-cigares", + "allume-feu", + "allume-feux", + "allume-gaz", + "allumette-bougie", + "allumettes-bougies", + "alpha-amylase", + "alpha-amylases", + "alpha-conversion", + "alpha-conversions", + "alpha-test", + "alpha-tests", + "alpha-tridymite", + "alpha-tridymites", + "alpha-variscite", + "alpha-variscites", + "alsacien-lorrain", + "alto-basso", + "alto-bassos", + "aluminium-épidote", + "aluminium-épidotes", + "alumu-tesu", + "aléseuse-fraiseuse", + "aléseuses-fraiseuses", + "ambre-gris", + "ambystome-tigre", + "ambystomes-tigres", + "ami-ami", + "amiante-ciment", + "amino-acide", + "amino-acides", + "amino-acétique", + "amour-propre", + "amours-propres", + "ampli-syntoniseur", + "ampère-heure", + "ampères-heures", + "amuse-bouche", + "amuse-bouches", + "amuse-gueule", + "amuse-gueules", + "analyste-programmeur", + "analystes-programmeurs", + "ananas-bois", + "anarcho-capitalisme", + "anarcho-capitalismes", + "anarcho-fasciste", + "anarcho-fascistes", + "anarcho-punk", + "anarcho-punks", + "anarcho-syndicalisme", + "anarcho-syndicalismes", + "anarcho-syndicaliste", + "anarcho-syndicalistes", + "anatomo-pathologie", + "anatomo-pathologies", + "anatomo-pathologique", + "anatomo-pathologiques", + "andrézien-bouthéonnais", + "andrézienne-bouthéonnaise", + "andréziennes-bouthéonnaises", + "andréziens-bouthéonnais", + "anguille-spaghetti", + "animal-garou", + "animalier-soigneur", + "animaux-garous", + "année-homme", + "année-lumière", + "années-homme", + "années-hommes", + "années-lumière", + "ano-génital", + "ano-génitale", + "ano-génitales", + "ano-génitaux", + "ansbach-triesdorfer", + "ante-bois", + "ante-meridiem", + "ante-meridiems", + "ante-mortem", + "ante-mortems", + "antenne-relais", + "antennes-radar", + "antennes-relais", + "anthropo-gammamétrie", + "anthropo-gammamétries", + "anthropo-toponyme", + "anthropo-toponymes", + "anthropo-zoomorphe", + "anthropo-zoomorphes", + "antiguais-barbudien", + "antiguais-barbudiens", + "antiguaise-barbudienne", + "antiguaises-barbudiennes", + "antilope-chevreuil", + "anté-diluvien", + "anté-hypophyse", + "anté-hypophyses", + "anté-prédécesseur", + "anté-prédécesseurs", + "anté-pénultième", + "anté-pénultièmes", + "apico-alvéolaire", + "apico-dental", + "appartement-témoin", + "appartements-témoins", + "appel-contre-appel", + "appels-contre-appels", + "apprenti-sorcellerie", + "apprenti-sorcelleries", + "apprenti-sorcier", + "apprentie-sorcière", + "apprenties-sorcières", + "apprentis-sorciers", + "appui-bras", + "appui-livres", + "appui-main", + "appui-mains", + "appui-pied", + "appui-pieds", + "appui-pot", + "appui-pots", + "appui-tête", + "appui-têtes", + "appuie-main", + "appuie-mains", + "appuie-tête", + "appuie-têtes", + "appuis-main", + "appuis-pot", + "appuis-tête", + "aqua-tinta", + "aqua-toffana", + "aquae-sextien", + "aquae-sextienne", + "aquae-sextiennes", + "aquae-sextiens", + "aquila-alba", + "araignée-crabe", + "araignée-loup", + "araignées-crabes", + "araignées-loups", + "aralo-caspien", + "aralo-caspienne", + "arbre-à-la-fièvre", + "arbres-refuges", + "arcado-chypriote", + "arcado-chypriotes", + "arcado-cypriote", + "arcado-cypriotes", + "ardennite-(As)", + "ardennite-(As)s", + "ardi-gasna", + "argent-métal", + "argentite-β", + "argentite-βs", + "argento-analcime", + "argento-analcimes", + "argento-perrylite", + "argento-perrylites", + "argilo-calcaire", + "argilo-calcaires", + "argilo-gréseuse", + "argilo-gréseuses", + "argilo-gréseux", + "argilo-loessique", + "argilo-loessiques", + "argilo-siliceuse", + "argilo-siliceuses", + "argilo-siliceux", + "arginine-vasopressine", + "ariaco-dompierrois", + "ariaco-dompierroise", + "ariaco-dompierroises", + "aristo-bourgeoisie", + "aristo-bourgeoisies", + "aristotélico-thomiste", + "aristotélico-thomistes", + "arivey-lingeois", + "arivey-lingeoise", + "arivey-lingeoises", + "armançon-martinois", + "armançon-martinoise", + "armançon-martinoises", + "armbouts-cappellois", + "armbouts-cappelloise", + "armbouts-cappelloises", + "arnaud-guilhémois", + "arnaud-guilhémoise", + "arnaud-guilhémoises", + "arrache-clou", + "arrache-clous", + "arrache-pied", + "arrache-sonde", + "arrow-root", + "arrêt-buffet", + "arrêt-court", + "arrête-boeuf", + "arrête-bœuf", + "arrête-bœufs", + "arrêts-buffet", + "arrêts-courts", + "ars-laquenexois", + "ars-laquenexoise", + "ars-laquenexoises", + "art-thérapie", + "art-thérapies", + "artisan-créateur", + "artisans-créateurs", + "artério-sclérose", + "artério-scléroses", + "assa-foetida", + "assemble-nuages", + "assis-debout", + "assurance-chômage", + "assurance-chômages", + "assurance-emploi", + "assurance-vie", + "assurances-chômage", + "assurances-vie", + "assyro-chaldéen", + "astronome-astrologue", + "astronomes-astrologues", + "astur-léonais", + "ataxie-télangiectasie", + "attache-bossette", + "attache-bossettes", + "attache-doudou", + "attache-doudous", + "attaché-case", + "attaché-cases", + "attachés-cases", + "attentat-suicide", + "attentats-suicides", + "atto-ohm", + "atto-ohms", + "attrape-couillon", + "attrape-couillons", + "attrape-minette", + "attrape-minettes", + "attrape-minon", + "attrape-minons", + "attrape-mouche", + "attrape-mouches", + "attrape-nigaud", + "attrape-nigauds", + "attrape-rêves", + "attrape-tout", + "attrape-vilain", + "au-dedans", + "au-dehors", + "au-delà", + "au-delàs", + "au-dessous", + "au-dessus", + "au-devant", + "au-deçà", + "au-lof", + "au-tour", + "aube-vigne", + "aujourd'hui", + "aulnaie-frênaie", + "aulnaies-frênaies", + "auloi-jumeaux", + "auriculo-ventriculaire", + "auriculo-ventriculaires", + "aurum-musivum", + "aussi-tost", + "aussi-tôt", + "australo-américain", + "austro-asiatique", + "austro-asiatiques", + "austro-hongrois", + "austro-hongroise", + "austro-hongroises", + "austro-occidental", + "austro-occidentale", + "austro-occidentales", + "austro-occidentaux", + "auteur-compositeur", + "auteure-compositrice", + "auteures-compositrices", + "auteurs-compositeurs", + "autos-caravanes", + "autos-mitrailleuses", + "autos-scooters", + "autos-tamponnantes", + "autos-tamponneuses", + "autre-littérature", + "autre-églisois", + "avale-tout", + "avale-tout-cru", + "avale-touts", + "avants-centres", + "avants-postes", + "averno-méditerranéen", + "averno-méditerranéenne", + "averno-méditerranéennes", + "averno-méditerranéens", + "aveugle-né", + "aveugle-née", + "aveugles-nés", + "avions-cargos", + "axo-missien", + "axo-missienne", + "axo-missiennes", + "axo-missiens", + "ayant-cause", + "ayant-droit", + "ayants-cause", + "ayants-droit", + "aye-aye", + "ayes-ayes", + "ayur-veda", + "azinphos-méthyl", + "azinphos-éthyl", + "aï-aï", + "b-a-ba", + "b.a.-ba", + "baa'thisa", + "baa'thisai", + "baa'thisaient", + "baa'thisais", + "baa'thisait", + "baa'thisant", + "baa'thisas", + "baa'thisasse", + "baa'thisassent", + "baa'thisasses", + "baa'thisassiez", + "baa'thisassions", + "baa'thise", + "baa'thisent", + "baa'thiser", + "baa'thisera", + "baa'thiserai", + "baa'thiseraient", + "baa'thiserais", + "baa'thiserait", + "baa'thiseras", + "baa'thiserez", + "baa'thiseriez", + "baa'thiserions", + "baa'thiserons", + "baa'thiseront", + "baa'thises", + "baa'thisez", + "baa'thisiez", + "baa'thisions", + "baa'thisons", + "baa'thisâmes", + "baa'thisât", + "baa'thisâtes", + "baa'thisèrent", + "baa'thisé", + "baa'thisée", + "baa'thisées", + "baa'thisés", + "babil's", + "babine-witsuwit'en", + "baby-beef", + "baby-beefs", + "baby-boom", + "baby-boomer", + "baby-boomers", + "baby-boomeur", + "baby-boomeurs", + "baby-boomeuse", + "baby-boomeuses", + "baby-foot", + "baby-foots", + "baby-sitter", + "baby-sitters", + "baby-sitting", + "baby-sittings", + "bachat-long", + "bachat-longs", + "bachi-bouzouck", + "bachi-bouzoucks", + "bachi-bouzouk", + "bachi-bouzouks", + "bahá'í", + "bahá'íe", + "bahá'íes", + "bahá'ís", + "baie-mahaultien", + "baie-mahaultienne", + "baie-mahaultiennes", + "baie-mahaultiens", + "baille-blé", + "bain-douche", + "bain-marie", + "bains-douches", + "bains-marie", + "baise-main", + "bal-musette", + "balai-brosse", + "balais-brosses", + "baleine-pilote", + "baleines-pilotes", + "ball-trap", + "balle-molle", + "balle-queue", + "ballon-panier", + "ballon-sonde", + "ballon-volant", + "ballons-panier", + "ballons-paniers", + "ballons-sondes", + "bals-musette", + "bana-bana", + "bana-banas", + "banana-split", + "banana-splits", + "bande-annonce", + "bande-son", + "bandes-annonces", + "bank-note", + "bank-notes", + "bar-tabac", + "bar-tabacs", + "barium-adulaire", + "barium-adulaires", + "barium-anorthite", + "barium-anorthites", + "barium-phlogopite", + "barium-phlogopites", + "barium-sanidine", + "barium-sanidines", + "barré-bandé", + "barrés-bandés", + "bars-tabacs", + "baryton-basse", + "barytons-basses", + "baryum-orthose", + "baryum-orthoses", + "basco-béarnaise", + "basco-navarrais", + "base-ball", + "base-balls", + "base-jump", + "base-jumpeur", + "base-jumpeurs", + "base-jumpeuse", + "base-jumpeuses", + "basi-sphénoïdal", + "basket-ball", + "basket-balls", + "baso-cellulaire", + "baso-cellulaires", + "basque-uruguayen", + "basset-hound", + "bassi-colica", + "bassi-colicas", + "bassin-versant", + "bassins-versants", + "bat-flanc", + "bat-flancs", + "bat-l'eau", + "bats-l'eau", + "battant-l'oeil", + "battant-l'œil", + "battants-l'oeil", + "battants-l'œil", + "batte-lessive", + "batte-mare", + "batte-plate", + "batte-queue", + "battes-plates", + "baussery-montain", + "baussery-montaine", + "baussery-montaines", + "baussery-montains", + "bay-ice", + "bay-ices", + "beach-volley", + "beach-volleys", + "beagle-harrier", + "beaujolais-villages", + "beaux-arts", + "beaux-dabes", + "beaux-enfants", + "beaux-esprits", + "beaux-fils", + "beaux-frères", + "beaux-oncles", + "beaux-parents", + "beaux-petits-fils", + "beaux-pères", + "becque-cornu", + "becques-cornus", + "becs-cornus", + "becs-courbes", + "becs-d'argent", + "becs-d'oie", + "becs-d'âne", + "becs-durs", + "becs-ouverts", + "becs-plats", + "becs-pointus", + "becs-ronds", + "becs-tranchants", + "bedlington-terrier", + "behā'ī", + "bekkō-amé", + "bel-enfant", + "bel-esprit", + "bel-oncle", + "bel-outil", + "bel-étage", + "belgo-hollandais", + "bels-outils", + "ben-ahinois", + "benne-kangourou", + "bensulfuron-méthyle", + "benzoylprop-éthyl", + "berd'huisien", + "berd'huisienne", + "berd'huisiennes", + "berd'huisiens", + "bernard-l'ermite", + "bernard-l'hermite", + "bernico-montois", + "bernico-montoise", + "bernico-montoises", + "bette-marine", + "bettes-marines", + "beun'aise", + "beurre-frais", + "biche-cochon", + "biens-fonds", + "big-endian", + "bil-ka", + "bil-kas", + "bin's", + "bin-bin", + "bin-bins", + "bissau-guinéen", + "bistro-brasserie", + "bistro-brasseries", + "bit-el-mal", + "bitter-pit", + "bière-pong", + "bla-bla", + "bla-bla-bla", + "black-bass", + "black-blanc-beur", + "black-bottom", + "black-bottoms", + "black-rot", + "blanche-coiffe", + "blanche-queue", + "blanche-raie", + "blanches-coiffes", + "blancs-becs", + "blancs-bocs", + "blancs-bois", + "blancs-d'Espagne", + "blancs-estocs", + "blancs-mangers", + "blancs-manteaux", + "blancs-raisins", + "blancs-seings", + "blancs-signés", + "blancs-étocs", + "bleu-bite", + "bleu-manteau", + "bleu-merle", + "bleus-manteaux", + "blies-ebersingeois", + "blies-ebersingeoise", + "blies-ebersingeoises", + "blies-ébersingeois", + "blies-ébersingeoise", + "blies-ébersingeoises", + "bling-bling", + "bling-blings", + "bloc-cylindres", + "bloc-eau", + "bloc-film", + "bloc-films", + "bloc-moteur", + "bloc-moteurs", + "bloc-note", + "bloc-notes", + "block-système", + "blocs-eau", + "blocs-films", + "blocs-notes", + "blu-ray", + "blue-jean", + "blue-jeans", + "blue-lias", + "boat-people", + "bobby-soxer", + "bobby-soxers", + "body-building", + "boeuf-carotte", + "boissy-maugien", + "boissy-maugienne", + "boissy-maugiennes", + "boissy-maugiens", + "boit-sans-soif", + "bolivo-paraguayen", + "bombardier-torpilleur", + "bombardiers-torpilleurs", + "bonnet-chinois", + "bonnet-rouge", + "bonnet-vert", + "bonnets-chinois", + "bonnets-verts", + "boogie-woogie", + "boogie-woogies", + "bord-opposé", + "borde-plats", + "border-terrier", + "bore-out", + "bore-outs", + "borne-couteau", + "borne-fontaine", + "borne-fusible", + "borne-fusibles", + "bornes-couteaux", + "bornes-fontaines", + "bosc-guérardais", + "bosc-guérardaise", + "bosc-guérardaises", + "bosc-renoulthien", + "bosc-renoulthienne", + "bosc-renoulthiennes", + "bosc-renoulthiens", + "bosno-serbe", + "bosno-serbes", + "botte-chaussettes", + "bottom-up", + "bouche-nez", + "bouche-trou", + "bouche-trous", + "bouffe-curé", + "bouffe-curés", + "bouffe-galette", + "boui-boui", + "bouig-bouig", + "bouillon-blanc", + "bouis-bouis", + "boulay-morinois", + "boulay-morinoise", + "boulay-morinoises", + "boule-dogue", + "boules-dogues", + "boum-boum", + "bourgeois-bohème", + "bourgeois-bohèmes", + "bourgeoise-bohème", + "bourgeoises-bohèmes", + "bourgue-épine", + "bourgues-épines", + "bourre-chrétien", + "bourre-goule", + "bourre-goules", + "bourre-noix", + "bourre-pif", + "bourre-pifs", + "bout-avant", + "bout-d'aile", + "bout-d'argent", + "bout-dehors", + "bout-du-pont-de-l'arnais", + "bout-du-pont-de-l'arnaise", + "bout-du-pont-de-l'arnaises", + "bout-rimé", + "bout-saigneux", + "boute-charge", + "boute-dehors", + "boute-feu", + "boute-hache", + "boute-hors", + "boute-joie", + "boute-lof", + "boute-selle", + "boute-selles", + "boute-tout-cuire", + "bouton-d'or", + "bouton-poussoir", + "bouton-pression", + "boutons-d'or", + "boutons-pression", + "bouts-avant", + "bouts-d'aile", + "bouts-d'argent", + "bouts-dehors", + "bouts-rimés", + "bouts-saigneux", + "bow-string", + "bow-strings", + "bow-window", + "bow-windows", + "box-calf", + "box-office", + "box-offices", + "boxer-short", + "boxer-shorts", + "boy-scout", + "boy-scouts", + "bracelet-montre", + "bracelets-montres", + "brachio-céphalique", + "brachio-céphaliques", + "brachio-radial", + "branc-ursine", + "branc-ursines", + "branche-ursine", + "branches-ursines", + "brancs-ursines", + "branle-bas", + "branle-gai", + "branle-long", + "branle-queue", + "branles-bas", + "branles-gais", + "branles-longs", + "branque-ursine", + "bras-d'assien", + "bras-d'assienne", + "bras-d'assiennes", + "bras-d'assiens", + "brash-ice", + "brash-ices", + "brasse-camarade", + "brasse-camarades", + "bray-dunois", + "bray-dunoise", + "bray-dunoises", + "brazza-congolais", + "bredi-breda", + "brelic-breloque", + "brelique-breloque", + "breuil-bernardin", + "breuil-bernardine", + "breuil-bernardines", + "breuil-bernardins", + "brick-goélette", + "brigadier-chef", + "brigadiers-chefs", + "brillat-savarin", + "brillet-pontin", + "brillet-pontine", + "brillet-pontines", + "brillet-pontins", + "brin-d'amour", + "brin-d'estoc", + "brins-d'amour", + "brins-d'estoc", + "bris-d'huis", + "brise-bise", + "brise-bises", + "brise-burnes", + "brise-cou", + "brise-cous", + "brise-fer", + "brise-fers", + "brise-flots", + "brise-glace", + "brise-glaces", + "brise-image", + "brise-images", + "brise-lame", + "brise-lames", + "brise-lunette", + "brise-mariage", + "brise-motte", + "brise-mottes", + "brise-mur", + "brise-murs", + "brise-os", + "brise-pierre", + "brise-pierres", + "brise-raison", + "brise-raisons", + "brise-roche", + "brise-roches", + "brise-scellé", + "brise-scellés", + "brise-soleil", + "brise-tout", + "brise-vent", + "brise-vents", + "bromophos-éthyl", + "broncho-pneumonie", + "broncho-pneumonies", + "broncho-pulmonaire", + "broncho-pulmonaires", + "brou-brou", + "broue-pub", + "broue-pubs", + "brouille-blanche", + "brouille-blanches", + "broute-minou", + "broute-minous", + "brown-nosers", + "brown-out", + "broût-vernetois", + "broût-vernetoise", + "broût-vernetoises", + "bruesme-d'auffe", + "bruesmes-d'auffe", + "brule-gueule", + "brule-gueules", + "brule-maison", + "brule-maisons", + "brule-parfum", + "brule-parfums", + "brun-suisse", + "brut-ingénu", + "brute-bonne", + "bruts-ingénus", + "brèche-dent", + "brèche-dents", + "brécy-brièrois", + "brécy-brièroise", + "brécy-brièroises", + "brûle-amorce", + "brûle-bout", + "brûle-gueule", + "brûle-gueules", + "brûle-maison", + "brûle-maisons", + "brûle-parfum", + "brûle-parfums", + "brûle-pourpoint", + "brûle-queue", + "brûle-tout", + "buccin-marin", + "buccins-marins", + "bucco-dentaire", + "bucco-dentaires", + "bucco-génital", + "bucco-génitale", + "bucco-génitales", + "bucco-génitaux", + "bucco-labial", + "bucco-pharyngé", + "bucco-pharyngée", + "bucco-pharyngées", + "bucco-pharyngés", + "buck-béan", + "buck-béans", + "buen-retiro", + "buenos-airien", + "buis-prévenchais", + "buis-prévenchaise", + "buis-prévenchaises", + "buisson-ardent", + "buissons-ardents", + "bull-dogs", + "bull-mastiff", + "bull-terrier", + "bull-terriers", + "bungee-jumping", + "bungy-jumping", + "bureau-chef", + "burg-reulandais", + "buste-reliquaire", + "bustes-reliquaires", + "butter-oil", + "bye-bye", + "bèque-fleur", + "bèque-fleurs", + "bébé-bulle", + "bébé-bus", + "bébé-médicament", + "bébé-nageur", + "bébé-éprouvette", + "bébés-bulles", + "bébés-médicament", + "bébés-nageurs", + "bébés-éprouvette", + "bégler-beg", + "béglier-beg", + "béni-non-non", + "béni-oui-oui", + "bény-bocain", + "bény-bocaine", + "bény-bocaines", + "bény-bocains", + "béta-cyfluthrine", + "béta-gal", + "bêque-bois", + "bœuf-carotte", + "bœuf-carottes", + "bœuf-garou", + "c'est-à-dire", + "c'que", + "c'qui", + "c'te", + "c-commanda", + "c-commandai", + "c-commandaient", + "c-commandais", + "c-commandait", + "c-commandant", + "c-commandas", + "c-commandasse", + "c-commandassent", + "c-commandasses", + "c-commandassiez", + "c-commandassions", + "c-commande", + "c-commandent", + "c-commander", + "c-commandera", + "c-commanderai", + "c-commanderaient", + "c-commanderais", + "c-commanderait", + "c-commanderas", + "c-commanderez", + "c-commanderiez", + "c-commanderions", + "c-commanderons", + "c-commanderont", + "c-commandes", + "c-commandez", + "c-commandiez", + "c-commandions", + "c-commandons", + "c-commandâmes", + "c-commandât", + "c-commandâtes", + "c-commandèrent", + "c-commandé", + "c-commandée", + "c-commandées", + "c-commandés", + "c.-à-d.", + "cabane-roulotte", + "cabanes-roulottes", + "cacasse-à-cul-nu", + "cacasses-à-cul-nu", + "cadrage-débordement", + "caf'conc", + "café-bar", + "café-bistro", + "café-calva", + "café-comptoir", + "café-concert", + "café-crème", + "café-filtre", + "café-théâtre", + "cafés-bars", + "cafés-concerts", + "cafés-crèmes", + "cafés-filtre", + "cafés-théâtres", + "cage-théâtre", + "cages-théâtres", + "cague-braille", + "cague-brailles", + "cahin-caha", + "cail-cédra", + "cail-cédras", + "cail-cédrin", + "cail-cédrins", + "caille-lait", + "caille-laits", + "cailleu-tassart", + "caillot-rosat", + "caillots-rosats", + "caillé-blanc", + "caillés-blancs", + "caisse-outre", + "caisse-palette", + "caisses-outres", + "caisses-palettes", + "cake-walk", + "cake-walks", + "calcite-rhodochrosite", + "calcites-rhodochrosites", + "calcium-autunite", + "calcium-autunites", + "calcium-pyromorphite", + "calcium-pyromorphites", + "calcium-rhodochrosite", + "calcium-rhodochrosites", + "cale-bas", + "cale-dos", + "cale-hauban", + "cale-haubans", + "cale-pied", + "cale-pieds", + "caleçon-combinaison", + "caleçons-combinaisons", + "call-girl", + "call-girls", + "calo-moulinotin", + "calo-moulinotine", + "calo-moulinotines", + "calo-moulinotins", + "came-cruse", + "camion-bélier", + "camion-citerne", + "camion-cuisine", + "camion-cuisines", + "camion-poubelle", + "camions-bennes", + "camions-béliers", + "camions-citernes", + "camions-poubelles", + "camp-volant", + "campanulo-infundibiliforme", + "campanulo-infundibiliformes", + "camping-car", + "camping-cars", + "camping-gaz", + "campo-haltien", + "campo-haltienne", + "campo-haltiennes", + "campo-haltiens", + "campo-laïcien", + "campo-laïcienne", + "campo-laïciennes", + "campo-laïciens", + "camps-volants", + "caméra-lucida", + "caméra-piéton", + "caméra-piétons", + "canadien-français", + "canapé-lit", + "canapés-lits", + "candau-casteidois", + "candau-casteidoise", + "candau-casteidoises", + "cani-joering", + "cani-rando", + "canne-épée", + "cannes-épées", + "cannib's", + "canon-revolver", + "canons-revolvers", + "canoë-kayak", + "canoë-kayaks", + "capelle-filismontin", + "capelle-filismontine", + "capelle-filismontines", + "capelle-filismontins", + "capi-aga", + "capi-agas", + "capigi-bassi", + "capigi-bassis", + "capital-risque", + "capital-risques", + "capital-risqueur", + "capital-risqueurs", + "capitan-pacha", + "capitan-pachas", + "capitaux-risqueurs", + "caporal-chef", + "caporaux-chefs", + "capsule-congé", + "capsules-congés", + "caput-mortuum", + "capélo-hugonais", + "capélo-hugonaise", + "capélo-hugonaises", + "caque-denier", + "car-ferries", + "car-ferry", + "car-ferrys", + "car-jacking", + "carbo-azotine", + "carbonate-apatite", + "carbonate-apatites", + "carbone-14", + "carbones-14", + "carcere-duro", + "cardio-chirurgien", + "cardio-chirurgienne", + "cardio-chirurgiennes", + "cardio-chirurgiens", + "cardio-kickboxing", + "cardio-kickboxings", + "cardio-thoracique", + "cardio-thoraciques", + "cardio-training", + "cardio-vasculaire", + "cardio-vasculaires", + "carfentrazone-éthyle", + "cargo-dortoir", + "cargos-dortoirs", + "caro-percyais", + "caro-percyaise", + "caro-percyaises", + "carré-bossu", + "carrée-bossue", + "carrées-bossues", + "carrés-bossus", + "carte-cadeau", + "carte-fille", + "carte-index", + "carte-lettre", + "carte-maximum", + "carte-mère", + "carte-soleil", + "carte-vue", + "cartes-cadeaux", + "cartes-filles", + "cartes-lettres", + "cartes-maximum", + "cartes-mères", + "cartes-vues", + "carton-index", + "carton-pierre", + "carton-pâte", + "cartons-pâte", + "carême-prenant", + "cas-limite", + "cas-limites", + "cash-back", + "cash-flow", + "cash-flows", + "cassi-ascher", + "cassi-aschers", + "castelnau-durbannais", + "castelnau-durbannaise", + "castelnau-durbannaises", + "castet-arrouyais", + "castet-arrouyaise", + "castet-arrouyaises", + "castillano-aragonais", + "cat-boat", + "catalan-valencien-baléare", + "catalase-positive", + "cato-cathartique", + "cato-cathartiques", + "caïque-bazar", + "caïques-bazars", + "cejourd'hui", + "celle-ci", + "celle-là", + "celles-ci", + "celles-là", + "celto-nordique", + "celto-nordiques", + "celui-ci", + "celui-là", + "cent-cinquante-cinq", + "cent-cinquante-cinquièmes", + "cent-garde", + "cent-gardes", + "cent-lances", + "cent-mille", + "cent-suisse", + "cent-suisses", + "centre-bourg", + "centre-droit", + "centre-gauche", + "centre-tir", + "centre-ville", + "centres-bourgs", + "centres-villes", + "cerf-veau", + "cerf-volant", + "cerf-voliste", + "cerfs-veaux", + "cerfs-volants", + "cerfs-volistes", + "certificat-cadeau", + "cesoird'hui", + "cession-bail", + "cesta-punta", + "ceux-ci", + "ceux-là", + "ch'kâra", + "ch'kâras", + "ch'ni", + "ch't'aime", + "ch'ti", + "ch'timi", + "ch'tis", + "ch.-l.", + "cha'ban", + "cha-cha", + "cha-cha-cha", + "cha-chas", + "chabada-bada", + "chabazite-Ca", + "chabazite-Cas", + "chabazite-Na", + "chabazite-Nas", + "chambolle-musigny", + "chamboule-tout", + "chamito-sémitique", + "chamito-sémitiques", + "champs-clos", + "changxing'ien", + "chanos-cursonnais", + "chanos-cursonnaise", + "chanos-cursonnaises", + "chantilly-tiffany", + "chapellois-fortinien", + "chapellois-fortiniens", + "chapelloise-fortinienne", + "chapelloises-fortiniennes", + "chapon-sérésien", + "chardon-Marie", + "chardon-Roland", + "chardons-Marie", + "chargeuse-pelleteuse", + "charme-houblon", + "charmes-houblons", + "charte-partie", + "chasses-parties", + "chasseur-bombardier", + "chasseur-cueilleur", + "chasseurs-bombardiers", + "chasseurs-cueilleurs", + "chassez-déchassez", + "chassez-huit", + "chassé-croisé", + "chassés-croisés", + "chauche-branche", + "chauche-branches", + "chauche-poule", + "chauffes-doubles", + "chausse-pied", + "chausse-pieds", + "chausse-trape", + "chausse-trapes", + "chausse-trappe", + "chausse-trappes", + "chauve-souriceau", + "chauve-souricelle", + "chauve-souricière", + "chauve-souricières", + "chauve-souris", + "chauve-souris-garou", + "chauves-souriceaux", + "chauves-souricelles", + "chauves-souris", + "chauves-souris-garous", + "chaux-azote", + "chaux-azotes", + "check-up", + "check-ups", + "cheese-cake", + "cheese-cakes", + "chef-boutonnais", + "chef-boutonnaise", + "chef-boutonnaises", + "chef-d'oeuvre", + "chef-d'œuvre", + "chef-lieu", + "chef-mets", + "chef-mois", + "chefs-d'oeuvre", + "chefs-d'œuvre", + "chefs-lieux", + "cherche-fiche", + "cherche-merde", + "cherche-midi", + "cherche-pointe", + "cheval-fondu", + "cheval-garou", + "cheval-heure", + "cheval-jupon", + "cheval-vapeur", + "chevau-léger", + "chevau-légers", + "chevaux-léger", + "chevaux-légers", + "chevaux-vapeur", + "cheveu-de-Marie-Madeleine", + "cheveux-de-Marie-Madeleine", + "chewing-gum", + "chewing-gums", + "chez-moi", + "chez-soi", + "chez-sois", + "chiche-face", + "chiche-kebab", + "chiche-kébab", + "chiches-faces", + "chiches-kebabs", + "chien-assis", + "chien-cerf", + "chien-chaud", + "chien-chauds", + "chien-garou", + "chien-loup", + "chien-nid", + "chien-rat", + "chienne-louve", + "chiennes-louves", + "chiens-assis", + "chiens-cerf", + "chiens-garous", + "chiens-loups", + "chiens-nids", + "chiens-rats", + "chiffre-taxe", + "chiffres-clés", + "chiffres-taxes", + "china-paya", + "chiotte-kès", + "chiottes-kès", + "chirurgien-dentiste", + "chirurgiens-dentistes", + "chloro-IPC", + "chlorpyriphos-méthyl", + "chlorpyriphos-éthyl", + "choano-organismes", + "choche-pierre", + "choche-poule", + "choux-choux", + "choux-fleurs", + "choux-navets", + "choux-palmistes", + "choux-raves", + "chow-chow", + "chow-chows", + "christe-marine", + "christes-marines", + "chrom-brugnatellite", + "chrom-brugnatellites", + "chrome-clinozoïsite", + "chrome-clinozoïsites", + "chrome-fluorite", + "chrome-fluorites", + "chrome-pistazite", + "chrome-pistazites", + "chrome-trémolite", + "chrome-trémolites", + "chrome-zoïsite", + "chrome-zoïsites", + "chrono-localisation", + "chrono-localisations", + "chrétiens-démocrates", + "chuteur-op", + "chuteurs-ops", + "châssis-support", + "châssis-supports", + "châtaigne-d'eau", + "châtaignes-d'eau", + "châteaux-forts", + "chèque-cadeau", + "chèque-repas", + "chèque-restaurant", + "chèque-vacances", + "chèques-cadeaux", + "chèques-repas", + "chèques-restaurants", + "chèques-vacances", + "chèvre-choutiste", + "chèvre-choutistes", + "chèvre-feuille", + "chèvre-pied", + "chèvre-pieds", + "chèvres-feuilles", + "chéry-chartreuvois", + "chéry-chartreuvoise", + "chéry-chartreuvoises", + "chêne-gomme", + "chêne-liège", + "chêne-marin", + "chêne-pommier", + "chênes-gommes", + "chênes-lièges", + "chênes-marins", + "ci-après", + "ci-attaché", + "ci-contre", + "ci-delez", + "ci-dessous", + "ci-dessus", + "ci-devant", + "ci-gisent", + "ci-git", + "ci-gît", + "ci-haut", + "ci-hauts", + "ci-incluse", + "ci-incluses", + "ci-joint", + "ci-jointe", + "ci-jointes", + "ci-joints", + "ci-plus-bas", + "ci-plus-haut", + "cia-cia", + "cinq-cents", + "cinq-dix-quinze", + "cinq-huitième", + "cinq-marsien", + "cinq-marsienne", + "cinq-marsiennes", + "cinq-marsiens", + "cinq-mâts", + "cinq-quatre-un", + "cinq-six", + "cinquante-cinq", + "cinquante-cinquante", + "cinquante-deux", + "cinquante-huit", + "cinquante-neuf", + "cinquante-quatre", + "cinquante-sept", + "cinquante-six", + "cinquante-trois", + "ciné-club", + "ciné-clubs", + "ciné-parc", + "cinéma-dinatoire", + "cinéma-dinatoires", + "circolo-mezzo", + "circonscriptions-clés", + "circum-aural", + "circum-continental", + "cire-pompe", + "cire-pompes", + "cirque-ménagerie", + "cirque-théâtre", + "cirques-ménageries", + "cirques-théâtres", + "cis-gangétique", + "cis-gangétiques", + "cis-verbénol", + "citizen-band", + "citron-pays", + "citrons-pays", + "cité-dortoir", + "cité-État", + "cités-dortoirs", + "cités-États", + "clac-clac", + "clac-clacs", + "claque-merde", + "claque-oreille", + "claque-oreilles", + "claque-patin", + "claque-patins", + "clavi-cylindre", + "clavi-harpe", + "clavi-lyre", + "clic-clac", + "client-cible", + "client-cibles", + "client-serveur", + "cligne-musette", + "climato-sceptique", + "climato-sceptiques", + "clin-foc", + "clin-focs", + "cloche-pied", + "cloche-pieds", + "cloche-plaque", + "clodinafop-propargyl", + "clopin-clopant", + "cloquintocet-mexyl", + "clos-fontainois", + "clos-fontainoise", + "clos-fontainoises", + "clos-masure", + "clos-masures", + "clos-vougeot", + "clos-vougeots", + "club-house", + "clubs-houses", + "clématite-viorne", + "clématites-viornes", + "clérico-nationaliste", + "clérico-nationalistes", + "coat-méalien", + "coat-méalienne", + "coat-méaliennes", + "coat-méaliens", + "cobalt-gris", + "cobalt-mica", + "cobalt-ochre", + "cobalto-sphaérosidérite", + "cobalto-sphaérosidérites", + "cobalto-épsomite", + "cobalto-épsomites", + "cobalts-gris", + "cobalts-micas", + "cobalts-ochres", + "cochon-garou", + "cochons-garous", + "coco-fesses", + "cocotte-minute", + "codes-barres", + "codes-clés", + "coffre-fort", + "coffres-forts", + "coin-coin", + "coin-coins", + "col-nu", + "col-vert", + "col-verts", + "colin-maillard", + "colin-tampon", + "colis-route", + "colis-routes", + "collant-pipette", + "collant-pipettes", + "collet-monté", + "colloid-calcite", + "colloid-calcites", + "collé-serré", + "collés-serrés", + "cols-nus", + "cols-verts", + "colville-okanagan", + "com'com", + "combi-short", + "combi-shorts", + "comble-lacune", + "comble-lacunes", + "come-back", + "commis-voyageur", + "commis-voyageurs", + "commissaire-priseur", + "commissaires-priseurs", + "compositeur-typographe", + "compositeur-typographes", + "comptes-rendus", + "compère-loriot", + "compères-loriot", + "comédie-ballet", + "comédies-ballets", + "concavo-concave", + "concavo-convexe", + "conforte-main", + "conférences-débats", + "congo-kinois", + "congolo-kinois", + "congolo-kinoise", + "congolo-kinoises", + "conseil-général", + "contra-latéral", + "contrat-cadre", + "contrats-cadres", + "contrôle-commande", + "convexo-concave", + "copia-colla", + "copiable-collable", + "copiables-collables", + "copiage-collage", + "copiages-collages", + "copiai-collai", + "copiaient-collaient", + "copiais-collais", + "copiait-collait", + "copiant-collant", + "copias-collas", + "copiasse-collasse", + "copiassent-collassent", + "copiasses-collasses", + "copiassiez-collassiez", + "copiassions-collassions", + "copie-colle", + "copie-lettres", + "copient-collent", + "copier-coller", + "copier-collers", + "copiera-collera", + "copierai-collerai", + "copieraient-colleraient", + "copierais-collerais", + "copierait-collerait", + "copieras-colleras", + "copierez-collerez", + "copieriez-colleriez", + "copierions-collerions", + "copierons-collerons", + "copieront-colleront", + "copies-colles", + "copiez-collez", + "copiez-colliez", + "copions-collions", + "copions-collons", + "copiâmes-collâmes", + "copiât-collât", + "copiâtes-collâtes", + "copièrent-collèrent", + "copié-collé", + "copié-collés", + "copiée-collée", + "copiées-collées", + "copiés-collés", + "coq-héron", + "coq-souris", + "coquel'œil", + "coquel'œils", + "coral-rag", + "corbeau-pêcheur", + "corbeaux-pêcheurs", + "corbeil-essonnois", + "corbeil-essonnoise", + "corbeil-essonnoises", + "cordons-bleus", + "corn-flake", + "corn-flakes", + "corned-beef", + "corned-beefs", + "corps-mort", + "corps-morts", + "cortico-cortical", + "cortico-corticale", + "cortico-corticales", + "cortico-corticaux", + "cortil-noirmontois", + "costa-ricien", + "costa-ricienne", + "costa-riciennes", + "costa-riciens", + "costard-cravate", + "costards-cravates", + "costo-claviculaire", + "costo-sternal", + "costo-thoracique", + "costo-vertébral", + "costo-vertébrale", + "costo-vertébrales", + "costo-vertébraux", + "cosy-corner", + "cosy-corners", + "coton-poudre", + "coton-poudres", + "coton-tige", + "cotons-poudres", + "cotons-tiges", + "cotte-hardie", + "cottes-hardies", + "cou-jaune", + "cou-nu", + "couble-soiffière", + "couche-culotte", + "couche-point", + "couche-points", + "couche-tard", + "couche-tôt", + "couches-culottes", + "couci-couci", + "couci-couça", + "coude-pied", + "coule-sang", + "couper-coller", + "coupon-réponse", + "coupons-réponses", + "coupé-cabriolet", + "coupé-collé", + "coupé-décalé", + "coupé-lit", + "coupés-cabriolets", + "coupés-collés", + "coupés-décalés", + "coupés-lits", + "cour-masure", + "courant-jet", + "courants-jets", + "coure-vite", + "course-poursuite", + "courses-poursuites", + "cous-cous", + "cous-jaunes", + "cout'donc", + "couteau-scie", + "couteaux-scie", + "couvreur-zingueur", + "cover-girl", + "cover-girls", + "cow-boy", + "cow-boys", + "coxa-retrorsa", + "coxo-fémoral", + "crabe-araignée", + "crabes-araignées", + "crac-crac", + "crachouillot-thérapeute", + "craignant-Dieu", + "cran-gevrien", + "cran-gevrienne", + "cran-gevriennes", + "cran-gevriens", + "cranio-facial", + "cranves-salien", + "cranves-saliens", + "cranves-saliène", + "cranves-saliènes", + "crapaud-buffle", + "crapauds-buffles", + "crapet-soleil", + "crayon-feutre", + "crayon-souris", + "crayons-feutre", + "crayons-feutres", + "crest-volantain", + "crest-volantaine", + "crest-volantaines", + "crest-volantains", + "crevette-mante", + "crevettes-mantes", + "cri-cri", + "cri-cris", + "cric-crac", + "crico-trachéal", + "crico-trachéale", + "crico-trachéales", + "crico-trachéaux", + "cristallo-électrique", + "cristallo-électriques", + "criste-marine", + "croad-langshan", + "croiseur-école", + "croiseurs-écoles", + "croix-caluois", + "croix-caluoise", + "croix-caluoises", + "croix-pile", + "croix-roussien", + "croix-roussienne", + "croix-roussiennes", + "croix-roussiens", + "cromlec'h", + "cromlec'hs", + "croque-abeilles", + "croque-lardon", + "croque-lardons", + "croque-madame", + "croque-madames", + "croque-mademoiselle", + "croque-mademoiselles", + "croque-messieurs", + "croque-mitaine", + "croque-mitaines", + "croque-monsieur", + "croque-monsieurs", + "croque-mort", + "croque-morts", + "croque-moutons", + "croque-noisette", + "croque-noisettes", + "croque-noix", + "croque-note", + "crossing-over", + "crown-glass", + "cruci-capétien", + "cruci-capétienne", + "cruci-capétiennes", + "cruci-capétiens", + "cruci-falgardien", + "cruci-falgardienne", + "cruci-falgardiennes", + "cruci-falgardiens", + "crud-ammoniac", + "crypto-communiste", + "crypto-luthérien", + "crypto-luthérienne", + "crypto-luthériennes", + "crypto-luthériens", + "crypto-monnaie", + "crypto-monnaies", + "crève-chassis", + "crève-chien", + "crève-chiens", + "crève-coeur", + "crève-coeurs", + "crève-cœur", + "crève-cœurs", + "crève-vessie", + "crève-vessies", + "créateur-typographe", + "crédit-bail", + "crédit-temps", + "crédits-bail", + "crédits-bails", + "crédits-baux", + "crédits-temps", + "crête-marine", + "crêtes-marines", + "cubito-carpien", + "cubito-carpienne", + "cubito-carpiennes", + "cubito-carpiens", + "cubo-prismatique", + "cubo-prismatiques", + "cueille-essaim", + "cueille-fruits", + "cueilleur-égreneur", + "cueilleurs-égreneurs", + "cueilleuse-égreneuse", + "cueilleuse-épanouilleuse", + "cueilleuses-égreneuses", + "cueilleuses-épanouilleuses", + "cui-cui", + "cuir-laine", + "cuiry-houssien", + "cuiry-houssienne", + "cuiry-houssiennes", + "cuiry-houssiens", + "cuisse-madame", + "cuisse-madames", + "cuit-poires", + "cuit-pommes", + "cuit-vapeur", + "cuit-vapeurs", + "cul-bas", + "cul-blanc", + "cul-brun", + "cul-bénit", + "cul-cul", + "cul-culs", + "cul-de-basse-fosse", + "cul-doré", + "cul-levé", + "cul-rouge", + "cul-rousselet", + "cul-terreux", + "culit-api", + "culs-blancs", + "culs-bénits", + "culs-de-basse-fosse", + "culs-levés", + "culs-rouges", + "culs-terreux", + "cultivateur-tasseur", + "cultivateurs-tasseurs", + "culturo-scientifique", + "culturo-scientifiques", + "cumulo-nimbus", + "cunéo-scaphoïdien", + "cupro-allophane", + "cupro-allophanes", + "cupro-aluminium", + "cupro-aluminiums", + "cupro-ammoniacal", + "cupro-elbaïte", + "cupro-elbaïtes", + "cupro-fraipontite", + "cupro-fraipontites", + "cupro-nickel", + "cupro-nickels", + "cure-dent", + "cure-dents", + "cure-feu", + "cure-feux", + "cure-langue", + "cure-langues", + "cure-môle", + "cure-ongle", + "cure-ongles", + "cure-oreille", + "cure-oreilles", + "cure-pied", + "cure-pieds", + "cure-pipe", + "cure-pipes", + "curti-marignacais", + "curti-marignacaise", + "curti-marignacaises", + "custodi-nos", + "cycle-car", + "cycle-cars", + "cyclo-bus", + "cyclo-cross", + "cyclo-draisine", + "cyclo-draisines", + "cyclo-nomade", + "cyclo-nomades", + "cyclo-octyl-diméthylurée", + "cyclo-pousse", + "cyclo-pousses", + "cyhalofop-butyl", + "cylindro-conique", + "cyth's", + "cyto-architectonie", + "cyto-architectonies", + "cyto-architectonique", + "cyto-architectoniques", + "câblo-opérateur", + "câblo-opérateurs", + "cèleri-rave", + "cèleri-raves", + "céleri-rave", + "céleris-raves", + "céléri-rave", + "céphalo-pharyngien", + "céphalo-pharyngienne", + "céphalo-pharyngiennes", + "céphalo-pharyngiens", + "céphalo-rachidien", + "cérébro-lésion", + "cérébro-lésions", + "cérébro-rachidien", + "cérébro-rachidienne", + "cérébro-rachidiennes", + "cérébro-rachidiens", + "cérébro-spinal", + "cérébro-spinale", + "cérébro-spinales", + "cérébro-spinaux", + "césaro-papisme", + "césaro-papismes", + "césaro-papiste", + "césaro-papistes", + "césium-analcime", + "césium-analcimes", + "d-amphétamine", + "dalai-lama", + "dalai-lamas", + "dalaï-lama", + "dalaï-lamas", + "dame-aubert", + "dame-d'onze-heures", + "dame-jeanne", + "dame-pipi", + "dame-ronde", + "dames-d'onze-heures", + "dames-jeannes", + "dames-pipi", + "dames-rondes", + "danse-poteau", + "dare-dare", + "dead-line", + "dead-lines", + "dena'ina", + "dena'inas", + "dento-facial", + "dermato-allergologue", + "dermato-allergologues", + "dernier-né", + "dernier-nés", + "derniers-nés", + "dernière-née", + "dessinateur-typographe", + "deux-cent-vingt-et-un", + "deux-cents", + "deux-chaisois", + "deux-chaisoise", + "deux-chaisoises", + "deux-chevaux", + "deux-dents", + "deux-mille", + "deux-mâts", + "deux-peccable", + "deux-peccables", + "deux-pièces", + "deux-points", + "deux-ponts", + "deux-quatre", + "deux-roues", + "deux-temps", + "devrai-gondragnier", + "devrai-gondragniers", + "devrai-gondragnière", + "devrai-gondragnières", + "dextro-volubile", + "di-1-p-menthène", + "diam's", + "diastéréo-isomère", + "diastéréo-isomères", + "dichloro-diphényl-dichloroéthane", + "dichlorprop-p", + "diclofop-méthyl", + "diesel-électrique", + "diesels-électriques", + "digue-digue", + "dihydro-oxycodéinone", + "dik-dik", + "dik-diks", + "dikégulac-sodium", + "diméthyl-dixanthogène", + "diméthénamide-P", + "dining-room", + "dining-rooms", + "diola-kasa", + "diony-sapinois", + "diony-sapinoise", + "diony-sapinoises", + "diptéro-sodomie", + "diptéro-sodomies", + "disc-jockey", + "disc-jockeys", + "distance-temps", + "divergi-nervé", + "diésel-électrique", + "diésels-électriques", + "diéthyl-diphényl-dichloroéthane", + "djoumada-l-oula", + "djoumada-t-tania", + "doati-casteidois", + "doati-casteidoise", + "doati-casteidoises", + "docu-fiction", + "docu-fictions", + "documentaire-choc", + "documentaires-chocs", + "dodémorphe-acétate", + "dog-cart", + "dog-carts", + "dom-tomien", + "dom-tomienne", + "dom-tomiennes", + "dom-tomiens", + "dommage-intérêt", + "dommages-intérêts", + "dompte-venin", + "dompte-venins", + "don-juanisme", + "don-juanismes", + "don-quichottisme", + "don-quichottismes", + "donation-partage", + "donations-partages", + "donnant-donnant", + "donne-jour", + "doom-death", + "dorso-vélaire", + "dorso-vélaires", + "dos-d'âne", + "dou-l-hidjja", + "dou-l-qa'da", + "doubet-talibautier", + "doubet-talibautiers", + "doubet-talibautière", + "doubet-talibautières", + "douce-amère", + "douces-amères", + "doux-agnel", + "doux-amer", + "doux-amers", + "doux-ballon", + "doux-vert", + "dragonnet-lyre", + "drainage-taupe", + "draineuse-trancheuse", + "draineuses-trancheuses", + "drap-housse", + "drap-housses", + "drelin-drelin", + "drift-ice", + "drift-ices", + "dring-dring", + "drive-in", + "drive-ins", + "drive-way", + "drive-ways", + "droit-fil", + "droit-fils", + "drop-goal", + "drop-goals", + "drug-store", + "drug-stores", + "dry-tooleur", + "dry-tooleurs", + "dry-tooling", + "dual-core", + "dual-cores", + "duc-d'Albe", + "duc-d'albe", + "duché-pairie", + "duchés-pairies", + "ducs-d'Albe", + "ducs-d'albe", + "duffel-coat", + "duffel-coats", + "duffle-coat", + "duffle-coats", + "dum-dum", + "duo-tang", + "duo-tangs", + "duplicato-dentelé", + "dur-bec", + "dure-mère", + "dure-peau", + "dures-mères", + "dures-peaux", + "durs-becs", + "duty-free", + "dynamo-électrique", + "dynamo-électriques", + "dès-méshui", + "débat-spectacle", + "débauche-embauche", + "déca-ampère", + "déca-ampères", + "découd-vite", + "découpe-neige", + "découpes-neige", + "décrochez-moi-ça", + "déjà-vu", + "démocrate-chrétien", + "démocrate-chrétienne", + "démocrates-chrétiennes", + "démocrates-chrétiens", + "démonte-pneu", + "démonte-pneus", + "déméton-méthyl", + "dépose-minute", + "député-maire", + "députés-maires", + "dépôt-vente", + "dépôts-ventes", + "déséthyl-terbuméton", + "dîner-spectacle", + "dîners-spectacles", + "eau-bénitier", + "eau-bénitiers", + "eau-forte", + "eaux-bonnais", + "eaux-bonnaise", + "eaux-bonnaises", + "eaux-fortes", + "eaux-vannes", + "edit-a-thon", + "edit-a-thons", + "effet-bulle", + "effets-bulles", + "ego-document", + "ego-documents", + "el-âsker", + "elle-même", + "elles-mêmes", + "ello-rhénan", + "ello-rhénane", + "ello-rhénanes", + "ello-rhénans", + "emballage-bulle", + "emballage-coque", + "emballages-bulles", + "emballages-coques", + "emo-sexualité", + "emo-sexualités", + "emporte-pièce", + "emporte-pièces", + "en-avant", + "en-avants", + "en-but", + "en-buts", + "en-cas", + "en-cours", + "en-dessous", + "en-dessus", + "en-deçà", + "en-garant", + "en-tout-cas", + "en-tête", + "en-têtes", + "enfant-bulle", + "enfant-roi", + "enfant-soldat", + "enfants-bulles", + "enfants-robots", + "enfants-rois", + "enfants-soldats", + "enfile-aiguille", + "enfile-aiguilles", + "enfle-boeuf", + "enfle-boeufs", + "enfle-bœuf", + "enfle-bœufs", + "enquêtes-minute", + "enseignant-chercheur", + "enseignante-chercheuse", + "enseignantes-chercheuses", + "enseignants-chercheurs", + "entrer-coucher", + "entrée-sortie", + "entrées-sorties", + "entéro-colite", + "entéro-colites", + "entéro-cystocèle", + "entéro-hydrocèle", + "entéro-hydromphale", + "entéro-hémorrhagie", + "entéro-mérocèle", + "entéro-mésentérite", + "entéro-pneumatose", + "entéro-rénal", + "entéro-rénale", + "entéro-rénales", + "entéro-rénaux", + "entéro-sarcocèle", + "entéro-sarcocèles", + "entéro-sténose", + "entéro-sténoses", + "entéro-épiplocèle", + "entéro-épiplocèles", + "ep's", + "eskimau-aléoute", + "eskimo-aléoute", + "eskimo-aléoutes", + "espace-boutique", + "espace-temps", + "espace-vente", + "espaces-temps", + "espaces-ventes", + "espadon-voilier", + "esprit-fort", + "esprits-forts", + "esquimau-aléoute", + "esquimo-aléoute", + "essert-romanais", + "essert-romanaise", + "essert-romanaises", + "essuie-glace", + "essuie-glaces", + "essuie-main", + "essuie-mains", + "essuie-meuble", + "essuie-meubles", + "essuie-phare", + "essuie-phares", + "essuie-pied", + "essuie-pieds", + "essuie-plume", + "essuie-plumes", + "essuie-tout", + "essuie-touts", + "essuie-verre", + "essuie-verres", + "estrée-blanchois", + "estrée-blanchoise", + "estrée-blanchoises", + "estrée-cauchois", + "estrée-cauchoise", + "estrée-cauchoises", + "estrée-waminois", + "estrée-waminoise", + "estrée-waminoises", + "ethnico-religieux", + "euro-africain", + "euro-africaines", + "euro-asiatique", + "euro-asiatiques", + "euro-bashing", + "euro-manifestation", + "euro-manifestations", + "euro-obligation", + "euro-obligations", + "eusses-tu-cru", + "eux-mêmes", + "exa-ampère", + "exa-ampères", + "exa-octet", + "exa-octets", + "exa-électron-volt", + "exa-électron-volts", + "exaélectron-volt", + "exaélectron-volts", + "excito-nervin", + "excito-nervine", + "excito-nervines", + "excito-nervins", + "excusez-moi", + "exo-noyau", + "exo-noyaux", + "expert-comptable", + "extracto-chargeur", + "extracto-chargeurs", + "extracto-résine", + "extracto-résineux", + "extro-déterminé", + "extrêmes-droites", + "extrêmes-gauches", + "extrêmes-onctions", + "eye-liner", + "eye-liners", + "f'jer", + "f'jers", + "f'nêtre", + "f'nêtres", + "face-B", + "face-kini", + "face-kinis", + "face-sitting", + "face-sittings", + "faces-B", + "faches-thumesnilois", + "faches-thumesniloise", + "faches-thumesniloises", + "faim-valle", + "fair-play", + "fair-plays", + "faire-part", + "faire-savoir", + "faire-valoir", + "fait-divers", + "fait-diversier", + "fait-diversiers", + "fait-main", + "fait-tout", + "faits-divers", + "faits-diversier", + "faits-diversiers", + "fan-club", + "fan-clubs", + "fancy-fair", + "fancy-fairs", + "farcy-pontain", + "farcy-pontaine", + "farcy-pontaines", + "farcy-pontains", + "fast-food", + "fast-foods", + "fausse-braie", + "fausse-couche", + "fausse-limande", + "fausse-monnayeuse", + "fausse-porte", + "fausses-braies", + "fausses-couches", + "fausses-monnayeuses", + "fax-tractage", + "fax-tractages", + "fayl-billotin", + "fayl-billotine", + "fayl-billotines", + "fayl-billotins", + "fech-fech", + "feed-back", + "femelle-stérile", + "femelle-stériles", + "femme-enfant", + "femme-objet", + "femme-orchestre", + "femme-renarde", + "femmes-enfants", + "femmes-orchestres", + "femmes-renardes", + "femto-ohm", + "femto-ohms", + "fer-blanc", + "fer-chaud", + "ferme-bourse", + "ferme-circuit", + "ferme-circuits", + "ferme-porte", + "ferme-portes", + "fermes-hôtels", + "fermier-général", + "ferrando-forézienne", + "ferre-mule", + "ferro-axinite", + "ferro-axinites", + "ferro-magnésien", + "ferro-magnétisme", + "ferro-magnétismes", + "ferro-phlogopite", + "ferro-phlogopites", + "ferro-prussiate", + "ferro-prussiates", + "ferry-boat", + "ferry-boats", + "fers-blancs", + "fesh-fesh", + "fesse-cahier", + "fesse-mathieu", + "fesse-mathieus", + "fesse-mathieux", + "fesse-tonneau", + "fesse-tonneaux", + "fest-deiz", + "fest-noz", + "fest-nozs", + "feuille-caillou-ciseaux", + "feuille-morte", + "fibre-cellule", + "fibro-cartilage", + "fibro-cellulaire", + "fibro-cystique", + "fibro-cystiques", + "fibro-granulaire", + "fibro-muqueux", + "fibro-soyeux", + "fibro-séreux", + "fiche-échalas", + "fiducie-sûreté", + "fie-vïnnamide", + "fie-vïnnamides", + "fifty-fifty", + "figuier-mûrier", + "filet-poubelle", + "filets-poubelles", + "fille-mère", + "filles-mères", + "film-fleuve", + "films-annonces", + "filtre-presse", + "filtres-presses", + "fin-or", + "fine-metal", + "finno-ougrien", + "finno-ougrienne", + "finno-ougriennes", + "finno-ougriens", + "first-fit", + "fisse-larron", + "fisses-larrons", + "fist-fucking", + "fist-fuckings", + "fitz-jamois", + "fitz-jamoise", + "fitz-jamoises", + "fix-up", + "fixe-chaussette", + "fixe-chaussettes", + "fixe-fruit", + "fixe-fruits", + "fixe-longe", + "fixe-moustaches", + "fixe-ruban", + "fixe-rubans", + "fla-fla", + "fla-flas", + "flanc-garde", + "flanc-gardes", + "flanc-mou", + "flancs-gardes", + "flancs-mous", + "flash-back", + "flash-ball", + "flash-balls", + "flash-mob", + "flash-mobs", + "fleur-feuille", + "flic-flac", + "flint-glass", + "flip-flap", + "flirty-fishing", + "float-tube", + "float-tubes", + "flos-ferri", + "flos-ferré", + "flotte-tube", + "flotte-tubes", + "flou-flou", + "fluazifop-P-butyl", + "fluazifop-butyl", + "fluoro-phlogopite", + "fluoro-phlogopites", + "flupyrsulfuron-méthyle", + "fluroxypyr-meptyl", + "fluvio-marin", + "fly-over", + "fly-overs", + "fly-tox", + "foi-menti", + "foi-mentie", + "foire-exposition", + "foires-expositions", + "folk-lore", + "folk-lores", + "folle-avoine", + "folle-blanche", + "folle-verte", + "folles-avoines", + "fon-gbe", + "food-court", + "food-courts", + "food-truck", + "food-trucks", + "force-vivier", + "forge-mètre", + "formica-leo", + "formule-choc", + "formule-chocs", + "forsétyl-al", + "forte-piano", + "forte-pianos", + "forts-vêtu", + "forêt-clairière", + "forêt-climax", + "forêt-galerie", + "forêt-parc", + "forêts-clairières", + "forêts-climax", + "forêts-galeries", + "forêts-parcs", + "fosétyl-Al", + "fouette-cul", + "fouette-culs", + "fouette-queue", + "fouette-queues", + "fougère-aigle", + "fougères-aigles", + "fouille-merde", + "foule-crapaud", + "fourche-fière", + "fourmi-lion", + "fourmis-lions", + "fourre-tout", + "foué-toutrac", + "foué-toutracs", + "fox-hound", + "fox-hounds", + "fox-terrier", + "fox-terriers", + "fox-trot", + "fox-trott", + "foy-notre-damien", + "frais-chier", + "frappe-abord", + "frappe-babord", + "frappe-d'abord", + "frappe-devant", + "frappe-main", + "frappe-mains", + "frappe-plaque", + "frappe-plaques", + "free-lance", + "frein-vapeur", + "freins-vapeur", + "freyming-merlebachois", + "freyming-merlebachoise", + "freyming-merlebachoises", + "fric-frac", + "fric-fracs", + "fronto-iniaque", + "frou-frou", + "frou-frous", + "frous-frous", + "fuel-oil", + "fuel-oils", + "full-contact", + "full-stack", + "fulmi-coton", + "fulmi-cotons", + "fume-cigare", + "fume-cigares", + "fume-cigarette", + "fume-cigarettes", + "fumée-gelée", + "fusil-mitrailleur", + "fusilier-commando", + "fusilier-marin", + "fusiliers-commandos", + "fusiliers-marins", + "fusils-mitrailleurs", + "fusion-acquisition", + "fusée-sonde", + "fut's", + "fute-fute", + "futes-futes", + "futuna-aniwa", + "fémoro-tibial", + "fénoxaprop-P-éthyl", + "fénoxaprop-éthyl", + "féodo-vassalique", + "féodo-vassaliques", + "g-strophanthine", + "gabrielino-fernandeño", + "gadz'arts", + "gagnant-gagnant", + "gagnant-gagnant-gagnant", + "gagnante-gagnante", + "gagnante-gagnante-gagnante", + "gagnantes-gagnantes", + "gagnantes-gagnantes-gagnantes", + "gagnants-gagnants", + "gagnants-gagnants-gagnants", + "gagne-pain", + "gagne-pains", + "gagne-petit", + "gaillet-gratteron", + "gaillets-gratterons", + "gaine-culotte", + "gaines-culottes", + "galaïco-portugais", + "galeries-refuges", + "galette-saucisse", + "galette-saucisses", + "galvano-cautère", + "galvano-magnétique", + "galvano-magnétiques", + "galvano-magnétisme", + "galvano-magnétismes", + "galégo-portugais", + "gamma-1,2,3,4,5,6-hexachlorocyclohexane", + "gamma-HCH", + "gamma-hexachlorobenzène", + "gamma-hexachlorocyclohexane", + "garcette-goitre", + "garden-parties", + "garden-party", + "garden-partys", + "gas-oil", + "gas-oils", + "gauche-fer", + "gay-friendly", + "gays-friendly", + "gaz-cab", + "gaz-poivre", + "gazelle-girafe", + "gel-douche", + "gel-douches", + "gentleman-rider", + "gentlemen-riders", + "germanate-analcime", + "germanate-analcimes", + "germano-américain", + "germano-américaine", + "germano-américaines", + "germano-américains", + "germano-anglais", + "germano-anglaises", + "germano-iranien", + "germano-italo-japonais", + "germo-roburien", + "germo-roburienne", + "germo-roburiennes", + "germo-roburiens", + "gestalt-thérapie", + "gestalt-thérapies", + "giga-ampère", + "giga-ampères", + "giga-ohm", + "giga-ohms", + "giga-électron-volt", + "giga-électron-volts", + "gigabit-ethernet", + "gigaélectron-volt", + "gigaélectron-volts", + "gill-box", + "glabello-iniaque", + "glass-cord", + "glauco-ferrugineuse", + "glauco-ferrugineuses", + "glauco-ferrugineux", + "glisser-déposer", + "globe-trotter", + "globe-trotters", + "globe-trotteur", + "globe-trotteurs", + "globe-trotteuse", + "globe-trotteuses", + "glosso-pharyngien", + "glosso-staphylin", + "glosso-staphylins", + "glosso-épiglottique", + "glosso-épiglottiques", + "gloubi-boulga", + "gluco-corticoïde", + "gluco-corticoïdes", + "glufosinate-ammonium", + "glycosyl-phosphatidylinositol", + "glycéraldéhyde-3-phosphate", + "go-slow", + "goal-average", + "goal-averages", + "goal-ball", + "gobe-dieu", + "gobe-goujons", + "gobe-mouche", + "gobe-moucherie", + "gobe-moucherons", + "gobe-mouches", + "gobe-mouton", + "gode-ceinture", + "gode-miché", + "gode-michés", + "godes-ceintures", + "goma-dare", + "gomme-cogne", + "gomme-cognes", + "gomme-gutte", + "gomme-résine", + "gommo-résineux", + "gorge-bleue", + "gorge-fouille", + "gourdan-polignanais", + "gourdan-polignanaise", + "gourdan-polignanaises", + "gouris-taitien", + "gouris-taitienne", + "gouris-taitiennes", + "gouris-taitiens", + "goutte-rose", + "gouzi-gouzi", + "gouzis-gouzis", + "goyave-ananas", + "goyaves-ananas", + "gracieux-berluron", + "grain-d'orge", + "grandgousier-pélican", + "grano-lamellaire", + "grap-fruit", + "grap-fruits", + "grapho-moteur", + "grappe-fruit", + "gras-double", + "gras-doubles", + "gras-fondu", + "grattes-ciels", + "grave-cimens", + "grave-ciment", + "grave-ciments", + "graves-ciment", + "gravi-kora", + "grenadier-voltigeur", + "grenadiers-voltigeurs", + "grenouille-taureau", + "grenouilles-taureaux", + "grez-neuvillois", + "grez-neuvilloise", + "grez-neuvilloises", + "gri-gri", + "gri-gris", + "griche-dents", + "grille-midi", + "grille-pain", + "grille-pains", + "grippe-argent", + "grippe-chair", + "grippe-fromage", + "grippe-fromages", + "grippe-minaud", + "grippe-minauds", + "grippe-sou", + "grippe-sous", + "gris-farinier", + "gris-fariniers", + "gris-gris", + "gris-pendart", + "gris-pendarts", + "grise-bonne", + "grises-bonnes", + "grosse-gorge", + "grosso-modo", + "grâcieux-hollognois", + "guarasu'we", + "guerre-éclair", + "guet-apens", + "guet-appens", + "guets-apens", + "guette-chemin", + "gueule-bée", + "guide-fil", + "guide-fils", + "guide-main", + "guide-âne", + "guide-ânes", + "guigne-cul", + "guigne-culs", + "guilherandais-grangeois", + "guilherandaise-grangeoise", + "guilherandaises-grangeoises", + "guili-guili", + "guili-guilis", + "guillemet-apostrophe", + "guillemets-apostrophes", + "guit-guit", + "guitare-harpe", + "guitare-violoncelle", + "guitare-violoncelles", + "gulf-stream", + "gulf-streams", + "gusathion-méthyl", + "gusathion-éthyl", + "gut-komm", + "gutta-percha", + "gutturo-maxillaire", + "gué-d'allérien", + "gué-d'allérienne", + "gué-d'allériennes", + "gué-d'allériens", + "gwich'in", + "gâche-métier", + "gâte-bois", + "gâte-ménage", + "gâte-ménages", + "gâte-métier", + "gâte-métiers", + "gâte-papier", + "gâte-papiers", + "gâte-pâte", + "gâte-sauce", + "gâte-sauces", + "gère-bélestinois", + "gère-bélestinoise", + "gère-bélestinoises", + "gélatino-bromure", + "gélatino-bromures", + "génie-conseil", + "génies-conseils", + "génio-hyoïdien", + "génio-hyoïdienne", + "génio-hyoïdiennes", + "génio-hyoïdiens", + "génito-crural", + "génito-urinaire", + "génito-urinaires", + "gétah-lahoë", + "ha-ha", + "ha-has", + "hache-bâché", + "hache-légume", + "hache-légumes", + "hache-paille", + "hache-pailles", + "hache-écorce", + "hache-écorces", + "hagio-onomastique", + "hagio-onomastiques", + "hakko-ryu", + "hale-avans", + "hale-avant", + "hale-avants", + "hale-bas", + "hale-breu", + "hale-croc", + "hale-dedans", + "hale-dehors", + "half-and-half", + "half-pipe", + "half-pipes", + "half-track", + "half-tracks", + "halo-halo", + "halo-lunaire", + "halos-lunaires", + "haloxyfop-R", + "haloxyfop-éthoxyéthyl", + "halte-garderie", + "halte-garderies", + "halte-là", + "haltes-garderies", + "halvadji-bachi", + "ham-nalinnois", + "hames-boucrois", + "hames-boucroise", + "hames-boucroises", + "hamme-millois", + "handi-accessible", + "handi-accessibles", + "happe-chair", + "happe-chat", + "happe-foie", + "hara-kiri", + "hara-kiris", + "hara-kiriser", + "harai-goshi", + "haraï-goshi", + "hardi-petit", + "harpe-guitare", + "harpe-luth", + "has-been", + "has-beens", + "hausse-col", + "hausse-cols", + "hausse-pied", + "hausse-pieds", + "hausse-queue", + "hecto-ohm", + "hecto-ohms", + "hentai-gana", + "herbe-aux-femmes-battues", + "herbes-aux-femmes-battues", + "herd-book", + "heure-homme", + "heure-lumière", + "heures-hommes", + "heures-lumière", + "heurte-pot", + "hexa-core", + "hexa-cores", + "hexa-rotor", + "hexa-rotors", + "hi-fi", + "hi-han", + "high-life", + "high-tech", + "himène-plume", + "hippocampe-feuillu", + "hippocampes-feuillus", + "hispano-américain", + "hispano-américaine", + "hispano-américaines", + "hispano-américains", + "hispano-arabe", + "hispano-arabes", + "hispano-mauresque", + "hispano-moresque", + "hispano-moresques", + "histoire-géo", + "historico-culturelle", + "hit-parade", + "hit-parades", + "hitléro-trotskisme", + "hitléro-trotskiste", + "hoat-chi", + "hoche-cul", + "hoche-culs", + "hoche-queue", + "hokkaïdo-ken", + "hold-up", + "home-jacking", + "home-jackings", + "home-sitter", + "home-sitters", + "home-sitting", + "home-sittings", + "home-trainer", + "home-trainers", + "homme-animal", + "homme-chacal", + "homme-clé", + "homme-femme", + "homme-fourmi", + "homme-grenouille", + "homme-loup", + "homme-léopard", + "homme-mort", + "homme-morts", + "homme-objet", + "homme-orchestre", + "homme-robot", + "homme-sandwich", + "homme-tronc", + "hommes-chacals", + "hommes-clés", + "hommes-femmes", + "hommes-fourmis", + "hommes-grenouilles", + "hommes-loups", + "hommes-léopards", + "hommes-objets", + "hommes-orchestres", + "hommes-robots", + "hommes-sandwiches", + "hommes-sandwichs", + "hommes-troncs", + "homo-épitaxie", + "homo-épitaxies", + "hon-hergeois", + "hon-hergeoise", + "hon-hergeoises", + "honey-dew", + "hong-kongais", + "hong-kongaise", + "hong-kongaises", + "horo-kilométrique", + "horo-kilométriques", + "hors-bord", + "hors-bords", + "hors-champ", + "hors-concours", + "hors-d'oeuvre", + "hors-d'œuvre", + "hors-fonds", + "hors-jeu", + "hors-jeux", + "hors-ligne", + "hors-lignes", + "hors-norme", + "hors-piste", + "hors-pistes", + "hors-sac", + "hors-service", + "hors-sol", + "hors-sols", + "hors-sujet", + "hors-série", + "hors-séries", + "hors-temps", + "hors-texte", + "hors-textes", + "horse-ball", + "horse-guard", + "horse-guards", + "hostello-flavien", + "hostello-flavienne", + "hostello-flaviennes", + "hostello-flaviens", + "hot-dog", + "hot-dogs", + "hot-melt", + "hot-melts", + "hot-plug", + "houl'eau", + "house-boats", + "houx-frelon", + "houx-frelons", + "huis-clos", + "huit-marsiste", + "huit-marsistes", + "huit-pieds", + "huit-reflets", + "huit-ressorts", + "huitante-neuf", + "huitante-neuvième", + "huitante-neuvièmes", + "hume-vent", + "huppe-col", + "huron-wendat", + "hydrargyro-cyanate", + "hydrargyro-cyanates", + "hydraulico-pneumatique", + "hydro-aviation", + "hydro-aviations", + "hydro-avion", + "hydro-avions", + "hydro-ensemencement", + "hydro-ensemencements", + "hydro-météorologie", + "hydro-électricité", + "hydro-électricités", + "hydro-électrique", + "hydro-électriques", + "hyo-pharyngien", + "hyo-épiglottique", + "hyo-épiglottiques", + "hypo-centre", + "hypo-centres", + "hypo-iodeuse", + "hypo-iodeuses", + "hypo-iodeux", + "hypothético-déductif", + "hystéro-catalepsie", + "hystéro-catalepsies", + "hystéro-épilepsie", + "hystéro-épilepsies", + "hyène-garou", + "hyènes-garous", + "hâ-hâ", + "hâ-hâs", + "hémi-dodécaèdre", + "hémi-octaèdre", + "hémi-épiphyte", + "hémi-épiphytes", + "hépato-biliaire", + "hépato-cystique", + "hépato-cystiques", + "hépato-gastrique", + "hépato-gastrite", + "hépato-gastrites", + "héroï-comique", + "héroï-comiques", + "hétéro-céphalophorie", + "hétéro-céphalophories", + "hétéro-réparation", + "hétéro-réparations", + "hétéro-épitaxie", + "hétéro-évaluation", + "hétéro-évaluations", + "hôtel-Dieu", + "hôtellerie-restauration", + "hôtels-Dieu", + "i-butane", + "i-butanes", + "i.-e.", + "iatro-magique", + "iatro-magiques", + "ibéro-roman", + "ice-belt", + "ice-belts", + "ice-berg", + "ice-bergs", + "ice-blink", + "ice-blinks", + "ice-bloc", + "ice-blocs", + "ice-cream", + "ice-creams", + "ice-foot", + "ice-foots", + "ice-rapt", + "ice-rapts", + "ice-table", + "ice-tables", + "ici-bas", + "idio-électricité", + "idio-électrique", + "idio-électriques", + "idéal-type", + "idée-force", + "idée-maîtresse", + "idées-forces", + "idées-maîtresses", + "ifira-mele", + "ifira-meles", + "igny-marin", + "igny-marine", + "igny-marines", + "igny-marins", + "iliaco-fémoral", + "iliaco-musculaire", + "ilio-pectiné", + "ilio-pubien", + "ilio-scrotal", + "ilo-dionysien", + "ilo-dionysienne", + "ilo-dionysiennes", + "ilo-dionysiens", + "iléo-colique", + "iléo-coliques", + "iléo-cæcal", + "iléo-cæcale", + "iléo-cæcales", + "iléo-cæcaux", + "iléos-meldois", + "iléos-meldoise", + "iléos-meldoises", + "image-gradient", + "imazaméthabenz-méthyl", + "immuno-pharmacologie", + "immuno-pharmacologies", + "impari-nervié", + "impari-nervé", + "impari-penné", + "import-export", + "impératrice-mère", + "impératrices-mères", + "in-12", + "in-12º", + "in-16", + "in-16º", + "in-18", + "in-18º", + "in-32", + "in-4", + "in-4.º", + "in-4to", + "in-4º", + "in-6", + "in-6º", + "in-8", + "in-8.º", + "in-8vo", + "in-8º", + "in-cent-vingt-huit", + "in-dix-huit", + "in-douze", + "in-duodecimo", + "in-folio", + "in-fº", + "in-huit", + "in-manus", + "in-octavo", + "in-plano", + "in-plº", + "in-promptu", + "in-quarto", + "in-sedecimo", + "in-seize", + "in-six", + "in-trente-deux", + "in-vingt-quatre", + "in-vitro", + "inch'Allah", + "inch'allah", + "incito-moteur", + "incito-motricité", + "income-tax", + "indane-1,3-dione", + "inde-plate", + "india-océanisme", + "india-océanismes", + "info-ballon", + "info-ballons", + "info-bulle", + "info-bulles", + "ingénieur-conseil", + "ingénieur-docteur", + "ingénieur-maître", + "ingénieure-conseil", + "ingénieures-conseils", + "ingénieurs-conseils", + "ingénieurs-docteurs", + "ingénieurs-maîtres", + "injonction-bâillon", + "insecto-mortifère", + "insecto-mortifères", + "inspecteur-chef", + "inspecteurs-chefs", + "insulino-dépendant", + "insulino-dépendante", + "insulino-dépendantes", + "insulino-dépendants", + "interno-médial", + "interro-négatif", + "intervertébro-costal", + "inuit-aléoute", + "inuit-aléoutes", + "iodo-borique", + "iodo-chlorure", + "iodosulfuron-méthyl-sodium", + "iowa-oto", + "iowa-otos", + "ischio-anal", + "ischio-clitorien", + "ischio-fémoral", + "ischio-fémorale", + "ischio-fémorales", + "ischio-fémoraux", + "ischio-jambier", + "ischio-jambiers", + "ischio-jambière", + "ischio-jambières", + "ischio-périnéal", + "ischio-tibial", + "ischio-tibiaux", + "isoxadifen-éthyl", + "israélo-syrienne", + "istro-roumain", + "ivre-mort", + "ivre-morte", + "ivres-mortes", + "ivres-morts", + "j't'aime", + "jack-russell", + "jaguar-garou", + "jaguars-garous", + "jam-sessions", + "jambon-beurre", + "jaunay-clanais", + "jaunay-clanaise", + "jaunay-clanaises", + "jaï-alaï", + "jaï-alaïs", + "je-m'en-fichisme", + "je-m'en-fichismes", + "je-m'en-fichiste", + "je-m'en-fichistes", + "je-m'en-foutisme", + "je-m'en-foutismes", + "je-m'en-foutiste", + "je-m'en-foutistes", + "je-ne-sais-quoi", + "jet-stream", + "jet-streams", + "jette-bouts", + "jeu-malochois", + "jeu-malochoise", + "jeu-malochoises", + "jeu-parti", + "jiu-jitsu", + "joint-venture", + "joint-ventures", + "joli-bois", + "jour-homme", + "jour-lumière", + "jours-hommes", + "jours-lumière", + "ju-jitsu", + "ju-ju", + "judéo-allemand", + "judéo-alsacien", + "judéo-arabe", + "judéo-arabes", + "judéo-asiatique", + "judéo-bolchévisme", + "judéo-centrisme", + "judéo-christianisme", + "judéo-christiano-islamique", + "judéo-christiano-islamiques", + "judéo-christiano-musulman", + "judéo-chrétien", + "judéo-chrétienne", + "judéo-chrétiennes", + "judéo-chrétiens", + "judéo-espagnol", + "judéo-espagnole", + "judéo-espagnoles", + "judéo-espagnols", + "judéo-iranien", + "judéo-libyen", + "judéo-lybien", + "judéo-maçonnique", + "judéo-maçonniques", + "judéo-musulman", + "judéo-musulmans", + "judéo-nazi", + "judéo-nazis", + "juke-box", + "juke-boxes", + "jully-sarçois", + "jully-sarçoise", + "jully-sarçoises", + "junk-food", + "junk-foods", + "jupe-culotte", + "jupes-culottes", + "juridico-politique", + "juridico-politiques", + "jusque-là", + "juxta-position", + "juxta-positions", + "juǀ'hoan", + "jérôme-boschisme", + "jérôme-boschismes", + "k-voisinage", + "k-voisinages", + "k-way", + "k-ways", + "kali'na", + "kan-kan", + "kan-kans", + "kansai-ben", + "kara-gueuz", + "kara-kalpak", + "karachay-balkar", + "karafuto-ken", + "karatchaï-balkar", + "kem's", + "khambo-lama", + "khambo-lamas", + "khatti-chérif", + "khatti-chérifs", + "khi-carré", + "khi-carrés", + "khi-deux", + "kif-kif", + "kilo-ohm", + "kilo-ohms", + "kilo-électron-volt", + "kilo-électron-volts", + "kilo-électrons-volts", + "kilogramme-force", + "kilogramme-poids", + "kilogrammes-force", + "kilogrammes-poids", + "kilomètre-heure", + "kilomètres-heure", + "kiloélectron-volt", + "kiloélectron-volts", + "kiloélectrons-volts", + "kin-ball", + "kino-congolais", + "kip-kap", + "kip-kaps", + "kirsch-wasser", + "kirsch-wassers", + "kiss-in", + "kite-surf", + "kite-surfa", + "kite-surfai", + "kite-surfaient", + "kite-surfais", + "kite-surfait", + "kite-surfant", + "kite-surfas", + "kite-surfasse", + "kite-surfassent", + "kite-surfasses", + "kite-surfassiez", + "kite-surfassions", + "kite-surfe", + "kite-surfent", + "kite-surfer", + "kite-surfera", + "kite-surferai", + "kite-surferaient", + "kite-surferais", + "kite-surferait", + "kite-surferas", + "kite-surferez", + "kite-surferiez", + "kite-surferions", + "kite-surferons", + "kite-surferont", + "kite-surfers", + "kite-surfes", + "kite-surfez", + "kite-surfiez", + "kite-surfions", + "kite-surfons", + "kite-surfâmes", + "kite-surfât", + "kite-surfâtes", + "kite-surfèrent", + "kite-surfé", + "knicker-bocker", + "knicker-bockers", + "ko-soto-gake", + "kouan-hoa", + "kouign-aman", + "kouign-amann", + "kouign-amanns", + "kouign-amans", + "krav-naga", + "krésoxim-méthyl", + "kung-fu", + "kwan-li-so", + "kérato-pharyngien", + "kérato-staphylin", + "kérato-staphylins", + "l-amphétamine", + "la-fertois", + "la-fertoise", + "la-fertoises", + "lab-ferment", + "lab-ferments", + "lac-laque", + "lac-laques", + "lache-bras", + "lacrima-Christi", + "lacrima-christi", + "lacryma-Christi", + "lacryma-christi", + "lacto-végétarisme", + "lacto-végétarismes", + "laemmer-geier", + "laemmer-geiers", + "laisser-aller", + "laisser-allers", + "laisser-courre", + "laisser-faire", + "laissez-faire", + "laissez-passer", + "laissé-pour-compte", + "laissée-pour-compte", + "laissées-pour-compte", + "laissés-pour-compte", + "lambda-cyhalothrine", + "lampe-tempête", + "lampes-tempête", + "lampris-lune", + "lance-amarres", + "lance-balles", + "lance-bombe", + "lance-bombes", + "lance-flamme", + "lance-flammes", + "lance-fusée", + "lance-fusées", + "lance-grenade", + "lance-grenades", + "lance-missile", + "lance-missiles", + "lance-patates", + "lance-pierre", + "lance-pierres", + "lance-roquette", + "lance-roquettes", + "lance-torpille", + "lance-torpilles", + "land-ice", + "land-ices", + "langue-toit", + "langues-toit", + "lanne-soubiranais", + "lanne-soubiranaise", + "lanne-soubiranaises", + "lapin-garou", + "lapins-garous", + "lapis-lazuli", + "lapu-lapu", + "lau-balutin", + "lau-balutine", + "lau-balutines", + "lau-balutins", + "launay-villersois", + "launay-villersoise", + "launay-villersoises", + "laurier-cerise", + "laurier-rose", + "laurier-sauce", + "laurier-tarte", + "laurier-thym", + "laurier-tin", + "lauriers-cerises", + "lauriers-roses", + "lauriers-tins", + "lavans-quingeois", + "lavans-quingeoise", + "lavans-quingeoises", + "lave-auto", + "lave-autos", + "lave-glace", + "lave-linge", + "lave-linges", + "lave-main", + "lave-mains", + "lave-pont", + "lave-ponts", + "lave-tête", + "lave-têtes", + "lave-vaisselle", + "lave-vaisselles", + "laveuse-sécheuse", + "lazur-apatite", + "lazur-apatites", + "lease-back", + "leather-jacket", + "lecteur-graveur", + "lecteurs-graveurs", + "lemmer-geyer", + "lemmer-geyers", + "lepto-kurticité", + "lepto-kurticités", + "lepto-kurtique", + "lepto-kurtiques", + "lever-dieu", + "lgbti-friendly", + "lgbti-phobie", + "lgbti-phobies", + "liane-corail", + "lianes-corail", + "liberum-veto", + "libidino-calotin", + "libre-choix", + "libre-penseur", + "libre-penseuse", + "libre-service", + "libre-échange", + "libre-échangisme", + "libre-échangismes", + "libre-échangiste", + "libre-échangistes", + "libres-choix", + "libres-penseurs", + "libres-penseuses", + "libres-services", + "libyco-berbère", + "libyco-berbères", + "libéral-conservateur", + "libéral-conservatisme", + "lice-po", + "liche-casse", + "licol-drisse", + "licols-drisses", + "lieu-dit", + "lieutenant-colonel", + "lieutenant-gouverneur", + "lieutenant-général", + "lieutenants-colonels", + "lieux-dits", + "limande-sole", + "limande-soles", + "limandes-soles", + "lime-bois", + "lime-uranite", + "lime-uranites", + "linon-batiste", + "linon-batistes", + "lion-garou", + "lions-garous", + "lire-écrire", + "lit-cage", + "lit-clos", + "lits-cages", + "lits-clos", + "little-endian", + "living-room", + "living-rooms", + "livres-cassettes", + "livret-police", + "localité-type", + "location-financement", + "locoalo-mendonnais", + "locoalo-mendonnaise", + "locoalo-mendonnaises", + "locution-phrase", + "locutions-phrases", + "loemmer-geyer", + "loemmer-geyers", + "logan-berry", + "logan-berrys", + "logiciel-socle", + "logo-syllabique", + "logo-syllabiques", + "loi-cadre", + "loi-programme", + "loi-écran", + "lois-cadre", + "lois-programme", + "lois-écrans", + "lombo-costal", + "lombo-costo-trachélien", + "lombo-dorso-trachélien", + "lombo-huméral", + "lombo-sacré", + "lompénie-serpent", + "long-courrier", + "long-courriers", + "long-grain", + "long-jointé", + "long-jointée", + "long-métrage", + "long-temps", + "long-tems", + "longs-courriers", + "longs-métrages", + "longue-langue", + "longue-vue", + "longue-épine", + "longues-langues", + "longues-vues", + "longues-épines", + "loqu'du", + "loqu'due", + "loqu'dues", + "loqu'dus", + "lord-lieutenance", + "lord-lieutenances", + "lord-lieutenant", + "lord-lieutenants", + "lord-maire", + "louise-bonne", + "louises-bonnes", + "loup-cerve", + "loup-cervier", + "loup-garou", + "loups-cerves", + "loups-cerviers", + "loups-garous", + "lourd-léger", + "lourds-légers", + "lourouzien-bourbonnais", + "lourouzienne-bourbonnaise", + "lourouziennes-bourbonnaises", + "lourouziens-bourbonnais", + "louve-garelle", + "louve-garolle", + "louve-garou", + "louves-garelles", + "louves-garolles", + "louves-garous", + "louveteau-garou", + "louveteaux-garous", + "louvie-soubironnais", + "louvie-soubironnaise", + "louvie-soubironnaises", + "love-in", + "low-cost", + "low-costs", + "low-tech", + "ludo-sportif", + "ludo-sportifs", + "ludo-sportive", + "ludo-sportives", + "ludo-éducatif", + "lui-même", + "lumen-seconde", + "lumens-secondes", + "luni-solaire", + "luni-solaires", + "lyro-guitare", + "là-bas", + "là-contre", + "là-dedans", + "là-delez", + "là-dessous", + "là-dessus", + "là-haut", + "là-pour-ça", + "lâcher-tout", + "læmmer-geyer", + "læmmer-geyers", + "lèche-cul", + "lèche-culs", + "lèche-vitrine", + "lèche-vitrines", + "lèse-majesté", + "lèse-majestés", + "lève-cul", + "lève-culs", + "lève-gazon", + "lève-glace", + "lève-glaces", + "lève-tard", + "lève-tôt", + "lève-vitre", + "lève-vitres", + "légume-feuille", + "légume-fleur", + "légume-fruit", + "légume-racine", + "légume-tige", + "légumes-feuilles", + "légumes-fleurs", + "légumes-fruits", + "légumes-racines", + "légumes-tiges", + "léopard-garou", + "léopards-garous", + "lépisostée-alligator", + "lévi-straussien", + "lévi-straussienne", + "lévi-straussiennes", + "lévi-straussiens", + "lœmmer-geyer", + "lœmmer-geyers", + "m'amie", + "m'as", + "m'as-tu-vu", + "m'as-tu-vue", + "m'as-tu-vues", + "m'as-tu-vus", + "m'bororo", + "m'demma", + "m'enfin", + "m'halla", + "m'hallas", + "m'kahla", + "m'kahlas", + "m'sieur", + "m-commerce", + "m-paiement", + "m-paiements", + "ma'di", + "ma-jong", + "ma-jongs", + "mac-ferlane", + "mac-ferlanes", + "mac-kintosh", + "mac-kintoshs", + "machin-chose", + "machin-choses", + "machin-chouette", + "machine-outil", + "machines-outils", + "machins-chouettes", + "machon-gorgeon", + "magasin-pilote", + "magasins-pilotes", + "magnésio-anthophyllite", + "magnésio-anthophyllites", + "magnésio-axinite", + "magnésio-axinites", + "magnésio-calcite", + "magnésio-calcites", + "magnéto-optique", + "magnéto-optiques", + "magnéto-électrique", + "magnéto-électriques", + "mah-jong", + "mah-jongs", + "mahi-mahi", + "mail-coach", + "mailly-castellois", + "mailly-castelloise", + "mailly-castelloises", + "main-brune", + "main-courante", + "main-d'oeuvre", + "main-d'œuvre", + "main-forte", + "main-militaire", + "maine-anjou", + "mains-courantes", + "mains-d'oeuvre", + "mains-d'œuvre", + "maire-adjoint", + "maires-adjoints", + "maison-mère", + "maisons-mères", + "maitre-autel", + "maitre-chanteur", + "maitre-chien", + "maitre-nageur", + "maitre-nageuse", + "maitres-chiens", + "maitres-nageurs", + "maitres-nageuses", + "maitresse-nageuse", + "maitresses-nageuses", + "make-up", + "make-ups", + "making-of", + "makura-e", + "makura-es", + "malayo-polynésien", + "malayo-polynésienne", + "malayo-polynésiennes", + "malayo-polynésiens", + "malgré-nous", + "malle-poste", + "mals-peignées", + "mals-peignés", + "malécite-passamaquoddy", + "mam'selle", + "mam'selles", + "mam'zelle", + "mam'zelles", + "mamie-boomeuse", + "mamie-boomeuses", + "mamy-boomeuse", + "mamy-boomeuses", + "man-bun", + "man-buns", + "manco-liste", + "manco-listes", + "mandant-dépendant", + "mandat-carte", + "mandat-cash", + "mandat-lettre", + "mandat-poste", + "mandats-cartes", + "mandats-cash", + "mandats-lettres", + "mandats-poste", + "manganico-potassique", + "mangano-ankérite", + "mangano-ankérites", + "mangano-phlogopite", + "mangano-phlogopites", + "manganoso-ammonique", + "mange-Canayen", + "mange-debout", + "mange-disque", + "mange-disques", + "mange-merde", + "mange-piles", + "mange-tout", + "maniaco-dépressif", + "maniaco-dépressifs", + "maniaco-dépressive", + "maniaco-dépressives", + "mappe-monde", + "mappes-mondes", + "marche-palier", + "marché-gare", + "marché-gares", + "marco-lucanien", + "marco-lucanienne", + "marco-lucaniennes", + "marco-lucaniens", + "margarino-sulfurique", + "margis-chef", + "margis-chefs", + "mariage-sacrement", + "maries-salopes", + "marin-pêcheur", + "marins-pêcheurs", + "marka-dafing", + "marno-bitumineux", + "marno-calcaire", + "marno-calcaires", + "marque-ombrelle", + "marque-produit", + "marque-produits", + "marques-ombrelles", + "marte-piquant", + "marte-piquants", + "marteau-pilon", + "marteau-piqueur", + "marteaux-pilons", + "marteaux-piqueurs", + "martin-bâton", + "martin-bâtons", + "martin-chasseur", + "martin-pêcheur", + "martin-sec", + "martin-sire", + "martin-sucré", + "martins-chasseurs", + "martins-pêcheurs", + "martins-sires", + "martins-sucrés", + "martre-zibeline", + "martres-zibelines", + "marxisme-léninisme", + "marxiste-léniniste", + "marxistes-léninistes", + "maréchal-ferrant", + "maréchaux-ferrans", + "maréchaux-ferrants", + "mas-chélyen", + "mas-chélyenne", + "mas-chélyennes", + "mas-chélyens", + "mas-tençois", + "mas-tençoise", + "mas-tençoises", + "masa'il", + "masa'ils", + "mass-média", + "mass-médias", + "masseur-kinésithérapeute", + "masseurs-kinésithérapeutes", + "masseuse-kinésithérapeute", + "masseuses-kinésithérapeutes", + "materno-infantile", + "materno-infantiles", + "mathématico-informatique", + "mathématico-informatiques", + "matthéo-lucanien", + "matthéo-lucanienne", + "matthéo-lucaniennes", + "matthéo-lucaniens", + "mauritano-marocain", + "mauritano-sénégalais", + "maxillo-dentaire", + "maxillo-facial", + "maxillo-labial", + "maxillo-musculaire", + "maël-carhaisien", + "maël-carhaisienne", + "maël-carhaisiennes", + "maël-carhaisiens", + "maître-assistant", + "maître-autel", + "maître-bau", + "maître-chanteur", + "maître-chanteuse", + "maître-chien", + "maître-cylindre", + "maître-jacques", + "maître-mot", + "maître-nageur", + "maître-nageuse", + "maîtres-assistants", + "maîtres-autels", + "maîtres-chanteurs", + "maîtres-chanteuses", + "maîtres-chiens", + "maîtres-cylindres", + "maîtres-jacques", + "maîtres-mots", + "maîtres-nageurs", + "maîtres-nageuses", + "maîtresse-femme", + "maîtresse-nageuse", + "maîtresses-femmes", + "maîtresses-nageuses", + "mea-culpa", + "mele-fila", + "membrano-calcaire", + "menthe-coq", + "menuisier-moulurier", + "mercuroso-mercurique", + "merisier-pays", + "merisiers-pays", + "mets-en", + "metz-tesseran", + "metz-tesseranne", + "metz-tesserannes", + "metz-tesserans", + "meuse-rhin-yssel", + "mezzo-soprano", + "mezzo-sopranos", + "mezzo-termine", + "mezzo-tinto", + "meâ-culpâ", + "miam-miam", + "miaou-miaou", + "michel-angélesque", + "michel-angélesques", + "microélectron-volt", + "microélectron-volts", + "midi-chlorien", + "midi-chloriens", + "midi-pelle", + "midi-pelles", + "midi-pyrénéen", + "mieux-disant", + "mieux-disante", + "mieux-disantes", + "mieux-disants", + "mieux-être", + "militaro-bureaucratique", + "militaro-bureaucratiques", + "militaro-industriel", + "militaro-industrielle", + "militaro-industrielles", + "militaro-industriels", + "milk-bar", + "milk-bars", + "milk-shake", + "milk-shakes", + "mille-canton", + "mille-feuille", + "mille-feuilles", + "mille-fleurs", + "mille-pattes", + "mille-pertuis", + "mille-pieds", + "mille-points", + "milli-ohm", + "milli-ohms", + "milli-électron-volt", + "milli-électron-volts", + "milliampère-heure", + "milliampères-heures", + "milliélectron-volt", + "milliélectron-volts", + "mime-acrobate", + "ministre-présidence", + "ministre-présidences", + "ministre-président", + "ministres-présidents", + "minn'gotain", + "minn'gotaine", + "minn'gotaines", + "minn'gotains", + "minus-habens", + "minute-lumière", + "minutes-lumière", + "mire-oeuf", + "mire-oeufs", + "mire-œuf", + "mire-œufs", + "miro-miro", + "mixed-border", + "mixti-unibinaire", + "mobil-home", + "mobil-homes", + "modern-style", + "modèle-vue-contrôleur", + "mofu-gudur", + "moi-même", + "moins-disant", + "moins-disants", + "moins-que-rien", + "moins-value", + "moins-values", + "mois-homme", + "mois-hommes", + "mois-lumière", + "moissonner-battre", + "moissonneuse-batteuse", + "moissonneuse-lieuse", + "moissonneuses-batteuses", + "moissonneuses-lieuses", + "moite-moite", + "moitié-moitié", + "mojeño-ignaciano", + "mojeño-javierano", + "mojeño-loretano", + "mojeño-trinitario", + "mollo-mollo", + "moment-clé", + "moment-clés", + "moments-clés", + "monsieur-dame", + "monte-charge", + "monte-charges", + "monte-courroie", + "monte-courroies", + "monte-escalier", + "monte-escaliers", + "monte-jus", + "monte-lait", + "monte-meuble", + "monte-meubles", + "monte-pente", + "monte-pentes", + "monte-plat", + "monte-plats", + "monti-corcellois", + "monti-corcelloise", + "monti-corcelloises", + "montis-fagussin", + "montis-fagussine", + "montis-fagussines", + "montis-fagussins", + "montre-bracelet", + "montre-chronomètre", + "montres-bracelets", + "montres-chronomètres", + "montréalo-centrisme", + "moque-dieu", + "mords-cheval", + "morphine-base", + "mort-bois", + "mort-chien", + "mort-dieu", + "mort-né", + "mort-née", + "mort-nées", + "mort-nés", + "mort-plain", + "mort-plains", + "mort-terrain", + "mort-vivant", + "morte-eau", + "morte-paye", + "morte-payes", + "morte-saison", + "morte-vivante", + "mortes-eaux", + "mortes-payes", + "mortes-saisons", + "mortes-vivantes", + "morts-bois", + "morts-chiens", + "morts-flats", + "morts-terrains", + "morts-vivants", + "moteur-fusée", + "moteurs-fusées", + "moto-cross", + "moto-crotte", + "moto-crottes", + "moto-réducteur", + "moto-réducteurs", + "moto-école", + "moto-écoles", + "mouche-araignée", + "mouche-sans-raison", + "mouche-scorpion", + "mouches-sans-raison", + "mouches-scorpions", + "mouille-bouche", + "moule-bite", + "moule-burnes", + "moule-fesses", + "moules-burnes", + "moulin-mageois", + "moulin-mageoise", + "moulin-mageoises", + "moustique-tigre", + "moustiques-tigres", + "mouve-chaux", + "moyens-ducs", + "mu'ugalavyáni", + "mu-métal", + "muco-pus", + "mud-minnow", + "mule-jenny", + "mull-jenny", + "multiplate-forme", + "multiplates-formes", + "mur-rideau", + "murnau-werdenfels", + "murs-rideaux", + "musculo-cutané", + "musettes-repas", + "music-hall", + "music-hallesque", + "music-hallesques", + "music-halls", + "mâche-bouchons", + "mâche-dru", + "mâche-laurier", + "mâle-stérile", + "mâle-stériles", + "mâles-stériles", + "mère-grand", + "mères-grand", + "mètre-ruban", + "mètres-ruban", + "mécoprop-P", + "médecine-ball", + "médecine-balls", + "médiéval-fantastique", + "médiévale-fantastique", + "médiévales-fantastiques", + "médiévaux-fantastiques", + "méduse-boite", + "méduse-boîte", + "méduses-boites", + "méduses-boîtes", + "méfenpyr-diéthyl", + "méga-ampère", + "méga-ampères", + "méga-herbivore", + "méga-herbivores", + "méga-océan", + "méga-océans", + "méga-ohm", + "méga-ohms", + "méga-église", + "méga-églises", + "méga-électron-volt", + "méga-électron-volts", + "mégalo-martyr", + "mégalo-martyrs", + "mégaélectron-volt", + "mégaélectron-volts", + "mégléno-roumain", + "méli-mélo", + "mélis-mélos", + "ménil-annellois", + "ménil-annelloise", + "ménil-annelloises", + "ménil-gondoyen", + "ménil-gondoyenne", + "ménil-gondoyennes", + "ménil-gondoyens", + "méningo-encéphalite", + "méningo-gastrique", + "méningo-gastriques", + "mépiquat-chlorure", + "mérier-blanc", + "mériers-blancs", + "méso-américain", + "méso-américaine", + "méso-américaines", + "méso-américains", + "méso-diastolique", + "méso-diastoliques", + "méso-hygrophile", + "méso-hygrophiles", + "méso-systolique", + "méso-systoliques", + "mésosulfuron-méthyl-sodium", + "métacarpo-phalangien", + "métalaxyl-M", + "métam-sodium", + "métaphysico-théologo-cosmolo-nigologie", + "métaphysico-théologo-cosmolo-nigologies", + "métatarso-phalangien", + "méthyl-buténol", + "métirame-zinc", + "métro-boulot-dodo", + "météo-dépendant", + "météo-dépendante", + "météo-dépendantes", + "météo-dépendants", + "mêle-tout", + "mêli-mêlo", + "mêlis-mêlos", + "mêlé-cass", + "mêlé-casse", + "mêlé-casses", + "mêlé-cassis", + "n'dama", + "n'damas", + "n'srani", + "n-3", + "n-6", + "n-9", + "n-aire", + "n-aires", + "n-boule", + "n-boules", + "n-butane", + "n-butanes", + "n-butyle", + "n-cube", + "n-cubes", + "n-dimensionnel", + "n-gone", + "n-gones", + "n-gramme", + "n-grammes", + "n-ième", + "n-ièmes", + "n-octaèdre", + "n-octaèdres", + "n-polytope", + "n-polytopes", + "n-simplexe", + "n-simplexes", + "n-sphère", + "n-sphères", + "n-uple", + "n-uples", + "n-uplet", + "n-uplets", + "na-dené", + "na-déné", + "nam-nam", + "nam-nams", + "name-dropping", + "nano-ohm", + "nano-ohms", + "naphtoxy-2-acétamide", + "narco-guérilla", + "narco-guérillas", + "narco-trafiquant", + "narco-trafiquants", + "narco-État", + "narco-États", + "narcotico-âcre", + "naso-génien", + "naso-lobaire", + "naso-lobaires", + "naso-oculaire", + "naso-palatin", + "naso-palpébral", + "naso-sourcilier", + "naso-transversal", + "nat-gadaw", + "nat-gadaws", + "nat-kadaw", + "nat-kadaws", + "national-socialisme", + "national-socialiste", + "nationale-socialiste", + "nationales-socialistes", + "nationaux-socialistes", + "natro-feldspat", + "natro-feldspats", + "natu-majorité", + "nautico-estival", + "navarro-aragonais", + "navarro-labourdin", + "navire-citerne", + "navire-mère", + "navire-usine", + "navire-école", + "navires-citernes", + "navires-mères", + "navires-écoles", + "ne-m'oubliez-pas", + "negro-spiritual", + "negro-spirituals", + "neptuno-plutonien", + "neptuno-plutonienne", + "neptuno-plutoniens", + "nerf-ferrure", + "nerf-férure", + "net-citoyen", + "net-citoyens", + "nettoie-pipe", + "neuf-berquinois", + "neuf-berquinoise", + "neuf-berquinoises", + "neuf-cents", + "neutro-alcalin", + "neuve-chapellois", + "neuve-chapelloise", + "neuve-chapelloises", + "neuve-grangeais", + "neuve-grangeaise", + "neuve-grangeaises", + "neuvy-sautourien", + "neuvy-sautourienne", + "neuvy-sautouriennes", + "neuvy-sautouriens", + "newton-mètre", + "newtons-mètres", + "nez-percé", + "ngaï-ngaï", + "ngaï-ngaïs", + "ni-ni", + "nian-nian", + "niche-crédence", + "nickel-ankérite", + "nickel-ankérites", + "nickel-magnésite", + "nickel-magnésites", + "nickel-skuttérudite", + "nickel-skuttérudites", + "night-club", + "night-clubbing", + "night-clubs", + "nigéro-congolais", + "nilo-saharien", + "nilo-saharienne", + "nilo-sahariennes", + "nilo-sahariens", + "nin-nin", + "nippo-américain", + "nippo-américaine", + "nippo-américaines", + "nippo-américains", + "nique-douille", + "nique-douilles", + "nitro-cellulose", + "nitro-celluloses", + "nitro-hydrochlorique", + "nitro-hydrochloriques", + "nitrotal-isopropyl", + "niuafo'ou", + "niuafo'ous", + "nivo-glaciaire", + "nivo-glaciaires", + "nivo-pluvial", + "no-kill", + "no-kills", + "no-poo", + "noie-chien", + "noir-pie", + "noir-pioche", + "noir-pioches", + "noir-ployant", + "noisy-rudignonais", + "noisy-rudignonaise", + "noisy-rudignonaises", + "noli-me-tangere", + "nonante-cinq", + "nonante-deux", + "nonante-huit", + "nonante-neuf", + "nonante-quatre", + "nonante-sept", + "nonante-six", + "nonante-trois", + "nort-leulinghemois", + "nort-leulinghemoise", + "nort-leulinghemoises", + "nous-même", + "nous-mêmes", + "nouveau-gallois", + "nouveau-né", + "nouveau-née", + "nouveau-nées", + "nouveau-nés", + "nouveau-venu", + "nouveaux-nés", + "nouveaux-venus", + "nouvel-âgeuse", + "nouvel-âgeuses", + "nouvel-âgeux", + "nouvelle-née", + "nouvelle-venue", + "nouvelles-nées", + "nouvelles-venues", + "noyé-d'eau", + "nu-pied", + "nu-pieds", + "nu-propriétaire", + "nu-tête", + "nue-propriétaire", + "nue-propriété", + "nuer-dinka", + "nues-propriétaires", + "nues-propriétés", + "nuit-deboutiste", + "nuit-deboutistes", + "nuoc-mam", + "nuoc-mâm", + "nus-propriétaires", + "nègre-soie", + "nègres-soies", + "nègue-chien", + "nègue-fol", + "néfaste-food", + "néfaste-foods", + "néphro-angiosclérose", + "néphro-angioscléroses", + "néphro-gastrique", + "néphro-urétérectomie", + "néphro-urétérectomies", + "névro-mimosie", + "névro-mimosies", + "nœud-nœud", + "nœuds-nœuds", + "o-ring", + "o-rings", + "occipito-atloïdien", + "occipito-atloïdienne", + "occipito-atloïdiennes", + "occipito-atloïdiens", + "occipito-axoïdien", + "occipito-axoïdienne", + "occipito-axoïdiennes", + "occipito-axoïdiens", + "occipito-cotyloïdien", + "occipito-cotyloïdienne", + "occipito-cotyloïdiennes", + "occipito-cotyloïdiens", + "occipito-frontal", + "occipito-méningien", + "occipito-pariétal", + "occipito-pétreuse", + "occipito-pétreuses", + "occipito-pétreux", + "occipito-sacro-iliaque", + "occipito-sacré", + "occitano-roman", + "octante-deux", + "octante-neuf", + "octo-core", + "octo-cores", + "octo-rotor", + "octo-rotors", + "oculo-motricité", + "oculo-motricités", + "oculo-musculaire", + "oculo-musculaires", + "oculo-zygomatique", + "odonto-stomatologie", + "oesophago-gastro-duodénoscopie", + "oesophago-gastro-duodénoscopies", + "off-market", + "off-shore", + "ogivo-cylindrique", + "ohm-mètre", + "ohms-mètres", + "oie-cygne", + "oiseau-chameau", + "oiseau-cloche", + "oiseau-lyre", + "oiseau-mouche", + "oiseau-papillon", + "oiseau-tonnerre", + "oiseau-trompette", + "oiseau-éléphant", + "oiseaux-chameaux", + "oiseaux-cloches", + "oiseaux-lyres", + "oiseaux-mouches", + "oiseaux-papillons", + "oiseaux-tonnerres", + "oiseaux-trompettes", + "old-ice", + "old-ices", + "oligo-élément", + "oligo-éléments", + "olla-podrida", + "olé-olé", + "oléo-calcaire", + "oléo-calcaires", + "omaha-ponca", + "omaha-poncas", + "omble-chevalier", + "ombre-chevalier", + "ombro-thermique", + "ombro-thermiques", + "omphalo-mésentérique", + "omphalo-mésentériques", + "omphalo-phlébite", + "omphalo-phlébites", + "oméga-3", + "oméga-6", + "oméga-9", + "on-dit", + "one-man-show", + "one-shot", + "one-step", + "one-steps", + "one-woman-show", + "oost-cappelois", + "oost-cappeloise", + "oost-cappeloises", + "opal-AN", + "open-source", + "open-space", + "open-spaces", + "opt-in", + "opt-out", + "opto-strié", + "opéra-comique", + "opéras-comiques", + "or-sol", + "orang-outan", + "orang-outang", + "orangs-outangs", + "orangs-outans", + "orbito-nasal", + "orbito-palpébral", + "oreille-d'abbé", + "oreille-d'ours", + "oreille-d'âne", + "oreilles-d'ours", + "oreilles-d'âne", + "organo-calcaire", + "organo-calcaires", + "organo-chloré", + "organo-chlorée", + "organo-chlorées", + "organo-chlorés", + "organo-halogéné", + "organo-halogénée", + "organo-halogénées", + "organo-halogénés", + "organo-phosphoré", + "organo-phosphorée", + "organo-phosphorées", + "organo-phosphorés", + "orienteur-marqueur", + "orienté-objet", + "orp-jauchois", + "ortho-sympathique", + "ortho-sympathiques", + "ossau-iraty", + "ossau-iratys", + "ostéo-arthrite", + "ostéo-arthrites", + "oto-rhino", + "oto-rhino-laryngologie", + "oto-rhino-laryngologies", + "oto-rhino-laryngologiste", + "oto-rhino-laryngologistes", + "oto-rhinos", + "ouaf-ouaf", + "oui-da", + "oui-non-bof", + "ouralo-altaïque", + "ouralo-altaïques", + "ours-garou", + "ours-garous", + "ouve-wirquinois", + "ouve-wirquinoise", + "ouve-wirquinoises", + "ouèche-ouèche", + "ouèches-ouèches", + "ouï-dire", + "ouïr-dire", + "ovo-lacto-végétarisme", + "ovo-lacto-végétarismes", + "ovo-urinaire", + "ovo-végétarisme", + "ovo-végétarismes", + "oxidéméton-méthyl", + "oxo-biodégradable", + "oxo-biodégradables", + "oxo-dégradable", + "oxo-dégradables", + "oxy-iodure", + "oxy-iodures", + "oxydo-réduction", + "oxydo-réductions", + "oxydéméton-méthyl", + "p'rlotte", + "p't-être", + "p'tain", + "p'tit", + "p'tite", + "p'tites", + "p'tits", + "p-acétylaminophénol", + "p-adique", + "p-adiques", + "p-graphe", + "p-graphes", + "p.-ê.", + "pH-mètre", + "pa'anga", + "pack-ice", + "pack-ices", + "package-deal", + "package-deals", + "pagano-chrétien", + "page-turner", + "pail-mail", + "pain-beurre", + "pain-d'épicier", + "pain-d'épiciers", + "pain-d'épicière", + "pain-d'épicières", + "pal-fer", + "palato-labial", + "palato-labiale", + "palato-pharyngien", + "palato-pharyngite", + "palato-pharyngites", + "palato-salpingien", + "palato-staphylin", + "palato-staphylins", + "palladico-potassique", + "palmier-chanvre", + "palmier-dattier", + "palmiers-chanvre", + "palmiers-dattiers", + "palpe-mâchoire", + "palu'e", + "palu'es", + "paléo-continental", + "paléo-lac", + "paléo-lacs", + "paléo-reconstruction", + "paléo-reconstructions", + "pama-nyungan", + "pan-européen", + "pan-européenne", + "pan-européennes", + "pan-européens", + "pan-lucanisme", + "pan-mandingue", + "pan-mandingues", + "panchen-lama", + "pancréatico-duodénal", + "panier-repas", + "paniers-repas", + "panpan-cucul", + "panthère-garou", + "panthères-garous", + "papa-gâteau", + "papas-gâteaux", + "papier-caillou-ciseaux", + "papier-calque", + "papier-cul", + "papier-filtre", + "papier-monnaie", + "papiers-calque", + "papy-boom", + "papy-boomer", + "papy-boomers", + "papy-boomeur", + "papy-boomeurs", + "paquet-cadeau", + "paquets-cadeaux", + "par-cœur", + "par-dehors", + "par-delà", + "par-derrière", + "par-dessous", + "par-dessus", + "par-devant", + "par-devers", + "para-acétyl-amino-phénol", + "para-continental", + "para-dichlorobenzène", + "para-légal", + "para-légale", + "para-légales", + "para-légaux", + "parachute-frein", + "parachutes-freins", + "parathion-méthyl", + "parathion-éthyl", + "parc-d'anxtotais", + "parc-d'anxtotaise", + "parc-d'anxtotaises", + "parking-relais", + "parler-pour-ne-rien-dire", + "parotido-auriculaire", + "parotido-auriculaires", + "parti-pris", + "participation-pari", + "particule-dieu", + "particules-dieu", + "parva-pétricien", + "parva-pétricienne", + "parva-pétriciennes", + "parva-pétriciens", + "pas-d'âne", + "pascal-seconde", + "pascals-secondes", + "paso-doble", + "paso-dobles", + "passif-agressif", + "passifs-agressifs", + "passing-shot", + "passing-shots", + "patronnier-gradeur", + "patronniers-gradeurs", + "patronnière-gradeuse", + "patronnières-gradeuses", + "patte-d'oie", + "patte-pelu", + "patte-pelus", + "pattes-d'oie", + "pauci-relationnel", + "pauci-relationnelle", + "pauci-relationnelles", + "pauci-relationnels", + "pauci-spécifique", + "pauci-spécifiques", + "pause-café", + "pause-carrière", + "pause-santé", + "pauses-café", + "pauses-carrière", + "pauses-santé", + "pay-per-view", + "pay-to-win", + "pays-bas", + "payé-emporté", + "pc-banking", + "peau-bleue", + "peau-rouge", + "peaux-rouges", + "peer-to-peer", + "peigne-cul", + "peigne-culs", + "peigne-zizi", + "peis-coua", + "pele-ata", + "pelle-pioche", + "pelles-bêches", + "pelles-pioches", + "pelure-d'oignon", + "pelvi-crural", + "pelvi-trochantérien", + "pelvi-trochantérienne", + "pelvi-trochantériennes", + "pelvi-trochantériens", + "pen-testeur", + "pen-testeurs", + "pen-testeuse", + "pen-testeuses", + "pen-ty", + "pencak-silat", + "penn-ty", + "pense-bête", + "pense-bêtes", + "penta-continental", + "penta-core", + "penta-cores", + "penta-cœur", + "penta-cœurs", + "perche-brochet", + "perche-soleil", + "perd-sa-queue", + "perd-tout", + "perdant-perdant", + "perdante-perdante", + "perdantes-perdantes", + "perdants-perdants", + "perfo-vérif", + "perroquet-hibou", + "perroquets-hiboux", + "perruche-moineau", + "perruches-moineaux", + "pesco-végétarien", + "pet'che", + "pet-d'âne", + "peul-peul", + "peut-être", + "pharyngo-laryngite", + "pharyngo-laryngites", + "pharyngo-staphylin", + "philosopho-théologique", + "philosopho-théologiques", + "phonético-symbolique", + "phoque-garou", + "phoque-léopard", + "phoques-garous", + "phosphate-allophane", + "phosphate-allophanes", + "phoséthyl-Al", + "phosétyl-Al", + "photos-finish", + "phragmito-scirpaie", + "phragmito-scirpaies", + "phrase-clé", + "phrases-clés", + "phréno-glottisme", + "phréno-glottismes", + "physico-chimie", + "physico-chimies", + "physico-chimique", + "physico-chimiques", + "physico-mathématique", + "physico-mathématiques", + "physio-pathologie", + "physio-pathologies", + "phénico-punique", + "phénico-puniques", + "pian's", + "piane-piane", + "piano-bar", + "piano-bars", + "piano-forte", + "piano-fortes", + "piano-manivelle", + "pic-vert", + "pic-verts", + "pichot-chêne", + "pichots-chênes", + "pick-up", + "pick-ups", + "pico-condensateur", + "pico-condensateurs", + "pico-ohm", + "pico-ohms", + "pics-verts", + "pidgin-english", + "pie-grièche", + "pie-mère", + "pie-noir", + "pie-noire", + "pie-noires", + "pie-noirs", + "pie-rouge", + "pierres-qui-virent", + "pies-grièches", + "pies-mères", + "pile-poil", + "pilo-sébacé", + "pin's", + "pin-pon", + "pin-up", + "pince-balle", + "pince-balles", + "pince-fesse", + "pince-fesses", + "pince-lisière", + "pince-maille", + "pince-mailles", + "pince-monseigneur", + "pince-nez", + "pince-notes", + "pince-oreille", + "pince-oreilles", + "pince-sans-rire", + "pince-érigne", + "pince-érignes", + "pinces-monseigneur", + "ping-pong", + "ping-pongs", + "pino-balméen", + "pino-balméenne", + "pino-balméennes", + "pino-balméens", + "pins-justarétois", + "pins-justarétoise", + "pins-justarétoises", + "piou-piou", + "piou-pious", + "pipe-line", + "pipe-lines", + "piqueur-suceur", + "pirimiphos-méthyl", + "pirimiphos-éthyl", + "pis-aller", + "pis-allers", + "pisse-chien", + "pisse-chiens", + "pisse-copie", + "pisse-copies", + "pisse-debout", + "pisse-froid", + "pisse-mémère", + "pisse-mémé", + "pisse-sang", + "pisse-trois-gouttes", + "pisse-vinaigre", + "pisse-vinaigres", + "pisse-z-yeux", + "pissy-pôvillais", + "pissy-pôvillaise", + "pissy-pôvillaises", + "pistillo-staminé", + "pistolet-mitrailleur", + "pistolets-mitrailleurs", + "pit-bulls", + "pixie-bob", + "piétin-verse", + "piétin-échaudage", + "piézo-électricité", + "piézo-électricités", + "piézo-électrique", + "piézo-électriques", + "plachy-buyonnais", + "plachy-buyonnaise", + "plachy-buyonnaises", + "plain-chant", + "plain-pied", + "plains-chants", + "plains-pieds", + "plan-masse", + "plan-plan", + "plan-planisme", + "plan-planismes", + "plan-séquence", + "plan-séquences", + "planches-contacts", + "plans-masses", + "plans-séquences", + "plante-crayon", + "plante-éponge", + "plantes-crayons", + "plaque-bière", + "plaque-tonnerre", + "plat-bord", + "plat-cul", + "plat-culs", + "plate-bande", + "plate-bière", + "plate-face", + "plate-forme", + "plate-longe", + "plateau-repas", + "plateaux-repas", + "plates-bandes", + "plates-formes", + "plates-longes", + "platinico-ammonique", + "plats-bords", + "play-back", + "play-backs", + "play-boy", + "play-boys", + "play-off", + "play-offs", + "plaît-il", + "plein-cintre", + "plein-emploi", + "pleine-fougerais", + "pleine-fougeraise", + "pleine-fougeraises", + "pleins-cintres", + "pleu-pleu", + "pleure-misère", + "pleure-misères", + "pleuro-péricardite", + "pleuronecte-guitare", + "plieuse-inséreuse", + "plieuses-inséreuses", + "plongée-spéléo", + "plongées-spéléo", + "plouezoc'hois", + "plouezoc'hoise", + "plouezoc'hoises", + "ploulec'hois", + "ploulec'hoise", + "ploulec'hoises", + "plounéour-trezien", + "plounéour-trezienne", + "plounéour-treziennes", + "plounéour-treziens", + "ploye-ressort", + "plui-plui", + "plum-cake", + "plum-cakes", + "plum-pudding", + "plumbo-aragonite", + "plumbo-aragonites", + "plume-couteau", + "plumes-couteaux", + "pluri-continental", + "pluri-interprétable", + "pluri-interprétables", + "pluri-journalier", + "pluri-modal", + "pluri-national", + "pluri-nationale", + "pluri-nationales", + "pluri-nationaux", + "plus-d'atouts", + "plus-disant", + "plus-part", + "plus-payé", + "plus-produit", + "plus-produits", + "plus-pétition", + "plus-que-parfait", + "plus-que-parfaits", + "plus-value", + "plus-values", + "pluto-neptunien", + "pluvier-hirondelle", + "plû-part", + "poche-cuiller", + "poche-revolver", + "poches-revolver", + "pochette-surprise", + "pochettes-surprise", + "pochettes-surprises", + "podio-régalien", + "podio-régalienne", + "podio-régaliennes", + "podio-régaliens", + "podo-orthésiste", + "podo-orthésistes", + "poggio-mezzanais", + "poggio-mezzanaise", + "poggio-mezzanaises", + "pogne-cul", + "pogne-culs", + "poids-lourd", + "poids-lourds", + "point-arrière", + "point-col", + "point-milieu", + "point-selle", + "point-virgule", + "point-voyelle", + "pointe-noirais", + "pointe-noiraise", + "pointe-noiraises", + "points-virgules", + "points-voyelles", + "poissonnier-écailler", + "poitevin-saintongeais", + "poivre-sel", + "poix-résine", + "poka-yoké", + "politico-idéologique", + "politico-idéologiques", + "politico-médiatique", + "politico-religieuse", + "politico-religieuses", + "politico-religieux", + "politico-économique", + "politico-économiques", + "pollueur-payeur", + "pollueurs-payeurs", + "poly-articulaire", + "poly-articulaires", + "poly-insaturé", + "poly-insaturée", + "poly-insaturées", + "poly-insaturés", + "poly-sexuel", + "poly-sexuelle", + "polychlorodibenzo-p-dioxine", + "polychlorodibenzo-p-dioxines", + "pomme-grenade", + "pompage-turbinage", + "pompages-turbinages", + "ponts-bascules", + "ponts-canaux", + "ponts-levis", + "ponts-neufs", + "pop-corn", + "pop-in", + "pop-ins", + "pop-punk", + "pop-up", + "pop-ups", + "popa'a", + "porc-épic", + "porcs-épics", + "portes-fenêtres", + "portes-tambour", + "porto-vecchiais", + "porto-vecchiaise", + "porto-vecchiaises", + "portrait-charge", + "portrait-robot", + "portraits-charges", + "portraits-robots", + "pose-tubes", + "post-11-Septembre", + "posé-décollé", + "posé-décollés", + "pot-beurrier", + "pot-bouille", + "pot-pourri", + "potassico-ammonique", + "potassico-mercureux", + "poto-poto", + "potron-jacquet", + "potron-minet", + "pots-pourris", + "pouce-pied", + "pouces-pieds", + "poudre-éclair", + "poudres-éclair", + "poudres-éclairs", + "pour-boire", + "pour-cent", + "pourri-gâté", + "poursuite-bâillon", + "pouy-roquelain", + "pouy-roquelaine", + "pouy-roquelaines", + "pouy-roquelains", + "pouët-pouët", + "pow-wow", + "pow-wows", + "poët-lavalien", + "poët-lavalienne", + "poët-lavaliennes", + "poët-lavaliens", + "premier-né", + "premiers-nés", + "presqu'accident", + "presqu'accidents", + "presqu'ile", + "presqu'iles", + "presqu'île", + "presqu'îles", + "press-book", + "press-books", + "presse-agrume", + "presse-agrumes", + "presse-ail", + "presse-artère", + "presse-artères", + "presse-citron", + "presse-citrons", + "presse-fruits", + "presse-légumes", + "presse-papier", + "presse-papiers", + "presse-purée", + "presse-purées", + "presse-urètre", + "presse-urètres", + "presse-étoffe", + "presse-étoffes", + "presse-étoupe", + "presse-étoupes", + "pressignaco-vicois", + "pressignaco-vicoise", + "pressignaco-vicoises", + "preux-romanien", + "preux-romanienne", + "preux-romaniennes", + "preux-romaniens", + "prie-Dieu", + "prim'holstein", + "prima-mensis", + "prime-sautier", + "prince-président", + "prince-sans-rire", + "prince-édouardien", + "prince-édouardienne", + "prince-édouardiennes", + "prince-édouardiens", + "prince-électeur", + "princes-présidents", + "princes-électeurs", + "prisons-écoles", + "privat-docent", + "privat-docentisme", + "privat-docentismes", + "prix-choc", + "prix-chocs", + "programme-cadre", + "programmes-cadres", + "prohexadione-calcium", + "promis-juré", + "promis-jurée", + "promis-jurées", + "promis-jurés", + "promène-couillon", + "promène-couillons", + "pronom-adjectif", + "pronoms-adjectifs", + "prostato-péritonéal", + "prostato-péritonéale", + "prostato-péritonéales", + "prostato-péritonéaux", + "prout-prout", + "prout-proute", + "prout-proutes", + "prout-prouts", + "prunier-cerise", + "pruniers-cerises", + "prés-bois", + "prés-salés", + "prés-vergers", + "président-candidat", + "présidente-candidate", + "présidentes-candidates", + "présidents-candidats", + "présidents-directeurs", + "prête-nom", + "prête-noms", + "prêtres-ouvriers", + "prône-misère", + "pschitt-pschitt", + "psycho-physiologique", + "psycho-physiologiques", + "psycho-physique", + "psycho-physiques", + "psycho-pop", + "ptérygo-pharyngien", + "pub-restaurant", + "pub-restaurants", + "puce-chique", + "puces-chiques", + "puis-je", + "puiset-doréen", + "puiset-doréenne", + "puiset-doréennes", + "puiset-doréens", + "pull-buoy", + "pull-buoys", + "pull-over", + "pull-overs", + "pull-up", + "pulmo-aortique", + "pulso-réacteurs", + "pulvérisateur-mélangeur", + "punaise-mouche", + "punaises-mouches", + "punching-ball", + "punching-balls", + "punkah-wallah", + "pur-sang", + "pur-sangs", + "pure-laine", + "purge-mariage", + "purge-mariages", + "purs-sangs", + "push-back", + "push-up", + "putot-bessinois", + "putot-bessinoise", + "putot-bessinoises", + "pyraflufen-éthyl", + "pyrimiphos-méthyl", + "pyrimiphos-éthyl", + "pyro-électricité", + "pyro-électricités", + "pyro-électrique", + "pyro-électriques", + "pâtissier-chocolatier", + "pèse-acide", + "pèse-acides", + "pèse-alcool", + "pèse-alcools", + "pèse-bébé", + "pèse-bébés", + "pèse-esprit", + "pèse-esprits", + "pèse-lait", + "pèse-laits", + "pèse-lettre", + "pèse-lettres", + "pèse-liqueur", + "pèse-liqueurs", + "pèse-mout", + "pèse-mouts", + "pèse-moût", + "pèse-moûts", + "pèse-nitre", + "pèse-nitres", + "pèse-personne", + "pèse-personnes", + "pèse-sel", + "pèse-sels", + "pèse-sirop", + "pèse-sirops", + "pèse-vernis", + "pète-sec", + "pète-secs", + "pète-sèche", + "pète-sèches", + "pédal'eau", + "pédicure-podologue", + "pédicures-podologues", + "pénicillino-résistance", + "pénicillino-résistances", + "pénicillino-sensibilité", + "pénicillino-sensibilités", + "péronéo-calcanéen", + "péronéo-malléolaire", + "péronéo-malléolaires", + "péronéo-phalangien", + "péronéo-tibial", + "péta-ampère", + "péta-ampères", + "péta-électron-volt", + "péta-électron-volts", + "pétaélectron-volt", + "pétaélectron-volts", + "pétro-monarchie", + "pétro-monarchies", + "pétro-occipital", + "pétro-salpingo-staphylin", + "pétro-salpingo-staphylins", + "pétro-staphylin", + "pétrolier-minéralier", + "pétrus-colien", + "pétrus-colienne", + "pétrus-coliennes", + "pétrus-coliens", + "pêche-bernard", + "pêche-bernards", + "q'anjob'al", + "qu-in-situ", + "quad-core", + "quad-cores", + "quadri-accélération", + "quadri-accélérationnellement", + "quadri-ailé", + "quadri-couche", + "quadri-couches", + "quadri-courant", + "quadri-dimensionnel", + "quadri-dimensionnelle", + "quadri-dimensionnelles", + "quadri-dimensionnels", + "quadri-rotor", + "quadri-rotors", + "quadruple-croche", + "quadruples-croches", + "quarante-cinq", + "quarante-deux", + "quarante-douze", + "quarante-huit", + "quarante-huitard", + "quarante-huitarde", + "quarante-huitardes", + "quarante-huitards", + "quarante-huitième", + "quarante-huitièmes", + "quarante-langues", + "quarante-neuf", + "quarante-neuvième", + "quarante-neuvièmes", + "quarante-quatre", + "quarante-sept", + "quarante-six", + "quarante-trois", + "quarante-vingt", + "quart-arrière", + "quart-biscuité", + "quart-d'heure", + "quart-monde", + "quart-temps", + "quarte-fagot", + "quartier-général", + "quartier-maitre", + "quartier-maitres", + "quartier-maître", + "quartier-mestre", + "quartiers-maîtres", + "quarts-arrières", + "quat'z'arts", + "quatorze-marsiste", + "quatorze-marsistes", + "que'ques", + "quelqu'un", + "quelqu'une", + "quelques-unes", + "quelques-uns", + "questche-wasser", + "question-piège", + "question-tag", + "questions-pièges", + "questions-réponses", + "questions-tags", + "queue-d'aronde", + "queue-d'hironde", + "queue-d'oison", + "queue-d'or", + "queue-fourchue", + "queue-rouge", + "queues-d'aronde", + "queues-d'hironde", + "queues-d'or", + "qui-va-là", + "qui-vive", + "quick-and-dirty", + "quintuple-croche", + "quintuples-croches", + "quinze-vingt", + "quinze-vingts", + "quizalofop-P-éthyl", + "quizalofop-p-éthyl", + "quizalofop-éthyl", + "quote-part", + "quotes-parts", + "rabat-eau", + "rabat-eaux", + "rabat-joie", + "rabat-joies", + "rabi'-oul-aououal", + "rabi'-out-tani", + "racine-blanche", + "racines-blanches", + "rad'soc", + "rad'socs", + "rad-soc", + "rad-socs", + "radar-tronçon", + "radars-tronçons", + "radical-socialisme", + "radical-socialismes", + "radical-socialiste", + "radicale-socialiste", + "radicales-socialistes", + "radicaux-socialistes", + "rag-time", + "rag-times", + "rahat-lokoum", + "rahat-lokoums", + "rahat-loukoum", + "rahat-loukoums", + "raid-aventure", + "raie-aigle", + "raie-guitare", + "raie-papillon", + "raies-aigles", + "raies-papillons", + "rail-road", + "rail-route", + "rallie-papier", + "rallonge-bouton", + "rallonge-boutons", + "ralé-poussé", + "ramasse-bourrier", + "ramasse-bourriers", + "ramasse-couvert", + "ramasse-couverts", + "ramasse-miette", + "ramasse-miettes", + "ramasse-monnaie", + "ramasse-poussière", + "ramasse-poussières", + "ramasse-ton-bras", + "ramasseuse-presse", + "ramasseuses-presses", + "rase-motte", + "rase-mottes", + "rase-pet", + "rase-pets", + "rat-baillet", + "rat-bayard", + "rat-garou", + "rat-taupe", + "rat-trompette", + "ratisse-caisse", + "rats-garous", + "ray-grass", + "re'em", + "re'ems", + "ready-made", + "reality-show", + "reality-shows", + "rebrousse-poil", + "recourbe-cils", + "recto-vaginal", + "recto-verso", + "redresse-seins", + "regardez-moi", + "reine-claude", + "reine-marguerite", + "reines-claudes", + "reines-marguerites", + "relève-gravure", + "relève-gravures", + "relève-moustache", + "relève-moustaches", + "relève-quartier", + "relève-quartiers", + "relève-selle", + "relève-selles", + "remettez-vous", + "remilly-wirquinois", + "remilly-wirquinoise", + "remilly-wirquinoises", + "remonte-pente", + "remonte-pentes", + "remue-ménage", + "remue-ménages", + "remue-méninge", + "remue-méninges", + "remue-queue", + "remue-queues", + "renard-garou", + "renarde-garou", + "rendez-vous", + "rennes-robots", + "renouée-bambou", + "rentre-dedans", + "renvoi-instruire", + "repose-pied", + "repose-pieds", + "repose-poignet", + "repose-poignets", + "repose-tête", + "repose-têtes", + "requins-baleines", + "requins-citrons", + "requins-crocodiles", + "requins-garous", + "requins-hâ", + "requins-marteaux", + "requins-taupes", + "requins-tigres", + "rest-o-pack", + "restaurant-bar", + "restaurant-bistro", + "restaurant-brasserie", + "restaurant-pub", + "restaurants-bistros", + "reste-avec", + "resto-bar", + "resto-bistro", + "resto-brasserie", + "resto-pub", + "retraite-chapeau", + "retraites-chapeaux", + "retroussons-nos-manches", + "revenant-bon", + "revenants-bons", + "revenez-y", + "rex-castor", + "rex-castors", + "rez-mur", + "rhodesian-ridgeback", + "rhéo-fluidifiant", + "rhéo-fluidifiante", + "rhéo-fluidifiantes", + "rhéo-fluidifiants", + "rhéo-épaississant", + "rhéo-épaississante", + "rhéo-épaississantes", + "rhéo-épaississants", + "rhéto-roman", + "rhéto-romane", + "rhéto-romanes", + "rhéto-romans", + "ria-sirachois", + "ria-sirachoise", + "ria-sirachoises", + "ric-rac", + "rigaud-montain", + "rigaud-montaine", + "rigaud-montaines", + "rigaud-montains", + "rigny-usséen", + "rigny-usséenne", + "rigny-usséennes", + "rigny-usséens", + "rince-bouche", + "rince-bouches", + "rince-bouteille", + "rince-bouteilles", + "rince-doigt", + "rince-doigts", + "risque-tout", + "riz-pain-sel", + "road-book", + "road-books", + "roast-beef", + "roast-beefs", + "robe-chandail", + "robe-housse", + "robert-messin", + "robert-messine", + "robert-messines", + "robert-messins", + "robes-chandails", + "robes-housses", + "robot-chien", + "robots-chiens", + "roche-blanchais", + "roche-blanchaise", + "roche-blanchaises", + "roche-mère", + "roche-papier-ciseaux", + "roches-mères", + "rock'n'roll", + "rock-a-billy", + "rocking-chair", + "rocking-chairs", + "roge-bougeron", + "roge-bougeronne", + "roge-bougeronnes", + "roge-bougerons", + "roger-bontemps", + "rogne-cul", + "rogne-pied", + "rogne-pieds", + "rogne-salaires", + "roll-out", + "roll-outs", + "roller-derby", + "roller-derbys", + "roman-feuilleton", + "roman-fleuve", + "roman-photo", + "roman-photos", + "romans-feuilletons", + "romans-fleuves", + "romans-photos", + "rompt-pierre", + "rompt-pierres", + "ron-ron", + "rond-point", + "ronde-bosse", + "ronde-bosses", + "rondes-bosses", + "ronds-points", + "ronge-bois", + "ronge-maille", + "rongo-rongo", + "roost-warendinois", + "roost-warendinoise", + "roost-warendinoises", + "rose-croix", + "rose-marine", + "roses-marines", + "rosti-montois", + "rosti-montoise", + "rosti-montoises", + "rouge-aile", + "rouge-bord", + "rouge-brun", + "rouge-flasher", + "rouge-gorge", + "rouge-herbe", + "rouge-herbes", + "rouge-noir", + "rouge-pie", + "rouge-queue", + "rouges-ailes", + "rouges-gorges", + "rouges-queues", + "rouget-barbet", + "rouget-grondin", + "roul-sa-bosse", + "roulage-décollage", + "roule-goupille", + "roule-goupilles", + "roule-ta-bosse", + "rouler-bouler", + "roullet-stéphanois", + "roullet-stéphanoise", + "roullet-stéphanoises", + "roulé-boulé", + "roulé-saucisse", + "roulés-boulés", + "rousse-tête", + "rousses-têtes", + "roux-mirien", + "rufino-sulfurique", + "rufino-sulfuriques", + "ruine-babine", + "ruine-babines", + "russo-allemand", + "russo-allemande", + "russo-allemandes", + "russo-allemands", + "russo-américain", + "russo-japonaise", + "russo-polonaise", + "râlé-poussé", + "réal-politique", + "réal-politiques", + "réception-cadeaux", + "récipient-mesure", + "récipient-mesures", + "régis-borgien", + "régis-borgienne", + "régis-borgiennes", + "régis-borgiens", + "rémy-montais", + "rémy-montaise", + "rémy-montaises", + "répondeur-enregistreur", + "répondeur-enregistreurs", + "résino-gommeux", + "réunion-bilan", + "réunions-bilan", + "réveil-matin", + "réveille-matin", + "réveille-matins", + "rêve-creux", + "rü'üsá", + "sa'ban", + "sabre-peuple", + "sac-jacking", + "sac-poubelle", + "saccharo-glycose", + "sacro-iliaques", + "sacro-lombaire", + "sacro-saint", + "sacro-sainte", + "sacro-saintement", + "sacro-saintes", + "sacro-saints", + "sacro-vertébral", + "sacré-coeur", + "sacré-cœur", + "sacs-poubelle", + "sacs-poubelles", + "sado-maso", + "sado-masochisme", + "sado-masochiste", + "sado-masochistes", + "safari-parc", + "safari-parcs", + "sage-femme", + "sage-homme", + "sages-femmes", + "sahélo-saharien", + "sahélo-saharienne", + "sahélo-sahariennes", + "sahélo-sahariens", + "saigne-nez", + "sain-belois", + "sain-beloise", + "sain-beloises", + "sain-bois", + "sain-foin", + "saisie-arrêt", + "saisie-attribution", + "saisie-brandon", + "saisie-exécution", + "saisie-gagerie", + "saisie-revendication", + "saisies-arrêts", + "saisies-attributions", + "saisies-brandons", + "saisies-exécutions", + "saisies-gageries", + "saisies-revendications", + "saisir-arrêter", + "saisir-brandonner", + "saisir-exécuter", + "saisir-gager", + "saisir-revendiquer", + "salafo-sioniste", + "salaire-coût", + "salaire-coûts", + "salamandre-tigre", + "salle-prunetais", + "salle-prunetaise", + "salle-prunetaises", + "salles-sourçois", + "salles-sourçoise", + "salles-sourçoises", + "salpingo-pharyngien", + "salve-d'honneur", + "salves-d'honneur", + "sam'suffit", + "sam'suffits", + "sancto-bénédictin", + "sancto-bénédictine", + "sancto-bénédictines", + "sancto-bénédictins", + "sancto-julianais", + "sancto-julianaise", + "sancto-julianaises", + "sancto-prixin", + "sancto-prixine", + "sancto-prixines", + "sancto-prixins", + "sang-froid", + "sang-gris", + "sang-mêlé", + "sang-mêlés", + "sankaku-jime", + "santi-johanien", + "santi-johanienne", + "santi-johaniennes", + "santi-johaniens", + "santoline-cyprès", + "sapeur-pompier", + "sapeurs-pompiers", + "sapeuse-pompière", + "sapeuses-pompières", + "sarclo-buttage", + "sarclo-buttages", + "sarco-hydrocèle", + "sarco-hydrocèles", + "sarco-épiplocèle", + "sarco-épiplomphale", + "sarco-épiplomphales", + "sarre-unionnais", + "sarre-unionnaise", + "sarre-unionnaises", + "sart-dames-avelinois", + "sart-eustachois", + "sart-risbartois", + "satellites-espions", + "sati-drap", + "sauf-conduit", + "sauf-conduits", + "sault-rethelois", + "sault-retheloise", + "sault-retheloises", + "saute-bouchon", + "saute-bouchons", + "saute-mouton", + "saute-moutons", + "saute-ruisseau", + "saute-ruisseaux", + "sauve-l'honneur", + "sauve-qui-peut", + "sauve-rabans", + "sauve-vie", + "savez-vous", + "savoir-faire", + "savoir-vivre", + "scale-out", + "scale-up", + "scaphoïdo-astragalien", + "scaphoïdo-cuboïdien", + "sceau-cylindre", + "sceau-de-Notre-Dame", + "sceaux-cylindres", + "sceaux-de-Notre-Dame", + "schiste-carton", + "schistes-carton", + "scie-cloche", + "science-fictif", + "science-fiction", + "science-fictions", + "sciences-fiction", + "sciences-fictions", + "scies-cloches", + "scirpo-phragmitaie", + "scirpo-phragmitaies", + "scottish-terrier", + "scuto-sternal", + "scènes-clés", + "seconde-lumière", + "secondes-lumière", + "seine-portais", + "seine-portaise", + "seine-portaises", + "self-control", + "self-défense", + "self-government", + "self-governments", + "self-made-man", + "self-made-mans", + "self-made-men", + "self-made-woman", + "self-made-womans", + "self-made-women", + "self-service", + "self-services", + "selk'nam", + "semaine-lumière", + "semaines-lumière", + "semen-contra", + "semper-virens", + "sensori-moteur", + "sensori-moteurs", + "sensori-motrice", + "sensori-motrices", + "sensori-motricité", + "sent-bon", + "sept-et-le-va", + "sept-frèrien", + "sept-frèrienne", + "sept-frèriennes", + "sept-frèriens", + "sept-meulois", + "sept-meuloise", + "sept-meuloises", + "sept-mâts", + "sept-oeil", + "sept-oeils", + "sept-sortais", + "sept-sortaise", + "sept-sortaises", + "sept-ventais", + "sept-ventaise", + "sept-ventaises", + "sept-œil", + "sept-œils", + "septante-cinq", + "septante-deux", + "septante-huit", + "septante-neuf", + "septante-quatre", + "septante-sept", + "septante-six", + "septante-trois", + "septentrio-occidental", + "septentrio-occidentale", + "septentrio-occidentales", + "septentrio-occidentaux", + "serbo-croate", + "sergent-chef", + "sergent-major", + "sergents-chefs", + "sergents-majors", + "serres-fines", + "serres-gastonnais", + "serres-gastonnaise", + "serres-gastonnaises", + "serres-morlanais", + "serres-morlanaise", + "serres-morlanaises", + "serri-sapinois", + "serri-sapinoise", + "serri-sapinoises", + "service-volée", + "services-volées", + "serviette-éponge", + "serviettes-éponges", + "servo-direction", + "servo-directions", + "servo-frein", + "servo-freins", + "servo-moteur", + "sex-appeal", + "sex-digital", + "sex-digitisme", + "sex-digitismes", + "sex-ratio", + "sex-ratios", + "sex-shop", + "sex-shops", + "sex-symbol", + "sex-symbols", + "sex-toy", + "sex-toys", + "sexe-ratio", + "shabu-shabu", + "shar-peï", + "shar-peïs", + "shikoku-inu", + "shipibo-conibo", + "shoot-'em-up", + "short-culotte", + "short-culottes", + "short-track", + "short-tracks", + "show-biz", + "show-business", + "sicilio-sarde", + "side-car", + "side-cariste", + "side-caristes", + "side-cars", + "sierra-léonais", + "sierra-léonaise", + "sierra-léonaises", + "sigma-additif", + "sigma-additivité", + "sigma-additivités", + "silicico-aluminique", + "silicico-aluminiques", + "silicico-cuivreux", + "silure-spatule", + "simili-cuir", + "simili-cuirs", + "singe-araignée", + "singe-chouette", + "singe-lion", + "singe-écureuil", + "singes-araignées", + "singes-chouettes", + "singes-lions", + "singes-écureuils", + "sister-ship", + "sister-ships", + "sit-in", + "sit-ins", + "sit-up", + "sit-ups", + "six-cent-soixante-six", + "six-cent-soixante-sixième", + "six-cent-soixante-sixièmes", + "six-cents", + "six-clefs", + "six-coups", + "six-doigts", + "six-fournais", + "six-fournaise", + "six-fournaises", + "six-mâts", + "six-vingts", + "siècle-lumière", + "siècles-lumière", + "ski-alpinisme", + "ski-alpinismes", + "ski-alpiniste", + "ski-alpinistes", + "sleeping-car", + "sloop-of-war", + "slop-tank", + "smaragdo-chalcite", + "smaragdo-chalcites", + "snack-bar", + "snack-bars", + "snow-boot", + "snow-boots", + "soap-opéra", + "soaps-opéras", + "sociaux-démocrates", + "sociaux-traitres", + "sociaux-traîtres", + "société-écran", + "sociétés-écrans", + "soda-spodumenes", + "sodo-calcique", + "sodo-calciques", + "soi-disamment", + "soi-disant", + "soi-même", + "soit-communiqué", + "sole-ruardon", + "solliès-pontois", + "solliès-pontoise", + "solliès-pontoises", + "solliès-villain", + "solliès-villaine", + "solliès-villaines", + "solliès-villains", + "somato-psychique", + "somato-psychiques", + "somme-leuzien", + "somme-suippas", + "somme-suippase", + "somme-suippases", + "songe-creux", + "songe-malice", + "songhaï-zarma", + "songhaï-zarmas", + "sot-l'y-laisse", + "sotto-voce", + "sou-chong", + "sou-chongs", + "soudano-tchado-lybien", + "souffre-douleur", + "souffre-douleurs", + "soufre-sélénifère", + "soum-soum", + "soupe-tout-seul", + "sourd-muet", + "sourd-parlant", + "sourde-muette", + "sourdes-muettes", + "sourds-muets", + "souris-chauve", + "souris-chauves", + "souris-crayon", + "souris-crayons", + "souris-opossums", + "souris-stylo", + "souris-stylos", + "soutien-gorge", + "soutien-loloches", + "soutiens-gorge", + "souvenez-vous-de-moi", + "souveraineté-association", + "souï-manga", + "soŋay-zarma", + "soŋay-zarmas", + "sparring-partner", + "spatio-temporel", + "spatio-temporelle", + "spatio-temporelles", + "spatio-temporels", + "speed-dating", + "sphinx-bourdon", + "sphéno-temporal", + "spin-off", + "spin-offs", + "spina-bifida", + "spina-ventosa", + "spiro-bloc", + "spiro-blocs", + "sport-étude", + "sportivo-financier", + "sports-études", + "spruce-beer", + "squale-grogneur", + "sri-lankais", + "sri-lankaise", + "sri-lankaises", + "st'at'imc", + "stage-coach", + "stage-coachs", + "stand-by", + "stand-up", + "stannoso-potassique", + "star-système", + "star-systèmes", + "start-up", + "start-upeur", + "starting-block", + "starting-blocks", + "starting-gate", + "station-service", + "stations-service", + "stations-services", + "statue-menhir", + "statues-menhirs", + "steam-boat", + "steam-boats", + "steeple-chase", + "step-back", + "step-backs", + "sterno-claviculaire", + "sterno-claviculaires", + "sterno-clido-mastoïdien", + "sterno-clido-mastoïdienne", + "sterno-clido-mastoïdiennes", + "sterno-clido-mastoïdiens", + "sterno-cléido-mastoïdien", + "sterno-cléido-mastoïdiens", + "sterno-huméral", + "sterno-hyoïdien", + "sterno-pubien", + "stock-car", + "stock-cars", + "stock-option", + "stock-options", + "stock-tampon", + "stocks-tampons", + "stomo-gastrique", + "stomo-gastriques", + "stop-ski", + "stop-skis", + "story-board", + "story-boards", + "strauss-kahnien", + "strauss-kahniens", + "street-artiste", + "street-artistes", + "street-gadz", + "stroke-play", + "strom-apparat", + "struggle-for-life", + "struggle-for-lifes", + "stud-book", + "stuffing-box", + "stylo-bille", + "stylo-billes", + "stylo-feutre", + "stylo-glosse", + "stylo-gomme", + "stylo-pistolet", + "stylo-plume", + "stylo-souris", + "stylos-feutres", + "stylos-gommes", + "stylos-plume", + "stylos-souris", + "sténo-dactylographe", + "sténo-dactylographes", + "sténo-méditerranéen", + "sténo-méditerranéenne", + "sténo-méditerranéennes", + "sténo-méditerranéens", + "stéphano-carladésien", + "stéphano-carladésienne", + "stéphano-carladésiennes", + "stéphano-carladésiens", + "stéréo-isomère", + "stéréo-isomères", + "su-sucre", + "su-sucres", + "subrogé-tuteur", + "subrogés-tuteurs", + "suce-boules", + "suce-bœuf", + "suce-fleur", + "suce-fleurs", + "suce-goulot", + "suce-goulots", + "suce-médailles", + "sudoro-algique", + "suivez-moi-jeune-homme", + "sulfo-margarique", + "suméro-akkadien", + "suméro-akkadienne", + "suméro-akkadiennes", + "suméro-akkadiens", + "super-8", + "support-chaussettes", + "supports-chaussettes", + "supra-axillaire", + "supra-axillaires", + "supra-caudal", + "supra-caudale", + "supra-caudales", + "supra-caudaux", + "supra-épineux", + "surdi-mutité", + "surdi-mutités", + "suro-pédieuse", + "suro-pédieuses", + "suro-pédieux", + "surprise-partie", + "surprise-parties", + "surprises-parties", + "surveillant-général", + "suédo-américain", + "suédo-américaine", + "suédo-américaines", + "suédo-américains", + "sweat-shirt", + "sweat-shirts", + "syndesmo-pharyngien", + "syro-chaldaïque", + "syro-chaldéen", + "syro-chaldéens", + "syro-saoudien", + "systèmes-clés", + "sèche-cheveu", + "sèche-cheveux", + "sèche-linge", + "séchoir-atomiseur", + "séchoir-atomiseurs", + "sénateur-maire", + "sénatus-consulte", + "sénatus-consultes", + "séro-sanguin", + "séro-sanguine", + "séro-sanguines", + "séro-sanguins", + "t'inquiète", + "t'occupe", + "t'oh", + "t-bone", + "t-bones", + "t-elle", + "t-il", + "t-on", + "t-shirt", + "t-shirts", + "tabagn's", + "table-bureau", + "tables-bureaux", + "tac-tac", + "tai-kadai", + "tailles-douces", + "taki-taki", + "talco-micacé", + "talco-quartzeux", + "talk-show", + "talkie-walkie", + "talkie-walkies", + "talkies-walkies", + "taly-pen", + "taly-pens", + "tam-tam", + "tam-tams", + "tambour-major", + "tambours-majors", + "tams-tams", + "tao-taï", + "tao-taïs", + "tape-beurre", + "tape-beurres", + "tape-cul", + "tape-culs", + "tape-dur", + "tape-durs", + "tapis-brosse", + "tapis-franc", + "tapis-francs", + "tapis-luge", + "tapis-luges", + "tapis-plain", + "tard-venus", + "tarso-métatarse", + "tarso-métatarsien", + "tarton-raire", + "tate-mono", + "tate-monos", + "tau-fluvalinate", + "taupe-grillon", + "taupes-grillons", + "taxi-auto", + "taxi-automobile", + "taxi-brousse", + "taxi-girl", + "taxi-girls", + "taxi-vélo", + "taxis-brousse", + "taxis-vélos", + "taï-kadaï", + "taï-le", + "taï-nüa", + "tchado-burkinabé", + "tchado-centrafricain", + "tchado-egyptien", + "tchado-lybien", + "tchado-soudano-lybien", + "tchin-tchin", + "tchou-tchou", + "tchéco-slovaque", + "tchéco-slovaques", + "teach-in", + "teach-ins", + "tee-shirt", + "tee-shirts", + "teen-ager", + "teen-agers", + "teint-vin", + "teint-vins", + "teinture-mère", + "temporo-conchinien", + "temporo-superficiel", + "tensio-actif", + "tente-abri", + "tente-ménagerie", + "tentes-ménageries", + "ter-ter", + "terno-annulaire", + "terra-cotta", + "terre-grièpe", + "terre-neuva", + "terre-neuvas", + "terre-neuve", + "terre-neuvien", + "terre-neuvienne", + "terre-neuviennes", + "terre-neuviens", + "terre-neuvier", + "terre-neuviers", + "terre-noix", + "terre-plein", + "terre-pleins", + "terret-bourret", + "terza-rima", + "test-match", + "test-matchs", + "test-objet", + "tette-chèvre", + "tette-chèvres", + "teufs-teufs", + "thifensulfuron-méthyle", + "thimistérien-clermontois", + "thiophanate-méthyl", + "thiophanate-éthyl", + "thon-samsonais", + "thoré-folléen", + "thoré-folléenne", + "thoré-folléennes", + "thoré-folléens", + "thoult-tronaisien", + "thoult-tronaisienne", + "thoult-tronaisiennes", + "thoult-tronaisiens", + "thraco-illyrienne", + "thuit-angevin", + "thuit-angevine", + "thuit-angevines", + "thuit-angevins", + "thuit-signolais", + "thuit-signolaise", + "thuit-signolaises", + "thuit-simérien", + "thuit-simérienne", + "thuit-simériennes", + "thuit-simériens", + "thun-episcopien", + "thun-épiscopien", + "thun-épiscopienne", + "thun-épiscopiennes", + "thun-épiscopiens", + "thézy-glimontois", + "thézy-glimontoise", + "thézy-glimontoises", + "thêta-jointure", + "thêta-jointures", + "ti-coune", + "ti-counes", + "ti-cul", + "ti-papoute", + "ti-punch", + "ti-punchs", + "tibio-malléolaire", + "tibéto-birman", + "tibéto-birmane", + "tibéto-birmanes", + "tibéto-birmans", + "tic-tac", + "tic-tac-toe", + "tic-tacs", + "ticket-restaurant", + "tie-break", + "tie-breaks", + "tierce-feuille", + "tierce-rime", + "tierces-rimes", + "tiger-kidnappeur", + "tiger-kidnapping", + "tiger-kidnappings", + "tigre-garou", + "tigres-garous", + "tiki-taka", + "tilleul-othonnais", + "tilleul-othonnaise", + "tilleul-othonnaises", + "tilt-shift", + "timbre-amende", + "timbre-poste", + "timbre-quittance", + "timbre-taxe", + "timbres-amende", + "timbres-poste", + "timbres-quittances", + "time-lapse", + "time-lapses", + "time-sharing", + "time-sharings", + "tiou-tiou", + "tiou-tious", + "tira-tutto", + "tiroir-caisse", + "tiroirs-caisses", + "tissu-éponge", + "tissus-éponges", + "titan-cotte", + "titanico-ammonique", + "titanico-ammoniques", + "titre-service", + "titres-services", + "toba-qom", + "toc-feu", + "toc-toc", + "toc-tocs", + "tohu-bohu", + "tohu-bohus", + "tohus-bohus", + "toi-même", + "toit-terrasse", + "toits-terrasses", + "tolclofos-méthyl", + "tom-pouce", + "tom-tom", + "tom-toms", + "tombe-cartouche", + "tonne-grenoir", + "tonne-mètre", + "top-down", + "top-model", + "top-models", + "top-modèle", + "top-modèles", + "top-secret", + "top-secrets", + "topo-guide", + "topo-guides", + "toque-feu", + "torche-cul", + "torche-culs", + "torche-fer", + "torche-pertuis", + "torche-pin", + "torche-pinceau", + "torche-pinceaux", + "torche-pins", + "tord-boyau", + "tord-boyaux", + "tord-nez", + "tori-i", + "torse-poil", + "torse-poils", + "tortue-alligator", + "tortue-boite", + "tortue-boîte", + "tortue-duc", + "tortues-alligators", + "tortues-boites", + "tortues-boîtes", + "tortues-ducs", + "tosa-inu", + "tote-bag", + "tote-bags", + "touch-and-go", + "touche-pipi", + "touche-touche", + "touille-boeuf", + "touille-boeufs", + "touille-bœuf", + "touille-bœufs", + "tour-minute", + "tour-opérateur", + "tour-opérateurs", + "tour-opératrice", + "tour-opératrices", + "tourneur-fraiseur", + "tourneurs-fraiseurs", + "tours-minute", + "tours-opérateurs", + "tours-opératrices", + "toxi-infectieux", + "toxi-infection", + "toxi-infections", + "toy-terrier", + "trace-bouche", + "trace-roulis", + "trace-sautereau", + "trace-vague", + "trachée-artère", + "trachélo-occipital", + "trachéo-bronchite", + "trachéo-bronchites", + "trade-union", + "trade-unionisme", + "trade-unionismes", + "trade-unions", + "tragi-comique", + "tragi-comiques", + "tragi-comédie", + "tragi-comédies", + "trains-trams", + "trait-d'union", + "trait-d'unioné", + "trait-track", + "tram-train", + "trams-trains", + "tran-tran", + "tranche-maçonné", + "tranche-montagne", + "tranche-montagnes", + "tranche-papier", + "tranche-tête", + "tranchées-abris", + "trench-coat", + "trench-coats", + "tribo-électricité", + "tribo-électricités", + "tribo-électrique", + "tribo-électriques", + "tribénuron-méthyle", + "tric-trac", + "tric-tracs", + "trichloro-nitrométhane", + "trichloro-trinitro-benzène", + "triflusulfuron-méthyle", + "trinexapac-éthyl", + "trinitro-cellulose", + "trinitro-celluloses", + "tripe-madame", + "triple-croche", + "triples-croches", + "trique-madame", + "tris-mal", + "tris-male", + "tris-males", + "tris-maux", + "trompe-cheval", + "trompe-couillon", + "trompe-l'oeil", + "trompe-l'œil", + "trompe-oreilles", + "trompe-valet", + "trop-bu", + "trop-payé", + "trop-payés", + "trop-perçu", + "trop-perçus", + "trop-plein", + "trop-pleins", + "trotte-chemin", + "trotte-menu", + "trouble-fête", + "trouble-fêtes", + "trousse-barre", + "trousse-barres", + "trousse-pet", + "trousse-pets", + "trousse-pied", + "trousse-pieds", + "trousse-pète", + "trousse-pètes", + "trousse-queue", + "trousse-queues", + "trousse-traits", + "très-chrétien", + "très-haut", + "tré-flip", + "tré-flips", + "tré-sept", + "trépan-benne", + "trépan-bennes", + "tsoin-tsoin", + "tsouin-tsouin", + "tss-tss", + "tsé-tsé", + "tsé-tsés", + "tta-kun", + "tta-kuns", + "ttun-ttun", + "ttun-ttuns", + "tu-tu-ban-ban", + "tubéro-infundibulaire", + "tubéro-infundibulaires", + "tue-brebis", + "tue-chien", + "tue-chiens", + "tue-diable", + "tue-diables", + "tue-l'amour", + "tue-loup", + "tue-loups", + "tue-mouche", + "tue-mouches", + "tue-poule", + "tue-teignes", + "tue-vent", + "tuniso-égypto-lybien", + "tupi-guarani", + "turbo-alternateur", + "turbo-alternateurs", + "turbo-capitalisme", + "turbo-capitalismes", + "turbo-compresseur", + "turbo-compresseurs", + "turbo-prof", + "turbo-profs", + "turco-coréen", + "turco-mongol", + "turco-persan", + "turco-syrien", + "turn-over", + "tutti-frutti", + "tux-zillertal", + "twin-set", + "twin-sets", + "tz'utujil", + "tâte-ferraille", + "tâte-poule", + "tâte-vin", + "tâte-vins", + "témoins-clés", + "téra-ampère", + "téra-ampères", + "téra-électron-volt", + "téra-électron-volts", + "térawatt-heure", + "térawatt-heures", + "térawatts-heures", + "téraélectron-volt", + "téraélectron-volts", + "tétra-atomique", + "tétrachloro-isophtalonitrile", + "tétrachlorodibenzo-p-dioxine", + "tétrachlorodibenzo-p-dioxines", + "tête-bleu", + "tête-bêche", + "tête-chèvre", + "tête-ronde", + "tête-verte", + "têtes-vertes", + "tôt-fait", + "tôt-faits", + "u-commerce", + "ukiyo-e", + "ukiyo-es", + "unda-maris", + "une-deux", + "uni-dimensionnel", + "uni-dimensionnelle", + "uni-dimensionnelles", + "uni-dimensionnels", + "uni-modal", + "uni-sonore", + "uni-sonores", + "unité-souris", + "unités-souris", + "univers-bloc", + "univers-île", + "univers-îles", + "upa-upa", + "urane-mica", + "uranes-micas", + "uro-génital", + "uro-génitale", + "uro-génitales", + "uro-génitaux", + "urétro-cystotomie", + "urétro-cystotomies", + "uto-aztèque", + "uto-aztèques", + "uva-ursi", + "uva-ursis", + "v'là", + "v'nir", + "v'nu", + "v's", + "va-de-la-gueule", + "va-nu-pieds", + "va-outre", + "va-t'en", + "va-t-en", + "va-t-en-guerre", + "va-te-laver", + "va-tout", + "vache-biche", + "vache-garou", + "vaches-biches", + "vaches-garous", + "vade-in-pace", + "vade-mecum", + "vaeakau-taumako", + "vaeakau-taumakos", + "vagino-vésical", + "vaine-pâture", + "valence-gramme", + "valence-grammes", + "valse-hésitation", + "valses-hésitations", + "vanity-case", + "vanity-cases", + "vas-y", + "vasculo-nerveux", + "vaterite-A", + "vaterite-As", + "veau-laq", + "veau-marin", + "velci-aller", + "venez-y-voir", + "ventre-madame", + "ver-coquin", + "verge-d'or", + "verges-d'or", + "vers-librisme", + "vers-librismes", + "vers-libriste", + "vers-libristes", + "vert-bois", + "vert-jaune", + "vert-monnier", + "vert-monniers", + "veston-cravate", + "vestons-cravates", + "vetula-domussien", + "vetula-domussienne", + "vetula-domussiennes", + "vetula-domussiens", + "vices-gouverneurs", + "victim-blaming", + "viens-poupoulerie", + "viens-poupouleries", + "vif-argent", + "vif-gage", + "vigne-blanche", + "vignes-blanches", + "village-rue", + "village-tas", + "villages-rue", + "villages-rues", + "villages-tas", + "villard-d'hérien", + "villard-d'hérienne", + "villard-d'hériennes", + "villard-d'hériens", + "villes-champignons", + "villes-clés", + "villes-provinces", + "villes-États", + "vino-benzoïque", + "vino-benzoïques", + "violet-évêque", + "viorne-tin", + "viornes-tin", + "vire-capot", + "vire-capots", + "vire-vire", + "visa-bourgien", + "visa-bourgienne", + "visa-bourgiennes", + "visa-bourgiens", + "visuo-spacial", + "visuo-spaciale", + "visuo-spaciales", + "visuo-spaciaux", + "vivaro-alpin", + "vivaro-alpins", + "vive-eau", + "vives-eaux", + "vivre-ensemble", + "voile-manteau", + "voile-manteaux", + "vois-tu", + "vol-bélier", + "vol-béliers", + "volley-ball", + "volley-balls", + "volt-ampère", + "volt-ampères", + "volte-face", + "volte-faces", + "vomito-negro", + "vomito-négro", + "vous-même", + "vous-mêmes", + "voyageur-kilomètre", + "voyageurs-kilomètres", + "voyez-vous", + "vrigne-meusien", + "vrigne-meusienne", + "vrigne-meusiennes", + "vrigne-meusiens", + "vu-arriver", + "végéto-sulfurique", + "vélo-rail", + "vélo-rails", + "vélo-taxi", + "vélo-école", + "vélo-écoles", + "vélos-taxis", + "vétéro-testamentaire", + "vétéro-testamentaires", + "w.-c.", + "wah-wah", + "walkie-talkie", + "walkies-talkies", + "wallon-cappelois", + "wallon-cappeloise", + "wallon-cappeloises", + "waray-waray", + "water-ballast", + "water-ballasts", + "water-closet", + "water-closets", + "water-polo", + "water-proof", + "water-proofs", + "wauthier-brainois", + "waux-hall", + "waux-halls", + "waza-ari", + "web-to-print", + "week-end", + "week-ends", + "wemaers-cappelois", + "wemaers-cappeloise", + "wemaers-cappeloises", + "wesh-wesh", + "west-cappelois", + "west-cappeloise", + "west-cappeloises", + "white-spirit", + "willy-willy", + "witsuwit'en", + "wuchiaping'ien", + "y'a", + "yacht-club", + "yacht-clubs", + "yin-yang", + "ylang-ylang", + "yo-yo", + "yocto-ohm", + "yocto-ohms", + "yotta-ampère", + "yotta-ampères", + "you-you", + "you-yous", + "young-ice", + "young-ices", + "yuki-onna", + "yuki-onnas", + "yé-yé", + "z'en", + "z'y", + "z'yeux", + "zapil's", + "zapil'ser", + "zayse-zergulla", + "zepto-ohm", + "zepto-ohms", + "zetta-ampère", + "zetta-ampères", + "zinc-blende", + "zinc-blendes", + "zizi-panpan", + "zones-clés", + "zoo-cinéma", + "zoo-cinémas", + "zoo-gymnaste", + "zoo-gymnastes", + "zulgo-gemzek", + "zuricho-montpelliérain", + "zuricho-montpelliéraine", + "zuricho-montpelliéraines", + "zuricho-montpelliérains", + "zy-va", + "zy-vas", + "zygomato-auriculaire", + "zygomato-labial", + "zygomato-maxillaire", + "zénith-secteur", + "zénith-secteurs", + "zéro-dimensionnel", + "Œuf-en-Ternois", + "Ében-Émael", + "Écalles-Alix", + "Écaussinnes-Lalaing", + "Écaussinnes-d'Enghien", + "Échenans-sous-Mont-Vaudois", + "Éclans-Nenon", + "Éclaron-Braucourt-Sainte-Livière", + "Éclusier-Vaux", + "École-Valentin", + "Écotay-l'Olme", + "Édouard-Josse", + "Église-Neuve-d'Issac", + "Église-Neuve-de-Vergt", + "Égliseneuve-d'Entraigues", + "Éguzon-Chantôme", + "Égée-Méridionale", + "Égée-Septentrionale", + "Éhein-bas", + "Éleu-dit-Leauwette", + "Élincourt-Sainte-Marguerite", + "Élise-Daucourt", + "Émilie-Romagne", + "Émilien-Romagnol", + "Émilienne-Romagnole", + "Émiliennes-Romagnoles", + "Émiliens-Romagnols", + "Énencourt-Léage", + "Épagne-Épagnette", + "Épaux-Bézu", + "Épi-Contois", + "Épi-Contoise", + "Épi-Contoises", + "Épiais-Rhus", + "Épinay-Champlâtreux", + "Épreville-près-le-Neubourg", + "Équatoria-Central", + "Équatoria-Occidental", + "Équatoria-Oriental", + "Équennes-Éramecourt", + "Équeurdreville-Hainneville", + "Équihen-Plage", + "Étang-Bertrand", + "Étang-Salé", + "Étang-Saléen", + "Étang-Saléenne", + "Étang-Saléennes", + "Étang-Saléens", + "Étang-Vergy", + "État-Major", + "État-nation", + "État-nounou", + "État-providence", + "États-Généraux", + "États-Majors", + "États-nounous", + "États-providence", + "Étinehem-Méricourt", + "Étival-Clairefontaine", + "Étival-lès-le-Mans", + "Étrelles-et-la-Montbleuse", + "Étricourt-Manancourt", + "Étricourt-Manancourtois", + "Étricourt-Manancourtoise", + "Étricourt-Manancourtoises", + "Étueffont-Bas", + "Évette-Salbert", + "Évin-Malmaison", + "Évry-Grégy-sur-Yerre", + "Évry-Petit-Bourg", + "Î.-P.-É.", + "Île-Bouchard", + "Île-Molène", + "Île-Rousse", + "Île-Tudiste", + "Île-Tudistes", + "Île-Tudy", + "Île-d'Aix", + "Île-d'Anticosti", + "Île-d'Arz", + "Île-d'Elle", + "Île-d'Houat", + "Île-d'Olonne", + "Île-d'Yeu", + "Île-du-Prince-Édouard", + "Îles-de-la-Madeleine", + "Îlo-Dionysien", + "Îlo-Dionysienne", + "Ölbronn-Dürrn", + "Übach-Palenberg", + "Ühlingen-Birkendorf", + "âme-sœur", + "âmes-sœurs", + "âne-zèbre", + "ânes-zèbres", + "ça-va-ça-vient", + "ça-voir", + "ça-voirs", + "çui-là", + "écart-type", + "écarts-types", + "écho-location", + "écho-locations", + "échos-radars", + "écorche-œil", + "écoute-s'il-pleut", + "écrase-merde", + "écrase-merdes", + "écurie-ménagerie", + "écuries-ménageries", + "église-halle", + "égypto-lybien", + "égypto-tchado-soudanais", + "éka-actinide", + "éka-actinides", + "éka-aluminium", + "éka-astate", + "éka-bismuth", + "éka-bore", + "éka-borium", + "éka-francium", + "éka-mercure", + "éka-plomb", + "éka-polonium", + "éka-prométhium", + "éka-silicium", + "électron-volt", + "électron-volts", + "élément-clé", + "éléments-clés", + "émetteur-récepteur", + "émetteur-récepteurs", + "émilien-romagnol", + "émilienne-romagnole", + "émiliennes-romagnoles", + "émiliens-romagnols", + "émission-débat", + "énargite-beta", + "énargite-betas", + "éoli-harpe", + "épargne-logement", + "épaulé-jeté", + "épaulés-jetés", + "épi-contois", + "épi-contoise", + "épi-contoises", + "épidote-gris", + "épinard-fraise", + "épine-fleurie", + "épine-vinette", + "épines-vinettes", + "épiplo-entérocèle", + "épiplo-ischiocèle", + "épiplo-mérocèle", + "épluche-légume", + "épluche-légumes", + "épuise-volante", + "épuises-volantes", + "équato-guinéen", + "équato-guinéenne", + "équato-guinéennes", + "équato-guinéens", + "éso-narthex", + "étalon-or", + "étang-saléen", + "étang-saléenne", + "étang-saléennes", + "étang-saléens", + "état-limite", + "état-major", + "états-civils", + "états-généraux", + "états-limites", + "états-majors", + "états-nations", + "étau-limeur", + "étaux-limeurs", + "éthane-1,2-diol", + "éthyl-benzène", + "éthéro-chloroforme", + "étouffe-chrétien", + "étouffe-chrétiens", + "étrangle-chat", + "étrangle-chien", + "étrangle-loup", + "étrangle-loups", + "étricourt-manancourtois", + "étricourt-manancourtoise", + "étricourt-manancourtoises", + "être-là", + "île-prison", + "île-tudiste", + "île-tudistes", + "île-État", + "îles-prisons", + "îles-États", + "îlo-dionysien", + "ôte-agrafes", + "über-célèbre", + "über-célèbres", + "Łutselk'e", + "œsophago-gastro-duodénoscopie", + "œsophago-gastro-duodénoscopies", + "œuf-coque", + "œufs-coque", + "α-D-glucofuranose", + "α-D-glucopyranose", + "α-D-ribofuranose", + "α-D-ribopyranose", + "α-Dahllite", + "α-Dahllites", + "α-L-ribofuranose", + "α-L-ribopyranose", + "β-D-glucofuranose", + "β-D-glucopyranose", + "β-D-ribofuranose", + "β-D-ribopyranose", + "β-Dahllite", + "β-Dahllites", + "β-L-ribofuranose", + "β-L-ribopyranose", + "β-galactosidase", + "β-lactamine", + "β-sitostérol", + "β-sitostérols", + "γ-Dahllite", + "γ-Dahllites", + "μ-métal", + "σ-additivité", + "σ-additivités", + "σ-compacité", + "σ-compact", + "σ-compacts", + "Abbie-Gaëlle", + "Abby-Gail", + "Aëlle-Lys", + "Agathe-Rose", + "Aimée-Alexandrine", + "Amy-Lee", + "Andrée-Ange", + "Anna-Elisa", + "Anna-Maria", + "Anna-Sophia", + "Anne-Aëlle", + "Anne-Aymone", + "Anne-Catherine", + "Anne-Cécile", + "Anne-Christelle", + "Anne-Claire", + "Anne-Clarisse", + "Anne-Clémence", + "Anne-Clotilde", + "Anne-Colombe", + "Anne-Eléonore", + "Anne-Elisabeth", + "Anne-Flore", + "Anne-Gabrielle", + "Anne-Gaëlle", + "Anne-Garance", + "Anne-Hélène", + "Anne-Hortense", + "Anne-Isabelle", + "Anne-Juliette", + "Anne-Laurence", + "Anne-Lise", + "Anne-Louise", + "Anne-Lucie", + "Anne-Lyse", + "Anne-Marceline", + "Anne-Marguerite", + "Anne-Mathilde", + "Anne-Pascale", + "Anne-Pénélope", + "Anne-Raphaëlle", + "Anne-Sixtine", + "Anne-Victoire", + "Anne-Yvonne", + "Annie-Claire", + "Annie-Claude", + "Annie-Christine", + "Annie-Flore", + "Annie-France", + "Annie-Françoise", + "Annie-Laure", + "Annie-Paule", + "Annie-Pierre", + "Annie-Rose", + "Annie-Thérèse", + "Audrey-Anne", + "Aure-Anne", + "Bérénice-Marie", + "Blanche-Castille", + "Brune-Hilde", + "Carla-Luna", + "Carla-Marie", + "Carole-Anne", + "Catherine-Amélie", + "Catherine-Josée", + "Catherine-Luce", + "Cécile-Liv", + "Cécilie-Anne", + "Edith-Marie", + "Eléa-Nore", + "Elée-Anne", + "Elisa-Maude", + "Elise-Marie", + "Elyse-Ea", + "Elysée-Anne", + "Emma-Line", + "Emma-Lou", + "Emma-Louise", + "Emma-Rose", + "Eugénie-Héloïse", + "Eva-Elle", + "Eva-Marie", + "Eve-Aëlle", + "Eve-Angéline", + "Eve-Rose", + "Eve-Charlotte", + "Eve-Marie", + "Gracie-Lou", + "Hélène-Sarah", + "Héloïse-Luce", + "Héloïse-Marie", + "Jeanne-Antide", + "Jeanne-Claire", + "Jeanne-Charlotte", + "Jeanne-Cécile", + "Jeanne-Colombe", + "Jeanne-Françoise", + "Jeanne-Hélène", + "Jeanne-Laure", + "Jeanne-Lise", + "Jeanne-Louise", + "Jeanne-Marie", + "Jeanne-Sixtine", + "Jeanne-Sophie", + "Julie-Anne", + "Julie-Maude", + "Julie-Michelle", + "Josée-Anne", + "Laure-Anne", + "Laure-Eléa", + "Laure-Élise", + "Laure-Lise", + "Laure-Lou", + "Laure-Marie", + "Laurie-Anne", + "Léa-Jade", + "Lily-Rose", + "Lisa-Marie", + "Lise-Anne", + "Liv-Helen", + "Lou-Anne", + "Lou-Eve", + "Louisa-May", + "Louise-Marie", + "Louise-Anne", + "Louise-Hélène", + "Maëlle-Anne", + "Maëlle-Lys", + "Maï-Lan", + "Marguerite-Marie", + "Mary-Beth", + "Mary-Lynn", + "Mathilde-Marie", + "Meg-Anne", + "Nathalie-Anne", + "Paule-Émeline", + "Paule-Marie", + "Pauline-Hortense", + "Pauline-Marie", + "Pénélope-Fiona", + "Rose-Anne", + "Rose-Hélène", + "Sarah-Anne", + "Sarah-Eve", + "Sarah-Jane", + "Sarah-Laure", + "Sarah-Line", + "Sarah-Lise", + "Sarah-Lou", + "Sarah-Louise", + "Sarah-Marie", + "Sarah-Myriam", + "Sixtine-Jade", + "Sophie-Anne", + "Sophie-Caroline", + "Sylvie-Anne", + "Valérie-Anne", + "Alain-Maxime", + "Alain-Michel", + "Alain-Pierre", + "Ambroise-Polycarpe", + "André-Ferdinand", + "André-Jean", + "André-Marie", + "Ange-Marie", + "Antoine-Daniel", + "Antoine-Marie", + "Auguste-Eugène", + "Benoît-Joseph", + "Bernard-Marie", + "Camille-Raphaël", + "Carl-Éric", + "Carl-Edwin", + "Carl-Philippe", + "Charles-Aubin", + "Charles-Eric", + "Charles-Edouard", + "Charles-Étienne", + "Charles-Henri", + "Charles-Marie", + "Charles-Olivier", + "Charles-Orland", + "Christian-Jacques", + "Claude-Henri", + "Claude-Pierre", + "Côme-Edouard", + "David-Frédéric", + "David-Vincent", + "Don-Yves", + "Eric-Antoine", + "Etienne-Henri", + "Félix-Antoine", + "Félix-Marie", + "François-Alexandre", + "François-Charles", + "François-David", + "François-Éric", + "François-Ferdinand", + "François-Guillaume", + "François-Henri", + "François-Jérôme", + "François-Joseph", + "François-Julien", + "François-Louis", + "François-Marie", + "François-Nicolas", + "François-Pierre", + "François-Régis", + "François-René", + "François-Xavier", + "Frédéric-François", + "Gonzague-Edouard", + "Guy-Marie", + "Guy-Noël", + "Henri-Brice", + "Henri-Claude", + "Henri-Jean", + "Henri-Jules", + "Henri-Louis", + "Henri-Michel", + "Henri-Paul", + "Henri-Pierre", + "Henri-Xavier", + "Ian-Aël", + "Jacques-Alexandre", + "Jacques-Antoine", + "Jacques-Édouard", + "Jacques-Étienne", + "Jacques-Henri", + "Jacques-Olivier", + "Jacques-Pierre", + "Jacques-Yves", + "Jérôme-André", + "Joël-Francis", + "Joseph-Antoine", + "Joseph-Désiré", + "Joseph-Marie", + "Jules-Edouard", + "Jules-Henri", + "Julien-Pierre", + "Léo-Paul", + "Loup-Léis", + "Luc-Henri", + "Marc-André", + "Marc-Antoine", + "Marc-Aurèle", + "Marc-Emmanuel", + "Marc-Olivier", + "Michael-Corneille", + "Paul-Albert", + "Paul-Alexandre", + "Paul-André", + "Paul-Antoine", + "Paul-Armand", + "Paul-Arthur", + "Paul-Elie", + "Paul-Eloi", + "Paul-Emile", + "Paul-Étienne", + "Paul-Henri", + "Paul-Louis", + "Paul-Loup", + "Paul-Marie", + "Paul-Saintange", + "Paul-Vincent", + "Philippe-Alexandre", + "Philippe-Auguste", + "Rémi-Etienne", + "René-Charles", + "René-Marc", + "René-Jean", + "René-Paul", + "René-Pierre", + "René-Yves", + "Thibaud-Marie", + "Vincent-Xavier", + "Xavier-Alexandre", + "Xavier-François", + "Xavier-Marie", + "Xavier-Pierre", + "Yann-Aël", + "Yann-Alrick", + "Yann-Éric", + "Yann-Gaël", + "Yann-Yves", + "Yann-Vari", + "Yves-Alain", + "Yves-Alexandre", + "Yves-André", + "Yves-Éric", + "Yves-Henri", + "Yves-Laurent", + "Yves-Marie", + "Yves-Michel", + "Yves-Olivier", + "Yves-Pierre", + "abdomino-génital", + "abdomino-thoracique", + "aberdeen-angus", + "abri-bus", + "abri-jardin", + "abris-bus", + "abris-jardins", + "acido-alcalinité", + "acido-butyrométrie", + "acido-cétone", + "acido-cétones", + "aconit-napel", + "acqua-tofana", + "adjudante-chef", + "adjudantes-chefs", + "adénosine-phosphate", + "affleureuse-défonceuse", + "affût-truck", + "affûts-trucks", + "agent-voyer", + "aide soignante", + "aide-cuisinier", + "aide-cuisinière", + "aide-jardinier", + "aide-maçon", + "aide-moniteur", + "aide-ménagère", + "aides monitrices", + "aides soignantes", + "aides-comptables", + "aides-cuisiniers", + "aides-jardiniers", + "aides-maçons", + "ajusteur-mécanicien", + "ajusteuse mécanicienne", + "ajusteuses-mécaniciennes", + "album-photos", + "album-souvenir", + "albums-photos", + "aldéhyde-alcool", + "alexie-agraphie", + "allocation-chômage", + "allocation-logement", + "allocation-maternité", + "allocations maternité", + "allocations-maternité", + "allume-cigarette", + "allume-cigarettes", + "allume-pipe", + "alsace-lorraine", + "altitude-cabine", + "alvéolo-dentaire", + "ami-adversaire", + "amiantes-ciment", + "amie-adversaire", + "amino-indole", + "amino-indoles", + "ampli-tuner", + "ampère-tour", + "ampères-tours", + "analyste-programmeuse", + "ancre-chaîne", + "ancres-chaînes", + "anesthésiste-réanimateur", + "anesthésistes-réanimateurs", + "animal-machine", + "antéro-inférieur", + "antéro-interne", + "apprenti-maçon", + "apprentis-maçons", + "appui-coude", + "appui-nuque", + "appuie-bras", + "appuie-nuque", + "appuis-livres", + "arbitre-rapporteur", + "arbitres rapporteurs", + "arbre-manivelle", + "arbre-témoin", + "arbres-manivelles", + "arch-tube", + "arch-tubes", + "architecte-conseil", + "architecte-urbaniste", + "architectes urbanistes", + "architectes-conseils", + "archiviste-paléographe", + "arcs formerets", + "argilo-sableux", + "argilo-sablonneux", + "armoire-bibliothèque", + "armoire-penderie", + "armoire-toile", + "armoires-penderies", + "arrache-bouchon", + "arrache-moyeu", + "arrache-racine", + "arrache-tube", + "arrache-tubes", + "arrache-étai", + "arrow-roots", + "artiste-peintre", + "artistes-peintres", + "aspirant-pilote", + "aspirantes-pilotes", + "aspirants-pilotes", + "assistance-record", + "associé-gérant", + "associés gérants", + "associés-gérants", + "assurance-auto", + "assurance-crédit", + "assurance-décès", + "assurance-invalidité", + "assurance-maladie", + "assurance-maternité", + "assurance-qualité", + "assurance-vieillesse", + "assurances-auto", + "assurances-décès", + "atelier-théâtre", + "atome-gramme", + "atomes-grammes", + "attache-lettre", + "attaque-suicide", + "attaque-surprise", + "attaque-éclair", + "attaques surprises", + "attaques-surprises", + "attrape-con", + "auriculo-cardiaque", + "auriculo-temporal", + "austro-bavarois", + "auteur-interprète", + "auteurs-interprètes", + "autos tamponneuses", + "avance-pétard", + "avions-canons", + "avions-citernes", + "avions-espions", + "avions-météos", + "avions-radars", + "avions-stops", + "avions-taxis", + "avions-éclaireurs", + "aviso-escorteur", + "avisos escorteurs", + "avocat-conseil", + "avocats-conseils", + "baba-cool", + "babas-cool", + "baby-booms", + "baby-test", + "baby-tests", + "bad-lands", + "bague-agrafe", + "bague-allonge", + "bagues-agrafes", + "baguette-levier", + "baguettes-leviers", + "baise-pied", + "baladeur-radio", + "balai-éponge", + "balais-éponges", + "balance-étalon", + "balancier-compensateur", + "balancier-spiral", + "balanciers-spiraux", + "balcon-terrasse", + "balcons-terrasses", + "ballet-pantomime", + "ballet-théâtre", + "ballon-pilote", + "bancs-balances", + "bancs-titres", + "bande-amorce", + "bande-culotte", + "bande-image", + "bande-témoin", + "bande-étalon", + "bandes-amorces", + "bandes-culottes", + "bandes-son", + "bandes-étalons", + "bang-lang", + "bang-langs", + "banlieue-dortoir", + "banlieues-dortoirs", + "banquette-coffre", + "banquettes-coffres", + "bar-buffet", + "bar-mitsva", + "bar-mode", + "bar-restaurant", + "baratte-malaxeur", + "barattes-malaxeurs", + "barrage-coupole", + "barrage-gravité", + "barrage-réservoir", + "barrage-voûte", + "barrages-coupoles", + "barrages-réservoirs", + "barren-grounds", + "bars-mode", + "basket-balleur", + "basket-balleuses", + "bataillon-école", + "bataillons-écoles", + "batterie-tampon", + "batteur-dégraisseur", + "batteur-mélangeur", + "batteurs-mélangeurs", + "battle-dress", + "beach-boy", + "beach-boys", + "beach-soccer", + "becs d'âne", + "becs-croisés", + "becs-noueurs", + "bel-homme", + "belgo-luxembourgeois", + "benne-drague", + "bennes-dragues", + "bip-bip", + "bip-bips", + "bloc-correspondance", + "bloc-cylindre", + "bloc-diagramme", + "bloc-évier", + "blocs-correspondances", + "blocs-diagrammes", + "blocs-moteurs", + "blocs-éviers", + "boeuf-mode", + "bossa-nova", + "bossas-novas", + "bottier-chausseur", + "bottière-chausseuse", + "bouche-bouteille", + "bouche-bouteilles", + "boucherie-charcuterie", + "bouchon-allumeur", + "bouchons allumeurs", + "bouchons-allumeurs", + "boulanger-pâtissier", + "boulangère-pâtissière", + "bout-filtre", + "bouton-boule", + "bouts-filtres", + "bouée-culotte", + "bouées-culottes", + "boy-friend", + "boy-scoutisme", + "brain-trust", + "branle-queues", + "briquet-tempête", + "brise-balle", + "brise-béton", + "brise-coeur", + "brise-ménage", + "brise-tourteau", + "brise-tourteaux", + "brise-vue", + "broche-queue", + "broderie-application", + "bronchiolo-alvéolaire", + "bronchiolo-alvéolite", + "bronchiolo-alvéolites", + "broncho-pneumopathie", + "broncho-pneumopathies", + "brûle-bouts", + "bucco-nasal", + "bucco-pharyngien", + "buenos-aires", + "bulletin-réponse", + "bureau-commode", + "bureau-ministre", + "bus-bibliothèque", + "butte-témoin", + "bâtonnet-registre", + "bâtonnets-registres", + "bébé-cadum", + "bébés-cadum", + "bécasseau-maubèche", + "bécasseau-sanderling", + "bécasseau-échasse", + "bécasseaux-sanderlings", + "cabinet-conseil", + "cadeau-surprise", + "café-brasserie", + "café-hôtel", + "café-restaurant", + "café-tabac", + "cafés-hôtels", + "cafés-restaurants", + "caissier-comptable", + "caissiers-comptables", + "cale-porte", + "cale-pot", + "cale-pots", + "camion-grue", + "camion-navette", + "caméra-stylo", + "caméras-stylos", + "canal-tunnel", + "canaux-tunnels", + "canne-parapluie", + "canne-siège", + "cannes-parapluies", + "capitaine-expert", + "capital-décès", + "carcino-embryonnaire", + "cardio-auditif", + "cardio-circulatoire", + "cardio-respiratoire", + "cardio-rénal", + "cardio-tubérositaire", + "carnet-bloc", + "carnets-blocs", + "carolo-macérien", + "carte-crédit", + "cartes-crédit", + "carton-amiante", + "carton-cuir", + "cartons-cuirs", + "cartons-pierres", + "case-work", + "ceinture-piège", + "centre-africain", + "centre-européen", + "centres-tirs", + "chaise-lit", + "chaland-citerne", + "chalet-refuge", + "chalets-refuges", + "champagne-ardenne", + "champs-contrechamps", + "chapeau-claque", + "chapeau-cloche", + "chapeau-melon", + "chapeaux-claques", + "chariot-grue", + "chariots-grues", + "check-list", + "check-lists", + "chef-barre", + "chefs-barres", + "cheiro-oral", + "chemise-culotte", + "cherche-fil", + "cherche-pôles", + "chevaux-jupons", + "chez-nous", + "chez-toi", + "chien-dauphin", + "chien-lion", + "chiens-lions", + "chirurgien-barbier", + "chirurgien-major", + "chirurgiennes-majors", + "chirurgiens-barbiers", + "choke-bore", + "choke-bores", + "choréo-athétosique", + "choux-turneps", + "chris-craft", + "chrome-nickel", + "chrome-tungstène", + "chromes-tungstène", + "chrétien-démocrate", + "cité-jardin", + "coton-cordonnet", + "coton-graine", + "cotons-graines", + "cou-coupé", + "cou-rouge", + "cou-tors", + "couchage-couette", + "couchages-couette", + "country-music", + "courroie-guide", + "course-croisière", + "course-trame", + "courses-trames", + "courts métrages", + "cous rouges", + "cous-coupés", + "cous-tors", + "couteau-hélice", + "couteau-poignard", + "couteaux-poignards", + "cover-boy", + "cover-boys", + "cover-crop", + "cow-girl", + "cow-girls", + "crayon-lèvre", + "cro-magnon", + "croix-rouge", + "croyez-en", + "crève-tonneau", + "crédit-fournisseur", + "crédit-vacance", + "crétino-goitreux", + "cueille-fleurs", + "cul-béni", + "culs bénis", + "culs de bouteille", + "culs de bus", + "culs-bruns", + "cultivateur-herse", + "cure-casserole", + "cure-môles", + "cut-back", + "cylindre-sceau", + "cylindre-équerre", + "cylindres-équerres", + "cylindro-ogival", + "cyto-hormonal", + "câble-chaîne", + "céleri-navet", + "céleris-navets", + "d'arrache-pied", + "d'avant-garde", + "dame-blanche", + "dame-honteuse", + "dames-blanches", + "descente-homme", + "dessinateur-cartographe", + "dessinateurs-cartographes", + "diesel-hydraulique", + "diesels-hydrauliques", + "discours-choc", + "discours-fleuve", + "discours-fleuves", + "discussion-marathon", + "disons-le", + "disque-jockey", + "disque-lotion", + "disques-lotion", + "divan-coffre", + "doigt-indicateur", + "doigts-indicateurs", + "dom-tom", + "dorso-palatal", + "doubles fonds", + "dry-farming", + "dynamo-démarreur", + "dynamos-démarreur", + "débat-fleuve", + "décor-praticable", + "décret-loi", + "décrets-lois", + "défaut-congé", + "défauts-congés", + "défense-recours", + "dépannage-remorquage", + "dîner-débat", + "dîners-débats", + "eau-mère", + "eaux-mères", + "ecce-homo", + "effet-boomerang", + "en-soi", + "encaisse-or", + "encaisses-or", + "endo-urétral", + "endocrino-musculaire", + "entreprise-pilote", + "entreprises-pilotes", + "entéro-hépatique", + "espace-enfant", + "espaces-enfants", + "essuie-vitre", + "essuie-vitres", + "etats-généraux", + "euro-emprunt", + "euro-emprunts", + "excito-moteur", + "experts-comptables", + "facteur-chance", + "faim-calle", + "faits divers", + "fanion-signal", + "fauteuil-brouette", + "fauteuil-vase", + "fauteuils-brouettes", + "faux culs", + "feld maréchaux", + "feld-maréchal", + "femme-agent", + "femme-grenouille", + "ferme-auberge", + "fermes-auberges", + "ferro-alliage", + "ferro-alliages", + "fesse-maille", + "film-pack", + "films-packs", + "filon-couche", + "filons-couches", + "filtre-bouchon", + "fish-eye", + "fish-eyes", + "fixe-bouchon", + "fixe-bouchons", + "fixe-serviette", + "fixe-tube", + "fixe-tubes", + "flock-book", + "flock-books", + "fluvio-deltaïque", + "fluvio-lacustre", + "fluvio-maritime", + "folliculo-stimulant", + "fourgon-pompe", + "franche-comté", + "free-jazz", + "free-martinisme", + "frette-tourillon", + "frettes-tourillons", + "frics-fracs", + "frise-beurre", + "fuso-spirillaire", + "fuso-spirillaires", + "fusée-détonateur", + "fusée-parachute", + "fusées-parachutes", + "fémoro-cutané", + "gage-contre", + "gage-pleige", + "gages-pleiges", + "gagne-denier", + "gagne-deniers", + "gaine-combinaison", + "gaines-combinaisons", + "gardes champêtres", + "gardien-chef", + "gardiens-chefs", + "garçon-boucher", + "gentleman-cambrioleur", + "gentleman-farmer", + "gentlemen farmers", + "germano-britannique", + "germano-soviétique", + "gin-rami", + "gin-tonic", + "gin-tonics", + "ginger-beer", + "ginger-beers", + "gingivo-stomatite", + "girl-friend", + "girl-scout", + "girl-scouts", + "glosso-labié", + "glucido-lipidique", + "glucido-protidique", + "glycéro-phospho-amino-lipide", + "glycéro-phospho-amino-lipides", + "gobe-baleine", + "gobe-lune", + "gobe-or", + "gobe-spectre", + "gold-point", + "gomme-ammoniaque", + "gomme-ester", + "gomme-laque", + "gommes-ammoniaques", + "gommes-laques", + "goûte-vin", + "grainier-fleuriste", + "grainières-fleuristes", + "gram-négatif", + "gram-positif", + "gramme-poids", + "grammes-poids", + "grec-melkite", + "grecque-melkite", + "grecques-melkites", + "grecs melkites", + "grecs-melkites", + "grill-express", + "grill-room", + "grille-marron", + "grille-écran", + "grippe-coquin", + "groupe-cible", + "gueuze-lambic", + "gueuzes-lambics", + "guide-coke", + "guide-greffe", + "guide-greffes", + "guide-ligne", + "guide-lime", + "guide-limes", + "guide-poil", + "guinée-bissau", + "guinée-equatoriale", + "guttas-perchas", + "gâte-cuir", + "gâte-filles", + "gâte-maison", + "génito-spinal", + "géranium-lierre", + "géraniums-lierres", + "hache-sarments", + "hale-crocs", + "hale-haut", + "hand-ball", + "hand-balleur", + "hand-balleuse", + "happe-bourse", + "happe-chairs", + "happe-lopin", + "happe-lopins", + "hard-bop", + "hard-core", + "hard-rock", + "hard-rocks", + "hard-top", + "hauts fourneaux", + "heure-lampe", + "heures-lampes", + "hexacoralliaire-madréporaire", + "hexacoralliaires-madréporaires", + "hispano-mexicain", + "histo-incompatible", + "hoche-queues", + "home-studio", + "homme-caoutchouc", + "homme-dieu", + "homme-oiseau", + "homme-singe", + "hommes-oiseaux", + "hormono-dépendant", + "hors-bilan", + "hors-caste", + "hors-cote", + "hors-marché", + "horse-pox", + "hospitalo-universitaire", + "hydro-injecteur", + "hydro-électrolytique", + "hypo-iodite", + "hypothalamo-hypophysaire", + "hystéro-épileptique", + "hépato-entérostomie", + "hépato-toxique", + "hépato-vésiculaire", + "hôtel-restaurant", + "hôtels-restaurants", + "ice-boat", + "ictéro-ascitique", + "ilang-ilang", + "ilangs-ilangs", + "ilio-fémoral", + "ilio-lombaire", + "iléo-caecal", + "iléo-colite", + "iléo-colostomie", + "iléo-colostomies", + "iléo-iléostomie", + "iléo-transversostomie", + "immuno-effecteur", + "immuno-enzymatique", + "immuno-inhibiteur", + "impulsion-obsession", + "indicateur-jauge", + "industrie-clef", + "industrie-clé", + "industries-clefs", + "inféro-postérieur", + "instruction-machine", + "instructions-machine", + "interro-emphatique", + "invitation-prestige", + "iodo-ioduré", + "iodo-organique", + "ion-gramme", + "iso-immunisation", + "israélo-arabe", + "israélo-libanais", + "israélo-palestinien", + "israélo-syrien", + "j'm'en-foutisme", + "jam-session", + "jazz-band", + "jet-society", + "jeu-concours", + "jeux-concours", + "jin-seng", + "joule-seconde", + "joules-seconde", + "judéo-français", + "judéo-françaises", + "judéo-marxiste", + "juge-arbitre", + "juge-consul", + "jumbo-jet", + "jumbo-jets", + "juré-vendeur", + "jurées-venderesses", + "kilo-octet", + "kilo-octets", + "kilowatt-heure", + "knock-down", + "know-how", + "l'après-midi", + "l'avant-veille", + "laisser-passer", + "lampe-projecteur", + "lampe-torche", + "lampes-projecteurs", + "lampes-torches", + "lance-amarre", + "lance-engins", + "lance-grenades-sous-marines", + "lance-harpon", + "lance-harpons", + "lance-poudre", + "langue-d'agneau", + "langues-d'agneau", + "laurier-camphrier", + "laurier-tulipier", + "lauriers camphriers", + "lauriers tulipiers", + "lauriers-camphriers", + "lauriers-tulipiers", + "lave-dos", + "lave-pieds", + "lave-tonneau", + "lave-tonneaux", + "lave-vitre", + "lawn-tennis", + "lemon-grass", + "lexico-grammatical", + "liberty-ship", + "liberty-ships", + "libres échangistes", + "libéro-ligneux", + "ligne-bloc", + "link-trainer", + "link-trainers", + "lipo-albuminique", + "lipo-nucléoprotéique", + "lit-bateau", + "lits-bateaux", + "livre-album", + "livres-albums", + "livret-portefeuille", + "livrets-portefeuilles", + "loco-régional", + "lofing-match", + "logement-foyer", + "logico-mathématique", + "lois-cadres", + "long-bois", + "longs métrages", + "longs-bois", + "lord-protecteur", + "lords-lieutenants", + "lumen-heure", + "luo-test", + "luo-tests", + "lupo-érythémato-viscérite", + "lupo-érythémato-viscérites", + "luso-brésilien", + "lyre-cithare", + "lève-cadre", + "lève-ligne", + "lève-lignes", + "lève-palette", + "lève-palettes", + "maculo-papuleux", + "macédo-roumain", + "magasin-souvenir", + "magasins-souvenirs", + "magico-religieux", + "mail-coaches", + "main-éponge", + "mains-éponges", + "maison-blanche", + "major-général", + "malle-cabine", + "malles-cabines", + "mandat-contribution", + "mange-mil", + "mao-spontex", + "marchand-diplomate", + "marchand-drapier", + "marchand-lunetier", + "marchande-drapière", + "marchandes-lunetières", + "marchands-diplomates", + "mass-media", + "massicotage-rognage", + "match-play", + "match-plays", + "math-élem", + "maths-sup", + "maître-queux", + "maîtres-queux", + "medicine-man", + "melting-pot", + "menu-gros", + "menu-vair", + "menus-vairs", + "messieurs-dames", + "mesure-choc", + "mieux-faisant", + "mieux-faisants", + "mille-trous", + "minéralier-pétrolier", + "minéraliers-pétroliers", + "miou-miou", + "mission-suicide", + "monte-ressort", + "monte-sac", + "monte-sacs", + "monteur-ajusteur", + "monteuses-ajusteuses", + "morpho-éco-éthologique", + "mortier-éprouvette", + "mortiers-éprouvettes", + "moteur-canon", + "motor-home", + "motor-homes", + "muco-purulent", + "mur-écran", + "murs-écrans", + "musculo-cartilagineux", + "musculo-membraneux", + "musculo-tendineux", + "mât-grue", + "mât-tour", + "mâts-grue", + "mère-poule", + "mères-poules", + "mètre-kilogramme", + "mètre-newton", + "mètre-semaine", + "mètres-newton", + "mécanicien-dentiste", + "mécaniciennes-dentistes", + "médecin-capitaine", + "médecin-chef", + "médecin-colonel", + "médecin-commandant", + "médecin-conseil", + "médecin-lieutenant", + "médecin-spécialiste", + "médecins-capitaines", + "médecins-chefs", + "médecins-colonels", + "médecins-conseils", + "médullo-surrénalome", + "méga-uretère", + "méga-uretères", + "mélange-maître", + "mélangeur-décanteur", + "mélano-indien", + "mélano-indiens", + "méningo-encéphalites", + "méningo-myélite", + "méthyl-mercure", + "métreur-vérificateur", + "métreuses-vérificatrices", + "narco-analyste", + "narco-dollar", + "narco-dollars", + "navire-atelier", + "navire-base", + "navire-hôpital", + "navire-jumeau", + "navire-major", + "navires-bases", + "navires-majors", + "negro spirituals", + "new-look", + "nitrate-fuel", + "nitrates-fuels", + "nodo-hissien", + "non alignée", + "non alignés", + "non combattante", + "non combattantes", + "non combattants", + "non comparants", + "non engagées", + "non engagés", + "non inscrite", + "non inscrites", + "non inscrits", + "non résidente", + "non résidents", + "non violente", + "non violents", + "noo-analeptique", + "norme-plancher", + "normes-planchers", + "nouvelle-ecosse", + "nouvelle-guinée", + "négro-africain", + "négro-américain", + "objet-souvenir", + "obsession-impulsion", + "obstétricien-accoucheur", + "obstétriciennes-accoucheuses", + "occipito-pubien", + "oculo-cardiaque", + "oculo-palpébral", + "oesophago-salivaire", + "oestro-progestatif", + "officier-conseil", + "officier-marinier", + "officiers-conseils", + "officiers-mariniers", + "oiseau-boucher", + "oiseau-fleur", + "oiseau-ombrelle", + "oiseau-satin", + "oiseau-serpent", + "oiseau-souris", + "oiseau-tempête", + "oiseaux-bouchers", + "oiseaux-satin", + "oiseaux-souris", + "oméga-complétude", + "oméga-inconsistance", + "opto-électronique", + "opéra-ballet", + "opéra-bouffe", + "opéra-oratorio", + "opéras-ballets", + "opéras-oratorios", + "oreille-d'homme", + "oreille-d'éléphant", + "oreilles d'âne", + "oreilles de judas", + "oreilles de loup", + "oreilles-d'homme", + "oreilles-d'éléphant", + "orfèvre-bijoutier", + "oro-hydrographique", + "orthopédiste-bandagiste", + "ostéo-articulaire", + "ostéo-dermopathique", + "ostéo-tendineux", + "oto-rhino-laryngologue", + "ouvrier-façonnier", + "ouvriers-façonniers", + "ouvrière-façonnière", + "ouvrières-façonnières", + "palma-christi", + "paléo-anthropologie", + "paléo-chrétien", + "pan-africain", + "pan-allemand", + "pan-américain", + "panneau-façade", + "par-terre", + "partie-cycle", + "pascals-seconde", + "passé-maître", + "patte-nageoire", + "pattes-nageoires", + "peigne-zizis", + "peintre-cartonnier", + "peintre-graveur", + "peintre-pistoleur", + "peintre-verrier", + "peintres graveurs", + "peintres pistoleurs", + "peintres-graveurs", + "peintres-verriers", + "permanentiste-coloriste", + "permanentistes-coloristes", + "permo-carbonifère", + "permo-trias", + "petits fours", + "ph-métrie", + "philosophico-religieux", + "philosophico-social", + "photos-robots", + "phéro-hormone", + "phéro-hormones", + "pie-mérien", + "pieds de nez", + "pigeon-hirondelle", + "pillow-lava", + "pilote-major", + "pilotes-majors", + "pince-cul", + "pince-culs", + "pince-fil", + "pince-jupe", + "pince-jupes", + "pince-pantalon", + "pince-étoffe", + "piper-cub", + "piper-cubs", + "plat-joint", + "plat-pont", + "plate-cuve", + "plate-tombe", + "plates-cuves", + "plats-joints", + "plein-jeu", + "plein-temps", + "plein-vent", + "pleine-croix", + "pleins-vents", + "pleuropneumonia-like", + "pli-faille", + "plombier-zingueur", + "plombiers-zingueurs", + "pluri-disciplinaire", + "plus-bas", + "plus-haut", + "plus-pétiteur", + "plus-pétiteurs", + "pluvio-nival", + "pneumo-choc", + "poche-poitrine", + "pochette-disco", + "pochettes-disco", + "poids-coq", + "poids-mouche", + "poignée-bandoulière", + "pole-position", + "politique-fiction", + "poll-tax", + "polypointe-onde", + "polypointes-ondes", + "polyuro-polydipsique", + "pomme-cannelle", + "ponton-grue", + "ponts-aqueducs", + "ponts-routes", + "pop-corns", + "portrait-type", + "pour-soi", + "premier-mai", + "première-née", + "presse-bouchons", + "presse-bouton", + "presse-garniture", + "presse-garnitures", + "presse-viande", + "prie-dieu", + "prix-étalon", + "prix-étalons", + "proche-orient", + "proche-oriental", + "produit-maison", + "produits-maison", + "propane-nitrile", + "prospecteur-placier", + "prospecteurs-placiers", + "prospectrices-placières", + "provinces-unies", + "publi-information", + "public-relations", + "punctum-maximum", + "pyélo-urétéral", + "pèse-jus", + "pèse-vin", + "pénicillino-résistant", + "queues de cheval", + "queues de pie", + "queues de poisson", + "queues-fourchues", + "quinze-août", + "quôc-ngu", + "raboteuse-dégauchisseuse", + "raboteuses-dégauchisseuses", + "racer-racer", + "racers-racers", + "racle-denier", + "racle-deniers", + "radiculo-cordonal", + "ramasse-plis", + "ramasseuse-dépanouilleuse", + "ramasseuse-égreneuse", + "ramasseuses-dépanouilleuses", + "recto-urétral", + "reine-mère", + "relève-rail", + "repose-bras", + "rhino-bronchite", + "rhino-pneumonie", + "rhino-pneumonies", + "roche-réservoir", + "roches-réservoirs", + "roi-pontife", + "roi-prêtre", + "roi-soleil", + "romans-photo", + "rose-cran", + "roseau-massue", + "roseaux-massues", + "roses-crans", + "russo-japonais", + "réduction-couple", + "réductions-couple", + "réfrigérateur-congélateur", + "régisseur-comptable", + "régisseurs-comptables", + "régisseuse-comptable", + "régisseuses-comptables", + "réticulo-endothélial", + "réticulo-endothéliose", + "réticulo-endothélioses", + "réticulo-filamenteux", + "réticulo-péritonite", + "révérend-père", + "rôle-titre", + "rôles-titres", + "sabre-baïonnette", + "sabre-poignard", + "sabres-baïonnettes", + "sacro-coccygien", + "sacro-coxalgie", + "sacro-coxalgies", + "sacro-iléite", + "sacro-sciatique", + "safari-photo", + "saisir-engager", + "salon-bibliothèque", + "salon-boudoir", + "salon-fumoir", + "salons-bibliothèques", + "sang-bleu", + "sang-dragon", + "sapeur-aérostier", + "sapeur-mineur", + "sapeurs-aérostiers", + "scapulo-humoral", + "scapulo-thoracique", + "sceau-matrice", + "sceaux-matrices", + "scotch-terrier", + "scout-car", + "script-girl", + "sea-line", + "sea-lines", + "secret-défense", + "secrets-défense", + "secrétaire-greffier", + "secrétaires-greffiers", + "secrétariat-greffe", + "secrétariats-greffes", + "secteur-clé", + "secteurs-clés", + "self-acting", + "self-actings", + "self-excitation", + "self-induction", + "self-trimmer", + "self-trimming", + "self-trimmings", + "sellerie-bourrellerie", + "sellerie-maroquinerie", + "selleries-maroquineries", + "sellier-bourrelier", + "sellier-maroquinier", + "selliers-maroquiniers", + "sensitivo-moteur", + "sensitivo-sensitif", + "sensitivo-sensoriel", + "sensori-tonique", + "shadow-boxing", + "shake-hand", + "shake-hands", + "show-room", + "show-rooms", + "shuttle-car", + "silico-alumineux", + "six-chiens", + "siège-auto", + "sièges-autos", + "skinn-effect", + "skip-cage", + "skip-cages", + "skye-terrier", + "sociaux-chrétiens", + "somato-moteur", + "somato-sensible", + "somato-sensitif", + "soude-sac", + "soude-sacs", + "sparring-partners", + "spino-cellulaire", + "spino-cérébelleux", + "spino-thalamique", + "sri-lanka", + "star-system", + "station-wagon", + "stations-wagons", + "stato-fusée", + "statu-quo", + "steam-cracking", + "steam-crackings", + "steeple-chases", + "stimulus-signe", + "stimulus-signes", + "street-dancer", + "street-dancers", + "strip-line", + "strip-lines", + "stylo-hyoïdien", + "stylo-mastoïdien", + "stylo-pharyngien", + "stylos-bille", + "sténo-ionique", + "stéréo-isomérie", + "suméro-babylonien", + "supra-conducteur", + "supra-national", + "supra-occipital", + "supra-optique", + "surdi-cécité", + "surf-casting", + "surprise-party", + "sylvo-cynégétique", + "syro-libanais", + "syro-malabare", + "syro-mésopotamien", + "système-expert", + "systèmes experts", + "systèmes-experts", + "séance-marathon", + "séro-immunologique", + "table-valise", + "tables-valises", + "tailleur-couturier", + "tailleur-pantalon", + "tailleurs-couturiers", + "talk-shows", + "tampon-dateur", + "tampons-dateurs", + "tangara-hirondelle", + "tangaras-hirondelles", + "tape-marteau", + "tape-marteaux", + "tapis-brosses", + "tarte-maison", + "tartes-maison", + "technico-commercial", + "technico-commerciale", + "technico-commerciaux", + "technico-scientifique", + "techno-industriel", + "teddy-bear", + "teddy-bears", + "temporo-buccal", + "temporo-massétérin", + "temporo-mastoïdien", + "temporo-pariétal", + "tennis-barbe", + "tennis-elbow", + "tentes-abris", + "test-minute", + "tests-minute", + "thalamo-cortical", + "thalamo-strié", + "thio-indigo", + "thio-uracile", + "thio-uraciles", + "thoraco-phéno-laparotomie", + "thrombo-angéite", + "thrombo-angéites", + "thrombo-élastogramme", + "thyro-aryténoïdien", + "thyro-hyoïdien", + "tibio-péronier", + "tibio-tarsien", + "tige-poussoir", + "tiroir-lit", + "tiroirs-lits", + "titre-ligne", + "titres-lignes", + "toiture-terrasse", + "toitures-terrasses", + "torche-pot", + "tord-fil", + "tord-fils", + "tour-operator", + "tourelle-balise", + "tourelles-balises", + "trachéo-bronchique", + "trachéo-laryngotomie", + "trains-ferries", + "tranche-gazon", + "tranche-lard", + "tranche-lards", + "trotte-bébé", + "trouble-paix", + "trousse-galant", + "trousse-galants", + "trésorier-payeur", + "trésoriers-payeurs", + "trésorière-payeuse", + "trésorières-payeuses", + "tube-canon", + "turco-mongoles", + "turco-tatar", + "turco-égyptien", + "tétra-iodofluorescéine", + "tétras-lyre", + "urico-inhibiteur", + "urétéro-vésical", + "valences-grammes", + "vente-réclame", + "venture-capital", + "version-lit", + "vertébro-basilaire", + "vertébro-vertébral", + "veste-fantaisie", + "vestes-fantaisie", + "vestibulo-cochléaire", + "vive-arête", + "viêt-minh", + "viêt-nam", + "voiturier-transporteur", + "voiturières-transporteuses", + "vomi-purgatif", + "vomis-purgatifs", + "voyage-marathon", + "voyageur-placier", + "voyageuse-placière", + "vraquier-pétrolier", + "vraquiers-pétroliers", + "vulgum-pecus", + "vélo-palatin", + "vérificateur-comptable", + "walk-over", + "water-drive", + "water-flooding", + "water-floodings", + "water-mangle", + "water-polos", + "watt-heure", + "watts-heures", + "ylangs-ylangs", + "échangeur-diffuseur", + "échantillon-type", + "échantillon-témoin", + "échantillons-types", + "échelle-observatoire", + "églises-halles", + "égypto-araméen", + "égypto-israélien", + "élève-brigadier", + "élève-caporal", + "élève-caporale", + "élève-maître", + "élèves-brigadières", + "élèves-maîtres", + "élément-bar", + "éléments-bars", + "émetteurs-récepteurs", + "épicier-droguiste", + "épée-baïonnette", + "étalon-deutschmark", + "état-membre", + "état-tampon", + "états-membres", + "évite-molettes", +] diff --git a/spacy/lang/fr/examples.py b/spacy/lang/fr/examples.py new file mode 100644 index 0000000..759de56 --- /dev/null +++ b/spacy/lang/fr/examples.py @@ -0,0 +1,21 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.fr.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars", + "Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs", + "San Francisco envisage d'interdire les robots coursiers sur les trottoirs", + "Londres est une grande ville du Royaume-Uni", + "L’Italie choisit ArcelorMittal pour reprendre la plus grande aciérie d’Europe", + "Apple lance HomePod parce qu'il se sent menacé par l'Echo d'Amazon", + "La France ne devrait pas manquer d'électricité cet été, même en cas de canicule", + "Nouvelles attaques de Trump contre le maire de Londres", + "Où es-tu ?", + "Qui est le président de la France ?", + "Où est la capitale des États-Unis ?", + "Quand est né Barack Obama ?", +] diff --git a/spacy/lang/fr/lemmatizer.py b/spacy/lang/fr/lemmatizer.py new file mode 100644 index 0000000..a7cbe0b --- /dev/null +++ b/spacy/lang/fr/lemmatizer.py @@ -0,0 +1,87 @@ +from typing import List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class FrenchLemmatizer(Lemmatizer): + """ + French language lemmatizer applies the default rule based lemmatization + procedure with some modifications for better French language support. + + The parts of speech 'ADV', 'PRON', 'DET', 'ADP' and 'AUX' are added to use + the rule-based lemmatization. As a last resort, the lemmatizer checks in + the lookup table. + """ + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "rule": + required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"] + return (required, []) + else: + return super().get_lookups_config(mode) + + def rule_lemmatize(self, token: Token) -> List[str]: + cache_key = (token.orth, token.pos) + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + univ_pos = token.pos_.lower() + if univ_pos in ("", "eol", "space"): + return [string.lower()] + elif "lemma_rules" not in self.lookups or univ_pos not in ( + "noun", + "verb", + "adj", + "adp", + "adv", + "aux", + "cconj", + "det", + "pron", + "punct", + "sconj", + ): + return self.lookup_lemmatize(token) + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + lookup_table = self.lookups.get_table("lemma_lookup", {}) + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, []) + string = string.lower() + forms = [] + # first try lookup in table based on upos + if string in index: + forms.append(string) + self.cache[cache_key] = forms + return forms + + # then add anything in the exceptions table + forms.extend(exceptions.get(string, [])) + + # if nothing found yet, use the rules + oov_forms = [] + if not forms: + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + pass + elif form in index or not form.isalpha(): + forms.append(form) + else: + oov_forms.append(form) + + # if still nothing, add the oov forms from rules + if not forms: + forms.extend(oov_forms) + + # use lookups, which fall back to the token itself + if not forms: + forms.append(lookup_table.get(string, [string])[0]) + forms = list(dict.fromkeys(forms)) + self.cache[cache_key] = forms + return forms diff --git a/spacy/lang/fr/lex_attrs.py b/spacy/lang/fr/lex_attrs.py new file mode 100644 index 0000000..9cf508a --- /dev/null +++ b/spacy/lang/fr/lex_attrs.py @@ -0,0 +1,43 @@ +from ...attrs import LIKE_NUM + +_num_words = set( + """ +zero un une deux trois quatre cinq six sept huit neuf dix +onze douze treize quatorze quinze seize dix-sept dix-huit dix-neuf +vingt trente quarante cinquante soixante soixante-dix septante quatre-vingt huitante quatre-vingt-dix nonante +cent mille mil million milliard billion quadrillion quintillion +sextillion septillion octillion nonillion decillion +""".split() +) + +_ordinal_words = set( + """ +premier première deuxième second seconde troisième quatrième cinquième sixième septième huitième neuvième dixième +onzième douzième treizième quatorzième quinzième seizième dix-septième dix-huitième dix-neuvième +vingtième trentième quarantième cinquantième soixantième soixante-dixième septantième quatre-vingtième huitantième quatre-vingt-dixième nonantième +centième millième millionnième milliardième billionnième quadrillionnième quintillionnième +sextillionnième septillionnième octillionnième nonillionnième decillionnième +""".split() +) + + +def like_num(text): + # Might require more work? + # See this discussion: https://github.com/explosion/spaCy/pull/1161 + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/fr/punctuation.py b/spacy/lang/fr/punctuation.py new file mode 100644 index 0000000..a3b178a --- /dev/null +++ b/spacy/lang/fr/punctuation.py @@ -0,0 +1,56 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, + merge_chars, +) +from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES + +ELISION = "' ’".replace(" ", "") +HYPHENS = r"- – — ‐ ‑".replace(" ", "") +_prefixes_elision = "d l n" +_prefixes_elision += " " + _prefixes_elision.upper() +_hyphen_suffixes = "ce clés elle en il ils je là moi nous on t vous" +_hyphen_suffixes += " " + _hyphen_suffixes.upper() + + +_prefixes = TOKENIZER_PREFIXES + [ + r"(?:({pe})[{el}])(?=[{a}])".format( + a=ALPHA, el=ELISION, pe=merge_chars(_prefixes_elision) + ) +] + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", # °C. -> ["°C", "."] + r"(?<=[0-9])%", # 4% -> ["4", "%"] + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9{al}{e}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + r"(?<=[{a}])[{h}]({hs})".format( + a=ALPHA, h=HYPHENS, hs=merge_chars(_hyphen_suffixes) + ), + ] +) + +_infixes = TOKENIZER_INFIXES + [ + r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION) +] + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/fr/stop_words.py b/spacy/lang/fr/stop_words.py new file mode 100644 index 0000000..b32ee3d --- /dev/null +++ b/spacy/lang/fr/stop_words.py @@ -0,0 +1,84 @@ +STOP_WORDS = set( + """ +a à â abord afin ah ai aie ainsi ait allaient allons +alors anterieur anterieure anterieures antérieur antérieure antérieures +apres après as assez attendu au +aupres auquel aura auraient aurait auront +aussi autre autrement autres autrui aux auxquelles auxquels avaient +avais avait avant avec avoir avons ayant + +bas basee bat + +c' c’ ça car ce ceci cela celle celle-ci celle-la celle-là celles celles-ci celles-la celles-là +celui celui-ci celui-la celui-là cent cependant certain certaine certaines certains certes ces +cet cette ceux ceux-ci ceux-là chacun chacune chaque chez ci cinq cinquantaine cinquante +cinquantième cinquième combien comme comment compris concernant + +d' d’ da dans de debout dedans dehors deja dejà delà depuis derriere +derrière des desormais desquelles desquels dessous dessus deux deuxième +deuxièmement devant devers devra different differente differentes differents différent +différente différentes différents dire directe directement dit dite dits divers +diverse diverses dix dix-huit dix-neuf dix-sept dixième doit doivent donc dont +douze douzième du duquel durant dès déja déjà désormais + +effet egalement eh elle elle-meme elle-même elles elles-memes elles-mêmes en encore +enfin entre envers environ es ès est et etaient étaient etais étais etait était +etant étant etc etre être eu eux eux-mêmes exactement excepté également + +fais faisaient faisant fait facon façon feront font + +gens + +ha hem hep hi ho hormis hors hou houp hue hui huit huitième +hé i il ils importe + +j' j’ je jusqu jusque juste + +l' l’ la laisser laquelle le lequel les lesquelles lesquels leur leurs longtemps +lors lorsque lui lui-meme lui-même là lès + +m' m’ ma maint maintenant mais malgre malgré me meme memes merci mes mien +mienne miennes miens mille moi moi-meme moi-même moindres moins +mon même mêmes + +n' n’ na ne neanmoins neuvième ni nombreuses nombreux nos notamment +notre nous nous-mêmes nouveau nul néanmoins nôtre nôtres + +o ô on ont onze onzième or ou ouias ouste outre +ouvert ouverte ouverts où + +par parce parfois parle parlent parler parmi partant +pas pendant pense permet personne peu peut peuvent peux plus +plusieurs plutot plutôt possible possibles pour pourquoi +pourrais pourrait pouvait prealable precisement +premier première premièrement +pres procedant proche près préalable précisement pu puis puisque + +qu' qu’ quand quant quant-à-soi quarante quatorze quatre quatre-vingt +quatrième quatrièmement que quel quelconque quelle quelles quelqu'un quelque +quelques quels qui quiconque quinze quoi quoique + +relative relativement rend rendre restant reste +restent retour revoici revoila revoilà + +s' s’ sa sait sans sauf se seize selon semblable semblaient +semble semblent sent sept septième sera seraient serait seront ses seul seule +seulement seuls seules si sien sienne siennes siens sinon six sixième soi soi-meme soi-même soit +soixante son sont sous souvent specifique specifiques spécifique spécifiques stop +suffisant suffisante suffit suis suit suivant suivante +suivantes suivants suivre sur surtout + +t' t’ ta tant te tel telle tellement telles tels tenant tend tenir tente +tes tien tienne tiennes tiens toi toi-meme toi-même ton touchant toujours tous +tout toute toutes treize trente tres trois troisième troisièmement très +tu té + +un une unes uns + +va vais vas vers via vingt voici voila voilà vont vos +votre votres vous vous-mêmes vu vé vôtre vôtres + +y + +""".split() +) diff --git a/spacy/lang/fr/syntax_iterators.py b/spacy/lang/fr/syntax_iterators.py new file mode 100644 index 0000000..a6bf3d3 --- /dev/null +++ b/spacy/lang/fr/syntax_iterators.py @@ -0,0 +1,85 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "nsubj", + "nsubj:pass", + "obj", + "obl", + "obl:agent", + "obl:arg", + "obl:mod", + "nmod", + "pcomp", + "appos", + "ROOT", + ] + post_modifiers = ["flat", "flat:name", "flat:foreign", "fixed", "compound"] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = {doc.vocab.strings.add(label) for label in labels} + np_modifs = {doc.vocab.strings.add(modifier) for modifier in post_modifiers} + np_label = doc.vocab.strings.add("NP") + adj_label = doc.vocab.strings.add("amod") + det_label = doc.vocab.strings.add("det") + det_pos = doc.vocab.strings.add("DET") + adp_pos = doc.vocab.strings.add("ADP") + conj_label = doc.vocab.strings.add("conj") + conj_pos = doc.vocab.strings.add("CCONJ") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + right_childs = list(word.rights) + right_child = right_childs[0] if right_childs else None + + if right_child: + if ( + right_child.dep == adj_label + ): # allow chain of adjectives by expanding to right + right_end = right_child.right_edge + elif ( + right_child.dep == det_label and right_child.pos == det_pos + ): # cut relative pronouns here + right_end = right_child + elif right_child.dep in np_modifs: # Check if we can expand to right + right_end = word.right_edge + else: + right_end = word + else: + right_end = word + prev_end = right_end.i + + left_index = word.left_edge.i + left_index = left_index + 1 if word.left_edge.pos == adp_pos else left_index + + yield left_index, right_end.i + 1, np_label + elif word.dep == conj_label: + head = word.head + while head.dep == conj_label and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + + left_index = word.left_edge.i # eliminate left attached conjunction + left_index = ( + left_index + 1 if word.left_edge.pos == conj_pos else left_index + ) + yield left_index, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/fr/tokenizer_exceptions.py b/spacy/lang/fr/tokenizer_exceptions.py new file mode 100644 index 0000000..fa2062e --- /dev/null +++ b/spacy/lang/fr/tokenizer_exceptions.py @@ -0,0 +1,443 @@ +import re + +from ...symbols import ORTH +from ...util import update_exc +from ..char_classes import ALPHA, ALPHA_LOWER +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .punctuation import ELISION, HYPHENS + +# not using the large _tokenizer_exceptions_list by default as it slows down the tokenizer +# from ._tokenizer_exceptions_list import FR_BASE_EXCEPTIONS +FR_BASE_EXCEPTIONS = ["aujourd'hui", "Aujourd'hui"] + + +def upper_first_letter(text): + if len(text) == 0: + return text + if len(text) == 1: + return text.upper() + return text[0].upper() + text[1:] + + +def lower_first_letter(text): + if len(text) == 0: + return text + if len(text) == 1: + return text.lower() + return text[0].lower() + text[1:] + + +_exc = {"J.-C.": [{ORTH: "J."}, {ORTH: "-C."}]} + + +for exc_data in [ + {ORTH: "av."}, + {ORTH: "janv."}, + {ORTH: "févr."}, + {ORTH: "avr."}, + {ORTH: "juill."}, + {ORTH: "sept."}, + {ORTH: "oct."}, + {ORTH: "nov."}, + {ORTH: "déc."}, + {ORTH: "apr."}, + {ORTH: "Dr."}, + {ORTH: "M."}, + {ORTH: "Mr."}, + {ORTH: "Mme."}, + {ORTH: "Mlle."}, + {ORTH: "n°"}, + {ORTH: "d°"}, + {ORTH: "St."}, + {ORTH: "Ste."}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "après-midi", + "au-delà", + "au-dessus", + "celle-ci", + "celles-ci", + "celui-ci", + "cf.", + "ci-dessous", + "elle-même", + "en-dessous", + "etc.", + "jusque-là", + "lui-même", + "MM.", + "No.", + "peut-être", + "pp.", + "quelques-uns", + "rendez-vous", + "Vol.", +]: + _exc[orth] = [{ORTH: orth}] + + +for verb in [ + "a", + "est", + "semble", + "indique", + "moque", + "passe", +]: + for orth in [verb, verb.title()]: + for pronoun in ["elle", "il", "on"]: + token = f"{orth}-t-{pronoun}" + _exc[token] = [{ORTH: orth}, {ORTH: "-t"}, {ORTH: "-" + pronoun}] + +for verb in ["est"]: + for orth in [verb, verb.title()]: + _exc[f"{orth}-ce"] = [{ORTH: orth}, {ORTH: "-ce"}] + + +for pre in ["qu'", "n'"]: + for orth in [pre, pre.title()]: + _exc[f"{orth}est-ce"] = [{ORTH: orth}, {ORTH: "est"}, {ORTH: "-ce"}] + + +for verb, pronoun in [("est", "il"), ("EST", "IL")]: + _exc[f"{verb}-{pronoun}"] = [{ORTH: verb}, {ORTH: "-" + pronoun}] + + +for s, verb, pronoun in [("s", "est", "il"), ("S", "EST", "IL")]: + _exc[f"{s}'{verb}-{pronoun}"] = [ + {ORTH: s + "'"}, + {ORTH: verb}, + {ORTH: "-" + pronoun}, + ] + + +_infixes_exc = [] # type: ignore[var-annotated] +orig_elision = "'" +orig_hyphen = "-" + +# loop through the elison and hyphen characters, and try to substitute the ones that weren't used in the original list +for infix in FR_BASE_EXCEPTIONS: + variants_infix = {infix} + for elision_char in [x for x in ELISION if x != orig_elision]: + variants_infix.update( + [word.replace(orig_elision, elision_char) for word in variants_infix] + ) + for hyphen_char in [x for x in ["-", "‐"] if x != orig_hyphen]: + variants_infix.update( + [word.replace(orig_hyphen, hyphen_char) for word in variants_infix] + ) + variants_infix.update([upper_first_letter(word) for word in variants_infix]) + _infixes_exc.extend(variants_infix) + +for orth in _infixes_exc: + _exc[orth] = [{ORTH: orth}] + + +_hyphen_prefix = [ + "a[ée]ro", + "abat", + "a[fg]ro", + "after", + "aigues?", + "am[ée]ricano", + "anglo", + "anti", + "apr[èe]s", + "arabo", + "arcs?", + "archi", + "arrières?", + "audio", + "avant", + "avion", + "auto", + "banc", + "bas(?:ses?)?", + "bateaux?", + "bec?", + "belles?", + "beau", + "best", + "bio?", + "bien", + "blanc", + "bo[îi]te", + "bonn?e?s?", + "bois", + "bou(?:c|rg)", + "b[êe]ta", + "cache", + "cap(?:ello)?", + "casse", + "castel", + "champ", + "chapelle", + "ch[âa]teau(?:neuf)?", + "chasse", + "cha(?:ud|t)e?s?", + "chauffe", + "chou", + "chromo", + "claire?s?", + "co(?:de|ca)?", + "compte", + "contre", + "cordon", + "coupe?", + "courte?s?", + "couvre", + "crash", + "crise", + "croche", + "cross", + "cyber", + "côte", + "demi", + "di(?:sney)?", + "dix", + "d[ée]s?", + "dys", + "ex?", + "émirato", + "entre", + "est", + "ethno", + "ex", + "extra", + "extrême", + "[ée]co", + "faux", + "fil", + "fort", + "franco?s?", + "gallo", + "gardes?", + "gastro", + "grande?", + "gratte", + "gr[ée]co", + "gros", + "g[ée]o", + "haute?s?", + "homm?es?", + "hors", + "hyper", + "indo", + "infra", + "inter", + "intra", + "islamo", + "italo", + "jean", + "labio", + "latino", + "live", + "lot", + "louis", + "m[ai]cro", + "mal", + "médio", + "mesnil", + "mi(?:ni)?", + "mono", + "mont?s?", + "moyen", + "multi", + "m[ée]cano", + "m[ée]dico", + "m[ée]do", + "m[ée]ta", + "mots?", + "neuro", + "noix", + "non", + "nord", + "notre", + "n[ée]o", + "ouest", + "outre", + "ouvre", + "passe", + "perce", + "pharmaco", + "ph[oy]to", + "pieds?", + "pique", + "poissons?", + "ponce", + "pont", + "po[rs]t", + "pousse", + "primo", + "pro(?:cès|to)?", + "pare", + "petite?s?", + "plessis", + "porte", + "pré", + "prêchi", + "protège", + "pseudo", + "pêle", + "péri", + "puy", + "quasi", + "quatre", + "radio", + "recourt", + "rythmo", + "(?:re)?doubles?", + "r[ée]", + "r[ée]tro", + "requin", + "sans?", + "sa?inte?s?", + "semi", + "serre", + "sino", + "socio", + "sociale?s?", + "soixante", + "sous", + "su[bdrs]", + "super", + "taille", + "tire", + "thermo", + "tiers", + "tourne", + "toute?s?", + "tra[iî]ne?", + "trans", + "trente", + "trois", + "trousse", + "tr(?:i|ou)", + "t[ée]l[ée]", + "utéro", + "vaso", + "vi[cd]e", + "vid[ée]o", + "vie(?:ux|i?lles?|i?l)", + "vill(?:e|eneuve|ers|ette|iers|y)", + "vingt", + "voitures?", + "wagons?", + "ultra", + "à", + "[ée]lectro", + "[ée]qui", + "Fontaine", + "La Chapelle", + "Marie", + "Le Mesnil", + "Neuville", + "Pierre", + "Val", + "Vaux", +] + +_regular_exp = [ + "^a[{hyphen}]sexualis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^arginine[{hyphen}]méthyl[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^binge[{hyphen}]watch[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^black[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^bouche[{hyphen}]por[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^burn[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^by[{hyphen}]pass[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^ch[{elision}]tiis[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER), + "^chape[{hyphen}]chut[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^down[{hyphen}]load[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^[ée]tats[{hyphen}]uni[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^droits?[{hyphen}]de[{hyphen}]l'homm[{al}]+$".format( + hyphen=HYPHENS, al=ALPHA_LOWER + ), + "^fac[{hyphen}]simil[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^fleur[{hyphen}]bleuis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^flic[{hyphen}]flaqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^fox[{hyphen}]trott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^google[{hyphen}]is[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^hard[{hyphen}]discount[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^hip[{hyphen}]hop[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^jet[{hyphen}]set[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^knock[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^lèche[{hyphen}]bott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^litho[{hyphen}]typographi[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^lock[{hyphen}]out[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^lombri[{hyphen}]compost[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^mac[{hyphen}]adamis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^marque[{hyphen}]pag[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^mouton[{hyphen}]noiris[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^new[{hyphen}]york[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^pair[{hyphen}]programm[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^people[{hyphen}]is[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^plan[{hyphen}]socialis[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^premier[{hyphen}]ministr[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^prud[{elision}]hom[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER), + "^réarc[{hyphen}]bout[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^refox[{hyphen}]trott[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^remicro[{hyphen}]ond[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^repique[{hyphen}]niqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^repetit[{hyphen}]déjeun[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^rick[{hyphen}]roll[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^rond[{hyphen}]ponn[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^shift[{hyphen}]cliqu[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^soudo[{hyphen}]bras[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^stabilo[{hyphen}]boss[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^strip[{hyphen}]teas[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^terra[{hyphen}]form[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^teuf[{hyphen}]teuf[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^yo[{hyphen}]yo[{al}]+$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^zig[{hyphen}]zag[{al}]*$".format(hyphen=HYPHENS, al=ALPHA_LOWER), + "^z[{elision}]yeut[{al}]+$".format(elision=ELISION, al=ALPHA_LOWER), +] +# catching cases like faux-vampire +_regular_exp += [ + "^{prefix}[{hyphen}][{al}][{hyphen}{al}{elision}]*$".format( + prefix=p, + hyphen=HYPHENS, # putting the - first in the [] range avoids having to use a backslash + elision=ELISION, + al=ALPHA_LOWER, + ) + for p in _hyphen_prefix +] + +# catching cases like entr'abat +_elision_prefix = ["r?é?entr", "grande?s?", "r"] +_regular_exp += [ + "^{prefix}[{elision}][{al}][{hyphen}{al}{elision}]*$".format( + prefix=p, elision=ELISION, hyphen=HYPHENS, al=ALPHA_LOWER + ) + for p in _elision_prefix +] + +# catching cases like saut-de-ski, pet-en-l'air +_hyphen_combination = [ + "l[èe]s?", + "la", + "en", + "des?", + "d[eu]", + "sur", + "sous", + "aux?", + "à", + "et", + "près", + "saint", +] +_regular_exp += [ + "^[{a}]+[{hyphen}]{hyphen_combo}[{hyphen}](?:l[{elision}])?[{a}]+$".format( + hyphen_combo=hc, elision=ELISION, hyphen=HYPHENS, a=ALPHA + ) + for hc in _hyphen_combination +] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) +TOKEN_MATCH = re.compile( + "(?iu)" + "|".join("(?:{})".format(m) for m in _regular_exp) +).match diff --git a/spacy/lang/ga/__init__.py b/spacy/lang/ga/__init__.py new file mode 100644 index 0000000..6f9a27a --- /dev/null +++ b/spacy/lang/ga/__init__.py @@ -0,0 +1,33 @@ +from typing import Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import IrishLemmatizer +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class IrishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + stop_words = STOP_WORDS + + +class Irish(Language): + lang = "ga" + Defaults = IrishDefaults + + +@Irish.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={"model": None, "mode": "pos_lookup", "overwrite": False}, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, model: Optional[Model], name: str, mode: str, overwrite: bool +): + return IrishLemmatizer(nlp.vocab, model, name, mode=mode, overwrite=overwrite) + + +__all__ = ["Irish"] diff --git a/spacy/lang/ga/lemmatizer.py b/spacy/lang/ga/lemmatizer.py new file mode 100644 index 0000000..cffcf1d --- /dev/null +++ b/spacy/lang/ga/lemmatizer.py @@ -0,0 +1,162 @@ +from typing import List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class IrishLemmatizer(Lemmatizer): + # This is a lookup-based lemmatiser using data extracted from + # BuNaMo (https://github.com/michmech/BuNaMo) + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "pos_lookup": + # fmt: off + required = [ + "lemma_lookup_adj", "lemma_lookup_adp", + "lemma_lookup_noun", "lemma_lookup_verb" + ] + # fmt: on + return (required, []) + else: + return super().get_lookups_config(mode) + + def pos_lookup_lemmatize(self, token: Token) -> List[str]: + univ_pos = token.pos_ + string = unponc(token.text) + if univ_pos not in ["PROPN", "ADP", "ADJ", "NOUN", "VERB"]: + return [string.lower()] + demutated = demutate(string) + secondary = "" + if string[0:1].lower() == "h" and string[1:2].lower() in "aáeéiíoóuú": + secondary = string[1:] + lookup_pos = univ_pos.lower() + if univ_pos == "PROPN": + lookup_pos = "noun" + if token.has_morph(): + # TODO: lookup is actually required for the genitive forms, but + # this is not in BuNaMo, and would not be of use with IDT. + if univ_pos == "NOUN" and ( + "VerbForm=Vnoun" in token.morph or "VerbForm=Inf" in token.morph + ): + hpref = "Form=HPref" in token.morph + return [demutate(string, hpref).lower()] + elif univ_pos == "ADJ" and "VerbForm=Part" in token.morph: + return [demutate(string).lower()] + lookup_table = self.lookups.get_table("lemma_lookup_" + lookup_pos, {}) + + def to_list(value): + if value is None: + value = [] + elif not isinstance(value, list): + value = [value] + return value + + if univ_pos == "ADP": + return to_list(lookup_table.get(string, string.lower())) + ret = [] + if univ_pos == "PROPN": + ret.extend(to_list(lookup_table.get(demutated))) + ret.extend(to_list(lookup_table.get(secondary))) + else: + ret.extend(to_list(lookup_table.get(demutated.lower()))) + ret.extend(to_list(lookup_table.get(secondary.lower()))) + if len(ret) == 0: + ret = [string.lower()] + return ret + + +def demutate(word: str, is_hpref: bool = False) -> str: + UVOWELS = "AÁEÉIÍOÓUÚ" + LVOWELS = "aáeéiíoóuú" + lc = word.lower() + # remove eclipsis + if lc.startswith("bhf"): + word = word[2:] + elif lc.startswith("mb"): + word = word[1:] + elif lc.startswith("gc"): + word = word[1:] + elif lc.startswith("nd"): + word = word[1:] + elif lc.startswith("ng"): + word = word[1:] + elif lc.startswith("bp"): + word = word[1:] + elif lc.startswith("dt"): + word = word[1:] + elif word[0:1] == "n" and word[1:2] in UVOWELS: + word = word[1:] + elif lc.startswith("n-") and word[2:3] in LVOWELS: + word = word[2:] + # non-standard eclipsis + elif lc.startswith("bh-f"): + word = word[3:] + elif lc.startswith("m-b"): + word = word[2:] + elif lc.startswith("g-c"): + word = word[2:] + elif lc.startswith("n-d"): + word = word[2:] + elif lc.startswith("n-g"): + word = word[2:] + elif lc.startswith("b-p"): + word = word[2:] + elif lc.startswith("d-t"): + word = word[2:] + + # t-prothesis + elif lc.startswith("ts"): + word = word[1:] + elif lc.startswith("t-s"): + word = word[2:] + + # h-prothesis, if known to be present + elif is_hpref and word[0:1] == "h": + word = word[1:] + # h-prothesis, simple case + # words can also begin with 'h', but unlike eclipsis, + # a hyphen is not used, so that needs to be handled + # elsewhere + elif word[0:1] == "h" and word[1:2] in UVOWELS: + word = word[1:] + + # lenition + # this breaks the previous if, to handle super-non-standard + # text where both eclipsis and lenition were used. + if lc[0:1] in "bcdfgmpst" and lc[1:2] == "h": + word = word[0:1] + word[2:] + + return word + + +def unponc(word: str) -> str: + # fmt: off + PONC = { + "ḃ": "bh", + "ċ": "ch", + "ḋ": "dh", + "ḟ": "fh", + "ġ": "gh", + "ṁ": "mh", + "ṗ": "ph", + "ṡ": "sh", + "ṫ": "th", + "Ḃ": "BH", + "Ċ": "CH", + "Ḋ": "DH", + "Ḟ": "FH", + "Ġ": "GH", + "Ṁ": "MH", + "Ṗ": "PH", + "Ṡ": "SH", + "Ṫ": "TH" + } + # fmt: on + buf = [] + for ch in word: + if ch in PONC: + buf.append(PONC[ch]) + else: + buf.append(ch) + return "".join(buf) diff --git a/spacy/lang/ga/stop_words.py b/spacy/lang/ga/stop_words.py new file mode 100644 index 0000000..4ef052c --- /dev/null +++ b/spacy/lang/ga/stop_words.py @@ -0,0 +1,43 @@ +STOP_WORDS = set( + """ +a ach ag agus an aon ar arna as + +ba beirt bhúr + +caoga ceathair ceathrar chomh chuig chun cois céad cúig cúigear + +daichead dar de deich deichniúr den dhá do don dtí dá dár dó + +faoi faoin faoina faoinár fara fiche + +gach gan go gur + +haon hocht + +i iad idir in ina ins inár is + +le leis lena lenár + +mar mo muid mé + +na nach naoi naonúr ná ní níor nó nócha + +ocht ochtar ochtó os + +roimh + +sa seacht seachtar seachtó seasca seisear siad sibh sinn sna sé sí + +tar thar thú triúr trí trína trínár tríocha tú + +um + +ár + +é éis + +í + +ó ón óna ónár +""".split() +) diff --git a/spacy/lang/ga/tokenizer_exceptions.py b/spacy/lang/ga/tokenizer_exceptions.py new file mode 100644 index 0000000..eb4b413 --- /dev/null +++ b/spacy/lang/ga/tokenizer_exceptions.py @@ -0,0 +1,71 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = { + "'acha'n": [{ORTH: "'ach", NORM: "gach"}, {ORTH: "a'n", NORM: "aon"}], + "dem'": [{ORTH: "de", NORM: "de"}, {ORTH: "m'", NORM: "mo"}], + "ded'": [{ORTH: "de", NORM: "de"}, {ORTH: "d'", NORM: "do"}], + "lem'": [{ORTH: "le", NORM: "le"}, {ORTH: "m'", NORM: "mo"}], + "led'": [{ORTH: "le", NORM: "le"}, {ORTH: "d'", NORM: "do"}], + "théis": [{ORTH: "th", NORM: "tar"}, {ORTH: "éis", NORM: "éis"}], + "tréis": [{ORTH: "tr", NORM: "tar"}, {ORTH: "éis", NORM: "éis"}], +} + +for exc_data in [ + {ORTH: "'gus", NORM: "agus"}, + {ORTH: "'ach", NORM: "gach"}, + {ORTH: "ao'", NORM: "aon"}, + {ORTH: "'niar", NORM: "aniar"}, + {ORTH: "'níos", NORM: "aníos"}, + {ORTH: "'ndiu", NORM: "inniu"}, + {ORTH: "'nocht", NORM: "anocht"}, + {ORTH: "m'"}, + {ORTH: "Aib."}, + {ORTH: "Ath."}, + {ORTH: "Beal."}, + {ORTH: "a.C.n."}, + {ORTH: "m.sh."}, + {ORTH: "M.F."}, + {ORTH: "M.Fómh."}, + {ORTH: "D.F."}, + {ORTH: "D.Fómh."}, + {ORTH: "r.C."}, + {ORTH: "R.C."}, + {ORTH: "r.Ch."}, + {ORTH: "r.Chr."}, + {ORTH: "R.Ch."}, + {ORTH: "R.Chr."}, + {ORTH: "⁊rl."}, + {ORTH: "srl."}, + {ORTH: "Co."}, + {ORTH: "Ean."}, + {ORTH: "Feab."}, + {ORTH: "gCo."}, + {ORTH: ".i."}, + {ORTH: "B'"}, + {ORTH: "b'"}, + {ORTH: "lch."}, + {ORTH: "Lch."}, + {ORTH: "lgh."}, + {ORTH: "Lgh."}, + {ORTH: "Lún."}, + {ORTH: "Már."}, + {ORTH: "Meith."}, + {ORTH: "Noll."}, + {ORTH: "Samh."}, + {ORTH: "tAth."}, + {ORTH: "tUas."}, + {ORTH: "teo."}, + {ORTH: "Teo."}, + {ORTH: "Uas."}, + {ORTH: "uimh."}, + {ORTH: "Uimh."}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +for orth in ["d'", "D'"]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/gd/__init__.py b/spacy/lang/gd/__init__.py new file mode 100644 index 0000000..048a3a0 --- /dev/null +++ b/spacy/lang/gd/__init__.py @@ -0,0 +1,18 @@ +from typing import Optional + +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class ScottishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + stop_words = STOP_WORDS + + +class Scottish(Language): + lang = "gd" + Defaults = ScottishDefaults + + +__all__ = ["Scottish"] diff --git a/spacy/lang/gd/stop_words.py b/spacy/lang/gd/stop_words.py new file mode 100644 index 0000000..d5132c3 --- /dev/null +++ b/spacy/lang/gd/stop_words.py @@ -0,0 +1,386 @@ +STOP_WORDS = set( + """ +'ad +'ar +'d # iad +'g # ag +'ga +'gam +'gan +'gar +'gur +'m # am +'n # an +'n seo +'na +'nad +'nam +'nan +'nar +'nuair +'nur +'s +'sa +'san +'sann +'se +'sna +a +a' +a'd # agad +a'm # agam +a-chèile +a-seo +a-sin +a-siud +a chionn +a chionn 's +a chèile +a chéile +a dh' +a h-uile +a seo +ac' # aca +aca +aca-san +acasan +ach +ag +agad +agad-sa +agads' +agadsa +agaibh +agaibhse +againn +againne +agam +agam-sa +agams' +agamsa +agus +aice +aice-se +aicese +aig +aig' # aige +aige +aige-san +aigesan +air +air-san +air neo +airsan +am +an +an seo +an sin +an siud +an uair +ann +ann a +ann a' +ann a shin +ann am +ann an +annad +annam +annam-s' +annamsa +anns +anns an +annta +aon +ar +as +asad +asda +asta +b' +bho +bhon +bhuaidhe # bhuaithe +bhuainn +bhuaipe +bhuaithe +bhuapa +bhur +brì +bu +c'à +car son +carson +cha +chan +chionn +choir +chon +chun +chèile +chéile +chòir +cia mheud +ciamar +co-dhiubh +cuide +cuin +cuin' +cuine +cà +cà' +càil +càit +càit' +càite +cò +cò mheud +có +d' +da +de +dh' +dha +dhaibh +dhaibh-san +dhaibhsan +dhan +dhasan +dhe +dhen +dheth +dhi +dhiom +dhiot +dhith +dhiubh +dhomh +dhomh-s' +dhomhsa +dhu'sa # dhut-sa +dhuibh +dhuibhse +dhuinn +dhuinne +dhuit +dhut +dhutsa +dhut-sa +dhà +dhà-san +dhàsan +dhòmhsa +diubh +do +docha +don +dà +dè +dè mar +dé +dé mar +dòch' +dòcha +e +eadar +eatarra +eatorra +eile +esan +fa +far +feud +fhad +fheudar +fhearr +fhein +fheudar +fheàrr +fhèin +fhéin +fhìn +fo +fodha +fodhainn +foipe +fon +fèin +ga +gach +gam +gan +ge brith +ged +gu +gu dè +gu ruige +gun +gur +gus +i +iad +iadsan +innte +is +ise +le +leam +leam-sa +leamsa +leat +leat-sa +leatha +leatsa +leibh +leis +leis-san +leoth' +leotha +leotha-san +linn +m' +m'a +ma +mac +man +mar +mas +mathaid +mi +mis' +mise +mo +mu +mu 'n +mun +mur +mura +mus +na +na b' +na bu +na iad +nach +nad +nam +nan +nar +nas +neo +no +nuair +o +o'n +oir +oirbh +oirbh-se +oirnn +oirnne +oirre +on +orm +orm-sa +ormsa +orra +orra-san +orrasan +ort +os +r' +ri +ribh +rinn +ris +rithe +rithe-se +rium +rium-sa +riums' +riumsa +riut +riuth' +riutha +riuthasan +ro +ro'n +roimh +roimhe +romhainn +romham +romhpa +ron +ruibh +ruinn +ruinne +sa +san +sann +se +seach +seo +seothach +shin +sibh +sibh-se +sibhse +sin +sineach +sinn +sinne +siod +siodach +siud +siudach +sna # ann an +sè +t' +tarsaing +tarsainn +tarsuinn +thar +thoigh +thro +thu +thuc' +thuca +thugad +thugaibh +thugainn +thugam +thugamsa +thuice +thuige +thus' +thusa +timcheall +toigh +toil +tro +tro' # troimh +troimh +troimhe +tron +tu +tusa +uair +ud +ugaibh +ugam-s' +ugam-sa +uice +uige +uige-san +umad +unnta # ann an +ur +urrainn +à +às +àsan +á +ás +è +ì +ò +ó +""".split("\n") +) diff --git a/spacy/lang/gd/tokenizer_exceptions.py b/spacy/lang/gd/tokenizer_exceptions.py new file mode 100644 index 0000000..b655841 --- /dev/null +++ b/spacy/lang/gd/tokenizer_exceptions.py @@ -0,0 +1,1981 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +""" + All rules and exceptions were taken from the "Gaelic Orthographic Conventions +of 2009" (GOC) and from the "Annotated Reference Corpus of Scottish Gaelic" (ARCOSG). I did +my best to ensure this tokenizer would lead to text as close as possible to the +tokenization of the ARCOSG and the conventions in the GOC. + + +ARCOSG: https://github.com/Gaelic-Algorithmic-Research-Group/ARCOSG +GOC: https://www.gaidhlig.scot/wp-content/uploads/2021/03/GOC-2009-English.pdf +""" + +# Compound words +_exc = { + "càil": [{ORTH: "cà", NORM: "càite"}, {ORTH: "il", NORM: "bheil"}], + "sna": [{ORTH: "s", NORM: "anns"}, {ORTH: "na", NORM: "na"}], + "orra": [{ORTH: "orr", NORM: "air"}, {ORTH: "a", NORM: "do"}], + "fiùs": [{ORTH: "fiù", NORM: "fiù"}, {ORTH: "s", NORM: "'s"}], +} + + +# Hyphenations that are alternative forms of words +for exc_data in [ + {ORTH: "fa-near", NORM: "fainear"}, + {ORTH: "Fa-near", NORM: "Fainear"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Abreviations and shortened words +for exc_data in [ + {ORTH: "'", NORM: "a"}, + {ORTH: "'S", NORM: "Agus"}, + {ORTH: "'s", NORM: "agus"}, + {ORTH: "B'", NORM: "Bu"}, + {ORTH: "b'", NORM: "bu"}, + {ORTH: "D'", NORM: "Do"}, + {ORTH: "d'", NORM: "do"}, + {ORTH: "'M", NORM: "Am"}, + {ORTH: "'m", NORM: "am"}, + {ORTH: "M'", NORM: "Mo"}, + {ORTH: "m'", NORM: "mo"}, + {ORTH: "'n", NORM: "an"}, + {ORTH: "'N", NORM: "An"}, + {ORTH: "Th'", NORM: "Tha"}, + {ORTH: "th'", NORM: "tha"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Words with a leading apostrophe +for orth in """ + 'ac + 'Ac + 'ad + 'Ad + 'ar + 'Ar + 'bhuannachd + 'Bhuannachd + 'd + 'D + 'eil + 'Eil + 'eug + 'Eug + 'g + 'G + 'ga + 'Ga + 'gad + 'Gad + 'gam + 'Gam + 'gan + 'Gan + 'gar + 'Gar + 'gur + 'Gur + 'ic + 'Ic + 'il + 'Il + 'ill' + 'Ill' + 'ille + 'Ille + 'illean + 'Illean + 'iodh + 'Iodh + 'l + 'L + 'm + 'M + 'n + 'N + 'na + 'Na + 'nad + 'Nad + 'nam + 'Nam + 'nan + 'Nan + 'nar + 'Nar + 'neil + 'Neil + 'nise + 'Nise + 'nuair + 'Nuair + 'nur + 'Nur + 's + 'S + 'sa + 'Sa + 'sa' + 'Sa' + 'san + 'San + 'sann + 'Sann + 'se + 'Se + 'sna + 'Sna + 'son + 'Son + 'urchaidh + 'Urchaidh + """.split(): + _exc[orth] = [{ORTH: orth}] + +# Words with a trailing or middling apostrophe +for orth in """ + a' + A' + a'd + A'd + a'm + A'm + a's + A's + ac' + Ac' + agads' + Agads' + agams' + Agams' + aig' + Aig' + annams' + Annams' + ars' + Ars' + b' + B' + ball' + Ball' + bioraicht' + Bioraicht' + bh' + Bh' + bhail' + Bhail' + bhall' + Bhall' + bheath' + Bheath' + bhliadhn' + Bhliadhn' + bliadhn' + Bliadhn' + bonnant' + Bonnant' + brist' + Brist' + bàt' + Bàt' + c'à + C'à + camp' + Camp' + chalp' + Chalp' + champ' + Champ' + chomhairl' + Chomhairl' + chual' + Chual' + chuimhn' + Chuimhn' + colaisd' + Colaisd' + comhl' + Comhl' + comhairl' + Comhairl' + creids' + Creids' + cual' + Cual' + cuimhn' + Cuimhn' + cuin' + Cuin' + cà' + Cà' + càit' + Càit' + d' + D' + d'readh + D'readh + d'reaghadh + D'reaghadh + daoin' + Daoin' + deimhinn' + Deimhinn' + de'n + De'n + dh' + Dh' + dhaib' + Dhaib' + dhaoin' + Dhaoin' + dhòmhs' + Dhòmhs' + dhu'sa + Dhu'sa + dhuin' + Dhuin' + do'n + Do'n + duin' + Duin' + dòch' + Dòch' + dùint' + Dùint' + eil' + Eil' + f'a + F'a + fac' + Fac' + fad' + Fad' + fhac' + Fhac' + fhad' + Fhad' + fhaid' + Fhaid' + fhaisg' + Fhaisg' + fhiosd' + Fhiosd' + fàilt' + Fàilt' + g' + G' + gàir' + Gàir' + ghill' + Ghill' + gill' + Gill' + inns' + Inns' + innt' + Innt' + ionnsaicht' + Ionnsaicht' + leams' + Leams' + leoth' + Leoth' + lobht' + Lobht' + m' + M' + m'a + M'a + m's + M's + mhuth' + Mhuth' + mhòr' + Mhòr' + mis' + Mis' + mu'n + Mu'n + mòr' + Mòr' + oirr' + Oirr' + o'n + O'n + phàp' + Phàp' + pàp' + Pàp' + pòs' + Pòs' + prionns' + Prionns' + r' + R' + riums' + Riums' + riuth' + Riuth' + ro'n + Ro'n + sa' + Sa' + sgoil' + Sgoil' + sgìr' + Sgìr' + sheòrs' + Sheòrs' + sin' + Sin' + stall' + Stall' + sìod' + Sìod' + sònraicht' + Sònraicht' + t' + T' + taigh' + Taigh' + tein' + Tein' + teoth' + Teoth' + th' + Th' + thoilicht' + Thoilicht' + thuc' + Thuc' + thuigs' + Thuigs' + thus' + Thus' + thàna' + Thàna' + toilicht' + Toilicht' + tro' + Tro' + uisg' + Uisg' + àit' + Àit' + òg' + Òg' + """.split(): + _exc[orth] = [{ORTH: orth}] + + +# Hyphenations that should remain as single tokens +for orth in """ +'n-dràsda +'N-dràsda +-bhliadhn' +-bhliadhn' +a-bhos +A-bhos +a-bhòn-dè +A-bhòn-dè +a-cheart +A-cheart +a-chèile +A-chèile +a-deas +A-deas +a-mach +A-mach +a-mhàin +A-mhàin +a-muigh +A-muigh +a-màireach +A-màireach +a-nall +A-nall +a-neist +A-neist +a-ni +A-ni +a-nis +A-nis +a-nisd +A-nisd +a-nise +A-nise +a-nist +A-nist +a-niste +A-niste +a-nochd +A-nochd +a-nuas +A-nuas +a-null +A-null +a-raoir +A-raoir +a-riamh +A-riamh +a-rithist +A-rithist +a-rèiste +A-rèiste +a-rìs +A-rìs +a-seo +A-seo +a-sin +A-sin +a-sineach +A-sineach +a-siud +A-siud +a-staigh +A-staigh +a-steach +A-steach +a-tuath +A-tuath +aca-san +Aca-san +agad-sa +Agad-sa +agam-sa +Agam-sa +aghaidh-bhualaich +Aghaidh-bhualaich +aice-se +Aice-se +aige-san +Aige-san +ainmeannan-àite +Ainmeannan-àite +air-san +Air-san +am-bliadhna +Am-bliadhna +am-màireach +Am-màireach +amp-head +Amp-head +an-diugh +An-diugh +an-dràsd +An-dràsd +an-dràsda +An-dràsda +an-dràst +An-dràst +an-dràsta +An-dràsta +an-dè +An-dè +an-dé +An-dé +an-nise +An-nise +an-nochd +An-nochd +an-raoir +An-raoir +an-uiridh +An-uiridh +an-àbhaisteach +An-àbhaisteach +an-àird +An-àird +an-àirde +An-àirde +an-àrda +An-àrda +ana-ceartas +Ana-ceartas +ana-seo +Ana-seo +ana-sin +Ana-sin +ana-siud +Ana-siud +annam-s' +Annam-s' +ao-coltach +Ao-coltach +aobhar-sa +Aobhar-sa +aois-léinn +Aois-léinn +aona-ghnothaich +Aona-ghnothaich +ar-a-mach +Ar-a-mach +ard-easbaig +Ard-easbaig +ard-luchd-poilitics +Ard-luchd-poilitics +ath-bhaile +Ath-bhaile +ath-bheòthachadh +Ath-bheòthachadh +ath-bhliadhna +Ath-bhliadhna +ath-ghiollachd +Ath-ghiollachd +ath-nuadhais +Ath-nuadhais +ath-sgrùdadh +Ath-sgrùdadh +ath-thriop +Ath-thriop +athair-san +Athair-san +baile-ciùird +Baile-ciùird +ball-coise +Ball-coise +ball-pàrlamaid +Ball-pàrlamaid +ball-sampaill +Ball-sampaill +balla-mara +Balla-mara +ban-chompanach +Ban-chompanach +ban-fhuamhaire +Ban-fhuamhaire +ban-ghillìosach +Ban-ghillìosach +ban-righ'nn +Ban-righ'nn +ban-rìgh +Ban-rìgh +bana-bhàird +Bana-bhàird +bana-chompanaich +Bana-chompanaich +bana-phòsda +Bana-phòsda +banas-taighe +Banas-taighe +beairt-fhuaigheil +Beairt-fhuaigheil +beairt-fuaigheil +Beairt-fuaigheil +bean-gairm +Bean-gairm +bean-phòsta +Bean-phòsta +bean-taighe +Bean-taighe +beul-aithris +Beul-aithris +beò-shlàint +Beò-shlàint +beò-shlàint' +Beò-shlàint' +beò-shlàinte +Beò-shlàinte +bhaga-sgoil +Bhaga-sgoil +bhall-pàrlamaid +Bhall-pàrlamaid +bhan-chompanach +Bhan-chompanach +bhan-dòmhnallach +Bhan-dòmhnallach +bhan-phrionnsa +Bhan-phrionnsa +bhan-righinn +Bhan-righinn +bhan-sheinneadair +Bhan-sheinneadair +bharr-iall +Bharr-iall +bhata-làidir +Bhata-làidir +bhath-room +Bhath-room +bheachd-sa +Bheachd-sa +bheachd-san +Bheachd-san +bheairt-fhighe +Bheairt-fhighe +bheairtean-fuaigheil +Bheairtean-fuaigheil +bheinn-sheilg +Bheinn-sheilg +bheul-aithris +Bheul-aithris +bheò-ghlacadh +Bheò-ghlacadh +bhith-beò +Bhith-beò +bhithinn-sa +Bhithinn-sa +bhogsa-chiùil +Bhogsa-chiùil +bhonn-stéidh +Bhonn-stéidh +bhràithrean-sa +Bhràithrean-sa +bhuain-mhòine +Bhuain-mhòine +bhun-sheòrsa +Bhun-sheòrsa +bhàn-righinn +Bhàn-righinn +bhàn-rinn +Bhàn-rinn +bhàn-rìgh +Bhàn-rìgh +bhàta-aiseig +Bhàta-aiseig +bhàta-sa +Bhàta-sa +bird-watcher +Bird-watcher +bith-beò +Bith-beò +bithinn-sa +Bithinn-sa +bliadhna-sa +Bliadhna-sa +bogha-saighead +Bogha-saighead +boma-peatroil +Boma-peatroil +bristeadh-a-mach +Bristeadh-a-mach +buidhean-cathrannais +Buidhean-cathrannais +buille-a-mach +Buille-a-mach +buille-shaor +Buille-shaor +bun-adhbharan +Bun-adhbharan +bun-chomharraidhean +Bun-chomharraidhean +bun-fhiosrachadh +Bun-fhiosrachadh +bun-sgoil +Bun-sgoil +bun-stèidh +Bun-stèidh +bàt-aiseig +Bàt-aiseig +bàta-aiseig +Bàta-aiseig +bàta-bathair +Bàta-bathair +cainnt-san +Cainnt-san +cal-mac +Cal-mac +carraighean-cuimhne +Carraighean-cuimhne +cead-telebhisean +Cead-telebhisean +ceann-cinnidh +Ceann-cinnidh +ceann-suidhe +Ceann-suidhe +chanain-sa +Chanain-sa +chaolas-arcach +Chaolas-arcach +charge-adh +Charge-adh +cheala-deug +Cheala-deug +chealla-deug +Chealla-deug +cheann-cinnidh +Cheann-cinnidh +cheann-feadhna +Cheann-feadhna +cheann-suidhe +Cheann-suidhe +chearc-fhraoich +Chearc-fhraoich +chearcall-meadhain +Chearcall-meadhain +chearcall-mheadhain +Chearcall-mheadhain +chlann-nighean +Chlann-nighean +chlàr-ama +Chlàr-ama +chlò-bhuaileadh +Chlò-bhuaileadh +chlò-bhualadh +Chlò-bhualadh +cho-chreutairean +Cho-chreutairean +cho-dhùin +Cho-dhùin +cho-dhùnadh +Cho-dhùnadh +cho-dhùnaidhean +Cho-dhùnaidhean +cho-fhaireachdainn +Cho-fhaireachdainn +cho-labhairt +Cho-labhairt +cho-obraiche +Cho-obraiche +cho-roinn +Cho-roinn +chom-pàirt +Chom-pàirt +chorra-ghritheach +Chorra-ghritheach +chrann-snàth +Chrann-snàth +chreach-s' +Chreach-s' +chrith-thalmhainn +Chrith-thalmhainn +chàch-a-chéile +Chàch-a-chéile +cinn-chuspair +Cinn-chuspair +cinn-iùil +Cinn-iùil +cion-doighe +Cion-doighe +clachan-meallain +Clachan-meallain +clann-sgoile +Clann-sgoile +claon-fhaireachdainn +Claon-fhaireachdainn +claon-shamhail +Claon-shamhail +cluicheadairean-meadhain +Cluicheadairean-meadhain +clàran-ama +Clàran-ama +cléir-seanchain +Cléir-seanchain +clò-bhualadair +Clò-bhualadair +clò-bhualadh +Clò-bhualadh +co-aimsireach +Co-aimsireach +co-bhanntachd +Co-bhanntachd +co-bhuannachd +Co-bhuannachd +co-buannachd +Co-buannachd +co-cheangail +Co-cheangail +co-cheangailte +Co-cheangailte +co-cheangal +Co-cheangal +co-chreutairean +Co-chreutairean +co-chruinneachadh +Co-chruinneachadh +co-dhiu +Co-dhiu +co-dhiubh +Co-dhiubh +co-dhiù +Co-dhiù +co-dhiùbh +Co-dhiùbh +co-dhùnadh +Co-dhùnadh +co-dhùnaidhean +Co-dhùnaidhean +co-fhaireachadh +Co-fhaireachadh +co-fhaireachdainn +Co-fhaireachdainn +co-impirean +Co-impirean +co-ionad +Co-ionad +co-ionann +Co-ionann +co-labhairt +Co-labhairt +co-labhairtean +Co-labhairtean +co-obrachadh +Co-obrachadh +co-sheirm +Co-sheirm +co-theacs +Co-theacs +coimeas-meudachd +Coimeas-meudachd +cola-deug +Cola-deug +com-pàirt +Com-pàirt +cope-adh +Cope-adh +crann-aodaich +Crann-aodaich +crann-snàth +Crann-snàth +crann-tarsainn +Crann-tarsainn +craobh-sgaoileadh +Craobh-sgaoileadh +crith-thalmhainn +Crith-thalmhainn +cruth-rannsachadh +Cruth-rannsachadh +cuid-eigin +Cuid-eigin +cumail-san +Cumail-san +cur-gu-buil +Cur-gu-buil +cur-seachad +Cur-seachad +cur-seachadan +Cur-seachadan +cìs-comhairle +Cìs-comhairle +cò-dhunadh +Cò-dhunadh +còmhlan-ciùil +Còmhlan-ciùil +cùis-lagh +Cùis-lagh +cùl-chàineadh +Cùl-chàineadh +cùl-shleamhnach +Cùl-shleamhnach +cùl-taic +Cùl-taic +da-rìreabh +Da-rìreabh +da-rìreadh +Da-rìreadh +da-rìribh +Da-rìribh +deagh-ghean +Deagh-ghean +dearg-fhuileach +Dearg-fhuileach +deireadh-sheachdain +Deireadh-sheachdain +deoch-làidir +Deoch-làidir +dha-rìreabh +Dha-rìreabh +dha-rìribh +Dha-rìribh +dhaibh-san +Dhaibh-san +dhe-salin-adh +Dhe-salin-adh +dhe-salt-adh +Dhe-salt-adh +dheidhinn-sa +Dheidhinn-sa +dhol-sìos +Dhol-sìos +dhomh-s' +Dhomh-s' +dhuine-dubh +Dhuine-dubh +dhà-san +Dhà-san +dhòigh-beatha +Dhòigh-beatha +di-sathairne +Di-sathairne +dian-amharc +Dian-amharc +dlùth-cheangal +Dlùth-cheangal +do-chreidsinneach +Do-chreidsinneach +do-labhairt +Do-labhairt +do-sheachant' +Do-sheachant' +dol-a-mach +Dol-a-mach +dol-air-adhart +Dol-air-adhart +dubh-chàineadh +Dubh-chàineadh +dubh-ghorm +Dubh-ghorm +dà-chultarach +Dà-chultarach +dà-reug +Dà-reug +dàn-mòr +Dàn-mòr +dì-moladh +Dì-moladh +dòigh-beatha +Dòigh-beatha +dòighean-beatha +Dòighean-beatha +e-mail +E-mail +eadar-dhealachadh +Eadar-dhealachadh +eadar-dhealachaidhean +Eadar-dhealachaidhean +eadar-dhealaichte +Eadar-dhealaichte +eadar-nàiseanta +Eadar-nàiseanta +earbainn-s +Earbainn-s +eàrr-ràdh +Eàrr-ràdh +eòrp-innseanach +Eòrp-innseanach +fa-leth +Fa-leth +fa-near +Fa-near +fad-as +Fad-as +fad-thréimhseach +Fad-thréimhseach +feadaig-mhonaidh +Feadaig-mhonaidh +fealla-dhà +Fealla-dhà +fear-a-ropa +Fear-a-ropa +fear-ceasnachaidh +Fear-ceasnachaidh +fear-faire +Fear-faire +fear-gairm +Fear-gairm +fear-glèidhidh +Fear-glèidhidh +fear-labhairt +Fear-labhairt +fear-naidheachd +Fear-naidheachd +fear-pòsta +Fear-pòsta +fear-sgrùdaidh +Fear-sgrùdaidh +fear-teagaisg +Fear-teagaisg +fear-trèinidh +Fear-trèinidh +fear-éisteachd +Fear-éisteachd +feed-adh +Feed-adh +fhear-ghlèidhidh +Fhear-ghlèidhidh +fhear-gleidhidh +Fhear-gleidhidh +fhear-glèidhidh +Fhear-glèidhidh +fhear-labhairt +Fhear-labhairt +fhear-leughaidh +Fhear-leughaidh +fhear-sa +Fhear-sa +fhear-sgrùdaidh +Fhear-sgrùdaidh +fhir-cinnidh +Fhir-cinnidh +fhéin-ìomhaigh +Fhéin-ìomhaigh +fhìor-luachmhor +Fhìor-luachmhor +fois-fhòirneirt +Fois-fhòirneirt +fàs-bheairtean +Fàs-bheairtean +féin-mhisneachd +Féin-mhisneachd +féin-mholadh +Féin-mholadh +fìor-thàbhachdach +Fìor-thàbhachdach +ge-ta +Ge-ta +ge-tà +Ge-tà +ged-tà +Ged-tà +geàrr-chunntais +Geàrr-chunntais +geàrr-chunntas +Geàrr-chunntas +geàrr-thréimhseach +Geàrr-thréimhseach +ghuth-thàmh +Ghuth-thàmh +glain'-amhairc +Glain'-amhairc +glas-ghuib +Glas-ghuib +gnàth-bhriathrachas +Gnàth-bhriathrachas +gàrradh-crìche +Gàrradh-crìche +h- +H- +h-ana-miannaibh +H-ana-miannaibh +h-uile +H-uile +hó-ró +Hó-ró +iar-mhinistear +Iar-mhinistear +inneal-spreadhaidh +Inneal-spreadhaidh +ionad-còmhnaidh +Ionad-còmhnaidh +join-adh +Join-adh +latha-an-diugh +Latha-an-diugh +leam-sa +Leam-sa +leas-adh +Leas-adh +lease-adh +Lease-adh +leat-sa +Leat-sa +leotha-san +Leotha-san +leth-char +Leth-char +leth-cheud +Leth-cheud +leth-ghàidhealtachd +Leth-ghàidhealtachd +leth-pocannan +Leth-pocannan +leth-sgeulan +Leth-sgeulan +leth-uair +Leth-uair +leughadh-ne +Leughadh-ne +lighiche-sprèidh +Lighiche-sprèidh +linn-an-òir +Linn-an-òir +litir-aonta +Litir-aonta +loma-làn +Loma-làn +lost-s' +Lost-s' +luchd-altram +Luchd-altram +luchd-altruim +Luchd-altruim +luchd-amhairc +Luchd-amhairc +luchd-ciùil +Luchd-ciùil +luchd-cruinneachaidh +Luchd-cruinneachaidh +luchd-dìon +Luchd-dìon +luchd-ealain +Luchd-ealain +luchd-einnseanaraidh +Luchd-einnseanaraidh +luchd-glèidhteachais +Luchd-glèidhteachais +luchd-gnìomhachais +Luchd-gnìomhachais +luchd-iomairt +Luchd-iomairt +luchd-lagh +Luchd-lagh +luchd-lagha +Luchd-lagha +luchd-leanmhainn +Luchd-leanmhainn +luchd-litreachais +Luchd-litreachais +luchd-obrach +Luchd-obrach +luchd-reic +Luchd-reic +luchd-sgrùdaidh +Luchd-sgrùdaidh +luchd-teagaisg +Luchd-teagaisg +luchd-turais +Luchd-turais +luchd-éisdeachd +Luchd-éisdeachd +luchd-éisteachd +Luchd-éisteachd +là-an-diugh +Là-an-diugh +làmh-chuideachaidh +Làmh-chuideachaidh +làmh-sgrìobhainn +Làmh-sgrìobhainn +làmh-sgrìobhainnean +Làmh-sgrìobhainnean +làmh-sgrìobhta +Làmh-sgrìobhta +làn-bheachd +Làn-bheachd +làn-ghàidhealtachd +Làn-ghàidhealtachd +làn-thuigse +Làn-thuigse +làn-ùine +Làn-ùine +làrna-mhàireach +Làrna-mhàireach +lìn-bheaga +Lìn-bheaga +lùth-chleasan +Lùth-chleasan +ma-ta +Ma-ta +ma-tha +Ma-tha +ma-thà +Ma-thà +ma-tà +Ma-tà +mac-an-duine +Mac-an-duine +mac-léinn +Mac-léinn +mac-meanmna +Mac-meanmna +maighstir-sgoile +Maighstir-sgoile +maor-chladaich +Maor-chladaich +maor-fearainn +Maor-fearainn +mar-thà +Mar-thà +marbh-riaghailt +Marbh-riaghailt +meadhan-aoiseil +Meadhan-aoiseil +meadhan-latha +Meadhan-latha +meadhan-oidhche +Meadhan-oidhche +meal-an-naidheachd +Meal-an-naidheachd +mean-fhàs +Mean-fhàs +mhac-meanmna +Mhac-meanmna +mheadhain-latha +Mheadhain-latha +mheadhain-oidhche +Mheadhain-oidhche +mheadhan-oidhche +Mheadhan-oidhche +mheantraiginn-sa +Mheantraiginn-sa +mhi-rùn +Mhi-rùn +mhic-an-duine +Mhic-an-duine +mhoraltachd-sa +Mhoraltachd-sa +mhuir-làn +Mhuir-làn +mhuir-sgèin +Mhuir-sgèin +mhàthair-san +Mhàthair-san +mhì-chinnt +Mhì-chinnt +mhì-chneasda +Mhì-chneasda +mhì-chòrdadh +Mhì-chòrdadh +mhì-riaraichte +Mhì-riaraichte +mhì-shocair +Mhì-shocair +mhòr-chuid +Mhòr-chuid +mhòr-shluagh +Mhòr-shluagh +mhòr-shluaigh +Mhòr-shluaigh +mhór-amharas +Mhór-amharas +mhór-chuid +Mhór-chuid +mhór-shluaigh +Mhór-shluaigh +mi-chneasda +Mi-chneasda +mi-rùn +Mi-rùn +mic-léinn +Mic-léinn +mion-chànain +Mion-chànain +mion-fhios +Mion-fhios +mion-fhiosrach +Mion-fhiosrach +mion-sgrùdadh +Mion-sgrùdadh +muir-meadhon-thireach +Muir-meadhon-thireach +mèinnean-talmhainn +Mèinnean-talmhainn +mì-chinnt +Mì-chinnt +mì-choltach +Mì-choltach +mì-dhòigh +Mì-dhòigh +mì-fhair +Mì-fhair +mì-fhortanach +Mì-fhortanach +mì-laghail +Mì-laghail +mì-nàdarra +Mì-nàdarra +mì-nàdarrach +Mì-nàdarrach +mì-rùin +Mì-rùin +mì-shealbhach +Mì-shealbhach +mì-thlachd +Mì-thlachd +mòr-shluagh +Mòr-shluagh +mór-bhuannachd +Mór-bhuannachd +mór-chuid +Mór-chuid +mór-roinn +Mór-roinn +n- +N- +neach-casaid +Neach-casaid +neach-cathrach +Neach-cathrach +neach-gairm +Neach-gairm +neo-chiontach +Neo-chiontach +neo-eisimeileach +Neo-eisimeileach +neo-iomlan +Neo-iomlan +neo-àbhaisteach +Neo-àbhaisteach +nua-bhàrdachd +Nua-bhàrdachd +nì-eigin +Nì-eigin +obair-sa +Obair-sa +oifigear-stiùiridh +Oifigear-stiùiridh +oirbh-se +Oirbh-se +ola-thruis +Ola-thruis +orm-sa +Orm-sa +orra-san +Orra-san +phiuthar-chéile +Phiuthar-chéile +phort-adhair +Phort-adhair +phump-adh +Phump-adh +phàipeir-naidheachd +Phàipeir-naidheachd +phòcaid-thòine +Phòcaid-thòine +pole-aichean +Pole-aichean +port-adhair +Port-adhair +proove-adh +Proove-adh +pàipear-naidheachd +Pàipear-naidheachd +pàipearan-naidheachd +Pàipearan-naidheachd +radio-beò +Radio-beò +rithe-se +Rithe-se +rium-sa +Rium-sa +ro-chumhang +Ro-chumhang +ro-eòlach +Ro-eòlach +ro-innleachd +Ro-innleachd +ro-làimh +Ro-làimh +ro-shealladh +Ro-shealladh +roth-thoisich +Roth-thoisich +rèidio-beò +Rèidio-beò +rùm-cùil +Rùm-cùil +sadadh-a-steach +Sadadh-a-steach +samhradh-a-chaidh +Samhradh-a-chaidh +saor-làithean +Saor-làithean +sead-fhighe +Sead-fhighe +sean-ghnàthas +Sean-ghnàthas +seana-bhliadhn' +Seana-bhliadhn' +seirbhis-aisig +Seirbhis-aisig +seòl-mara +Seòl-mara +seòmar-cadail +Seòmar-cadail +sgeulachdan-gaisge +Sgeulachdan-gaisge +sgoil-marcaidheachd +Sgoil-marcaidheachd +sgìr-easbaig +Sgìr-easbaig +sgìre-easbaig +Sgìre-easbaig +sheann-fhasanta +Sheann-fhasanta +shlatan-connaidh +Shlatan-connaidh +shon-sa +Shon-sa +shàr-sgoilear +Shàr-sgoilear +sibh-se +Sibh-se +snodha-gàire +Snodha-gàire +so-labhairt +So-labhairt +soch-mhalairteach +Soch-mhalairteach +spor-gunna +Spor-gunna +sàr-bheachdan +Sàr-bheachdan +sìor-dhol +Sìor-dhol +sùil-air-ais +Sùil-air-ais +sùil-mhara +Sùil-mhara +t- +T- +taigh-cuibhle +Taigh-cuibhle +taigh-céilidh +Taigh-céilidh +taigh-sa +Taigh-sa +taigh-sheinnse +Taigh-sheinnse +taigh-tasgaidh +Taigh-tasgaidh +taigh-tughaidh +Taigh-tughaidh +taigh-òsda +Taigh-òsda +taigh-òsta +Taigh-òsta +taighean-aoigheachd +Taighean-aoigheachd +taobh-sa +Taobh-sa +teachd-an-tìr +Teachd-an-tìr +teaghlach-chànanan +Teaghlach-chànanan +thaicean-airgid +Thaicean-airgid +thaighean-altraim +Thaighean-altraim +thonn-gheal +Thonn-gheal +thuigse-san +Thuigse-san +tigh-croiteir +Tigh-croiteir +tigh-còmhnaidh +Tigh-còmhnaidh +tigh-seinnse +Tigh-seinnse +tigh-sheinnse +Tigh-sheinnse +tighearnan-fearainn +Tighearnan-fearainn +togail-cridhe +Togail-cridhe +travel-adh +Travel-adh +triob-sa +Triob-sa +tro-chèile +Tro-chèile +troimh-a-chéile +Troimh-a-chéile +troimh-chèile +Troimh-chèile +troimhe-chéile +Troimhe-chéile +tuathanas-éisg +Tuathanas-éisg +tè-labhairt +Tè-labhairt +tìr-mhóir +Tìr-mhóir +tìr-mòr +Tìr-mòr +ugam-s' +Ugam-s' +ugam-sa +Ugam-sa +uige-san +Uige-san +uile-gu-lèir +Uile-gu-lèir +uile-tuigseach +Uile-tuigseach +use-agadh +Use-agadh +watch-adh +Watch-adh +weld-adh +Weld-adh +àrd-cheannard +Àrd-cheannard +àrd-chomhairliche +Àrd-chomhairliche +àrd-chonstabal +Àrd-chonstabal +àrd-dhuine +Àrd-dhuine +àrd-ionmhair +Àrd-ionmhair +àrd-oifigear +Àrd-oifigear +àrd-oifigeir +Àrd-oifigeir +àrd-sgoil +Àrd-sgoil +àrd-ìre +Àrd-ìre +àrd-ùrlair +Àrd-ùrlair +àrd-ùrlar +Àrd-ùrlar +às-creideach +Às-creideach +àtha-cheilpe +Àtha-cheilpe +ìre-sa +Ìre-sa +ìre-se +Ìre-se +òg-mhios +Òg-mhios +òige-sa +Òige-sa +òrd-mhòr +Òrd-mhòr""".split(): + _exc[orth] = [{ORTH: orth}] + +# Multiple words that should remain as one token +for orth in """'n diugh +'N diugh +'n dà +'N dà +'n iar +'N iar +'n seo +'N seo +'n uairsin +'N uairsin +a a sineach +A a sineach +a b' +A b' +a bhos +A bhos +a bhàn +A bhàn +a bhòn raoir +A bhòn raoir +a bhòn uiridh +A bhòn uiridh +a bhòn-dè +A bhòn-dè +a bhòn-raoir +A bhòn-raoir +a bhòn-uiridh +A bhòn-uiridh +a bu' +A bu' +a chaoidh +A chaoidh +a cheana +A cheana +a chionn +A chionn +a chionn 's +A chionn 's +a chuile +A chuile +a chèil +A chèil +a chèile +A chèile +a chéile +A chéile +a deas +A deas +a dh' +A dh' +a h-uile +A h-uile +a mach +A mach +a muigh +A muigh +a màireach +A màireach +a nall +A nall +a neisd +A neisd +a nis +A nis +a nisd +A nisd +a nise +A nise +a niste +A niste +a nochd +A nochd +a nuas +A nuas +a null +A null +a raoir +A raoir +a riamh +A riamh +a rithist +A rithist +a s +A s +a seo +A seo +a seothach +A seothach +a shineach +A shineach +a sin +A sin +a sineach +A sineach +a staidh +A staidh +a staigh +A staigh +a steach +A steach +a stigh +A stigh +a tuath +A tuath +a uiridh +A uiridh +a' diugh +A' diugh +a' s +A' s +air bith +Air bith +air choireigin +Air choireigin +air choireigin-ach +Air choireigin-ach +air choreigin +Air choreigin +air dheireadh +Air dheireadh +air falbh +Air falbh +air neo +Air neo +air thùs +Air thùs +am a màireach muigh +Am a màireach muigh +am bliadhna +Am bliadhna +am muigh +Am muigh +an am +An am +an aodann bàn +An aodann bàn +an ath bhliadhna +An ath bhliadhna +an ath oidhch' +An ath oidhch' +an ath oidhche +An ath oidhche +an ath sheachdain +An ath sheachdain +an ath sheachdainn +An ath sheachdainn +an ath-bhliadhna +An ath-bhliadhna +an ath-oidhch' +An ath-oidhch' +an ath-oidhche +An ath-oidhche +an ath-sheachdain +An ath-sheachdain +an ath-sheachdainn +An ath-sheachdainn +an ceart-uair +An ceart-uair +an ceartuair +An ceartuair +an còmhnaidh +An còmhnaidh +an de +An de +an deas +An deas +an diugh +An diugh +an dràsda +An dràsda +an dràsta +An dràsta +an dè +An dè +an ear +An ear +an earair +An earair +an earar +An earar +an earras +An earras +an iar +An iar +an iaras +An iaras +an làrna-mhàireach +An làrna-mhàireach +an raoir +An raoir +an sean +An sean +an seo +An seo +an seothach +An seothach +an sin +An sin +an sineach +An sineach +an siod +An siod +an siud +An siud +an siudach +An siudach +an toiseach +An toiseach +an uair +An uair +an uair sin +An uair sin +an uairsin +An uairsin +an uirigh +An uirigh +an àird +An àird +an àirde +An àirde +an ìre mhath +An ìre mhath +ana nàdarra +Ana nàdarra +ann a +Ann a +ann a sheo +Ann a sheo +ann a sheothach +Ann a sheothach +ann a shin +Ann a shin +ann a shineach +Ann a shineach +ann a shiodach +Ann a shiodach +ann a shiud +Ann a shiud +ann a shiudach +Ann a shiudach +ann a' +Ann a' +ann a' shiudach +Ann a' shiudach +ann a-seo +Ann a-seo +ann a-seothach +Ann a-seothach +ann a-sin +Ann a-sin +ann a-sineach +Ann a-sineach +ann a-siud +Ann a-siud +ann am +Ann am +ann an +Ann an +ann an seo +Ann an seo +ann an shin +Ann an shin +ann an shiud +Ann an shiud +ann an sin +Ann an sin +ann an siud +Ann an siud +ann seo +Ann seo +anns a' bhad +Anns a' bhad +anns an +Anns an +ath-oidhch' +Ath-oidhch' +ban-righ 'nn +Ban-righ 'nn +bho thoiseach +Bho thoiseach +bhon 'n +Bhon 'n +bhon a' +Bhon a' +bhon an +Bhon an +bhrist ' +Bhrist ' +buille a-mach +Buille a-mach +bun os cionn +Bun os cionn +car son +Car son +ceann a tuath +Ceann a tuath +cia mheud +Cia mheud +coille chaoil +Coille chaoil +cò mheud +Cò mheud +có dhiubh +Có dhiubh +d' rachadh +D' rachadh +dhen an +Dhen an +do n +Do n +dè mar +Dè mar +dé mar +Dé mar +eilean tiridhe +Eilean tiridhe +fa leth +Fa leth +fad as +Fad as +fo dheireadh +Fo dheireadh +fon a' +Fon a' +fon an +Fon an +gar bith +Gar bith +gar bith có +Gar bith có +ge 's bith +Ge 's bith +ge b' e air bith +Ge b' e air bith +ge be +Ge be +ge brith +Ge brith +ge brì +Ge brì +gleann dail +Gleann dail +gleann ois +Gleann ois +gu bè gu dè +Gu bè gu dè +gu dè +Gu dè +gu dé +Gu dé +gu ruige +Gu ruige +ho ro gheallaidh +Ho ro gheallaidh +ma dheireadh +Ma dheireadh +ma dheireadh thall +Ma dheireadh thall +ma sgaoil +Ma sgaoil +ma tha +Ma tha +mar an ceudna +Mar an ceudna +mar bu trice +Mar bu trice +mar tha +Mar tha +meadhan aoiseil +Meadhan aoiseil +mu 'n +Mu 'n +mu chuairt +Mu chuairt +mu dheas +Mu dheas +mu dheireadh +Mu dheireadh +mu dheireadh thall +Mu dheireadh thall +mu n +Mu n +mu thràth +Mu thràth +mun a' +Mun a' +mun an +Mun an +na b' +Na b' +na bu +Na bu +na iad +Na iad +nach maireann +Nach maireann +o'n uairsin +O'n uairsin +oidhch ' +Oidhch ' +on a' +On a' +on an +On an +pholl a' ghrùthain +Pholl a' ghrùthain +roinn eorpa +Roinn eorpa +ron a' +Ron a' +ron an +Ron an +ruaidh mhònaidh +Ruaidh mhònaidh +ruith thairis +Ruith thairis +sa bhad +Sa bhad +sadadh a-mach +Sadadh a-mach +sadadh a-steach +Sadadh a-steach +sam bidh +Sam bidh +sam bith +Sam bith +srath chluaidh +Srath chluaidh +taobh a-muigh +Taobh a-muigh +taobh an ear +Taobh an ear +taobh an iar +Taobh an iar +tria san ngaoidhilcc nalbanaigh +Tria san ngaoidhilcc nalbanaigh +tron a' +Tron a' +tron an +Tron an +tuilleadh 's a chòir +Tuilleadh 's a chòir +tuilleadh sa chòir +Tuilleadh sa chòir""".split("\n"): + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/grc/__init__.py b/spacy/lang/grc/__init__.py new file mode 100644 index 0000000..ed742f4 --- /dev/null +++ b/spacy/lang/grc/__init__.py @@ -0,0 +1,22 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class AncientGreekDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class AncientGreek(Language): + lang = "grc" + Defaults = AncientGreekDefaults + + +__all__ = ["AncientGreek"] diff --git a/spacy/lang/grc/examples.py b/spacy/lang/grc/examples.py new file mode 100644 index 0000000..51ec8f8 --- /dev/null +++ b/spacy/lang/grc/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.grc.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "ἐρᾷ μὲν ἁγνὸς οὐρανὸς τρῶσαι χθόνα, ἔρως δὲ γαῖαν λαμβάνει γάμου τυχεῖν·", + "εὐδαίμων Χαρίτων καὶ Μελάνιππος ἔφυ, θείας ἁγητῆρες ἐφαμερίοις φιλότατος.", + "ὃ μὲν δὴ ἀπόστολος ἐς τὴν Μίλητον ἦν.", + "Θρασύβουλος δὲ σαφέως προπεπυσμένος πάντα λόγον καὶ εἰδὼς τὰ Ἀλυάττης μέλλοι ποιήσειν μηχανᾶται τοιάδε.", + "φιλόπαις δ' ἦν ἐκμανῶς καὶ Ἀλέξανδρος ὁ βασιλεύς.", + "Ἀντίγονος ὁ βασιλεὺς ἐπεκώμαζε τῷ Ζήνωνι", + "αὐτὰρ ὃ δεύτατος ἦλθεν ἄναξ ἀνδρῶν Ἀγαμέμνων ἕλκος ἔχων", +] diff --git a/spacy/lang/grc/lex_attrs.py b/spacy/lang/grc/lex_attrs.py new file mode 100644 index 0000000..33cfca0 --- /dev/null +++ b/spacy/lang/grc/lex_attrs.py @@ -0,0 +1,313 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + # CARDINALS + "εἷς", + "ἑνός", + "ἑνί", + "ἕνα", + "μία", + "μιᾶς", + "μιᾷ", + "μίαν", + "ἕν", + "δύο", + "δυοῖν", + "τρεῖς", + "τριῶν", + "τρισί", + "τρία", + "τέτταρες", + "τεττάρων", + "τέτταρσι", + "τέτταρα", + "τέτταρας", + "πέντε", + "ἕξ", + "ἑπτά", + "ὀκτώ", + "ἐννέα", + "δέκα", + "ἕνδεκα", + "δώδεκα", + "πεντεκαίδεκα", + "ἑκκαίδεκα", + "ἑπτακαίδεκα", + "ὀκτωκαίδεκα", + "ἐννεακαίδεκα", + "εἴκοσι", + "τριάκοντα", + "τετταράκοντα", + "πεντήκοντα", + "ἑξήκοντα", + "ἑβδομήκοντα", + "ὀγδοήκοντα", + "ἐνενήκοντα", + "ἑκατόν", + "διακόσιοι", + "διακοσίων", + "διακοσιᾶν", + "διακοσίους", + "διακοσίοις", + "διακόσια", + "διακόσιαι", + "διακοσίαις", + "διακοσίαισι", + "διηκόσιοι", + "διηκοσίων", + "διηκοσιέων", + "διακοσίας", + "διηκόσια", + "διηκόσιαι", + "διηκοσίας", + "τριακόσιοι", + "τριακοσίων", + "τριακοσιᾶν", + "τριακοσίους", + "τριακοσίοις", + "τριακόσια", + "τριακόσιαι", + "τριακοσίαις", + "τριακοσίαισι", + "τριακοσιέων", + "τριακοσίας", + "τριηκόσια", + "τριηκοσίας", + "τριηκόσιοι", + "τριηκοσίοισιν", + "τριηκοσίους", + "τριηκοσίων", + "τετρακόσιοι", + "τετρακοσίων", + "τετρακοσιᾶν", + "τετρακοσίους", + "τετρακοσίοις", + "τετρακόσια", + "τετρακόσιαι", + "τετρακοσίαις", + "τετρακοσίαισι", + "τετρακοσιέων", + "τετρακοσίας", + "πεντακόσιοι", + "πεντακοσίων", + "πεντακοσιᾶν", + "πεντακοσίους", + "πεντακοσίοις", + "πεντακόσια", + "πεντακόσιαι", + "πεντακοσίαις", + "πεντακοσίαισι", + "πεντακοσιέων", + "πεντακοσίας", + "ἑξακόσιοι", + "ἑξακοσίων", + "ἑξακοσιᾶν", + "ἑξακοσίους", + "ἑξακοσίοις", + "ἑξακόσια", + "ἑξακόσιαι", + "ἑξακοσίαις", + "ἑξακοσίαισι", + "ἑξακοσιέων", + "ἑξακοσίας", + "ἑπτακόσιοι", + "ἑπτακοσίων", + "ἑπτακοσιᾶν", + "ἑπτακοσίους", + "ἑπτακοσίοις", + "ἑπτακόσια", + "ἑπτακόσιαι", + "ἑπτακοσίαις", + "ἑπτακοσίαισι", + "ἑπτακοσιέων", + "ἑπτακοσίας", + "ὀκτακόσιοι", + "ὀκτακοσίων", + "ὀκτακοσιᾶν", + "ὀκτακοσίους", + "ὀκτακοσίοις", + "ὀκτακόσια", + "ὀκτακόσιαι", + "ὀκτακοσίαις", + "ὀκτακοσίαισι", + "ὀκτακοσιέων", + "ὀκτακοσίας", + "ἐνακόσιοι", + "ἐνακοσίων", + "ἐνακοσιᾶν", + "ἐνακοσίους", + "ἐνακοσίοις", + "ἐνακόσια", + "ἐνακόσιαι", + "ἐνακοσίαις", + "ἐνακοσίαισι", + "ἐνακοσιέων", + "ἐνακοσίας", + "χίλιοι", + "χιλίων", + "χιλιῶν", + "χιλίους", + "χιλίοις", + "χίλιαι", + "χιλίας", + "χιλίαις", + "χίλια", + "χίλι", + "δισχίλιοι", + "δισχιλίων", + "δισχιλιῶν", + "δισχιλίους", + "δισχιλίοις", + "δισχίλιαι", + "δισχιλίας", + "δισχιλίαις", + "δισχίλια", + "δισχίλι", + "τρισχίλιοι", + "τρισχιλίων", + "τρισχιλιῶν", + "τρισχιλίους", + "τρισχιλίοις", + "τρισχίλιαι", + "τρισχιλίας", + "τρισχιλίαις", + "τρισχίλια", + "τρισχίλι", + "μύριοι", + "μύριοί", + "μυρίων", + "μυρίοις", + "μυρίους", + "μύριαι", + "μυρίαις", + "μυρίας", + "μύρια", + "δισμύριοι", + "δισμύριοί", + "δισμυρίων", + "δισμυρίοις", + "δισμυρίους", + "δισμύριαι", + "δισμυρίαις", + "δισμυρίας", + "δισμύρια", + "δεκακισμύριοι", + "δεκακισμύριοί", + "δεκακισμυρίων", + "δεκακισμυρίοις", + "δεκακισμυρίους", + "δεκακισμύριαι", + "δεκακισμυρίαις", + "δεκακισμυρίας", + "δεκακισμύρια", + # ANCIENT GREEK NUMBERS (1-100) + "α", + "β", + "γ", + "δ", + "ε", + "ϛ", + "ζ", + "η", + "θ", + "ι", + "ια", + "ιβ", + "ιγ", + "ιδ", + "ιε", + "ιϛ", + "ιζ", + "ιη", + "ιθ", + "κ", + "κα", + "κβ", + "κγ", + "κδ", + "κε", + "κϛ", + "κζ", + "κη", + "κθ", + "λ", + "λα", + "λβ", + "λγ", + "λδ", + "λε", + "λϛ", + "λζ", + "λη", + "λθ", + "μ", + "μα", + "μβ", + "μγ", + "μδ", + "με", + "μϛ", + "μζ", + "μη", + "μθ", + "ν", + "να", + "νβ", + "νγ", + "νδ", + "νε", + "νϛ", + "νζ", + "νη", + "νθ", + "ξ", + "ξα", + "ξβ", + "ξγ", + "ξδ", + "ξε", + "ξϛ", + "ξζ", + "ξη", + "ξθ", + "ο", + "οα", + "οβ", + "ογ", + "οδ", + "οε", + "οϛ", + "οζ", + "οη", + "οθ", + "π", + "πα", + "πβ", + "πγ", + "πδ", + "πε", + "πϛ", + "πζ", + "πη", + "πθ", + "ϟ", + "ϟα", + "ϟβ", + "ϟγ", + "ϟδ", + "ϟε", + "ϟϛ", + "ϟζ", + "ϟη", + "ϟθ", + "ρ", +] + + +def like_num(text): + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/grc/punctuation.py b/spacy/lang/grc/punctuation.py new file mode 100644 index 0000000..5903761 --- /dev/null +++ b/spacy/lang/grc/punctuation.py @@ -0,0 +1,57 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, +) + +_prefixes = ( + [ + "†", + "⸏", + "〈", + ] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + "†", + "⸎", + "〉", + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]", + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])—", + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/grc/stop_words.py b/spacy/lang/grc/stop_words.py new file mode 100644 index 0000000..cbb766a --- /dev/null +++ b/spacy/lang/grc/stop_words.py @@ -0,0 +1,61 @@ +STOP_WORDS = set( + """ +αὐτῷ αὐτοῦ αὐτῆς αὐτόν αὐτὸν αὐτῶν αὐτὸς αὐτὸ αὐτό αὐτός αὐτὴν αὐτοῖς αὐτοὺς αὔτ' αὐτὰ αὐτῇ αὐτὴ +αὐτὼ αὑταὶ καὐτὸς αὐτά αὑτός αὐτοῖσι αὐτοῖσιν αὑτὸς αὐτήν αὐτοῖσί αὐτοί αὐτοὶ αὐτοῖο αὐτάων αὐτὰς +αὐτέων αὐτώ αὐτάς αὐτούς αὐτή αὐταί αὐταὶ αὐτῇσιν τὠυτῷ τὠυτὸ ταὐτὰ ταύτῃ αὐτῇσι αὐτῇς αὐταῖς αὐτᾶς αὐτὰν ταὐτὸν + +γε γ' γέ γὰρ γάρ δαῖτα δαιτὸς δαιτὶ δαὶ δαιτί δαῖτ' δαΐδας δαΐδων δἰ διὰ διά δὲ δ' δέ δὴ δή εἰ εἴ κεἰ κεἴ αἴ αἲ εἲ αἰ + +ἐστί ἐστιν ὢν ἦν ἐστὶν ὦσιν εἶναι ὄντι εἰσιν ἐστι ὄντα οὖσαν ἦσαν ἔστι ὄντας ἐστὲ εἰσὶ εἶ ὤν ἦ οὖσαι ἔσται ἐσμὲν ἐστ' ἐστίν ἔστ' ὦ ἔσει ἦμεν εἰμι εἰσὶν ἦσθ' +ἐστὶ ᾖ οὖσ' ἔστιν εἰμὶ εἴμ' ἐσθ' ᾖς στί εἴην εἶναί οὖσα κἄστ' εἴη ἦσθα εἰμ' ἔστω ὄντ' ἔσθ' ἔμμεναι ἔω ἐὼν ἐσσι ἔσσεται ἐστὸν ἔσαν ἔστων ἐόντα ἦεν ἐοῦσαν ἔην +ἔσσομαι εἰσί ἐστόν ἔσκεν ἐόντ' ἐών ἔσσεσθ' εἰσ' ἐόντες ἐόντε ἐσσεῖται εἰμεν ἔασιν ἔσκε ἔμεναι ἔσεσθαι ἔῃ εἰμὲν εἰσι ἐόντας ἔστε εἰς ἦτε εἰμί ἔσσεαι ἔμμεν +ἐοῦσα ἔμεν ᾖσιν ἐστε ἐόντι εἶεν ἔσσονται ἔησθα ἔσεσθε ἐσσί ἐοῦσ' ἔασι ἔα ἦα ἐόν ἔσσεσθαι ἔσομαι ἔσκον εἴης ἔωσιν εἴησαν ἐὸν ἐουσέων ἔσσῃ ἐούσης ἔσονται +ἐούσας ἐόντων ἐόντος ἐσομένην ἔστωσαν ἔωσι ἔας ἐοῦσαι ἣν εἰσίν ἤστην ὄντες ὄντων οὔσας οὔσαις ὄντος οὖσι οὔσης ἔσῃ ὂν ἐσμεν ἐσμέν οὖσιν ἐσομένους ἐσσόμεσθα + +ἒς ἐς ἔς ἐν κεἰς εἲς κἀν ἔν κατὰ κατ' καθ' κατά κάτα κὰπ κὰκ κὰδ κὰρ κάρ κὰγ κὰμ καὶ καί μετὰ μεθ' μετ' μέτα μετά μέθ' μέτ' μὲν μέν μὴ + +μή μη οὐκ οὒ οὐ οὐχ οὐχὶ κοὐ κοὐχ οὔ κοὐκ οὐχί οὐκὶ οὐδὲν οὐδεὶς οὐδέν κοὐδεὶς κοὐδὲν οὐδένα οὐδενὸς οὐδέν' οὐδενός οὐδενὶ +οὐδεμία οὐδείς οὐδεμίαν οὐδὲ οὐδ' κοὐδ' οὐδέ οὔτε οὔθ' οὔτέ τε οὔτ' οὕτως οὕτω οὕτῶ χοὔτως οὖν ὦν ὧν τοῦτο τοῦθ' τοῦτον τούτῳ +τούτοις ταύτας αὕτη ταῦτα οὗτος ταύτης ταύτην τούτων ταῦτ' τοῦτ' τούτου αὗται τούτους τοῦτό ταῦτά τούτοισι χαὔτη ταῦθ' χοὖτοι +τούτοισιν οὗτός οὗτοι τούτω τουτέων τοῦτὸν οὗτοί τοῦτου οὗτοὶ ταύτῃσι ταύταις ταυτὶ παρὰ παρ' πάρα παρά πὰρ παραὶ πάρ' περὶ +πέρι περί πρὸς πρός ποτ' ποτὶ προτὶ προτί πότι + +σὸς σήν σὴν σὸν σόν σὰ σῶν σοῖσιν σός σῆς σῷ σαῖς σῇ σοῖς σοῦ σ' σὰν σά σὴ σὰς +σᾷ σοὺς σούς σοῖσι σῇς σῇσι σή σῇσιν σοὶ σου ὑμεῖς σὲ σύ σοι ὑμᾶς ὑμῶν ὑμῖν σε +σέ σὺ σέθεν σοί ὑμὶν σφῷν ὑμίν τοι τοὶ σφὼ ὔμμ' σφῶϊ σεῖο τ' σφῶϊν ὔμμιν σέο σευ σεῦ +ὔμμι ὑμέων τύνη ὑμείων τοί ὔμμες σεο τέ τεοῖο ὑμέας σὺν ξὺν σύν + +θ' τί τι τις τινες τινα τινος τινὸς τινὶ τινῶν τίς τίνες τινὰς τιν' τῳ του τίνα τοῦ τῷ τινί τινά τίνος τινι τινας τινὰ τινων +τίν' τευ τέο τινές τεο τινὲς τεῷ τέῳ τινός τεῳ τισὶ + +τοιαῦτα τοιοῦτον τοιοῦθ' τοιοῦτος τοιαύτην τοιαῦτ' τοιούτου τοιαῦθ' τοιαύτῃ τοιούτοις τοιαῦται τοιαῦτά τοιαύτη τοιοῦτοι τοιούτων τοιούτοισι +τοιοῦτο τοιούτους τοιούτῳ τοιαύτης τοιαύταις τοιαύτας τοιοῦτός τίνι τοῖσι τίνων τέων τέοισί τὰ τῇ τώ τὼ + +ἀλλὰ ἀλλ' ἀλλά ἀπ' ἀπὸ κἀπ' ἀφ' τἀπὸ κἀφ' ἄπο ἀπό τὠπὸ τἀπ' ἄλλων ἄλλῳ ἄλλη ἄλλης ἄλλους ἄλλοις ἄλλον ἄλλο ἄλλου τἄλλα ἄλλα +ἄλλᾳ ἄλλοισιν τἄλλ' ἄλλ' ἄλλος ἄλλοισι κἄλλ' ἄλλοι ἄλλῃσι ἄλλόν ἄλλην ἄλλά ἄλλαι ἄλλοισίν ὧλλοι ἄλλῃ ἄλλας ἀλλέων τἆλλα ἄλλως +ἀλλάων ἄλλαις τἆλλ' + +ἂν ἄν κἂν τἂν ἃν κεν κ' κέν κέ κε χ' ἄρα τἄρα ἄρ' τἄρ' ἄρ ῥα ῥά ῥ τὰρ ἄρά ἂρ + +ἡμᾶς με ἐγὼ ἐμὲ μοι κἀγὼ ἡμῶν ἡμεῖς ἐμοὶ ἔγωγ' ἁμοὶ ἡμῖν μ' ἔγωγέ ἐγώ ἐμοί ἐμοῦ κἀμοῦ ἔμ' κἀμὲ ἡμὶν μου ἐμέ ἔγωγε νῷν νὼ χἠμεῖς ἁμὲ κἀγώ κἀμοὶ χἠμᾶς +ἁγὼ ἡμίν κἄμ' ἔμοιγ' μοί τοὐμὲ ἄμμε ἐγὼν ἐμεῦ ἐμεῖο μευ ἔμοιγε ἄμμι μέ ἡμέας νῶϊ ἄμμιν ἧμιν ἐγών νῶΐ ἐμέθεν ἥμιν ἄμμες νῶι ἡμείων ἄμμ' ἡμέων ἐμέο +ἐκ ἔκ ἐξ κἀκ κ ἃκ κἀξ ἔξ εξ Ἐκ τἀμὰ ἐμοῖς τοὐμόν ἐμᾶς τοὐμὸν ἐμῶν ἐμὸς ἐμῆς ἐμῷ τὠμῷ ἐμὸν τἄμ' ἐμὴ ἐμὰς ἐμαῖς ἐμὴν ἐμόν ἐμὰ ἐμός ἐμοὺς ἐμῇ ἐμᾷ +οὑμὸς ἐμοῖν οὑμός κἀμὸν ἐμαὶ ἐμή ἐμάς ἐμοῖσι ἐμοῖσιν ἐμῇσιν ἐμῇσι ἐμῇς ἐμήν + +ἔνι ἐνὶ εἰνὶ εἰν ἐμ ἐπὶ ἐπ' ἔπι ἐφ' κἀπὶ τἀπὶ ἐπί ἔφ' ἔπ' ἐὰν ἢν ἐάν ἤν ἄνπερ + +αὑτοῖς αὑτὸν αὑτῷ ἑαυτοῦ αὑτόν αὑτῆς αὑτῶν αὑτοῦ αὑτὴν αὑτοῖν χαὐτοῦ αὑταῖς ἑωυτοῦ ἑωυτῇ ἑωυτὸν ἐωυτῷ ἑωυτῆς ἑωυτόν ἑωυτῷ +ἑωυτάς ἑωυτῶν ἑωυτοὺς ἑωυτοῖσι ἑαυτῇ ἑαυτούς αὑτοὺς ἑαυτῶν ἑαυτοὺς ἑαυτὸν ἑαυτῷ ἑαυτοῖς ἑαυτὴν ἑαυτῆς + +ἔτι ἔτ' ἔθ' κἄτι ἢ ἤ ἠέ ἠὲ ἦε ἦέ ἡ τοὺς τὴν τὸ τῶν τὸν ὁ ἁ οἱ τοῖς ταῖς τῆς τὰς αἱ τό τὰν τᾶς τοῖσιν αἳ χὠ τήν τά τοῖν τάς ὅ +χοἰ ἣ ἥ χἠ τάν τᾶν ὃ οἳ οἵ τοῖο τόν τοῖιν τούς τάων ταὶ τῇς τῇσι τῇσιν αἵ τοῖό τοῖσίν ὅττί ταί Τὴν τῆ τῶ τάδε ὅδε τοῦδε τόδε τόνδ' +τάδ' τῆσδε τῷδε ὅδ' τῶνδ' τῇδ' τοῦδέ τῶνδε τόνδε τόδ' τοῦδ' τάσδε τήνδε τάσδ' τήνδ' ταῖσδέ τῇδε τῆσδ' τάνδ' τῷδ' τάνδε ἅδε τοῖσδ' ἥδ' +τᾷδέ τοῖσδε τούσδ' ἥδε τούσδε τώδ' ἅδ' οἵδ' τῶνδέ οἵδε τᾷδε τοῖσδεσσι τώδε τῇδέ τοῖσιδε αἵδε τοῦδὲ τῆδ' αἵδ' τοῖσδεσι ὃν ἃ ὃς ᾧ οὗ ἅπερ +οὓς ἧς οἷς ἅσπερ ᾗ ἅ χὦνπερ ὣ αἷς ᾇ ὅς ἥπερ ἃς ὅσπερ ὅνπερ ὧνπερ ᾧπερ ὅν αἷν οἷσι ἇς ἅς ὥ οὕς ἥν οἷσιν ἕης ὅου ᾗς οἷσί οἷσίν τοῖσί ᾗσιν οἵπερ αἷσπερ +ὅστις ἥτις ὅτου ὅτοισι ἥντιν' ὅτῳ ὅντιν' ὅττι ἅσσά ὅτεῳ ὅτις ὅτιν' ὅτευ ἥντινα αἵτινές ὅντινα ἅσσα ᾧτινι οἵτινες ὅτι ἅτις ὅτ' ὑμὴ +ὑμήν ὑμὸν ὑπὲρ ὕπερ ὑπέρτερον ὑπεὶρ ὑπέρτατος ὑπὸ ὑπ' ὑφ' ὕπο ὑπαὶ ὑπό ὕπ' ὕφ' + + ὣς ὡς ὥς ὧς ὥστ' ὥστε ὥσθ' ὤ ὢ + + """.split() +) diff --git a/spacy/lang/grc/tokenizer_exceptions.py b/spacy/lang/grc/tokenizer_exceptions.py new file mode 100644 index 0000000..86527ff --- /dev/null +++ b/spacy/lang/grc/tokenizer_exceptions.py @@ -0,0 +1,111 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for token in ["᾽Απ'", "᾽ΑΠ'", "ἀφ'", "᾽Αφ", "ἀπὸ"]: + _exc[token] = [{ORTH: token, NORM: "από"}] + +for token in ["᾽Αλλ'", "ἀλλ'", "ἀλλὰ"]: + _exc[token] = [{ORTH: token, NORM: "ἀλλά"}] + +for token in ["παρ'", "Παρ'", "παρὰ", "παρ"]: + _exc[token] = [{ORTH: token, NORM: "παρά"}] + +for token in ["καθ'", "Καθ'", "κατ'", "Κατ'", "κατὰ"]: + _exc[token] = [{ORTH: token, NORM: "κατά"}] + +for token in ["Ἐπ'", "ἐπ'", "ἐπὶ", "Εφ'", "εφ'"]: + _exc[token] = [{ORTH: token, NORM: "επί"}] + +for token in ["Δι'", "δι'", "διὰ"]: + _exc[token] = [{ORTH: token, NORM: "διά"}] + +for token in ["Ὑπ'", "ὑπ'", "ὑφ'"]: + _exc[token] = [{ORTH: token, NORM: "ὑπό"}] + +for token in ["Μετ'", "μετ'", "μεθ'", "μετὰ"]: + _exc[token] = [{ORTH: token, NORM: "μετά"}] + +for token in ["Μ'", "μ'", "μέ", "μὲ"]: + _exc[token] = [{ORTH: token, NORM: "με"}] + +for token in ["Σ'", "σ'", "σέ", "σὲ"]: + _exc[token] = [{ORTH: token, NORM: "σε"}] + +for token in ["Τ'", "τ'", "τέ", "τὲ"]: + _exc[token] = [{ORTH: token, NORM: "τε"}] + +for token in ["Δ'", "δ'", "δὲ"]: + _exc[token] = [{ORTH: token, NORM: "δέ"}] + + +_other_exc = { + "μὲν": [{ORTH: "μὲν", NORM: "μέν"}], + "μὴν": [{ORTH: "μὴν", NORM: "μήν"}], + "τὴν": [{ORTH: "τὴν", NORM: "τήν"}], + "τὸν": [{ORTH: "τὸν", NORM: "τόν"}], + "καὶ": [{ORTH: "καὶ", NORM: "καί"}], + "καὐτός": [{ORTH: "κ", NORM: "καί"}, {ORTH: "αὐτός"}], + "καὐτὸς": [{ORTH: "κ", NORM: "καί"}, {ORTH: "αὐτὸς", NORM: "αὐτός"}], + "κοὐ": [{ORTH: "κ", NORM: "καί"}, {ORTH: "οὐ"}], + "χἡ": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ἡ"}], + "χοἱ": [{ORTH: "χ", NORM: "καί"}, {ORTH: "οἱ"}], + "χἱκετεύετε": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ἱκετεύετε"}], + "κἀν": [{ORTH: "κ", NORM: "καί"}, {ORTH: "ἀν", NORM: "ἐν"}], + "κἀγὼ": [{ORTH: "κἀ", NORM: "καί"}, {ORTH: "γὼ", NORM: "ἐγώ"}], + "κἀγώ": [{ORTH: "κἀ", NORM: "καί"}, {ORTH: "γώ", NORM: "ἐγώ"}], + "ἁγώ": [{ORTH: "ἁ", NORM: "ἃ"}, {ORTH: "γώ", NORM: "ἐγώ"}], + "ἁγὼ": [{ORTH: "ἁ", NORM: "ἃ"}, {ORTH: "γὼ", NORM: "ἐγώ"}], + "ἐγᾦδα": [{ORTH: "ἐγ", NORM: "ἐγώ"}, {ORTH: "ᾦδα", NORM: "οἶδα"}], + "ἐγᾦμαι": [{ORTH: "ἐγ", NORM: "ἐγώ"}, {ORTH: "ᾦμαι", NORM: "οἶμαι"}], + "κἀς": [{ORTH: "κ", NORM: "καί"}, {ORTH: "ἀς", NORM: "ἐς"}], + "κᾆτα": [{ORTH: "κ", NORM: "καί"}, {ORTH: "ᾆτα", NORM: "εἶτα"}], + "κεἰ": [{ORTH: "κ", NORM: "καί"}, {ORTH: "εἰ"}], + "κεἰς": [{ORTH: "κ", NORM: "καί"}, {ORTH: "εἰς"}], + "χὤτε": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ὤτε", NORM: "ὅτε"}], + "χὤπως": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ὤπως", NORM: "ὅπως"}], + "χὤτι": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ὤτι", NORM: "ὅτι"}], + "χὤταν": [{ORTH: "χ", NORM: "καί"}, {ORTH: "ὤταν", NORM: "ὅταν"}], + "οὑμός": [{ORTH: "οὑ", NORM: "ὁ"}, {ORTH: "μός", NORM: "ἐμός"}], + "οὑμὸς": [{ORTH: "οὑ", NORM: "ὁ"}, {ORTH: "μὸς", NORM: "ἐμός"}], + "οὑμοί": [{ORTH: "οὑ", NORM: "οἱ"}, {ORTH: "μοί", NORM: "ἐμoί"}], + "οὑμοὶ": [{ORTH: "οὑ", NORM: "οἱ"}, {ORTH: "μοὶ", NORM: "ἐμoί"}], + "σοὔστι": [{ORTH: "σοὔ", NORM: "σοί"}, {ORTH: "στι", NORM: "ἐστι"}], + "σοὐστί": [{ORTH: "σοὐ", NORM: "σοί"}, {ORTH: "στί", NORM: "ἐστί"}], + "σοὐστὶ": [{ORTH: "σοὐ", NORM: "σοί"}, {ORTH: "στὶ", NORM: "ἐστί"}], + "μοὖστι": [{ORTH: "μοὖ", NORM: "μοί"}, {ORTH: "στι", NORM: "ἐστι"}], + "μοὔστι": [{ORTH: "μοὔ", NORM: "μοί"}, {ORTH: "στι", NORM: "ἐστι"}], + "τοὔνομα": [{ORTH: "τοὔ", NORM: "τό"}, {ORTH: "νομα", NORM: "ὄνομα"}], + "οὑν": [{ORTH: "οὑ", NORM: "ὁ"}, {ORTH: "ν", NORM: "ἐν"}], + "ὦνερ": [{ORTH: "ὦ", NORM: "ὦ"}, {ORTH: "νερ", NORM: "ἄνερ"}], + "ὦνδρες": [{ORTH: "ὦ", NORM: "ὦ"}, {ORTH: "νδρες", NORM: "ἄνδρες"}], + "προὔχων": [{ORTH: "προὔ", NORM: "πρό"}, {ORTH: "χων", NORM: "ἔχων"}], + "προὔχοντα": [{ORTH: "προὔ", NORM: "πρό"}, {ORTH: "χοντα", NORM: "ἔχοντα"}], + "ὥνεκα": [{ORTH: "ὥ", NORM: "οὗ"}, {ORTH: "νεκα", NORM: "ἕνεκα"}], + "θοἰμάτιον": [{ORTH: "θο", NORM: "τό"}, {ORTH: "ἰμάτιον"}], + "ὥνεκα": [{ORTH: "ὥ", NORM: "οὗ"}, {ORTH: "νεκα", NORM: "ἕνεκα"}], + "τὠληθές": [{ORTH: "τὠ", NORM: "τὸ"}, {ORTH: "ληθές", NORM: "ἀληθές"}], + "θἡμέρᾳ": [{ORTH: "θ", NORM: "τῇ"}, {ORTH: "ἡμέρᾳ"}], + "ἅνθρωπος": [{ORTH: "ἅ", NORM: "ὁ"}, {ORTH: "νθρωπος", NORM: "ἄνθρωπος"}], + "τἄλλα": [{ORTH: "τ", NORM: "τὰ"}, {ORTH: "ἄλλα"}], + "τἆλλα": [{ORTH: "τἆ", NORM: "τὰ"}, {ORTH: "λλα", NORM: "ἄλλα"}], + "ἁνήρ": [{ORTH: "ἁ", NORM: "ὁ"}, {ORTH: "νήρ", NORM: "ἀνήρ"}], + "ἁνὴρ": [{ORTH: "ἁ", NORM: "ὁ"}, {ORTH: "νὴρ", NORM: "ἀνήρ"}], + "ἅνδρες": [{ORTH: "ἅ", NORM: "οἱ"}, {ORTH: "νδρες", NORM: "ἄνδρες"}], + "ἁγαθαί": [{ORTH: "ἁ", NORM: "αἱ"}, {ORTH: "γαθαί", NORM: "ἀγαθαί"}], + "ἁγαθαὶ": [{ORTH: "ἁ", NORM: "αἱ"}, {ORTH: "γαθαὶ", NORM: "ἀγαθαί"}], + "ἁλήθεια": [{ORTH: "ἁ", NORM: "ἡ"}, {ORTH: "λήθεια", NORM: "ἀλήθεια"}], + "τἀνδρός": [{ORTH: "τ", NORM: "τοῦ"}, {ORTH: "ἀνδρός"}], + "τἀνδρὸς": [{ORTH: "τ", NORM: "τοῦ"}, {ORTH: "ἀνδρὸς", NORM: "ἀνδρός"}], + "τἀνδρί": [{ORTH: "τ", NORM: "τῷ"}, {ORTH: "ἀνδρί"}], + "τἀνδρὶ": [{ORTH: "τ", NORM: "τῷ"}, {ORTH: "ἀνδρὶ", NORM: "ἀνδρί"}], + "αὑτός": [{ORTH: "αὑ", NORM: "ὁ"}, {ORTH: "τός", NORM: "αὐτός"}], + "αὑτὸς": [{ORTH: "αὑ", NORM: "ὁ"}, {ORTH: "τὸς", NORM: "αὐτός"}], + "ταὐτοῦ": [{ORTH: "τ", NORM: "τοῦ"}, {ORTH: "αὐτοῦ"}], +} + +_exc.update(_other_exc) + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/gu/__init__.py b/spacy/lang/gu/__init__.py new file mode 100644 index 0000000..2f22034 --- /dev/null +++ b/spacy/lang/gu/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class GujaratiDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Gujarati(Language): + lang = "gu" + Defaults = GujaratiDefaults + + +__all__ = ["Gujarati"] diff --git a/spacy/lang/gu/examples.py b/spacy/lang/gu/examples.py new file mode 100644 index 0000000..e67b7ba --- /dev/null +++ b/spacy/lang/gu/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.gu.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "લોકશાહી એ સરકારનું એક એવું તંત્ર છે જ્યાં નાગરિકો મત દ્વારા સત્તાનો ઉપયોગ કરે છે.", + "તે ગુજરાત રાજ્યના ધરમપુર શહેરમાં આવેલું હતું", + "કર્ણદેવ પહેલો સોલંકી વંશનો રાજા હતો", + "તેજપાળને બે પત્ની હતી", + "ગુજરાતમાં ભારતીય જનતા પક્ષનો ઉદય આ સમયગાળા દરમિયાન થયો", + "આંદોલનકારીઓએ ચીમનભાઇ પટેલના રાજીનામાની માંગણી કરી.", + "અહિયાં શું જોડાય છે?", + "મંદિરનો પૂર્વાભિમુખ ભાગ નાના મંડપ સાથે થોડો લંબચોરસ આકારનો છે.", +] diff --git a/spacy/lang/gu/stop_words.py b/spacy/lang/gu/stop_words.py new file mode 100644 index 0000000..2c85968 --- /dev/null +++ b/spacy/lang/gu/stop_words.py @@ -0,0 +1,88 @@ +STOP_WORDS = set( + """ +એમ +આ +એ +રહી +છે +છો +હતા +હતું +હતી +હોય +હતો +શકે +તે +તેના +તેનું +તેને +તેની +તેઓ +તેમને +તેમના +તેમણે +તેમનું +તેમાં +અને +અહીં +થી +થઈ +થાય +જે + ને +કે +ના +ની +નો +ને +નું +શું +માં +પણ +પર +જેવા +જેવું +જાય +જેમ +જેથી +માત્ર +માટે +પરથી +આવ્યું +એવી +આવી +રીતે +સુધી +થાય +થઈ +સાથે +લાગે +હોવા +છતાં +રહેલા +કરી +કરે +કેટલા +કોઈ +કેમ +કર્યો +કર્યુ +કરે +સૌથી +ત્યારબાદ +તથા +દ્વારા +જુઓ +જાઓ +જ્યારે +ત્યારે +શકો +નથી +હવે +અથવા +થતો +દર +એટલો +પરંતુ +""".split() +) diff --git a/spacy/lang/he/__init__.py b/spacy/lang/he/__init__.py new file mode 100644 index 0000000..07084ac --- /dev/null +++ b/spacy/lang/he/__init__.py @@ -0,0 +1,17 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class HebrewDefaults(BaseDefaults): + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} + + +class Hebrew(Language): + lang = "he" + Defaults = HebrewDefaults + + +__all__ = ["Hebrew"] diff --git a/spacy/lang/he/examples.py b/spacy/lang/he/examples.py new file mode 100644 index 0000000..ee484e0 --- /dev/null +++ b/spacy/lang/he/examples.py @@ -0,0 +1,23 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.he.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "סין מקימה קרן של 440 מיליון דולר להשקעה בהייטק בישראל", + 'רה"מ הודיע כי יחרים טקס בחסותו', + "הכנסת צפויה לאשר איכון אוטומטי של שיחות למוקד 100", + "תוכנית לאומית תהפוך את ישראל למעצמה דיגיטלית", + "סע לשלום, המפתחות בפנים.", + "מלצר, פעמיים טורקי!", + "ואהבת לרעך כמוך.", + "היום נעשה משהו בלתי נשכח.", + "איפה הילד?", + "מיהו נשיא צרפת?", + "מהי בירת ארצות הברית?", + "איך קוראים בעברית לצ'ופצ'יק של הקומקום?", + "מה הייתה הדקה?", + "מי אומר שלום ראשון, זה שעולה או זה שיורד?", +] diff --git a/spacy/lang/he/lex_attrs.py b/spacy/lang/he/lex_attrs.py new file mode 100644 index 0000000..2cd6f63 --- /dev/null +++ b/spacy/lang/he/lex_attrs.py @@ -0,0 +1,95 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "אפס", + "אחד", + "אחת", + "שתיים", + "שתים", + "שניים", + "שנים", + "שלוש", + "שלושה", + "ארבע", + "ארבעה", + "חמש", + "חמישה", + "שש", + "שישה", + "שבע", + "שבעה", + "שמונה", + "תשע", + "תשעה", + "עשר", + "עשרה", + "אחד עשר", + "אחת עשרה", + "שנים עשר", + "שתים עשרה", + "שלושה עשר", + "שלוש עשרה", + "ארבעה עשר", + "ארבע עשרה", + "חמישה עשר", + "חמש עשרה", + "ששה עשר", + "שש עשרה", + "שבעה עשר", + "שבע עשרה", + "שמונה עשר", + "שמונה עשרה", + "תשעה עשר", + "תשע עשרה", + "עשרים", + "שלושים", + "ארבעים", + "חמישים", + "שישים", + "שבעים", + "שמונים", + "תשעים", + "מאה", + "אלף", + "מליון", + "מליארד", + "טריליון", +] + + +_ordinal_words = [ + "ראשון", + "שני", + "שלישי", + "רביעי", + "חמישי", + "שישי", + "שביעי", + "שמיני", + "תשיעי", + "עשירי", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + + if text in _num_words: + return True + + # Check ordinal number + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/he/stop_words.py b/spacy/lang/he/stop_words.py new file mode 100644 index 0000000..23bb517 --- /dev/null +++ b/spacy/lang/he/stop_words.py @@ -0,0 +1,222 @@ +STOP_WORDS = set( + """ +אני +את +אתה +אנחנו +אתן +אתם +הם +הן +היא +הוא +שלי +שלו +שלך +שלה +שלנו +שלכם +שלכן +שלהם +שלהן +לי +לו +לה +לנו +לכם +לכן +להם +להן +אותה +אותו +זה +זאת +אלה +אלו +תחת +מתחת +מעל +בין +עם +עד +על +אל +מול +של +אצל +כמו +אחר +אותו +בלי +לפני +אחרי +מאחורי +עלי +עליו +עליה +עליך +עלינו +עליכם +עליכן +עליהם +עליהן +כל +כולם +כולן +כך +ככה +כזה +כזאת +זה +אותי +אותה +אותם +אותך +אותו +אותן +אותנו +ואת +את +אתכם +אתכן +איתי +איתו +איתך +איתה +איתם +איתן +איתנו +איתכם +איתכן +יהיה +תהיה +הייתי +היתה +היה +להיות +עצמי +עצמו +עצמה +עצמם +עצמן +עצמנו +מי +מה +איפה +היכן +במקום שבו +אם +לאן +למקום שבו +מקום בו +איזה +מהיכן +איך +כיצד +באיזו מידה +מתי +בשעה ש +כאשר +כש +למרות +לפני +אחרי +מאיזו סיבה +הסיבה שבגללה +למה +מדוע +לאיזו תכלית +כי +יש +אין +אך +מנין +מאין +מאיפה +יכל +יכלה +יכלו +יכול +יכולה +יכולים +יכולות +יוכלו +יוכל +מסוגל +לא +רק +אולי +אין +לאו +אי +כלל +בעד +נגד +אם +עם +אל +אלה +אלו +אף +על +מעל +מתחת +מצד +בשביל +לבין +באמצע +בתוך +דרך +מבעד +באמצעות +למעלה +למטה +מחוץ +מן +לעבר +מכאן +כאן +הנה +הרי +פה +שם +אך +ברם +שוב +אבל +מבלי +בלי +מלבד +רק +בגלל +מכיוון +עד +אשר +ואילו +למרות +כמו +כפי +אז +אחרי +כן +לכן +לפיכך +עז +מאוד +מעט +מעטים +במידה +שוב +יותר +מדי +גם +כן +נו +אחר +אחרת +אחרים +אחרות +אשר +או +""".split() +) diff --git a/spacy/lang/hi/__init__.py b/spacy/lang/hi/__init__.py new file mode 100644 index 0000000..980dc31 --- /dev/null +++ b/spacy/lang/hi/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class HindiDefaults(BaseDefaults): + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Hindi(Language): + lang = "hi" + Defaults = HindiDefaults + + +__all__ = ["Hindi"] diff --git a/spacy/lang/hi/examples.py b/spacy/lang/hi/examples.py new file mode 100644 index 0000000..f3196c5 --- /dev/null +++ b/spacy/lang/hi/examples.py @@ -0,0 +1,19 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.hi.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "एप्पल 1 अरब डॉलर के लिए यू.के. स्टार्टअप खरीदने पर विचार कर रहा है।", + "स्वायत्त कारें निर्माताओं की ओर बीमा दायित्व रखतीं हैं।", + "सैन फ्रांसिस्को फुटपाथ वितरण रोबोटों पर प्रतिबंध लगाने का विचार कर रहा है।", + "लंदन यूनाइटेड किंगडम का विशाल शहर है।", + "आप कहाँ हो?", + "फ्रांस के राष्ट्रपति कौन हैं?", + "संयुक्त राज्यों की राजधानी क्या है?", + "बराक ओबामा का जन्म कब हुआ था?", + "जवाहरलाल नेहरू भारत के पहले प्रधानमंत्री हैं।", + "राजेंद्र प्रसाद, भारत के पहले राष्ट्रपति, दो कार्यकाल के लिए कार्यालय रखने वाले एकमात्र व्यक्ति हैं।", +] diff --git a/spacy/lang/hi/lex_attrs.py b/spacy/lang/hi/lex_attrs.py new file mode 100644 index 0000000..4ecd1db --- /dev/null +++ b/spacy/lang/hi/lex_attrs.py @@ -0,0 +1,188 @@ +from ...attrs import LIKE_NUM, NORM +from ..norm_exceptions import BASE_NORMS + +# fmt: off +_stem_suffixes = [ + ["ो", "े", "ू", "ु", "ी", "ि", "ा"], + ["कर", "ाओ", "िए", "ाई", "ाए", "ने", "नी", "ना", "ते", "ीं", "ती", "ता", "ाँ", "ां", "ों", "ें"], + ["ाकर", "ाइए", "ाईं", "ाया", "ेगी", "ेगा", "ोगी", "ोगे", "ाने", "ाना", "ाते", "ाती", "ाता", "तीं", "ाओं", "ाएं", "ुओं", "ुएं", "ुआं"], + ["ाएगी", "ाएगा", "ाओगी", "ाओगे", "एंगी", "ेंगी", "एंगे", "ेंगे", "ूंगी", "ूंगा", "ातीं", "नाओं", "नाएं", "ताओं", "ताएं", "ियाँ", "ियों", "ियां"], + ["ाएंगी", "ाएंगे", "ाऊंगी", "ाऊंगा", "ाइयाँ", "ाइयों", "ाइयां"] +] + +# reference 1: https://en.wikipedia.org/wiki/Indian_numbering_system +# reference 2: https://blogs.transparent.com/hindi/hindi-numbers-1-100/ +# reference 3: https://www.mindurhindi.com/basic-words-and-phrases-in-hindi/ + +_one_to_ten = [ + "शून्य", + "एक", + "दो", + "तीन", + "चार", + "पांच", "पाँच", + "छह", + "सात", + "आठ", + "नौ", + "दस", +] + +_eleven_to_beyond = [ + "ग्यारह", + "बारह", + "तेरह", + "चौदह", + "पंद्रह", + "सोलह", + "सत्रह", + "अठारह", + "उन्नीस", + "बीस", + "इकीस", "इक्कीस", + "बाईस", + "तेइस", + "चौबीस", + "पच्चीस", + "छब्बीस", + "सताइस", "सत्ताइस", + "अट्ठाइस", + "उनतीस", + "तीस", + "इकतीस", "इकत्तीस", + "बतीस", "बत्तीस", + "तैंतीस", + "चौंतीस", + "पैंतीस", + "छतीस", "छत्तीस", + "सैंतीस", + "अड़तीस", + "उनतालीस", "उनत्तीस", + "चालीस", + "इकतालीस", + "बयालीस", + "तैतालीस", + "चवालीस", + "पैंतालीस", + "छयालिस", + "सैंतालीस", + "अड़तालीस", + "उनचास", + "पचास", + "इक्यावन", + "बावन", + "तिरपन", "तिरेपन", + "चौवन", "चउवन", + "पचपन", + "छप्पन", + "सतावन", "सत्तावन", + "अठावन", + "उनसठ", + "साठ", + "इकसठ", + "बासठ", + "तिरसठ", "तिरेसठ", + "चौंसठ", + "पैंसठ", + "छियासठ", + "सड़सठ", + "अड़सठ", + "उनहत्तर", + "सत्तर", + "इकहत्तर", + "बहत्तर", + "तिहत्तर", + "चौहत्तर", + "पचहत्तर", + "छिहत्तर", + "सतहत्तर", + "अठहत्तर", + "उन्नासी", "उन्यासी" + "अस्सी", + "इक्यासी", + "बयासी", + "तिरासी", + "चौरासी", + "पचासी", + "छियासी", + "सतासी", + "अट्ठासी", + "नवासी", + "नब्बे", + "इक्यानवे", + "बानवे", + "तिरानवे", + "चौरानवे", + "पचानवे", + "छियानवे", + "सतानवे", + "अट्ठानवे", + "निन्यानवे", + "सौ", + "हज़ार", + "लाख", + "करोड़", + "अरब", + "खरब", +] + +_num_words = _one_to_ten + _eleven_to_beyond + +_ordinal_words_one_to_ten = [ + "प्रथम", "पहला", + "द्वितीय", "दूसरा", + "तृतीय", "तीसरा", + "चौथा", + "पांचवाँ", + "छठा", + "सातवाँ", + "आठवाँ", + "नौवाँ", + "दसवाँ", +] +_ordinal_suffix = "वाँ" +# fmt: on + + +def norm(string): + # normalise base exceptions, e.g. punctuation or currency symbols + if string in BASE_NORMS: + return BASE_NORMS[string] + # set stem word as norm, if available, adapted from: + # http://computing.open.ac.uk/Sites/EACLSouthAsia/Papers/p6-Ramanathan.pdf + # http://research.variancia.com/hindi_stemmer/ + # https://github.com/taranjeet/hindi-tokenizer/blob/master/HindiTokenizer.py#L142 + for suffix_group in reversed(_stem_suffixes): + length = len(suffix_group[0]) + if len(string) <= length: + continue + for suffix in suffix_group: + if string.endswith(suffix): + return string[:-length] + return string + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + + # check ordinal numbers + # reference: http://www.englishkitab.com/Vocabulary/Numbers.html + if text in _ordinal_words_one_to_ten: + return True + if text.endswith(_ordinal_suffix): + if text[: -len(_ordinal_suffix)] in _eleven_to_beyond: + return True + return False + + +LEX_ATTRS = {NORM: norm, LIKE_NUM: like_num} diff --git a/spacy/lang/hi/stop_words.py b/spacy/lang/hi/stop_words.py new file mode 100644 index 0000000..475b07d --- /dev/null +++ b/spacy/lang/hi/stop_words.py @@ -0,0 +1,239 @@ +# Source: https://github.com/taranjeet/hindi-tokenizer/blob/master/stopwords.txt, https://data.mendeley.com/datasets/bsr3frvvjc/1#file-a21d5092-99d7-45d8-b044-3ae9edd391c6 + +STOP_WORDS = set( + """ +अंदर +अत +अदि +अप +अपना +अपनि +अपनी +अपने +अभि +अभी +अंदर +आदि +आप +अगर +इंहिं +इंहें +इंहों +इतयादि +इत्यादि +इन +इनका +इन्हीं +इन्हें +इन्हों +इस +इसका +इसकि +इसकी +इसके +इसमें +इसि +इसी +इसे +उंहिं +उंहें +उंहों +उन +उनका +उनकि +उनकी +उनके +उनको +उन्हीं +उन्हें +उन्हों +उस +उसके +उसि +उसी +उसे +एक +एवं +एस +एसे +ऐसे +ओर +और +कइ +कई +कर +करता +करते +करना +करने +करें +कहते +कहा +का +काफि +काफ़ी +कि +किंहें +किंहों +कितना +किन्हें +किन्हों +किया +किर +किस +किसि +किसी +किसे +की +कुछ +कुल +के +को +कोइ +कोई +कोन +कोनसा +कौन +कौनसा +गया +घर +जब +जहाँ +जहां +जा +जिंहें +जिंहों +जितना +जिधर +जिन +जिन्हें +जिन्हों +जिस +जिसे +जीधर +जेसा +जेसे +जैसा +जैसे +जो +तक +तब +तरह +तिंहें +तिंहों +तिन +तिन्हें +तिन्हों +तिस +तिसे +तो +था +थि +थी +थे +दबारा +दवारा +दिया +दुसरा +दुसरे +दूसरे +दो +द्वारा +न +नहिं +नहीं +ना +निचे +निहायत +नीचे +ने +पर +पहले +पुरा +पूरा +पे +फिर +बनि +बनी +बहि +बही +बहुत +बाद +बाला +बिलकुल +भि +भितर +भी +भीतर +मगर +मानो +मे +में +मैं +मुझको +मेरा +यदि +यह +यहाँ +यहां +यहि +यही +या +यिह +ये +रखें +रवासा +रहा +रहे +ऱ्वासा +लिए +लिये +लेकिन +व +वगेरह +वग़ैरह +वरग +वर्ग +वह +वहाँ +वहां +वहिं +वहीं +वाले +वुह +वे +वग़ैरह +संग +सकता +सकते +सबसे +सभि +सभी +साथ +साबुत +साभ +सारा +से +सो +संग +हि +ही +हुअ +हुआ +हुइ +हुई +हुए +हे +हें +है +हैं +हो +हूँ +होता +होति +होती +होते +होना +होने +""".split() +) diff --git a/spacy/lang/hr/__init__.py b/spacy/lang/hr/__init__.py new file mode 100644 index 0000000..fd7622a --- /dev/null +++ b/spacy/lang/hr/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class CroatianDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Croatian(Language): + lang = "hr" + Defaults = CroatianDefaults + + +__all__ = ["Croatian"] diff --git a/spacy/lang/hr/examples.py b/spacy/lang/hr/examples.py new file mode 100644 index 0000000..b28fb63 --- /dev/null +++ b/spacy/lang/hr/examples.py @@ -0,0 +1,15 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.hr.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Ovo je rečenica.", + "Kako se popravlja auto?", + "Zagreb je udaljen od Ljubljane svega 150 km.", + "Nećete vjerovati što se dogodilo na ovogodišnjem festivalu!", + "Budućnost Apple je upitna nakon dugotrajnog pada vrijednosti dionica firme.", + "Trgovina oružjem predstavlja prijetnju za globalni mir.", +] diff --git a/spacy/lang/hr/lemma_lookup_license.txt b/spacy/lang/hr/lemma_lookup_license.txt new file mode 100644 index 0000000..9cc003a --- /dev/null +++ b/spacy/lang/hr/lemma_lookup_license.txt @@ -0,0 +1,15 @@ +The list of Croatian lemmas was extracted from the reldi-tagger repository (https://github.com/clarinsi/reldi-tagger). +Reldi-tagger is licensed under the Apache 2.0 licence. + +@InProceedings{ljubesic16-new, + author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec}, + title = {New Inflectional Lexicons and Training Corpora for Improved Morphosyntactic Annotation of Croatian and Serbian}, + booktitle = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016)}, + year = {2016}, + date = {23-28}, + location = {Portorož, Slovenia}, + editor = {Nicoletta Calzolari (Conference Chair) and Khalid Choukri and Thierry Declerck and Sara Goggi and Marko Grobelnik and Bente Maegaard and Joseph Mariani and Helene Mazo and Asuncion Moreno and Jan Odijk and Stelios Piperidis}, + publisher = {European Language Resources Association (ELRA)}, + address = {Paris, France}, + isbn = {978-2-9517408-9-1} + } diff --git a/spacy/lang/hr/stop_words.py b/spacy/lang/hr/stop_words.py new file mode 100644 index 0000000..dd10f79 --- /dev/null +++ b/spacy/lang/hr/stop_words.py @@ -0,0 +1,344 @@ +# Source: https://github.com/stopwords-iso/stopwords-hr +STOP_WORDS = set( + """ +a +ah +aha +aj +ako +al +ali +arh +au +avaj +bar +baš +bez +bi +bih +bijah +bijahu +bijaše +bijasmo +bijaste +bila +bili +bilo +bio +bismo +biste +biti +brr +buć +budavši +bude +budimo +budite +budu +budući +bum +bumo +će +ćemo +ćeš +ćete +čijem +čijim +čijima +ću +da +daj +dakle +de +deder +dem +djelomice +djelomično +do +doista +dok +dokle +donekle +dosad +doskoro +dotad +dotle +dovečer +drugamo +drugdje +duž +e +eh +ehe +ej +eno +eto +evo +ga +gdjekakav +gdjekoje +gic +god +halo +hej +hm +hoće +hoćemo +hoćeš +hoćete +hoću +hop +htijahu +htijasmo +htijaste +htio +htjedoh +htjedoše +htjedoste +htjela +htjele +htjeli +hura +i +iako +ih +iju +ijuju +ikada +ikakav +ikakva +ikakve +ikakvi +ikakvih +ikakvim +ikakvima +ikakvo +ikakvog +ikakvoga +ikakvoj +ikakvom +ikakvome +ili +im +iz +ja +je +jedna +jedne +jedni +jedno +jer +jesam +jesi +jesmo +jest +jeste +jesu +jim +joj +još +ju +kada +kako +kao +koja +koje +koji +kojima +koju +kroz +lani +li +me +mene +meni +mi +mimo +moj +moja +moje +moji +moju +mu +na +nad +nakon +nam +nama +nas +naš +naša +naše +našeg +naši +ne +neće +nećemo +nećeš +nećete +neću +nego +neka +neke +neki +nekog +neku +nema +nešto +netko +ni +nije +nikoga +nikoje +nikoji +nikoju +nisam +nisi +nismo +niste +nisu +njega +njegov +njegova +njegovo +njemu +njezin +njezina +njezino +njih +njihov +njihova +njihovo +njim +njima +njoj +nju +no +o +od +odmah +on +ona +one +oni +ono +onu +onoj +onom +onim +onima +ova +ovaj +ovim +ovima +ovoj +pa +pak +pljus +po +pod +podalje +poimence +poizdalje +ponekad +pored +postrance +potajice +potrbuške +pouzdano +prije +s +sa +sam +samo +sasvim +sav +se +sebe +sebi +si +šic +smo +ste +što +šta +štogod +štagod +su +sva +sve +svi +svi +svog +svoj +svoja +svoje +svoju +svom +svu +ta +tada +taj +tako +te +tebe +tebi +ti +tim +tima +to +toj +tome +tu +tvoj +tvoja +tvoje +tvoji +tvoju +u +usprkos +utaman +uvijek +uz +uza +uzagrapce +uzalud +uzduž +valjda +vam +vama +vas +vaš +vaša +vaše +vašim +vašima +već +vi +vjerojatno +vjerovatno +vrh +vrlo +za +zaista +zar +zatim +zato +zbija +zbog +želeći +željah +željela +željele +željeli +željelo +željen +željena +željene +željeni +željenu +željeo +zimus +zum +""".split() +) diff --git a/spacy/lang/hsb/__init__.py b/spacy/lang/hsb/__init__.py new file mode 100644 index 0000000..e8b2ffc --- /dev/null +++ b/spacy/lang/hsb/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class UpperSorbianDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + + +class UpperSorbian(Language): + lang = "hsb" + Defaults = UpperSorbianDefaults + + +__all__ = ["UpperSorbian"] diff --git a/spacy/lang/hsb/examples.py b/spacy/lang/hsb/examples.py new file mode 100644 index 0000000..754011c --- /dev/null +++ b/spacy/lang/hsb/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.hsb.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "To běšo wjelgin raźone a jo se wót luźi derje pśiwzeło. Tak som dožywiła wjelgin", + "Jogo pśewóźowarce stej groniłej, až how w serbskich stronach njama Santa Claus nic pytaś.", + "A ten sobuźěłaśeŕ Statneje biblioteki w Barlinju jo pśimjeł drogotne knigły bźez rukajcowu z nagima rukoma!", + "Take wobchadanje z našym kulturnym derbstwom zewšym njejźo.", + "Wopśimjeśe drugich pśinoskow jo było na wusokem niwowje, ako pśecej.", +] diff --git a/spacy/lang/hsb/lex_attrs.py b/spacy/lang/hsb/lex_attrs.py new file mode 100644 index 0000000..5f300a7 --- /dev/null +++ b/spacy/lang/hsb/lex_attrs.py @@ -0,0 +1,106 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nul", + "jedyn", + "jedna", + "jedne", + "dwaj", + "dwě", + "tři", + "třo", + "štyri", + "štyrjo", + "pjeć", + "šěsć", + "sydom", + "wosom", + "dźewjeć", + "dźesać", + "jědnaće", + "dwanaće", + "třinaće", + "štyrnaće", + "pjatnaće", + "šěsnaće", + "sydomnaće", + "wosomnaće", + "dźewjatnaće", + "dwaceći", + "třiceći", + "štyrceći", + "pjećdźesat", + "šěsćdźesat", + "sydomdźesat", + "wosomdźesat", + "dźewjećdźesat", + "sto", + "tysac", + "milion", + "miliarda", + "bilion", + "biliarda", + "trilion", + "triliarda", +] + +_ordinal_words = [ + "prěni", + "prěnja", + "prěnje", + "druhi", + "druha", + "druhe", + "třeći", + "třeća", + "třeće", + "štwórty", + "štwórta", + "štwórte", + "pjaty", + "pjata", + "pjate", + "šěsty", + "šěsta", + "šěste", + "sydmy", + "sydma", + "sydme", + "wosmy", + "wosma", + "wosme", + "dźewjaty", + "dźewjata", + "dźewjate", + "dźesaty", + "dźesata", + "dźesate", + "jědnaty", + "jědnata", + "jědnate", + "dwanaty", + "dwanata", + "dwanate", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/hsb/stop_words.py b/spacy/lang/hsb/stop_words.py new file mode 100644 index 0000000..e6fedaf --- /dev/null +++ b/spacy/lang/hsb/stop_words.py @@ -0,0 +1,19 @@ +STOP_WORDS = set( + """ +a abo ale ani + +dokelž + +hdyž + +jeli jelizo + +kaž + +pak potom + +tež tohodla + +zo zoby +""".split() +) diff --git a/spacy/lang/hsb/tokenizer_exceptions.py b/spacy/lang/hsb/tokenizer_exceptions.py new file mode 100644 index 0000000..cd3bac9 --- /dev/null +++ b/spacy/lang/hsb/tokenizer_exceptions.py @@ -0,0 +1,18 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = dict() +for exc_data in [ + {ORTH: "mil.", NORM: "milion"}, + {ORTH: "wob.", NORM: "wobydler"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +for orth in [ + "resp.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ht/__init__.py b/spacy/lang/ht/__init__.py new file mode 100644 index 0000000..7f9feb0 --- /dev/null +++ b/spacy/lang/ht/__init__.py @@ -0,0 +1,55 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import HaitianCreoleLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tag_map import TAG_MAP +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class HaitianCreoleDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + tag_map = TAG_MAP + + +class HaitianCreole(Language): + lang = "ht" + Defaults = HaitianCreoleDefaults + + +@HaitianCreole.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return HaitianCreoleLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["HaitianCreole"] diff --git a/spacy/lang/ht/examples.py b/spacy/lang/ht/examples.py new file mode 100644 index 0000000..0afeb19 --- /dev/null +++ b/spacy/lang/ht/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ht.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple ap panse achte yon demaraj nan Wayòm Ini pou $1 milya dola", + "Machin otonòm fè responsablite asirans lan ale sou men fabrikan yo", + "San Francisco ap konsidere entèdi robo ki livre sou twotwa yo", + "Lond se yon gwo vil nan Wayòm Ini", + "Kote ou ye?", + "Kilès ki prezidan Lafrans?", + "Ki kapital Etazini?", + "Kile Barack Obama te fèt?", +] diff --git a/spacy/lang/ht/lemmatizer.py b/spacy/lang/ht/lemmatizer.py new file mode 100644 index 0000000..7687865 --- /dev/null +++ b/spacy/lang/ht/lemmatizer.py @@ -0,0 +1,50 @@ +from typing import List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class HaitianCreoleLemmatizer(Lemmatizer): + """ + Minimal Haitian Creole lemmatizer. + Returns a word's base form based on rules and lookup, + or defaults to the original form. + """ + + def is_base_form(self, token: Token) -> bool: + morph = token.morph.to_dict() + upos = token.pos_.lower() + + # Consider unmarked forms to be base + if upos in {"noun", "verb", "adj", "adv"}: + if not morph: + return True + if upos == "noun" and morph.get("Number") == "Sing": + return True + if upos == "verb" and morph.get("VerbForm") == "Inf": + return True + if upos == "adj" and morph.get("Degree") == "Pos": + return True + return False + + def rule_lemmatize(self, token: Token) -> List[str]: + string = token.text.lower() + pos = token.pos_.lower() + cache_key = (token.orth, token.pos) + if cache_key in self.cache: + return self.cache[cache_key] + + forms = [] + + # fallback rule: just return lowercased form + forms.append(string) + + self.cache[cache_key] = forms + return forms + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "rule": + required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"] + return (required, []) + return super().get_lookups_config(mode) diff --git a/spacy/lang/ht/lex_attrs.py b/spacy/lang/ht/lex_attrs.py new file mode 100644 index 0000000..ab1a39a --- /dev/null +++ b/spacy/lang/ht/lex_attrs.py @@ -0,0 +1,81 @@ +from ...attrs import LIKE_NUM, NORM + +# Cardinal numbers in Creole +_num_words = set( + """ +zewo youn en de twa kat senk sis sèt uit nèf dis +onz douz trèz katoz kenz sèz disèt dizwit diznèf +vent trant karant sinkant swasant swasann-dis +san mil milyon milya +""".split() +) + +# Ordinal numbers in Creole (some are French-influenced, some simplified) +_ordinal_words = set( + """ +premye dezyèm twazyèm katryèm senkyèm sizyèm sètvyèm uitvyèm nèvyèm dizyèm +onzèm douzyèm trèzyèm katozyèm kenzèm sèzyèm disetyèm dizwityèm diznèvyèm +ventyèm trantyèm karantyèm sinkantyèm swasantyèm +swasann-disyèm santyèm milyèm milyonnyèm milyadyèm +""".split() +) + +NORM_MAP = { + "'m": "mwen", + "'w": "ou", + "'l": "li", + "'n": "nou", + "'y": "yo", + "’m": "mwen", + "’w": "ou", + "’l": "li", + "’n": "nou", + "’y": "yo", + "m": "mwen", + "n": "nou", + "l": "li", + "y": "yo", + "w": "ou", + "t": "te", + "k": "ki", + "p": "pa", + "M": "Mwen", + "N": "Nou", + "L": "Li", + "Y": "Yo", + "W": "Ou", + "T": "Te", + "K": "Ki", + "P": "Pa", +} + + +def like_num(text): + text = text.strip().lower() + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + # Handle things like "3yèm", "10yèm", "25yèm", etc. + if text.endswith("yèm") and text[:-3].isdigit(): + return True + return False + + +def norm_custom(text): + return NORM_MAP.get(text, text.lower()) + + +LEX_ATTRS = { + LIKE_NUM: like_num, + NORM: norm_custom, +} diff --git a/spacy/lang/ht/punctuation.py b/spacy/lang/ht/punctuation.py new file mode 100644 index 0000000..c4a5d09 --- /dev/null +++ b/spacy/lang/ht/punctuation.py @@ -0,0 +1,58 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + merge_chars, +) + +ELISION = "'’".replace(" ", "") + +_prefixes_elision = "m n l y t k w" +_prefixes_elision += " " + _prefixes_elision.upper() + +TOKENIZER_PREFIXES = ( + LIST_PUNCT + + LIST_QUOTES + + [ + r"(?:({pe})[{el}])(?=[{a}])".format( + a=ALPHA, el=ELISION, pe=merge_chars(_prefixes_elision) + ) + ] +) + +TOKENIZER_SUFFIXES = ( + LIST_PUNCT + + LIST_QUOTES + + LIST_ELLIPSES + + [ + r"(?<=[0-9])%", # numbers like 10% + r"(?<=[0-9])(?:{h})".format(h=HYPHENS), # hyphens after numbers + r"(?<=[{a}])['’]".format(a=ALPHA), # apostrophes after letters + r"(?<=[{a}])['’][mwlnytk](?=\s|$)".format(a=ALPHA), # contractions + r"(?<=[{a}0-9])\)", # right parenthesis after letter/number + r"(?<=[{a}])\.(?=\s|$)".format( + a=ALPHA + ), # period after letter if space or end of string + r"(?<=\))[\.\?!]", # punctuation immediately after right parenthesis + ] +) + +TOKENIZER_INFIXES = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION), + ] +) diff --git a/spacy/lang/ht/stop_words.py b/spacy/lang/ht/stop_words.py new file mode 100644 index 0000000..50998e0 --- /dev/null +++ b/spacy/lang/ht/stop_words.py @@ -0,0 +1,49 @@ +STOP_WORDS = set( + """ +a ak an ankò ant apre ap atò avan avanlè +byen bò byenke + +chak + +de depi deja deja + +e en epi èske + +fò fòk + +gen genyen + +ki kisa kilès kote koukou konsa konbyen konn konnen kounye kouman + +la l laa le lè li lye lò + +m m' mwen + +nan nap nou n' + +ou oumenm + +pa paske pami pandan pito pou pral preske pwiske + +se selman si sou sòt + +ta tap tankou te toujou tou tan tout toutotan twòp tèl + +w w' wi wè + +y y' yo yon yonn + +non o oh eh + +sa san si swa si + +men mèsi oswa osinon + +""".split() +) + +# Add common contractions, with and without apostrophe variants +contractions = ["m'", "n'", "w'", "y'", "l'", "t'", "k'"] +for apostrophe in ["'", "’", "‘"]: + for word in contractions: + STOP_WORDS.add(word.replace("'", apostrophe)) diff --git a/spacy/lang/ht/syntax_iterators.py b/spacy/lang/ht/syntax_iterators.py new file mode 100644 index 0000000..44ff17f --- /dev/null +++ b/spacy/lang/ht/syntax_iterators.py @@ -0,0 +1,74 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse for Haitian Creole. + Works on both Doc and Span objects. + """ + + # Core nominal dependencies common in Haitian Creole + labels = [ + "nsubj", + "obj", + "obl", + "nmod", + "appos", + "ROOT", + ] + + # Modifiers to optionally include in chunk (to the right) + post_modifiers = ["compound", "flat", "flat:name", "fixed"] + + doc = doclike.doc + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + np_deps = {doc.vocab.strings.add(label) for label in labels} + np_mods = {doc.vocab.strings.add(mod) for mod in post_modifiers} + conj_label = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + adp_pos = doc.vocab.strings.add("ADP") + cc_pos = doc.vocab.strings.add("CCONJ") + + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + if word.left_edge.i <= prev_end: + continue + + if word.dep in np_deps: + right_end = word + # expand to include known modifiers to the right + for child in word.rights: + if child.dep in np_mods: + right_end = child.right_edge + elif child.pos == NOUN: + right_end = child.right_edge + + left_index = word.left_edge.i + # Skip prepositions at the start + if word.left_edge.pos == adp_pos: + left_index += 1 + + prev_end = right_end.i + yield left_index, right_end.i + 1, np_label + + elif word.dep == conj_label: + head = word.head + while head.dep == conj_label and head.head.i < head.i: + head = head.head + if head.dep in np_deps: + left_index = word.left_edge.i + if word.left_edge.pos == cc_pos: + left_index += 1 + prev_end = word.i + yield left_index, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/ht/tag_map.py b/spacy/lang/ht/tag_map.py new file mode 100644 index 0000000..a190984 --- /dev/null +++ b/spacy/lang/ht/tag_map.py @@ -0,0 +1,39 @@ +from spacy.symbols import ( + ADJ, + ADP, + ADV, + AUX, + CCONJ, + DET, + INTJ, + NOUN, + NUM, + PART, + PRON, + PROPN, + PUNCT, + SCONJ, + SYM, + VERB, + X, +) + +TAG_MAP = { + "NOUN": {"pos": NOUN}, + "VERB": {"pos": VERB}, + "AUX": {"pos": AUX}, + "ADJ": {"pos": ADJ}, + "ADV": {"pos": ADV}, + "PRON": {"pos": PRON}, + "DET": {"pos": DET}, + "ADP": {"pos": ADP}, + "SCONJ": {"pos": SCONJ}, + "CCONJ": {"pos": CCONJ}, + "PART": {"pos": PART}, + "INTJ": {"pos": INTJ}, + "NUM": {"pos": NUM}, + "PROPN": {"pos": PROPN}, + "PUNCT": {"pos": PUNCT}, + "SYM": {"pos": SYM}, + "X": {"pos": X}, +} diff --git a/spacy/lang/ht/tokenizer_exceptions.py b/spacy/lang/ht/tokenizer_exceptions.py new file mode 100644 index 0000000..deb152c --- /dev/null +++ b/spacy/lang/ht/tokenizer_exceptions.py @@ -0,0 +1,126 @@ +from spacy.symbols import NORM, ORTH + + +def make_variants(base, first_norm, second_orth, second_norm): + return { + base: [ + {ORTH: base.split("'")[0] + "'", NORM: first_norm}, + {ORTH: second_orth, NORM: second_norm}, + ], + base.capitalize(): [ + { + ORTH: base.split("'")[0].capitalize() + "'", + NORM: first_norm.capitalize(), + }, + {ORTH: second_orth, NORM: second_norm}, + ], + } + + +TOKENIZER_EXCEPTIONS = {"Dr.": [{ORTH: "Dr."}]} + +# Apostrophe forms +TOKENIZER_EXCEPTIONS.update(make_variants("m'ap", "mwen", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("n'ap", "nou", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("l'ap", "li", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("y'ap", "yo", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("m'te", "mwen", "te", "te")) +TOKENIZER_EXCEPTIONS.update(make_variants("m'pral", "mwen", "pral", "pral")) +TOKENIZER_EXCEPTIONS.update(make_variants("w'ap", "ou", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("k'ap", "ki", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("p'ap", "pa", "ap", "ap")) +TOKENIZER_EXCEPTIONS.update(make_variants("t'ap", "te", "ap", "ap")) + +# Non-apostrophe contractions (with capitalized variants) +TOKENIZER_EXCEPTIONS.update( + { + "map": [ + {ORTH: "m", NORM: "mwen"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Map": [ + {ORTH: "M", NORM: "Mwen"}, + {ORTH: "ap", NORM: "ap"}, + ], + "lem": [ + {ORTH: "le", NORM: "le"}, + {ORTH: "m", NORM: "mwen"}, + ], + "Lem": [ + {ORTH: "Le", NORM: "Le"}, + {ORTH: "m", NORM: "mwen"}, + ], + "lew": [ + {ORTH: "le", NORM: "le"}, + {ORTH: "w", NORM: "ou"}, + ], + "Lew": [ + {ORTH: "Le", NORM: "Le"}, + {ORTH: "w", NORM: "ou"}, + ], + "nap": [ + {ORTH: "n", NORM: "nou"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Nap": [ + {ORTH: "N", NORM: "Nou"}, + {ORTH: "ap", NORM: "ap"}, + ], + "lap": [ + {ORTH: "l", NORM: "li"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Lap": [ + {ORTH: "L", NORM: "Li"}, + {ORTH: "ap", NORM: "ap"}, + ], + "yap": [ + {ORTH: "y", NORM: "yo"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Yap": [ + {ORTH: "Y", NORM: "Yo"}, + {ORTH: "ap", NORM: "ap"}, + ], + "mte": [ + {ORTH: "m", NORM: "mwen"}, + {ORTH: "te", NORM: "te"}, + ], + "Mte": [ + {ORTH: "M", NORM: "Mwen"}, + {ORTH: "te", NORM: "te"}, + ], + "mpral": [ + {ORTH: "m", NORM: "mwen"}, + {ORTH: "pral", NORM: "pral"}, + ], + "Mpral": [ + {ORTH: "M", NORM: "Mwen"}, + {ORTH: "pral", NORM: "pral"}, + ], + "wap": [ + {ORTH: "w", NORM: "ou"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Wap": [ + {ORTH: "W", NORM: "Ou"}, + {ORTH: "ap", NORM: "ap"}, + ], + "kap": [ + {ORTH: "k", NORM: "ki"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Kap": [ + {ORTH: "K", NORM: "Ki"}, + {ORTH: "ap", NORM: "ap"}, + ], + "tap": [ + {ORTH: "t", NORM: "te"}, + {ORTH: "ap", NORM: "ap"}, + ], + "Tap": [ + {ORTH: "T", NORM: "Te"}, + {ORTH: "ap", NORM: "ap"}, + ], + } +) diff --git a/spacy/lang/hu/__init__.py b/spacy/lang/hu/__init__.py new file mode 100644 index 0000000..799e6d2 --- /dev/null +++ b/spacy/lang/hu/__init__.py @@ -0,0 +1,21 @@ +from ...language import BaseDefaults, Language +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKEN_MATCH, TOKENIZER_EXCEPTIONS + + +class HungarianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + token_match = TOKEN_MATCH + stop_words = STOP_WORDS + + +class Hungarian(Language): + lang = "hu" + Defaults = HungarianDefaults + + +__all__ = ["Hungarian"] diff --git a/spacy/lang/hu/examples.py b/spacy/lang/hu/examples.py new file mode 100644 index 0000000..c056c09 --- /dev/null +++ b/spacy/lang/hu/examples.py @@ -0,0 +1,12 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.hu.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Az Apple egy brit startup vásárlását tervezi 1 milliárd dollár értékben.", + "San Francisco vezetése mérlegeli a járdát használó szállító robotok betiltását.", + "London az Egyesült Királyság egy nagy városa.", +] diff --git a/spacy/lang/hu/punctuation.py b/spacy/lang/hu/punctuation.py new file mode 100644 index 0000000..dc97410 --- /dev/null +++ b/spacy/lang/hu/punctuation.py @@ -0,0 +1,61 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_ICONS, + CONCAT_QUOTES, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +# removing ° from the special icons to keep e.g. 99° as one token +_concat_icons = CONCAT_ICONS.replace("\u00b0", "") + +_currency = r"\$¢£€¥฿" +_quotes = CONCAT_QUOTES.replace("'", "") +_units = UNITS.replace("%", "") + +_prefixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + [_concat_icons] + + [r"[,.:](?=[{a}])".format(a=ALPHA)] +) + +_suffixes = ( + [r"\+"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + [_concat_icons] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:[{c}])".format(c=_currency), + r"(?<=[0-9])(?:{u})".format(u=_units), + r"(?<=[{al}{e}{q}(?:{c})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, c=_currency + ), + r"(?<=[{al})])-e".format(al=ALPHA_LOWER), + ] +) + +_infixes = ( + LIST_ELLIPSES + + [_concat_icons] + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=_quotes), + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/hu/stop_words.py b/spacy/lang/hu/stop_words.py new file mode 100644 index 0000000..e39a26d --- /dev/null +++ b/spacy/lang/hu/stop_words.py @@ -0,0 +1,62 @@ +STOP_WORDS = set( + """ +a abban ahhoz ahogy ahol aki akik akkor akár alatt amely amelyek amelyekben +amelyeket amelyet amelynek ami amikor amit amolyan amíg annak arra arról az +azok azon azonban azt aztán azután azzal azért + +be belül benne bár + +cikk cikkek cikkeket csak + +de + +e ebben eddig egy egyes egyetlen egyik egyre egyéb egész ehhez ekkor el ellen +elo eloször elott elso elég előtt emilyen ennek erre ez ezek ezen ezt ezzel +ezért + +fel felé + +ha hanem hiszen hogy hogyan hát + +ide igen ill ill. illetve ilyen ilyenkor inkább is ismét ison itt + +jobban jó jól + +kell kellett keressünk keresztül ki kívül között közül + +le legalább legyen lehet lehetett lenne lenni lesz lett + +ma maga magát majd meg mellett mely melyek mert mi miatt mikor milyen minden +mindenki mindent mindig mint mintha mit mivel miért mondta most már más másik +még míg + +nagy nagyobb nagyon ne nekem neki nem nincs néha néhány nélkül + +o oda ok oket olyan ott + +pedig persze például + +rá + +s saját sem semmi sok sokat sokkal stb. szemben szerint szinte számára szét + +talán te tehát teljes ti tovább továbbá több túl ugyanis + +utolsó után utána + +vagy vagyis vagyok valaki valami valamint való van vannak vele vissza viszont +volna volt voltak voltam voltunk + +által általában át + +én éppen és + +így + +ön össze + +úgy új újabb újra + +ő őket +""".split() +) diff --git a/spacy/lang/hu/tokenizer_exceptions.py b/spacy/lang/hu/tokenizer_exceptions.py new file mode 100644 index 0000000..3f79b02 --- /dev/null +++ b/spacy/lang/hu/tokenizer_exceptions.py @@ -0,0 +1,654 @@ +import re + +from ...symbols import ORTH +from ...util import update_exc +from ..punctuation import ALPHA_LOWER, CURRENCY +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for orth in [ + "-e", + "A.", + "AG.", + "AkH.", + "Aö.", + "B.", + "B.CS.", + "B.S.", + "B.Sc.", + "B.ú.é.k.", + "BE.", + "BEK.", + "BSC.", + "BSc.", + "BTK.", + "Bat.", + "Be.", + "Bek.", + "Bfok.", + "Bk.", + "Bp.", + "Bros.", + "Bt.", + "Btk.", + "Btke.", + "Btét.", + "C.", + "CSC.", + "Cal.", + "Cg.", + "Cgf.", + "Cgt.", + "Cia.", + "Co.", + "Colo.", + "Comp.", + "Copr.", + "Corp.", + "Cos.", + "Cs.", + "Csc.", + "Csop.", + "Cstv.", + "Ctv.", + "Ctvr.", + "D.", + "DR.", + "Dipl.", + "Dr.", + "Dsz.", + "Dzs.", + "E.", + "EK.", + "EU.", + "F.", + "Fla.", + "Folyt.", + "Fpk.", + "Főszerk.", + "G.", + "GK.", + "GM.", + "Gfv.", + "Gmk.", + "Gr.", + "Group.", + "Gt.", + "Gy.", + "H.", + "HKsz.", + "Hmvh.", + "I.", + "Ifj.", + "Inc.", + "Inform.", + "Int.", + "J.", + "Jr.", + "Jv.", + "K.", + "K.m.f.", + "KB.", + "KER.", + "KFT.", + "KRT.", + "Kb.", + "Ker.", + "Kft.", + "Kg.", + "Kht.", + "Kkt.", + "Kong.", + "Korm.", + "Kr.", + "Kr.e.", + "Kr.u.", + "Krt.", + "L.", + "LB.", + "Llc.", + "Ltd.", + "M.", + "M.A.", + "M.S.", + "M.SC.", + "M.Sc.", + "MA.", + "MH.", + "MSC.", + "MSc.", + "Mass.", + "Max.", + "Mlle.", + "Mme.", + "Mo.", + "Mr.", + "Mrs.", + "Ms.", + "Mt.", + "N.", + "N.N.", + "NB.", + "NBr.", + "Nat.", + "No.", + "Nr.", + "Ny.", + "Nyh.", + "Nyr.", + "Nyrt.", + "O.", + "OJ.", + "Op.", + "P.", + "P.H.", + "P.S.", + "PH.D.", + "PHD.", + "PROF.", + "Pf.", + "Ph.D", + "PhD.", + "Pk.", + "Pl.", + "Plc.", + "Pp.", + "Proc.", + "Prof.", + "Ptk.", + "R.", + "RT.", + "Rer.", + "Rt.", + "S.", + "S.B.", + "SZOLG.", + "Salg.", + "Sch.", + "Spa.", + "St.", + "Sz.", + "SzRt.", + "Szerk.", + "Szfv.", + "Szjt.", + "Szolg.", + "Szt.", + "Sztv.", + "Szvt.", + "Számv.", + "T.", + "TEL.", + "Tel.", + "Ty.", + "Tyr.", + "U.", + "Ui.", + "Ut.", + "V.", + "VB.", + "Vcs.", + "Vhr.", + "Vht.", + "Várm.", + "W.", + "X.", + "X.Y.", + "Y.", + "Z.", + "Zrt.", + "Zs.", + "a.C.", + "ac.", + "adj.", + "adm.", + "ag.", + "agit.", + "alez.", + "alk.", + "all.", + "altbgy.", + "an.", + "ang.", + "arch.", + "at.", + "atc.", + "aug.", + "b.a.", + "b.s.", + "b.sc.", + "bek.", + "belker.", + "berend.", + "biz.", + "bizt.", + "bo.", + "bp.", + "br.", + "bsc.", + "bt.", + "btk.", + "ca.", + "cc.", + "cca.", + "cf.", + "cif.", + "co.", + "corp.", + "cos.", + "cs.", + "csc.", + "csüt.", + "cső.", + "ctv.", + "dbj.", + "dd.", + "ddr.", + "de.", + "dec.", + "dikt.", + "dipl.", + "dj.", + "dk.", + "dl.", + "dny.", + "dolg.", + "dr.", + "du.", + "dzs.", + "ea.", + "ed.", + "eff.", + "egyh.", + "ell.", + "elv.", + "elvt.", + "em.", + "eng.", + "eny.", + "et.", + "etc.", + "ev.", + "ezr.", + "eü.", + "f.h.", + "f.é.", + "fam.", + "fb.", + "febr.", + "fej.", + "felv.", + "felügy.", + "ff.", + "ffi.", + "fhdgy.", + "fil.", + "fiz.", + "fm.", + "foglalk.", + "ford.", + "fp.", + "fr.", + "frsz.", + "fszla.", + "fszt.", + "ft.", + "fuv.", + "főig.", + "főisk.", + "főtörm.", + "főv.", + "gazd.", + "gimn.", + "gk.", + "gkv.", + "gmk.", + "gondn.", + "gr.", + "grav.", + "gy.", + "gyak.", + "gyártm.", + "gör.", + "hads.", + "hallg.", + "hdm.", + "hdp.", + "hds.", + "hg.", + "hiv.", + "hk.", + "hm.", + "ho.", + "honv.", + "hp.", + "hr.", + "hrsz.", + "hsz.", + "ht.", + "htb.", + "hv.", + "hőm.", + "i.e.", + "i.sz.", + "id.", + "ie.", + "ifj.", + "ig.", + "igh.", + "ill.", + "imp.", + "inc.", + "ind.", + "inform.", + "inic.", + "int.", + "io.", + "ip.", + "ir.", + "irod.", + "irod.", + "isk.", + "ism.", + "izr.", + "iá.", + "jan.", + "jav.", + "jegyz.", + "jgmk.", + "jjv.", + "jkv.", + "jogh.", + "jogt.", + "jr.", + "jvb.", + "júl.", + "jún.", + "karb.", + "kat.", + "kath.", + "kb.", + "kcs.", + "kd.", + "ker.", + "kf.", + "kft.", + "kht.", + "kir.", + "kirend.", + "kisip.", + "kiv.", + "kk.", + "kkt.", + "klin.", + "km.", + "korm.", + "kp.", + "krt.", + "kt.", + "ktsg.", + "kult.", + "kv.", + "kve.", + "képv.", + "kísérl.", + "kóth.", + "könyvt.", + "körz.", + "köv.", + "közj.", + "közl.", + "közp.", + "közt.", + "kü.", + "lat.", + "ld.", + "legs.", + "lg.", + "lgv.", + "loc.", + "lt.", + "ltd.", + "ltp.", + "luth.", + "m.a.", + "m.s.", + "m.sc.", + "ma.", + "mat.", + "max.", + "mb.", + "med.", + "megh.", + "met.", + "mf.", + "mfszt.", + "min.", + "miss.", + "mjr.", + "mjv.", + "mk.", + "mlle.", + "mme.", + "mn.", + "mozg.", + "mr.", + "mrs.", + "ms.", + "msc.", + "má.", + "máj.", + "márc.", + "mé.", + "mélt.", + "mü.", + "műh.", + "műsz.", + "műv.", + "művez.", + "nagyker.", + "nagys.", + "nat.", + "nb.", + "neg.", + "nk.", + "no.", + "nov.", + "nu.", + "ny.", + "nyilv.", + "nyrt.", + "nyug.", + "obj.", + "okl.", + "okt.", + "old.", + "olv.", + "orsz.", + "ort.", + "ov.", + "ovh.", + "pf.", + "pg.", + "ph.d", + "ph.d.", + "phd.", + "phil.", + "pjt.", + "pk.", + "pl.", + "plb.", + "plc.", + "pld.", + "plur.", + "pol.", + "polg.", + "poz.", + "pp.", + "proc.", + "prof.", + "prot.", + "pság.", + "ptk.", + "pu.", + "pü.", + "r.k.", + "rac.", + "rad.", + "red.", + "ref.", + "reg.", + "rer.", + "rev.", + "rf.", + "rkp.", + "rkt.", + "rt.", + "rtg.", + "röv.", + "s.b.", + "s.k.", + "sa.", + "sb.", + "sel.", + "sgt.", + "sm.", + "st.", + "stat.", + "stb.", + "strat.", + "stud.", + "sz.", + "szakm.", + "szaksz.", + "szakszerv.", + "szd.", + "szds.", + "szept.", + "szerk.", + "szf.", + "szimf.", + "szjt.", + "szkv.", + "szla.", + "szn.", + "szolg.", + "szt.", + "szubj.", + "szöv.", + "szül.", + "tanm.", + "tb.", + "tbk.", + "tc.", + "techn.", + "tek.", + "tel.", + "tf.", + "tgk.", + "ti.", + "tip.", + "tisztv.", + "titks.", + "tk.", + "tkp.", + "tny.", + "tp.", + "tszf.", + "tszk.", + "tszkv.", + "tv.", + "tvr.", + "ty.", + "törv.", + "tü.", + "ua.", + "ui.", + "unit.", + "uo.", + "uv.", + "vas.", + "vb.", + "vegy.", + "vh.", + "vhol.", + "vhr.", + "vill.", + "vizsg.", + "vk.", + "vkf.", + "vkny.", + "vm.", + "vol.", + "vs.", + "vsz.", + "vv.", + "vál.", + "várm.", + "vízv.", + "vö.", + "zrt.", + "zs.", + "Á.", + "Áe.", + "Áht.", + "É.", + "Épt.", + "Ész.", + "Új-Z.", + "ÚjZ.", + "Ún.", + "á.", + "ált.", + "ápr.", + "ásv.", + "é.", + "ék.", + "ény.", + "érk.", + "évf.", + "í.", + "ó.", + "össz.", + "ötk.", + "özv.", + "ú.", + "ú.n.", + "úm.", + "ún.", + "út.", + "üag.", + "üd.", + "üdv.", + "üe.", + "ümk.", + "ütk.", + "üv.", + "ű.", + "őrgy.", + "őrpk.", + "őrv.", +]: + _exc[orth] = [{ORTH: orth}] + + +_ord_num_or_date = r"([A-Z0-9]+[./-])*(\d+\.?)" +_num = r"[+\-]?\d+([,.]\d+)*" +_ops = r"[=<>+\-\*/^()÷%²]" +_suffixes = r"-[{al}]+".format(al=ALPHA_LOWER) +_numeric_exp = r"({n})(({o})({n}))*[%]?".format(n=_num, o=_ops) +_time_exp = r"\d+(:\d+)*(\.\d+)?" + +_nums = r"(({ne})|({t})|({on})|({c}))({s})?".format( + ne=_numeric_exp, t=_time_exp, on=_ord_num_or_date, c=CURRENCY, s=_suffixes +) + + +for u in "cfkCFK": + _exc[f"°{u}"] = [{ORTH: f"°{u}"}] + _exc[f"°{u}."] = [{ORTH: f"°{u}"}, {ORTH: "."}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) +TOKEN_MATCH = re.compile(r"^{n}$".format(n=_nums)).match diff --git a/spacy/lang/hy/__init__.py b/spacy/lang/hy/__init__.py new file mode 100644 index 0000000..e00d4fd --- /dev/null +++ b/spacy/lang/hy/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class ArmenianDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Armenian(Language): + lang = "hy" + Defaults = ArmenianDefaults + + +__all__ = ["Armenian"] diff --git a/spacy/lang/hy/examples.py b/spacy/lang/hy/examples.py new file mode 100644 index 0000000..9455396 --- /dev/null +++ b/spacy/lang/hy/examples.py @@ -0,0 +1,12 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.hy.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Լոնդոնը Միացյալ Թագավորության մեծ քաղաք է։", + "Ո՞վ է Ֆրանսիայի նախագահը։", + "Ո՞րն է Միացյալ Նահանգների մայրաքաղաքը։", + "Ե՞րբ է ծնվել Բարաք Օբաման։", +] diff --git a/spacy/lang/hy/lex_attrs.py b/spacy/lang/hy/lex_attrs.py new file mode 100644 index 0000000..4c96b8a --- /dev/null +++ b/spacy/lang/hy/lex_attrs.py @@ -0,0 +1,56 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "զրո", + "մեկ", + "երկու", + "երեք", + "չորս", + "հինգ", + "վեց", + "յոթ", + "ութ", + "ինը", + "տասը", + "տասնմեկ", + "տասներկու", + "տասներեք", + "տասնչորս", + "տասնհինգ", + "տասնվեց", + "տասնյոթ", + "տասնութ", + "տասնինը", + "քսան", + "երեսուն", + "քառասուն", + "հիսուն", + "վաթսուն", + "յոթանասուն", + "ութսուն", + "իննսուն", + "հարյուր", + "հազար", + "միլիոն", + "միլիարդ", + "տրիլիոն", + "քվինտիլիոն", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/hy/stop_words.py b/spacy/lang/hy/stop_words.py new file mode 100644 index 0000000..46d0f6b --- /dev/null +++ b/spacy/lang/hy/stop_words.py @@ -0,0 +1,107 @@ +STOP_WORDS = set( + """ +նա +ողջը +այստեղ +ենք +նա +էիր +որպես +ուրիշ +բոլորը +այն +այլ +նույնչափ +էի +մի +և +ողջ +ես +ոմն +հետ +նրանք +ամենքը +ըստ +ինչ-ինչ +այսպես +համայն +մի +նաև +նույնքան +դա +ովևէ +համար +այնտեղ +էին +որոնք +սույն +ինչ-որ +ամենը +նույնպիսի +ու +իր +որոշ +միևնույն +ի +այնպիսի +մենք +ամեն ոք +նույն +երբևէ +այն +որևէ +ին +այդպես +նրա +որը +վրա +դու +էինք +այդպիսի +էիք +յուրաքանչյուրը +եմ +պիտի +այդ +ամբողջը +հետո +եք +ամեն +այլ +կամ +այսքան +որ +այնպես +այսինչ +բոլոր +է +մեկնումեկը +այդչափ +այնքան +ամբողջ +երբևիցե +այնչափ +ամենայն +մյուս +այնինչ +իսկ +այդտեղ +այս +սա +են +ամեն ինչ +որևիցե +ում +մեկը +այդ +դուք +այսչափ +այդքան +այսպիսի +էր +յուրաքանչյուր +այս +մեջ +թ +""".split() +) diff --git a/spacy/lang/id/__init__.py b/spacy/lang/id/__init__.py new file mode 100644 index 0000000..93eb321 --- /dev/null +++ b/spacy/lang/id/__init__.py @@ -0,0 +1,24 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class IndonesianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + syntax_iterators = SYNTAX_ITERATORS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Indonesian(Language): + lang = "id" + Defaults = IndonesianDefaults + + +__all__ = ["Indonesian"] diff --git a/spacy/lang/id/_tokenizer_exceptions_list.py b/spacy/lang/id/_tokenizer_exceptions_list.py new file mode 100644 index 0000000..a0b35fa --- /dev/null +++ b/spacy/lang/id/_tokenizer_exceptions_list.py @@ -0,0 +1,3902 @@ +ID_BASE_EXCEPTIONS = set( + """ +aba-aba +abah-abah +abal-abal +abang-abang +abar-abar +abong-abong +abrit-abrit +abrit-abritan +abu-abu +abuh-abuhan +abuk-abuk +abun-abun +acak-acak +acak-acakan +acang-acang +acap-acap +aci-aci +aci-acian +aci-acinya +aco-acoan +ad-blocker +ad-interim +ada-ada +ada-adanya +ada-adanyakah +adang-adang +adap-adapan +add-on +add-ons +adik-adik +adik-beradik +aduk-adukan +after-sales +agak-agak +agak-agih +agama-agama +agar-agar +age-related +agut-agut +air-air +air-cooled +air-to-air +ajak-ajak +ajar-ajar +aji-aji +akal-akal +akal-akalan +akan-akan +akar-akar +akar-akaran +akhir-akhir +akhir-akhirnya +aki-aki +aksi-aksi +alah-mengalahi +alai-belai +alan-alan +alang-alang +alang-alangan +alap-alap +alat-alat +ali-ali +alif-alifan +alih-alih +aling-aling +aling-alingan +alip-alipan +all-electric +all-in-one +all-out +all-time +alon-alon +alt-right +alt-text +alu-alu +alu-aluan +alun-alun +alur-alur +alur-aluran +always-on +amai-amai +amatir-amatiran +ambah-ambah +ambai-ambai +ambil-mengambil +ambreng-ambrengan +ambring-ambringan +ambu-ambu +ambung-ambung +amin-amin +amit-amit +ampai-ampai +amprung-amprungan +amung-amung +anai-anai +anak-anak +anak-anakan +anak-beranak +anak-cucu +anak-istri +ancak-ancak +ancang-ancang +ancar-ancar +andang-andang +andeng-andeng +aneh-aneh +angan-angan +anggar-anggar +anggaran-red +anggota-anggota +anggung-anggip +angin-angin +angin-anginan +angkal-angkal +angkul-angkul +angkup-angkup +angkut-angkut +ani-ani +aning-aning +anjang-anjang +anjing-anjing +anjung-anjung +anjung-anjungan +antah-berantah +antar-antar +antar-mengantar +ante-mortem +antek-antek +anter-anter +antihuru-hara +anting-anting +antung-antung +anyam-menganyam +anyang-anyang +apa-apa +apa-apaan +apel-apel +api-api +apit-apit +aplikasi-aplikasi +apotek-apotek +aprit-apritan +apu-apu +apung-apung +arah-arah +arak-arak +arak-arakan +aram-aram +arek-arek +arem-arem +ari-ari +artis-artis +aru-aru +arung-arungan +asa-asaan +asal-asalan +asal-muasal +asal-usul +asam-asaman +asas-asas +aset-aset +asmaul-husna +asosiasi-asosiasi +asuh-asuh +asyik-asyiknya +atas-mengatasi +ati-ati +atung-atung +aturan-aturan +audio-video +audio-visual +auto-brightness +auto-complete +auto-focus +auto-play +auto-update +avant-garde +awan-awan +awan-berawan +awang-awang +awang-gemawang +awar-awar +awat-awat +awik-awik +awut-awutan +ayah-anak +ayak-ayak +ayam-ayam +ayam-ayaman +ayang-ayang +ayat-ayat +ayeng-ayengan +ayun-temayun +ayut-ayutan +ba-bi-bu +back-to-back +back-up +badan-badan +bade-bade +badut-badut +bagi-bagi +bahan-bahan +bahu-membahu +baik-baik +bail-out +bajang-bajang +baji-baji +balai-balai +balam-balam +balas-berbalas +balas-membalas +bale-bale +baling-baling +ball-playing +balon-balon +balut-balut +band-band +bandara-bandara +bangsa-bangsa +bangun-bangun +bangunan-bangunan +bank-bank +bantah-bantah +bantahan-bantahan +bantal-bantal +banyak-banyak +bapak-anak +bapak-bapak +bapak-ibu +bapak-ibunya +barang-barang +barat-barat +barat-daya +barat-laut +barau-barau +bare-bare +bareng-bareng +bari-bari +barik-barik +baris-berbaris +baru-baru +baru-batu +barung-barung +basa-basi +bata-bata +batalyon-batalyon +batang-batang +batas-batas +batir-batir +batu-batu +batuk-batuk +batung-batung +bau-bauan +bawa-bawa +bayan-bayan +bayang-bayang +bayi-bayi +bea-cukai +bedeng-bedeng +bedil-bedal +bedil-bedilan +begana-begini +bek-bek +bekal-bekalan +bekerdom-kerdom +bekertak-kertak +belang-belang +belat-belit +beliau-beliau +belu-belai +belum-belum +benar-benar +benda-benda +bengang-bengut +benggal-benggil +bengkal-bengkil +bengkang-bengkok +bengkang-bengkong +bengkang-bengkung +benteng-benteng +bentuk-bentuk +benua-benua +ber-selfie +berabad-abad +berabun-rabun +beracah-acah +berada-ada +beradik-berkakak +beragah-agah +beragak-agak +beragam-ragam +beraja-raja +berakit-rakit +beraku-akuan +beralu-aluan +beralun-alun +beramah-ramah +beramah-ramahan +beramah-tamah +beramai-ramai +berambai-ambai +berambal-ambalan +berambil-ambil +beramuk-amuk +beramuk-amukan +berandai-andai +berandai-randai +beraneh-aneh +berang-berang +berangan-angan +beranggap-anggapan +berangguk-angguk +berangin-angin +berangka-angka +berangka-angkaan +berangkai-rangkai +berangkap-rangkapan +berani-berani +beranja-anja +berantai-rantai +berapi-api +berapung-apung +berarak-arakan +beras-beras +berasak-asak +berasak-asakan +berasap-asap +berasing-asingan +beratus-ratus +berawa-rawa +berawas-awas +berayal-ayalan +berayun-ayun +berbagai-bagai +berbahas-bahasan +berbahasa-bahasa +berbaik-baikan +berbait-bait +berbala-bala +berbalas-balasan +berbalik-balik +berbalun-balun +berbanjar-banjar +berbantah-bantah +berbanyak-banyak +berbarik-barik +berbasa-basi +berbasah-basah +berbatu-batu +berbayang-bayang +berbecak-becak +berbeda-beda +berbedil-bedilan +berbega-bega +berbeka-beka +berbelah-belah +berbelakang-belakangan +berbelang-belang +berbelau-belauan +berbeli-beli +berbeli-belian +berbelit-belit +berbelok-belok +berbenang-benang +berbenar-benar +berbencah-bencah +berbencol-bencol +berbenggil-benggil +berbentol-bentol +berbentong-bentong +berberani-berani +berbesar-besar +berbidai-bidai +berbiduk-biduk +berbiku-biku +berbilik-bilik +berbinar-binar +berbincang-bincang +berbingkah-bingkah +berbintang-bintang +berbintik-bintik +berbintil-bintil +berbisik-bisik +berbolak-balik +berbolong-bolong +berbondong-bondong +berbongkah-bongkah +berbuai-buai +berbual-bual +berbudak-budak +berbukit-bukit +berbulan-bulan +berbunga-bunga +berbuntut-buntut +berbunuh-bunuhan +berburu-buru +berburuk-buruk +berbutir-butir +bercabang-cabang +bercaci-cacian +bercakap-cakap +bercakar-cakaran +bercamping-camping +bercantik-cantik +bercari-cari +bercari-carian +bercarik-carik +bercarut-carut +bercebar-cebur +bercepat-cepat +bercerai-berai +bercerai-cerai +bercetai-cetai +berciap-ciap +bercikun-cikun +bercinta-cintaan +bercita-cita +berciut-ciut +bercompang-camping +berconteng-conteng +bercoreng-coreng +bercoreng-moreng +bercuang-caing +bercuit-cuit +bercumbu-cumbu +bercumbu-cumbuan +bercura-bura +bercura-cura +berdada-dadaan +berdahulu-dahuluan +berdalam-dalam +berdalih-dalih +berdampung-dampung +berdebar-debar +berdecak-decak +berdecap-decap +berdecup-decup +berdecut-decut +berdedai-dedai +berdegap-degap +berdegar-degar +berdeham-deham +berdekah-dekah +berdekak-dekak +berdekap-dekapan +berdekat-dekat +berdelat-delat +berdembai-dembai +berdembun-dembun +berdempang-dempang +berdempet-dempet +berdencing-dencing +berdendam-dendaman +berdengkang-dengkang +berdengut-dengut +berdentang-dentang +berdentum-dentum +berdentung-dentung +berdenyar-denyar +berdenyut-denyut +berdepak-depak +berdepan-depan +berderai-derai +berderak-derak +berderam-deram +berderau-derau +berderik-derik +berdering-dering +berderung-derung +berderus-derus +berdesak-desakan +berdesik-desik +berdesing-desing +berdesus-desus +berdikit-dikit +berdingkit-dingkit +berdua-dua +berduri-duri +berduru-duru +berduyun-duyun +berebut-rebut +berebut-rebutan +beregang-regang +berek-berek +berembut-rembut +berempat-empat +berenak-enak +berencel-encel +bereng-bereng +berenggan-enggan +berenteng-renteng +beresa-esaan +beresah-resah +berfoya-foya +bergagah-gagahan +bergagap-gagap +bergagau-gagau +bergalur-galur +berganda-ganda +berganjur-ganjur +berganti-ganti +bergarah-garah +bergaruk-garuk +bergaya-gaya +bergegas-gegas +bergelang-gelang +bergelap-gelap +bergelas-gelasan +bergeleng-geleng +bergemal-gemal +bergembar-gembor +bergembut-gembut +bergepok-gepok +bergerek-gerek +bergesa-gesa +bergilir-gilir +bergolak-golak +bergolek-golek +bergolong-golong +bergores-gores +bergotong-royong +bergoyang-goyang +bergugus-gugus +bergulung-gulung +bergulut-gulut +bergumpal-gumpal +bergunduk-gunduk +bergunung-gunung +berhadap-hadapan +berhamun-hamun +berhandai-handai +berhanyut-hanyut +berhari-hari +berhati-hati +berhati-hatilah +berhektare-hektare +berhilau-hilau +berhormat-hormat +berhujan-hujan +berhura-hura +beri-beri +beri-memberi +beria-ia +beria-ria +beriak-riak +beriba-iba +beribu-ribu +berigi-rigi +berimpit-impit +berindap-indap +bering-bering +beringat-ingat +beringgit-ringgit +berintik-rintik +beriring-iring +beriring-iringan +berita-berita +berjabir-jabir +berjaga-jaga +berjagung-jagung +berjalan-jalan +berjalar-jalar +berjalin-jalin +berjalur-jalur +berjam-jam +berjari-jari +berjauh-jauhan +berjegal-jegalan +berjejal-jejal +berjela-jela +berjengkek-jengkek +berjenis-jenis +berjenjang-jenjang +berjilid-jilid +berjinak-jinak +berjingkat-jingkat +berjingkik-jingkik +berjingkrak-jingkrak +berjongkok-jongkok +berjubel-jubel +berjujut-jujutan +berjulai-julai +berjumbai-jumbai +berjumbul-jumbul +berjuntai-juntai +berjurai-jurai +berjurus-jurus +berjuta-juta +berka-li-kali +berkabu-kabu +berkaca-kaca +berkaing-kaing +berkait-kaitan +berkala-kala +berkali-kali +berkamit-kamit +berkanjar-kanjar +berkaok-kaok +berkarung-karung +berkasak-kusuk +berkasih-kasihan +berkata-kata +berkatak-katak +berkecai-kecai +berkecek-kecek +berkecil-kecil +berkecil-kecilan +berkedip-kedip +berkejang-kejang +berkejap-kejap +berkejar-kejaran +berkelar-kelar +berkelepai-kelepai +berkelip-kelip +berkelit-kelit +berkelok-kelok +berkelompok-kelompok +berkelun-kelun +berkembur-kembur +berkempul-kempul +berkena-kenaan +berkenal-kenalan +berkendur-kendur +berkeok-keok +berkepak-kepak +berkepal-kepal +berkeping-keping +berkepul-kepul +berkeras-kerasan +berkering-kering +berkeritik-keritik +berkeruit-keruit +berkerut-kerut +berketai-ketai +berketak-ketak +berketak-ketik +berketap-ketap +berketap-ketip +berketar-ketar +berketi-keti +berketil-ketil +berketuk-ketak +berketul-ketul +berkial-kial +berkian-kian +berkias-kias +berkias-kiasan +berkibar-kibar +berkilah-kilah +berkilap-kilap +berkilat-kilat +berkilau-kilauan +berkilo-kilo +berkimbang-kimbang +berkinja-kinja +berkipas-kipas +berkira-kira +berkirim-kiriman +berkisar-kisar +berkoak-koak +berkoar-koar +berkobar-kobar +berkobok-kobok +berkocak-kocak +berkodi-kodi +berkolek-kolek +berkomat-kamit +berkopah-kopah +berkoper-koper +berkotak-kotak +berkuat-kuat +berkuat-kuatan +berkumur-kumur +berkunang-kunang +berkunar-kunar +berkunjung-kunjungan +berkurik-kurik +berkurun-kurun +berkusau-kusau +berkusu-kusu +berkusut-kusut +berkuting-kuting +berkutu-kutuan +berlabun-labun +berlain-lainan +berlaju-laju +berlalai-lalai +berlama-lama +berlambai-lambai +berlambak-lambak +berlampang-lampang +berlanggar-langgar +berlapang-lapang +berlapis-lapis +berlapuk-lapuk +berlarah-larah +berlarat-larat +berlari-lari +berlari-larian +berlarih-larih +berlarik-larik +berlarut-larut +berlawak-lawak +berlayap-layapan +berlebih-lebih +berlebih-lebihan +berleha-leha +berlekas-lekas +berlekas-lekasan +berlekat-lekat +berlekuk-lekuk +berlempar-lemparan +berlena-lena +berlengah-lengah +berlenggak-lenggok +berlenggek-lenggek +berlenggok-lenggok +berleret-leret +berletih-letih +berliang-liuk +berlibat-libat +berligar-ligar +berliku-liku +berlikur-likur +berlimbak-limbak +berlimpah-limpah +berlimpap-limpap +berlimpit-limpit +berlinang-linang +berlindak-lindak +berlipat-lipat +berlomba-lomba +berlompok-lompok +berloncat-loncatan +berlopak-lopak +berlubang-lubang +berlusin-lusin +bermaaf-maafan +bermabuk-mabukan +bermacam-macam +bermain-main +bermalam-malam +bermalas-malas +bermalas-malasan +bermanik-manik +bermanis-manis +bermanja-manja +bermasak-masak +bermati-mati +bermegah-megah +bermemek-memek +bermenung-menung +bermesra-mesraan +bermewah-mewah +bermewah-mewahan +berminggu-minggu +berminta-minta +berminyak-minyak +bermuda-muda +bermudah-mudah +bermuka-muka +bermula-mula +bermuluk-muluk +bermulut-mulut +bernafsi-nafsi +bernaka-naka +bernala-nala +bernanti-nanti +berniat-niat +bernyala-nyala +berogak-ogak +beroleng-oleng +berolok-olok +beromong-omong +beroncet-roncet +beronggok-onggok +berorang-orang +beroyal-royal +berpada-pada +berpadu-padu +berpahit-pahit +berpair-pair +berpal-pal +berpalu-palu +berpalu-paluan +berpalun-palun +berpanas-panas +berpandai-pandai +berpandang-pandangan +berpangkat-pangkat +berpanjang-panjang +berpantun-pantun +berpasang-pasang +berpasang-pasangan +berpasuk-pasuk +berpayah-payah +berpeluh-peluh +berpeluk-pelukan +berpenat-penat +berpencar-pencar +berpendar-pendar +berpenggal-penggal +berperai-perai +berperang-perangan +berpesai-pesai +berpesta-pesta +berpesuk-pesuk +berpetak-petak +berpeti-peti +berpihak-pihak +berpijar-pijar +berpikir-pikir +berpikul-pikul +berpilih-pilih +berpilin-pilin +berpindah-pindah +berpintal-pintal +berpirau-pirau +berpisah-pisah +berpolah-polah +berpolok-polok +berpongah-pongah +berpontang-panting +berporah-porah +berpotong-potong +berpotong-potongan +berpuak-puak +berpual-pual +berpugak-pugak +berpuing-puing +berpukas-pukas +berpuluh-puluh +berpulun-pulun +berpuntal-puntal +berpura-pura +berpusar-pusar +berpusing-pusing +berpusu-pusu +berputar-putar +berrumpun-rumpun +bersaf-saf +bersahut-sahutan +bersakit-sakit +bersalah-salahan +bersalam-salaman +bersalin-salin +bersalip-salipan +bersama-sama +bersambar-sambaran +bersambut-sambutan +bersampan-sampan +bersantai-santai +bersapa-sapaan +bersarang-sarang +bersedan-sedan +bersedia-sedia +bersedu-sedu +bersejuk-sejuk +bersekat-sekat +berselang-selang +berselang-seli +berselang-seling +berselang-tenggang +berselit-selit +berseluk-beluk +bersembunyi-sembunyi +bersembunyi-sembunyian +bersembur-semburan +bersempit-sempit +bersenang-senang +bersenang-senangkan +bersenda-senda +bersendi-sendi +bersenggang-senggang +bersenggau-senggau +bersepah-sepah +bersepak-sepakan +bersepi-sepi +berserak-serak +berseri-seri +berseru-seru +bersesak-sesak +bersetai-setai +bersia-sia +bersiap-siap +bersiar-siar +bersih-bersih +bersikut-sikutan +bersilir-silir +bersimbur-simburan +bersinau-sinau +bersopan-sopan +bersorak-sorai +bersuap-suapan +bersudah-sudah +bersuka-suka +bersuka-sukaan +bersuku-suku +bersulang-sulang +bersumpah-sumpahan +bersungguh-sungguh +bersungut-sungut +bersunyi-sunyi +bersuruk-surukan +bersusah-susah +bersusuk-susuk +bersusuk-susukan +bersutan-sutan +bertabur-tabur +bertahan-tahan +bertahu-tahu +bertahun-tahun +bertajuk-tajuk +bertakik-takik +bertala-tala +bertalah-talah +bertali-tali +bertalu-talu +bertalun-talun +bertambah-tambah +bertanda-tandaan +bertangis-tangisan +bertangkil-tangkil +bertanya-tanya +bertarik-tarikan +bertatai-tatai +bertatap-tatapan +bertatih-tatih +bertawan-tawan +bertawar-tawaran +bertebu-tebu +bertebu-tebukan +berteguh-teguh +berteguh-teguhan +berteka-teki +bertelang-telang +bertelau-telau +bertele-tele +bertembuk-tembuk +bertempat-tempat +bertempuh-tempuh +bertenang-tenang +bertenggang-tenggangan +bertentu-tentu +bertepek-tepek +berterang-terang +berterang-terangan +berteriak-teriak +bertikam-tikaman +bertimbal-timbalan +bertimbun-timbun +bertimpa-timpa +bertimpas-timpas +bertingkah-tingkah +bertingkat-tingkat +bertinjau-tinjauan +bertiras-tiras +bertitar-titar +bertitik-titik +bertoboh-toboh +bertolak-tolak +bertolak-tolakan +bertolong-tolongan +bertonjol-tonjol +bertruk-truk +bertua-tua +bertua-tuaan +bertual-tual +bertubi-tubi +bertukar-tukar +bertukar-tukaran +bertukas-tukas +bertumpak-tumpak +bertumpang-tindih +bertumpuk-tumpuk +bertunda-tunda +bertunjuk-tunjukan +bertura-tura +berturut-turut +bertutur-tutur +beruas-ruas +berubah-ubah +berulang-alik +berulang-ulang +berumbai-rumbai +berundak-undak +berundan-undan +berundung-undung +berunggas-runggas +berunggun-unggun +berunggut-unggut +berungkur-ungkuran +beruntai-untai +beruntun-runtun +beruntung-untung +berunyai-unyai +berupa-rupa +berura-ura +beruris-uris +berurut-urutan +berwarna-warna +berwarna-warni +berwindu-windu +berwiru-wiru +beryang-yang +besar-besar +besar-besaran +betak-betak +beti-beti +betik-betik +betul-betul +biang-biang +biar-biar +biaya-biaya +bicu-bicu +bidadari-bidadari +bidang-bidang +bijak-bijaklah +biji-bijian +bila-bila +bilang-bilang +bincang-bincang +bincang-bincut +bingkah-bingkah +bini-binian +bintang-bintang +bintik-bintik +bio-oil +biri-biri +biru-biru +biru-hitam +biru-kuning +bisik-bisik +biti-biti +blak-blakan +blok-blok +bocah-bocah +bohong-bohong +bohong-bohongan +bola-bola +bolak-balik +bolang-baling +boleh-boleh +bom-bom +bomber-bomber +bonek-bonek +bongkar-bangkir +bongkar-membongkar +bongkar-pasang +boro-boro +bos-bos +bottom-up +box-to-box +boyo-boyo +buah-buahan +buang-buang +buat-buatan +buaya-buaya +bubun-bubun +bugi-bugi +build-up +built-in +built-up +buka-buka +buka-bukaan +buka-tutup +bukan-bukan +bukti-bukti +buku-buku +bulan-bulan +bulan-bulanan +bulang-baling +bulang-bulang +bulat-bulat +buli-buli +bulu-bulu +buluh-buluh +bulus-bulus +bunga-bunga +bunga-bungaan +bunuh-membunuh +bunyi-bunyian +bupati-bupati +bupati-wakil +buru-buru +burung-burung +burung-burungan +bus-bus +business-to-business +busur-busur +butir-butir +by-pass +bye-bye +cabang-cabang +cabik-cabik +cabik-mencabik +cabup-cawabup +caci-maki +cagub-cawagub +caing-caing +cakar-mencakar +cakup-mencakup +calak-calak +calar-balar +caleg-caleg +calo-calo +calon-calon +campang-camping +campur-campur +capres-cawapres +cara-cara +cari-cari +cari-carian +carut-marut +catch-up +cawali-cawawali +cawe-cawe +cawi-cawi +cebar-cebur +celah-celah +celam-celum +celangak-celinguk +celas-celus +celedang-celedok +celengkak-celengkok +celingak-celinguk +celung-celung +cemas-cemas +cenal-cenil +cengar-cengir +cengir-cengir +cengis-cengis +cengking-mengking +centang-perenang +cepat-cepat +ceplas-ceplos +cerai-berai +cerita-cerita +ceruk-menceruk +ceruk-meruk +cetak-biru +cetak-mencetak +cetar-ceter +check-in +check-ins +check-up +chit-chat +choki-choki +cingak-cinguk +cipika-cipiki +ciri-ciri +ciri-cirinya +cirit-birit +cita-cita +cita-citaku +close-up +closed-circuit +coba-coba +cobak-cabik +cobar-cabir +cola-cala +colang-caling +comat-comot +comot-comot +compang-camping +computer-aided +computer-generated +condong-mondong +congak-cangit +conggah-canggih +congkah-cangkih +congkah-mangkih +copak-capik +copy-paste +corak-carik +corat-coret +coreng-moreng +coret-coret +crat-crit +cross-border +cross-dressing +crypto-ransomware +cuang-caing +cublak-cublak +cubung-cubung +culik-culik +cuma-cuma +cumi-cumi +cungap-cangip +cupu-cupu +dabu-dabu +daerah-daerah +dag-dag +dag-dig-dug +daging-dagingan +dahulu-mendahului +dalam-dalam +dali-dali +dam-dam +danau-danau +dansa-dansi +dapil-dapil +dapur-dapur +dari-dari +daru-daru +dasar-dasar +datang-datang +datang-mendatangi +daun-daun +daun-daunan +dawai-dawai +dayang-dayang +dayung-mayung +debak-debuk +debu-debu +deca-core +decision-making +deep-lying +deg-degan +degap-degap +dekak-dekak +dekat-dekat +dengar-dengaran +dengking-mendengking +departemen-departemen +depo-depo +deputi-deputi +desa-desa +desa-kota +desas-desus +detik-detik +dewa-dewa +dewa-dewi +dewan-dewan +dewi-dewi +dial-up +diam-diam +dibayang-bayangi +dibuat-buat +diiming-imingi +dilebih-lebihkan +dimana-mana +dimata-matai +dinas-dinas +dinul-Islam +diobok-obok +diolok-olok +direksi-direksi +direktorat-direktorat +dirjen-dirjen +dirut-dirut +ditunggu-tunggu +divisi-divisi +do-it-yourself +doa-doa +dog-dog +doggy-style +dokok-dokok +dolak-dalik +dor-doran +dorong-mendorong +dosa-dosa +dress-up +drive-in +dua-dua +dua-duaan +dua-duanya +dubes-dubes +duduk-duduk +dugaan-dugaan +dulang-dulang +duri-duri +duta-duta +dwi-kewarganegaraan +e-arena +e-billing +e-budgeting +e-cctv +e-class +e-commerce +e-counting +e-elektronik +e-entertainment +e-evolution +e-faktur +e-filing +e-fin +e-form +e-government +e-govt +e-hakcipta +e-id +e-info +e-katalog +e-ktp +e-leadership +e-lhkpn +e-library +e-loket +e-m1 +e-money +e-news +e-nisn +e-npwp +e-paspor +e-paten +e-pay +e-perda +e-perizinan +e-planning +e-polisi +e-power +e-punten +e-retribusi +e-samsat +e-sport +e-store +e-tax +e-ticketing +e-tilang +e-toll +e-visa +e-voting +e-wallet +e-warong +ecek-ecek +eco-friendly +eco-park +edan-edanan +editor-editor +editor-in-chief +efek-efek +ekonomi-ekonomi +eksekutif-legislatif +ekspor-impor +elang-elang +elemen-elemen +emak-emak +embuh-embuhan +empat-empat +empek-empek +empet-empetan +empok-empok +empot-empotan +enak-enak +encal-encal +end-to-end +end-user +endap-endap +endut-endut +endut-endutan +engah-engah +engap-engap +enggan-enggan +engkah-engkah +engket-engket +entah-berentah +enten-enten +entry-level +equity-linked +erang-erot +erat-erat +erek-erek +ereng-ereng +erong-erong +esek-esek +ex-officio +exchange-traded +exercise-induced +extra-time +face-down +face-to-face +fair-play +fakta-fakta +faktor-faktor +fakultas-fakultas +fase-fase +fast-food +feed-in +fifty-fifty +file-file +first-leg +first-team +fitur-fitur +fitur-fiturnya +fixed-income +flip-flop +flip-plop +fly-in +follow-up +foto-foto +foya-foya +fraksi-fraksi +free-to-play +front-end +fungsi-fungsi +gaba-gaba +gabai-gabai +gada-gada +gading-gading +gadis-gadis +gado-gado +gail-gail +gajah-gajah +gajah-gajahan +gala-gala +galeri-galeri +gali-gali +gali-galian +galing-galing +galu-galu +gamak-gamak +gambar-gambar +gambar-menggambar +gamit-gamitan +gampang-gampangan +gana-gini +ganal-ganal +ganda-berganda +ganjal-mengganjal +ganjil-genap +ganteng-ganteng +gantung-gantung +gapah-gopoh +gara-gara +garah-garah +garis-garis +gasak-gasakan +gatal-gatal +gaun-gaun +gawar-gawar +gaya-gayanya +gayang-gayang +ge-er +gebyah-uyah +gebyar-gebyar +gedana-gedini +gedebak-gedebuk +gedebar-gedebur +gedung-gedung +gelang-gelang +gelap-gelapan +gelar-gelar +gelas-gelas +gelembung-gelembungan +geleng-geleng +geli-geli +geliang-geliut +geliat-geliut +gembar-gembor +gembrang-gembreng +gempul-gempul +gempur-menggempur +gendang-gendang +gengsi-gengsian +genjang-genjot +genjot-genjotan +genjrang-genjreng +genome-wide +geo-politik +gerabak-gerubuk +gerak-gerik +gerak-geriknya +gerakan-gerakan +gerbas-gerbus +gereja-gereja +gereng-gereng +geriak-geriuk +gerit-gerit +gerot-gerot +geruh-gerah +getak-getuk +getem-getem +geti-geti +gial-gial +gial-giul +gila-gila +gila-gilaan +gilang-gemilang +gilap-gemilap +gili-gili +giling-giling +gilir-bergilir +ginang-ginang +girap-girap +girik-girik +giring-giring +go-auto +go-bills +go-bluebird +go-box +go-car +go-clean +go-food +go-glam +go-jek +go-kart +go-mart +go-massage +go-med +go-points +go-pulsa +go-ride +go-send +go-shop +go-tix +go-to-market +goak-goak +goal-line +gol-gol +golak-galik +gondas-gandes +gonjang-ganjing +gonjlang-ganjling +gonta-ganti +gontok-gontokan +gorap-gorap +gorong-gorong +gotong-royong +gresek-gresek +gua-gua +gual-gail +gubernur-gubernur +gudu-gudu +gula-gula +gulang-gulang +gulung-menggulung +guna-ganah +guna-guna +gundala-gundala +guntang-guntang +gunung-ganang +gunung-gemunung +gunung-gunungan +guru-guru +habis-habis +habis-habisan +hak-hak +hak-hal +hakim-hakim +hal-hal +halai-balai +half-time +hama-hama +hampir-hampir +hancur-hancuran +hancur-menghancurkan +hands-free +hands-on +hang-out +hantu-hantu +happy-happy +harap-harap +harap-harapan +hard-disk +harga-harga +hari-hari +harimau-harimau +harum-haruman +hasil-hasil +hasta-wara +hat-trick +hati-hati +hati-hatilah +head-mounted +head-to-head +head-up +heads-up +heavy-duty +hebat-hebatan +hewan-hewan +hexa-core +hidup-hidup +hidup-mati +hila-hila +hilang-hilang +hina-menghinakan +hip-hop +hiru-biru +hiru-hara +hiruk-pikuk +hitam-putih +hitung-hitung +hitung-hitungan +hormat-menghormati +hot-swappable +hotel-hotel +how-to +hubar-habir +hubaya-hubaya +hukum-red +hukuman-hukuman +hula-hoop +hula-hula +hulu-hilir +humas-humas +hura-hura +huru-hara +ibar-ibar +ibu-anak +ibu-ibu +icak-icak +icip-icip +idam-idam +ide-ide +igau-igauan +ikan-ikan +ikut-ikut +ikut-ikutan +ilam-ilam +ilat-ilatan +ilmu-ilmu +imbang-imbangan +iming-iming +imut-imut +inang-inang +inca-binca +incang-incut +industri-industri +ingar-bingar +ingar-ingar +ingat-ingat +ingat-ingatan +ingau-ingauan +inggang-inggung +injak-injak +input-output +instansi-instansi +instant-on +instrumen-instrumen +inter-governmental +ira-ira +irah-irahan +iras-iras +iring-iringan +iris-irisan +isak-isak +isat-bb +iseng-iseng +istana-istana +istri-istri +isu-isu +iya-iya +jabatan-jabatan +jadi-jadian +jagoan-jagoan +jaja-jajaan +jaksa-jaksa +jala-jala +jalan-jalan +jali-jali +jalin-berjalin +jalin-menjalin +jam-jam +jamah-jamahan +jambak-jambakan +jambu-jambu +jampi-jampi +janda-janda +jangan-jangan +janji-janji +jarang-jarang +jari-jari +jaring-jaring +jarum-jarum +jasa-jasa +jatuh-bangun +jauh-dekat +jauh-jauh +jawi-jawi +jebar-jebur +jebat-jebatan +jegal-jegalan +jejak-jejak +jelang-menjelang +jelas-jelas +jelur-jelir +jembatan-jembatan +jenazah-jenazah +jendal-jendul +jenderal-jenderal +jenggar-jenggur +jenis-jenis +jenis-jenisnya +jentik-jentik +jerah-jerih +jinak-jinak +jiwa-jiwa +joli-joli +jolong-jolong +jongkang-jangking +jongkar-jangkir +jongkat-jangkit +jor-joran +jotos-jotosan +juak-juak +jual-beli +juang-juang +julo-julo +julung-julung +julur-julur +jumbai-jumbai +jungkang-jungkit +jungkat-jungkit +jurai-jurai +kabang-kabang +kabar-kabari +kabir-kabiran +kabruk-kabrukan +kabu-kabu +kabupaten-kabupaten +kabupaten-kota +kaca-kaca +kacang-kacang +kacang-kacangan +kacau-balau +kadang-kadang +kader-kader +kades-kades +kadis-kadis +kail-kail +kain-kain +kait-kait +kakak-adik +kakak-beradik +kakak-kakak +kakek-kakek +kakek-nenek +kaki-kaki +kala-kala +kalau-kalau +kaleng-kalengan +kali-kalian +kalimat-kalimat +kalung-kalung +kalut-malut +kambing-kambing +kamit-kamit +kampung-kampung +kampus-kampus +kanak-kanak +kanak-kanan +kanan-kanak +kanan-kiri +kangen-kangenan +kanwil-kanwil +kapa-kapa +kapal-kapal +kapan-kapan +kapolda-kapolda +kapolres-kapolres +kapolsek-kapolsek +kapu-kapu +karang-karangan +karang-mengarang +kareseh-peseh +karut-marut +karya-karya +kasak-kusuk +kasus-kasus +kata-kata +katang-katang +kava-kava +kawa-kawa +kawan-kawan +kawin-cerai +kawin-mawin +kayu-kayu +kayu-kayuan +ke-Allah-an +keabu-abuan +kearab-araban +keasyik-asyikan +kebarat-baratan +kebasah-basahan +kebat-kebit +kebata-bataan +kebayi-bayian +kebelanda-belandaan +keberlarut-larutan +kebesar-hatian +kebiasaan-kebiasaan +kebijakan-kebijakan +kebiru-biruan +kebudak-budakan +kebun-kebun +kebut-kebutan +kecamatan-kecamatan +kecentang-perenangan +kecil-kecil +kecil-kecilan +kecil-mengecil +kecokelat-cokelatan +kecomak-kecimik +kecuh-kecah +kedek-kedek +kedekak-kedekik +kedesa-desaan +kedubes-kedubes +kedutaan-kedutaan +keempat-empatnya +kegadis-gadisan +kegelap-gelapan +kegiatan-kegiatan +kegila-gilaan +kegirang-girangan +kehati-hatian +keheran-heranan +kehijau-hijauan +kehitam-hitaman +keinggris-inggrisan +kejaga-jagaan +kejahatan-kejahatan +kejang-kejang +kejar-kejar +kejar-kejaran +kejar-mengejar +kejingga-jinggaan +kejut-kejut +kejutan-kejutan +kekabur-kaburan +kekanak-kanakan +kekoboi-koboian +kekota-kotaan +kekuasaan-kekuasaan +kekuning-kuningan +kelak-kelik +kelak-keluk +kelaki-lakian +kelang-kelok +kelap-kelip +kelasah-kelusuh +kelek-kelek +kelek-kelekan +kelemak-kelemek +kelik-kelik +kelip-kelip +kelompok-kelompok +kelontang-kelantung +keluar-masuk +kelurahan-kelurahan +kelusuh-kelasah +kelut-melut +kemak-kemik +kemalu-maluan +kemana-mana +kemanja-manjaan +kemarah-marahan +kemasam-masaman +kemati-matian +kembang-kembang +kemenpan-rb +kementerian-kementerian +kemerah-merahan +kempang-kempis +kempas-kempis +kemuda-mudaan +kena-mengena +kenal-mengenal +kenang-kenangan +kencang-kencung +kencing-mengencingi +kencrang-kencring +kendang-kendang +kendang-kendangan +keningrat-ningratan +kentung-kentung +kenyat-kenyit +kepala-kepala +kepala-kepalaan +kepandir-pandiran +kepang-kepot +keperak-perakan +kepetah-lidahan +kepilu-piluan +keping-keping +kepucat-pucatan +kepuh-kepuh +kepura-puraan +keputih-putihan +kerah-kerahan +kerancak-rancakan +kerang-kerangan +kerang-keroh +kerang-kerot +kerang-keruk +kerang-kerung +kerap-kerap +keras-mengerasi +kercap-kercip +kercap-kercup +keriang-keriut +kerja-kerja +kernyat-kernyut +kerobak-kerabit +kerobak-kerobek +kerobak-kerobik +kerobat-kerabit +kerong-kerong +keropas-kerapis +kertak-kertuk +kertap-kertap +keruntang-pungkang +kesalahan-kesalahan +kesap-kesip +kesemena-menaan +kesenak-senakan +kesewenang-wenangan +kesia-siaan +kesik-kesik +kesipu-sipuan +kesu-kesi +kesuh-kesih +kesuk-kesik +ketakar-keteker +ketakutan-ketakutan +ketap-ketap +ketap-ketip +ketar-ketir +ketentuan-ketentuan +ketergesa-gesaan +keti-keti +ketidur-tiduran +ketiga-tiganya +ketir-ketir +ketua-ketua +ketua-tuaan +ketuan-tuanan +keungu-unguan +kewangi-wangian +ki-ka +kia-kia +kiai-kiai +kiak-kiak +kial-kial +kiang-kiut +kiat-kiat +kibang-kibut +kicang-kecoh +kicang-kicu +kick-off +kida-kida +kijang-kijang +kilau-mengilau +kili-kili +kilik-kilik +kincir-kincir +kios-kios +kira-kira +kira-kiraan +kiri-kanan +kirim-berkirim +kisah-kisah +kisi-kisi +kitab-kitab +kitang-kitang +kiu-kiu +klaim-klaim +klik-klikan +klip-klip +klub-klub +kluntang-klantung +knock-knock +knock-on +knock-out +ko-as +ko-pilot +koak-koak +koboi-koboian +kocah-kacih +kocar-kacir +kodam-kodam +kode-kode +kodim-kodim +kodok-kodok +kolang-kaling +kole-kole +koleh-koleh +kolong-kolong +koma-koma +komat-kamit +komisaris-komisaris +komisi-komisi +komite-komite +komoditas-komoditas +kongko-kongko +konsulat-konsulat +konsultan-konsultan +kontal-kantil +kontang-kanting +kontra-terorisme +kontrak-kontrak +konvensi-konvensi +kopat-kapit +koperasi-koperasi +kopi-kopi +koran-koran +koreng-koreng +kos-kosan +kosak-kasik +kota-kota +kota-wakil +kotak-katik +kotak-kotak +koyak-koyak +kuas-kuas +kuat-kuat +kubu-kubuan +kucar-kacir +kucing-kucing +kucing-kucingan +kuda-kuda +kuda-kudaan +kudap-kudap +kue-kue +kulah-kulah +kulak-kulak +kulik-kulik +kulum-kulum +kumat-kamit +kumpul-kumpul +kunang-kunang +kunar-kunar +kung-fu +kuning-hitam +kupat-kapit +kupu-kupu +kura-kura +kurang-kurang +kusat-mesat +kutat-kutet +kuti-kuti +kuwung-kuwung +kyai-kyai +laba-laba +labi-labi +labu-labu +laga-laga +lagi-lagi +lagu-lagu +laguh-lagah +lain-lain +laki-laki +lalu-lalang +lalu-lintas +lama-kelamaan +lama-lama +lamat-lamat +lambat-lambat +lampion-lampion +lampu-lampu +lancang-lancang +lancar-lancar +langak-longok +langgar-melanggar +langit-langit +langkah-langka +langkah-langkah +lanja-lanjaan +lapas-lapas +lapat-lapat +laporan-laporan +laptop-tablet +large-scale +lari-lari +lari-larian +laskar-laskar +lauk-pauk +laun-laun +laut-timur +lawah-lawah +lawak-lawak +lawan-lawan +lawi-lawi +layang-layang +layu-layuan +lebih-lebih +lecet-lecet +legak-legok +legum-legum +legup-legup +leha-leha +lekak-lekuk +lekap-lekup +lekas-lekas +lekat-lekat +lekuh-lekih +lekum-lekum +lekup-lekap +lembaga-lembaga +lempar-lemparan +lenggak-lenggok +lenggok-lenggok +lenggut-lenggut +lengket-lengket +lentam-lentum +lentang-lentok +lentang-lentung +lepa-lepa +lerang-lerang +lereng-lereng +lese-majeste +letah-letai +lete-lete +letuk-letuk +letum-letum +letup-letup +leyeh-leyeh +liang-liuk +liang-liut +liar-liar +liat-liut +lidah-lidah +life-toxins +liga-liga +light-emitting +lika-liku +lil-alamin +lilin-lilin +line-up +lintas-selat +lipat-melipat +liquid-cooled +lithium-ion +lithium-polymer +liuk-liuk +liung-liung +lobi-lobi +lock-up +locked-in +lokasi-lokasi +long-term +longak-longok +lontang-lanting +lontang-lantung +lopak-lapik +lopak-lopak +low-cost +low-density +low-end +low-light +low-multi +low-pass +lucu-lucu +luka-luka +lukisan-lukisan +lumba-lumba +lumi-lumi +luntang-lantung +lupa-lupa +lupa-lupaan +lurah-camat +maaf-memaafkan +mabuk-mabukan +mabul-mabul +macam-macam +macan-macanan +machine-to-machine +mafia-mafia +mahasiswa-mahasiswi +mahasiswa/i +mahi-mahi +main-main +main-mainan +main-mainlah +majelis-majelis +maju-mundur +makam-makam +makan-makan +makan-makanan +makanan-red +make-up +maki-maki +maki-makian +mal-mal +malai-malai +malam-malam +malar-malar +malas-malasan +mali-mali +malu-malu +mama-mama +man-in-the-middle +mana-mana +manajer-manajer +manik-manik +manis-manis +manis-manisan +marah-marah +mark-up +mas-mas +masa-masa +masak-masak +masalah-masalah +mash-up +masing-masing +masjid-masjid +masuk-keluar +mat-matan +mata-mata +match-fixing +mati-mati +mati-matian +maya-maya +mayat-mayat +mayday-mayday +media-media +mega-bintang +mega-tsunami +megal-megol +megap-megap +meger-meger +megrek-megrek +melak-melak +melambai-lambai +melambai-lambaikan +melambat-lambatkan +melaun-laun +melawak-lawak +melayang-layang +melayap-layap +melayap-layapkan +melebih-lebihi +melebih-lebihkan +melejang-lejangkan +melek-melekan +meleleh-leleh +melengah-lengah +melihat-lihat +melimpah-limpah +melincah-lincah +meliuk-liuk +melolong-lolong +melompat-lompat +meloncat-loncat +melonco-lonco +melongak-longok +melonjak-lonjak +memacak-macak +memada-madai +memadan-madan +memaki-maki +memaksa-maksa +memanas-manasi +memancit-mancitkan +memandai-mandai +memanggil-manggil +memanis-manis +memanjut-manjut +memantas-mantas +memasak-masak +memata-matai +mematah-matah +mematuk-matuk +mematut-matut +memau-mau +memayah-mayahkan +membaca-baca +membacah-bacah +membagi-bagikan +membalik-balik +membangkit-bangkit +membarut-barut +membawa-bawa +membayang-bayangi +membayang-bayangkan +membeda-bedakan +membelai-belai +membeli-beli +membelit-belitkan +membelu-belai +membenar-benar +membenar-benari +memberai-beraikan +membesar-besar +membesar-besarkan +membikin-bikin +membilah-bilah +membolak-balikkan +membongkar-bangkir +membongkar-bongkar +membuang-buang +membuat-buat +membulan-bulani +membunga-bungai +membungkuk-bungkuk +memburu-buru +memburu-burukan +memburuk-burukkan +memelintir-melintir +memencak-mencak +memencar-mencar +memercik-mercik +memetak-metak +memetang-metangkan +memetir-metir +memijar-mijar +memikir-mikir +memikir-mikirkan +memilih-milih +memilin-milin +meminang-minang +meminta-minta +memisah-misahkan +memontang-mantingkan +memorak-perandakan +memorak-porandakan +memotong-motong +memperamat-amat +memperamat-amatkan +memperbagai-bagaikan +memperganda-gandakan +memperganduh-ganduhkan +memperimpit-impitkan +memperkuda-kudakan +memperlengah-lengah +memperlengah-lengahkan +mempermacam-macamkan +memperolok-olok +memperolok-olokkan +mempersama-samakan +mempertubi-tubi +mempertubi-tubikan +memperturut-turutkan +memuja-muja +memukang-mukang +memulun-mulun +memundi-mundi +memundi-mundikan +memutar-mutar +memuyu-muyu +men-tweet +menagak-nagak +menakut-nakuti +menang-kalah +menanjur-nanjur +menanti-nanti +menari-nari +mencabik-cabik +mencabik-cabikkan +mencacah-cacah +mencaing-caing +mencak-mencak +mencakup-cakup +mencapak-capak +mencari-cari +mencarik-carik +mencarik-carikkan +mencarut-carut +mencengis-cengis +mencepak-cepak +mencepuk-cepuk +mencerai-beraikan +mencetai-cetai +menciak-ciak +menciap-ciap +menciar-ciar +mencita-citakan +mencium-cium +menciut-ciut +mencla-mencle +mencoang-coang +mencoba-coba +mencocok-cocok +mencolek-colek +menconteng-conteng +mencubit-cubit +mencucuh-cucuh +mencucuh-cucuhkan +mencuri-curi +mendecap-decap +mendegam-degam +mendengar-dengar +mendengking-dengking +mendengus-dengus +mendengut-dengut +menderai-deraikan +menderak-derakkan +menderau-derau +menderu-deru +mendesas-desuskan +mendesus-desus +mendetap-detap +mendewa-dewakan +mendudu-dudu +menduga-duga +menebu-nebu +menegur-neguri +menepak-nepak +menepak-nepakkan +mengabung-ngabung +mengaci-acikan +mengacu-acu +mengada-ada +mengada-ngada +mengadang-adangi +mengaduk-aduk +mengagak-agak +mengagak-agihkan +mengagut-agut +mengais-ngais +mengalang-alangi +mengali-ali +mengalur-alur +mengamang-amang +mengamat-amati +mengambai-ambaikan +mengambang-ambang +mengambung-ambung +mengambung-ambungkan +mengamit-ngamitkan +mengancai-ancaikan +mengancak-ancak +mengancar-ancar +mengangan-angan +mengangan-angankan +mengangguk-angguk +menganggut-anggut +mengangin-anginkan +mengangkat-angkat +menganjung-anjung +menganjung-anjungkan +mengap-mengap +mengapa-apai +mengapi-apikan +mengarah-arahi +mengarang-ngarang +mengata-ngatai +mengatup-ngatupkan +mengaum-aum +mengaum-aumkan +mengejan-ejan +mengejar-ngejar +mengejut-ngejuti +mengelai-ngelai +mengelepik-ngelepik +mengelip-ngelip +mengelu-elukan +mengelus-elus +mengembut-embut +mengempas-empaskan +mengenap-enapkan +mengendap-endap +mengenjak-enjak +mengentak-entak +mengentak-entakkan +mengepak-ngepak +mengepak-ngepakkan +mengepal-ngepalkan +mengerjap-ngerjap +mengerling-ngerling +mengertak-ngertakkan +mengesot-esot +menggaba-gabai +menggali-gali +menggalur-galur +menggamak-gamak +menggamit-gamitkan +menggapai-gapai +menggapai-gapaikan +menggaruk-garuk +menggebu-gebu +menggebyah-uyah +menggeleng-gelengkan +menggelepar-gelepar +menggelepar-geleparkan +menggeliang-geliutkan +menggelinding-gelinding +menggemak-gemak +menggembar-gemborkan +menggerak-gerakkan +menggerecak-gerecak +menggesa-gesakan +menggili-gili +menggodot-godot +menggolak-galikkan +menggorek-gorek +menggoreng-goreng +menggosok-gosok +menggoyang-goyangkan +mengguit-guit +menghalai-balaikan +menghalang-halangi +menghambur-hamburkan +menghinap-hinap +menghitam-memutihkan +menghitung-hitung +menghubung-hubungkan +menghujan-hujankan +mengiang-ngiang +mengibar-ngibarkan +mengibas-ngibas +mengibas-ngibaskan +mengidam-idamkan +mengilah-ngilahkan +mengilai-ilai +mengilat-ngilatkan +mengilik-ngilik +mengimak-imak +mengimbak-imbak +mengiming-iming +mengincrit-incrit +mengingat-ingat +menginjak-injak +mengipas-ngipas +mengira-ngira +mengira-ngirakan +mengiras-iras +mengiras-irasi +mengiris-iris +mengitar-ngitar +mengitik-ngitik +mengodol-odol +mengogok-ogok +mengolak-alik +mengolak-alikkan +mengolang-aling +mengolang-alingkan +mengoleng-oleng +mengolok-olok +mengombang-ambing +mengombang-ambingkan +mengongkang-ongkang +mengongkok-ongkok +mengonyah-anyih +mengopak-apik +mengorak-arik +mengorat-oret +mengorek-ngorek +mengoret-oret +mengorok-orok +mengotak-atik +mengotak-ngatikkan +mengotak-ngotakkan +mengoyak-ngoyak +mengoyak-ngoyakkan +mengoyak-oyak +menguar-nguarkan +menguar-uarkan +mengubah-ubah +mengubek-ubek +menguber-uber +mengubit-ubit +mengubrak-abrik +mengucar-ngacirkan +mengucek-ngucek +mengucek-ucek +menguik-uik +menguis-uis +mengulang-ulang +mengulas-ulas +mengulit-ulit +mengulum-ngulum +mengulur-ulur +menguman-uman +mengumbang-ambingkan +mengumpak-umpak +mengungkat-ungkat +mengungkit-ungkit +mengupa-upa +mengurik-urik +mengusil-usil +mengusil-usilkan +mengutak-atik +mengutak-ngatikkan +mengutik-ngutik +mengutik-utik +menika-nika +menimang-nimang +menimbang-nimbang +menimbun-nimbun +menimpang-nimpangkan +meningkat-ningkat +meniru-niru +menit-menit +menitar-nitarkan +meniup-niup +menjadi-jadi +menjadi-jadikan +menjedot-jedotkan +menjelek-jelekkan +menjengek-jengek +menjengit-jengit +menjerit-jerit +menjilat-jilat +menjungkat-jungkit +menko-menko +menlu-menlu +menonjol-nonjolkan +mentah-mentah +mentang-mentang +menteri-menteri +mentul-mentul +menuding-nuding +menumpah-numpahkan +menunda-nunda +menunduk-nunduk +menusuk-nusuk +menyala-nyala +menyama-nyama +menyama-nyamai +menyambar-nyambar +menyangkut-nyangkutkan +menyanjung-nyanjung +menyanjung-nyanjungkan +menyapu-nyapu +menyarat-nyarat +menyayat-nyayat +menyedang-nyedang +menyedang-nyedangkan +menyelang-nyelangkan +menyelang-nyeling +menyelang-nyelingkan +menyenak-nyenak +menyendi-nyendi +menyentak-nyentak +menyentuh-nyentuh +menyepak-nyepakkan +menyerak-nyerakkan +menyeret-nyeret +menyeru-nyerukan +menyetel-nyetel +menyia-nyiakan +menyibak-nyibak +menyobek-nyobek +menyorong-nyorongkan +menyungguh-nyungguhi +menyuruk-nyuruk +meraba-raba +merah-hitam +merah-merah +merambang-rambang +merangkak-rangkak +merasa-rasai +merata-ratakan +meraung-raung +meraung-raungkan +merayau-rayau +merayu-rayu +mercak-mercik +mercedes-benz +merek-merek +mereka-mereka +mereka-reka +merelap-relap +merem-merem +meremah-remah +meremas-remas +meremeh-temehkan +merempah-rempah +merempah-rempahi +merengek-rengek +merengeng-rengeng +merenik-renik +merenta-renta +merenyai-renyai +meresek-resek +merintang-rintang +merintik-rintik +merobek-robek +meronta-ronta +meruap-ruap +merubu-rubu +merungus-rungus +merungut-rungut +meta-analysis +metode-metode +mewanti-wanti +mewarna-warnikan +meyakin-yakini +mid-range +mid-size +miju-miju +mikro-kecil +mimpi-mimpi +minggu-minggu +minta-minta +minuman-minuman +mixed-use +mobil-mobil +mobile-first +mobile-friendly +moga-moga +mola-mola +momen-momen +mondar-mandir +monyet-monyet +morak-marik +morat-marit +move-on +muda-muda +muda-mudi +muda/i +mudah-mudahan +muka-muka +mula-mula +multiple-output +muluk-muluk +mulut-mulutan +mumi-mumi +mundur-mundur +muntah-muntah +murid-muridnya +musda-musda +museum-museum +muslim-muslimah +musuh-musuh +musuh-musuhnya +nabi-nabi +nada-nadanya +naga-naga +naga-naganya +naik-naik +naik-turun +nakal-nakalan +nama-nama +nanti-nantian +nanya-nanya +nasi-nasi +nasib-nasiban +near-field +negara-negara +negera-negara +negeri-negeri +negeri-red +neka-neka +nekat-nekat +neko-neko +nenek-nenek +neo-liberalisme +next-gen +next-generation +ngeang-ngeang +ngeri-ngeri +nggak-nggak +ngobrol-ngobrol +ngumpul-ngumpul +nilai-nilai +nine-dash +nipa-nipa +nong-nong +norma-norma +novel-novel +nyai-nyai +nyolong-nyolong +nyut-nyutan +ob-gyn +obat-obat +obat-obatan +objek-objek +obok-obok +obrak-abrik +octa-core +odong-odong +oedipus-kompleks +off-road +ogah-agih +ogah-ogah +ogah-ogahan +ogak-agik +ogak-ogak +ogoh-ogoh +olak-alik +olak-olak +olang-aling +olang-alingan +ole-ole +oleh-oleh +olok-olok +olok-olokan +olong-olong +om-om +ombang-ambing +omni-channel +on-board +on-demand +on-fire +on-line +on-off +on-premises +on-roll +on-screen +on-the-go +onde-onde +ondel-ondel +ondos-ondos +one-click +one-to-one +one-touch +one-two +oneng-oneng +ongkang-ongkang +ongol-ongol +online-to-offline +ontran-ontran +onyah-anyih +onyak-anyik +opak-apik +opsi-opsi +opt-in +orak-arik +orang-aring +orang-orang +orang-orangan +orat-oret +organisasi-organisasi +ormas-ormas +orok-orok +orong-orong +oseng-oseng +otak-atik +otak-otak +otak-otakan +over-heating +over-the-air +over-the-top +pa-pa +pabrik-pabrik +padi-padian +pagi-pagi +pagi-sore +pajak-pajak +paket-paket +palas-palas +palato-alveolar +paling-paling +palu-arit +palu-memalu +panas-dingin +panas-panas +pandai-pandai +pandang-memandang +panel-panel +pangeran-pangeran +panggung-panggung +pangkalan-pangkalan +panja-panja +panji-panji +pansus-pansus +pantai-pantai +pao-pao +para-para +parang-parang +parpol-parpol +partai-partai +paru-paru +pas-pasan +pasal-pasal +pasang-memasang +pasang-surut +pasar-pasar +pasu-pasu +paus-paus +paut-memaut +pay-per-click +paya-paya +pdi-p +pecah-pecah +pecat-pecatan +peer-to-peer +pejabat-pejabat +pekak-pekak +pekik-pekuk +pelabuhan-pelabuhan +pelacur-pelacur +pelajar-pelajar +pelan-pelan +pelangi-pelangi +pem-bully +pemain-pemain +pemata-mataan +pemda-pemda +pemeluk-pemeluknya +pemerintah-pemerintah +pemerintah-red +pemerintah-swasta +pemetang-metangan +pemilu-pemilu +pemimpin-pemimpin +peminta-minta +pemuda-pemuda +pemuda-pemudi +penanggung-jawab +pengali-ali +pengaturan-pengaturan +penggembar-gemboran +pengorak-arik +pengotak-ngotakan +pengundang-undang +pengusaha-pengusaha +pentung-pentungan +penyakit-penyakit +perak-perak +perang-perangan +peras-perus +peraturan-peraturan +perda-perda +perempat-final +perempuan-perempuan +pergi-pergi +pergi-pulang +perintang-rintang +perkereta-apian +perlahan-lahan +perlip-perlipan +permen-permen +pernak-pernik +pernik-pernik +pertama-tama +pertandingan-pertandingan +pertimbangan-pertimbangan +perudang-undangan +perundang-undangan +perundangan-undangan +perusahaan-perusahaan +perusahaan-perusahan +perwakilan-perwakilan +pesan-pesan +pesawat-pesawat +peta-jalan +petang-petang +petantang-petenteng +petatang-peteteng +pete-pete +piala-piala +piat-piut +pick-up +picture-in-picture +pihak-pihak +pijak-pijak +pijar-pijar +pijat-pijat +pikir-pikir +pil-pil +pilah-pilih +pilih-pilih +pilihan-pilihan +pilin-memilin +pilkada-pilkada +pina-pina +pindah-pindah +ping-pong +pinjam-meminjam +pintar-pintarlah +pisang-pisang +pistol-pistolan +piting-memiting +planet-planet +play-off +plin-plan +plintat-plintut +plonga-plongo +plug-in +plus-minus +plus-plus +poco-poco +pohon-pohonan +poin-poin +point-of-sale +point-of-sales +pokemon-pokemon +pokja-pokja +pokok-pokok +pokrol-pokrolan +polang-paling +polda-polda +poleng-poleng +polong-polongan +polres-polres +polsek-polsek +polwan-polwan +poma-poma +pondok-pondok +ponpes-ponpes +pontang-panting +pop-up +porak-parik +porak-peranda +porak-poranda +pos-pos +posko-posko +potong-memotong +praktek-praktek +praktik-praktik +produk-produk +program-program +promosi-degradasi +provinsi-provinsi +proyek-proyek +puing-puing +puisi-puisi +puji-pujian +pukang-pukang +pukul-memukul +pulang-pergi +pulau-pulai +pulau-pulau +pull-up +pulut-pulut +pundi-pundi +pungak-pinguk +punggung-memunggung +pura-pura +puruk-parak +pusar-pusar +pusat-pusat +push-to-talk +push-up +push-ups +pusing-pusing +puskesmas-puskesmas +putar-putar +putera-puteri +putih-hitam +putih-putih +putra-putra +putra-putri +putra/i +putri-putri +putus-putus +putusan-putusan +puvi-puvi +quad-core +raba-rabaan +raba-rubu +rada-rada +radio-frequency +ragu-ragu +rahasia-rahasiaan +raja-raja +rama-rama +ramai-ramai +ramalan-ramalan +rambeh-rambeh +rambu-rambu +rame-rame +ramu-ramuan +randa-rondo +rangkul-merangkul +rango-rango +rap-rap +rasa-rasanya +rata-rata +raun-raun +read-only +real-life +real-time +rebah-rebah +rebah-rebahan +rebas-rebas +red-eye +redam-redam +redep-redup +rehab-rekon +reja-reja +reka-reka +reka-rekaan +rekan-rekan +rekan-rekannya +rekor-rekor +relief-relief +remah-remah +remang-remang +rembah-rembah +rembah-rembih +remeh-cemeh +remeh-temeh +rempah-rempah +rencana-rencana +renyai-renyai +rep-repan +repot-repot +repuh-repuh +restoran-restoran +retak-retak +riang-riang +ribu-ribu +ribut-ribut +rica-rica +ride-sharing +rigi-rigi +rinai-rinai +rintik-rintik +ritual-ritual +robak-rabik +robat-rabit +robot-robot +role-play +role-playing +roll-on +rombang-rambing +romol-romol +rompang-romping +rondah-rondih +ropak-rapik +royal-royalan +royo-royo +ruak-ruak +ruba-ruba +rudal-rudal +ruji-ruji +ruku-ruku +rumah-rumah +rumah-rumahan +rumbai-rumbai +rumput-rumputan +runding-merunding +rundu-rundu +runggu-rangga +runner-up +runtang-runtung +rupa-rupa +rupa-rupanya +rusun-rusun +rute-rute +saat-saat +saban-saban +sabu-sabu +sabung-menyabung +sah-sah +sahabat-sahabat +saham-saham +sahut-menyahut +saing-menyaing +saji-sajian +sakit-sakitan +saksi-saksi +saku-saku +salah-salah +sama-sama +samar-samar +sambar-menyambar +sambung-bersambung +sambung-menyambung +sambut-menyambut +samo-samo +sampah-sampah +sampai-sampai +samping-menyamping +sana-sini +sandar-menyandar +sandi-sandi +sangat-sangat +sangkut-menyangkut +sapa-menyapa +sapai-sapai +sapi-sapi +sapu-sapu +saran-saran +sarana-prasarana +sari-sari +sarit-sarit +satu-dua +satu-satu +satu-satunya +satuan-satuan +saudara-saudara +sauk-menyauk +sauk-sauk +sayang-sayang +sayap-sayap +sayup-menyayup +sayup-sayup +sayur-mayur +sayur-sayuran +sci-fi +seagak-agak +seakal-akal +seakan-akan +sealak-alak +seari-arian +sebaik-baiknya +sebelah-menyebelah +sebentar-sebentar +seberang-menyeberang +seberuntung-beruntungnya +sebesar-besarnya +seboleh-bolehnya +sedalam-dalamnya +sedam-sedam +sedang-menyedang +sedang-sedang +sedap-sedapan +sedapat-dapatnya +sedikit-dikitnya +sedikit-sedikit +sedikit-sedikitnya +sedini-dininya +seelok-eloknya +segala-galanya +segan-menyegan +segan-menyegani +segan-segan +sehabis-habisnya +sehari-hari +sehari-harian +sehari-harinya +sejadi-jadinya +sekali-kali +sekali-sekali +sekenyang-kenyangnya +sekira-kira +sekolah-sekolah +sekonyong-konyong +sekosong-kosongnya +sektor-sektor +sekuasa-kuasanya +sekuat-kuatnya +sekurang-kurangnya +sel-sel +sela-menyela +sela-sela +selak-seluk +selama-lamanya +selambat-lambatnya +selang-seli +selang-seling +selar-belar +selat-latnya +selatan-tenggara +selekas-lekasnya +selentang-selenting +selepas-lepas +self-driving +self-esteem +self-healing +self-help +selir-menyelir +seloyong-seloyong +seluk-beluk +seluk-semeluk +sema-sema +semah-semah +semak-semak +semaksimal-maksimalnya +semalam-malaman +semang-semang +semanis-manisnya +semasa-masa +semata-mata +semau-maunya +sembunyi-sembunyi +sembunyi-sembunyian +sembur-sembur +semena-mena +semenda-menyemenda +semengga-mengga +semenggah-menggah +sementang-mentang +semerdeka-merdekanya +semi-final +semi-permanen +sempat-sempatnya +semu-semu +semua-muanya +semujur-mujurnya +semut-semutan +sen-senan +sendiri-sendiri +sengal-sengal +sengar-sengir +sengau-sengauan +senggak-sengguk +senggang-tenggang +senggol-menyenggol +senior-junior +senjata-senjata +senyum-senyum +seolah-olah +sepala-pala +sepandai-pandai +sepetang-petangan +sepoi-sepoi +sepraktis-praktisnya +sepuas-puasnya +serak-serak +serak-serik +serang-menyerang +serang-serangan +serangan-serangan +seraya-menyeraya +serba-serbi +serbah-serbih +serembah-serembih +serigala-serigala +sering-sering +serobot-serobotan +serong-menyerong +serta-menyertai +serta-merta +serta-serta +seru-seruan +service-oriented +sesak-menyesak +sesal-menyesali +sesayup-sayup +sesi-sesi +sesuang-suang +sesudah-sudah +sesudah-sudahnya +sesuka-suka +sesuka-sukanya +set-piece +setempat-setempat +setengah-setengah +setidak-tidaknya +setinggi-tingginya +seupaya-upaya +seupaya-upayanya +sewa-menyewa +sewaktu-waktu +sewenang-wenang +sewot-sewotan +shabu-shabu +short-term +short-throw +sia-sia +siang-siang +siap-siap +siapa-siapa +sibar-sibar +sibur-sibur +sida-sida +side-by-side +sign-in +siku-siku +sikut-sikutan +silah-silah +silang-menyilang +silir-semilir +simbol-simbol +simpan-pinjam +sinar-menyinar +sinar-seminar +sinar-suminar +sindir-menyindir +singa-singa +singgah-menyinggah +single-core +sipil-militer +sir-siran +sirat-sirat +sisa-sisa +sisi-sisi +siswa-siswa +siswa-siswi +siswa/i +siswi-siswi +situ-situ +situs-situs +six-core +six-speed +slintat-slintut +slo-mo +slow-motion +snap-on +sobek-sobekan +sodok-sodokan +sok-sokan +solek-menyolek +solid-state +sorak-sorai +sorak-sorak +sore-sore +sosio-ekonomi +soya-soya +spill-resistant +split-screen +sponsor-sponsor +sponsor-sponsoran +srikandi-srikandi +staf-staf +stand-by +stand-up +start-up +stasiun-stasiun +state-owned +striker-striker +studi-studi +suam-suam +suami-isteri +suami-istri +suami-suami +suang-suang +suara-suara +sudin-sudin +sudu-sudu +sudung-sudung +sugi-sugi +suka-suka +suku-suku +sulang-menyulang +sulat-sulit +sulur-suluran +sum-sum +sumber-sumber +sumpah-sumpah +sumpit-sumpit +sundut-bersundut +sungai-sungai +sungguh-sungguh +sungut-sungut +sunting-menyunting +super-damai +super-rahasia +super-sub +supply-demand +supply-side +suram-suram +surat-menyurat +surat-surat +suruh-suruhan +suruk-surukan +susul-menyusul +suwir-suwir +syarat-syarat +system-on-chip +t-shirt +t-shirts +tabar-tabar +tabir-mabir +tabrak-tubruk +tabuh-tabuhan +tabun-menabun +tahu-menahu +tahu-tahu +tahun-tahun +takah-takahnya +takang-takik +take-off +takut-takut +takut-takutan +tali-bertali +tali-tali +talun-temalun +taman-taman +tampak-tampak +tanak-tanakan +tanam-menanam +tanam-tanaman +tanda-tanda +tangan-menangan +tangan-tangan +tangga-tangga +tanggal-tanggal +tanggul-tanggul +tanggung-menanggung +tanggung-tanggung +tank-tank +tante-tante +tanya-jawab +tapa-tapa +tapak-tapak +tari-menari +tari-tarian +tarik-menarik +tarik-ulur +tata-tertib +tatah-tatah +tau-tau +tawa-tawa +tawak-tawak +tawang-tawang +tawar-menawar +tawar-tawar +tayum-temayum +tebak-tebakan +tebu-tebu +tedong-tedong +tegak-tegak +tegerbang-gerbang +teh-tehan +tek-tek +teka-teki +teknik-teknik +teman-teman +teman-temanku +temas-temas +tembak-menembak +temeh-temeh +tempa-menempa +tempat-tempat +tempo-tempo +temut-temut +tenang-tenang +tengah-tengah +tenggang-menenggang +tengok-menengok +teori-teori +teraba-raba +teralang-alang +terambang-ambang +terambung-ambung +terang-terang +terang-terangan +teranggar-anggar +terangguk-angguk +teranggul-anggul +terangin-angin +terangkup-angkup +teranja-anja +terapung-apung +terayan-rayan +terayap-rayap +terbada-bada +terbahak-bahak +terbang-terbang +terbata-bata +terbatuk-batuk +terbayang-bayang +terbeda-bedakan +terbengkil-bengkil +terbengong-bengong +terbirit-birit +terbuai-buai +terbuang-buang +terbungkuk-bungkuk +terburu-buru +tercangak-cangak +tercengang-cengang +tercilap-cilap +tercongget-congget +tercoreng-moreng +tercungap-cungap +terdangka-dangka +terdengih-dengih +terduga-duga +terekeh-ekeh +terembut-embut +terembut-rembut +terempas-empas +terengah-engah +teresak-esak +tergagap-gagap +tergagau-gagau +tergaguk-gaguk +tergapai-gapai +tergegap-gegap +tergegas-gegas +tergelak-gelak +tergelang-gelang +tergeleng-geleng +tergelung-gelung +tergerai-gerai +tergerenyeng-gerenyeng +tergesa-gesa +tergila-gila +tergolek-golek +tergontai-gontai +tergudik-gudik +tergugu-gugu +terguling-guling +tergulut-gulut +terhambat-hambat +terharak-harak +terharap-harap +terhengit-hengit +terheran-heran +terhinggut-hinggut +terigau-igau +terimpi-impi +terincut-incut +teringa-inga +teringat-ingat +terinjak-injak +terisak-isak +terjembak-jembak +terjerit-jerit +terkadang-kadang +terkagum-kagum +terkaing-kaing +terkakah-kakah +terkakak-kakak +terkampul-kampul +terkanjar-kanjar +terkantuk-kantuk +terkapah-kapah +terkapai-kapai +terkapung-kapung +terkatah-katah +terkatung-katung +terkecap-kecap +terkedek-kedek +terkedip-kedip +terkejar-kejar +terkekau-kekau +terkekeh-kekeh +terkekek-kekek +terkelinjat-kelinjat +terkelip-kelip +terkempul-kempul +terkemut-kemut +terkencar-kencar +terkencing-kencing +terkentut-kentut +terkepak-kepak +terkesot-kesot +terkesut-kesut +terkial-kial +terkijai-kijai +terkikih-kikih +terkikik-kikik +terkincak-kincak +terkindap-kindap +terkinja-kinja +terkirai-kirai +terkitar-kitar +terkocoh-kocoh +terkojol-kojol +terkokol-kokol +terkosel-kosel +terkotak-kotak +terkoteng-koteng +terkuai-kuai +terkumpal-kumpal +terlara-lara +terlayang-layang +terlebih-lebih +terlincah-lincah +terliuk-liuk +terlolong-lolong +terlongong-longong +terlunta-lunta +termangu-mangu +termanja-manja +termata-mata +termengah-mengah +termenung-menung +termimpi-mimpi +termonyong-monyong +ternanti-nanti +terngiang-ngiang +teroleng-oleng +terombang-ambing +terpalit-palit +terpandang-pandang +terpecah-pecah +terpekik-pekik +terpencar-pencar +terpereh-pereh +terpijak-pijak +terpikau-pikau +terpilah-pilah +terpinga-pinga +terpingkal-pingkal +terpingkau-pingkau +terpontang-panting +terpusing-pusing +terputus-putus +tersanga-sanga +tersaruk-saruk +tersedan-sedan +tersedih-sedih +tersedu-sedu +terseduh-seduh +tersendat-sendat +tersendeng-sendeng +tersengal-sengal +tersengguk-sengguk +tersengut-sengut +terseok-seok +tersera-sera +terserak-serak +tersetai-setai +tersia-sia +tersipu-sipu +tersoja-soja +tersungkuk-sungkuk +tersuruk-suruk +tertagak-tagak +tertahan-tahan +tertatih-tatih +tertegun-tegun +tertekan-tekan +terteleng-teleng +tertendang-tendang +tertimpang-timpang +tertitar-titar +terumbang-ambing +terumbang-umbang +terungkap-ungkap +terus-menerus +terus-terusan +tete-a-tete +text-to-speech +think-tank +think-thank +third-party +third-person +three-axis +three-point +tiap-tiap +tiba-tiba +tidak-tidak +tidur-tidur +tidur-tiduran +tie-dye +tie-in +tiga-tiganya +tikam-menikam +tiki-taka +tikus-tikus +tilik-menilik +tim-tim +timah-timah +timang-timangan +timbang-menimbang +time-lapse +timpa-menimpa +timu-timu +timun-timunan +timur-barat +timur-laut +timur-tenggara +tindih-bertindih +tindih-menindih +tinjau-meninjau +tinju-meninju +tip-off +tipu-tipu +tiru-tiruan +titik-titik +titik-titiknya +tiup-tiup +to-do +tokak-takik +toko-toko +tokoh-tokoh +tokok-menokok +tolak-menolak +tolong-menolong +tong-tong +top-level +top-up +totol-totol +touch-screen +trade-in +training-camp +trans-nasional +treble-winner +tri-band +trik-trik +triple-core +truk-truk +tua-tua +tuan-tuan +tuang-tuang +tuban-tuban +tubuh-tubuh +tujuan-tujuan +tuk-tuk +tukang-menukang +tukar-menukar +tulang-belulang +tulang-tulangan +tuli-tuli +tulis-menulis +tumbuh-tumbuhan +tumpang-tindih +tune-up +tunggang-tunggik +tunggang-tungging +tunggang-tunggit +tunggul-tunggul +tunjuk-menunjuk +tupai-tupai +tupai-tupaian +turi-turian +turn-based +turnamen-turnamen +turun-temurun +turut-menurut +turut-turutan +tuyuk-tuyuk +twin-cam +twin-turbocharged +two-state +two-step +two-tone +u-shape +uang-uangan +uar-uar +ubek-ubekan +ubel-ubel +ubrak-abrik +ubun-ubun +ubur-ubur +uci-uci +udang-undang +udap-udapan +ugal-ugalan +uget-uget +uir-uir +ujar-ujar +uji-coba +ujung-ujung +ujung-ujungnya +uka-uka +ukir-mengukir +ukir-ukiran +ula-ula +ulak-ulak +ulam-ulam +ulang-alik +ulang-aling +ulang-ulang +ulap-ulap +ular-ular +ular-ularan +ulek-ulek +ulu-ulu +ulung-ulung +umang-umang +umbang-ambing +umbi-umbian +umbul-umbul +umbut-umbut +uncang-uncit +undak-undakan +undang-undang +undang-undangnya +unduk-unduk +undung-undung +undur-undur +unek-unek +ungah-angih +unggang-anggit +unggat-unggit +unggul-mengungguli +ungkit-ungkit +unit-unit +universitas-universitas +unsur-unsur +untang-anting +unting-unting +untung-untung +untung-untungan +upah-mengupah +upih-upih +upside-down +ura-ura +uran-uran +urat-urat +uring-uringan +urup-urup +urup-urupan +urus-urus +usaha-usaha +user-user +user-useran +utak-atik +utang-piutang +utang-utang +utar-utar +utara-jauh +utara-selatan +uter-uter +utusan-utusan +v-belt +v-neck +value-added +very-very +video-video +visi-misi +visi-misinya +voa-islam +voice-over +volt-ampere +wajah-wajah +wajar-wajar +wake-up +wakil-wakil +walk-in +walk-out +wangi-wangian +wanita-wanita +wanti-wanti +wara-wara +wara-wiri +warna-warna +warna-warni +was-was +water-cooled +web-based +wide-angle +wilayah-wilayah +win-win +wira-wiri +wora-wari +work-life +world-class +yang-yang +yayasan-yayasan +year-on-year +yel-yel +yo-yo +zam-zam +zig-zag +""".split() +) diff --git a/spacy/lang/id/examples.py b/spacy/lang/id/examples.py new file mode 100644 index 0000000..17d1c5f --- /dev/null +++ b/spacy/lang/id/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.id.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Indonesia merupakan negara kepulauan yang kaya akan budaya.", + "Berapa banyak warga yang dibutuhkan saat kerja bakti?", + "Penyaluran pupuk berasal dari lima lokasi yakni Bontang, Kalimantan Timur, Surabaya, Banyuwangi, Semarang, dan Makassar.", + "PT Pupuk Kaltim telah menyalurkan 274.707 ton pupuk bersubsidi ke wilayah penyaluran di 14 provinsi.", + "Jakarta adalah kota besar yang nyaris tidak pernah tidur." + "Kamu ada di mana semalam?", + "Siapa yang membeli makanan ringan tersebut?", + "Siapa presiden pertama Republik Indonesia?", +] diff --git a/spacy/lang/id/lex_attrs.py b/spacy/lang/id/lex_attrs.py new file mode 100644 index 0000000..5952c4d --- /dev/null +++ b/spacy/lang/id/lex_attrs.py @@ -0,0 +1,66 @@ +import unicodedata + +from ...attrs import IS_CURRENCY, LIKE_NUM +from .punctuation import LIST_CURRENCY + +_num_words = [ + "nol", + "satu", + "dua", + "tiga", + "empat", + "lima", + "enam", + "tujuh", + "delapan", + "sembilan", + "sepuluh", + "sebelas", + "belas", + "puluh", + "ratus", + "ribu", + "juta", + "miliar", + "biliun", + "triliun", + "kuadriliun", + "kuintiliun", + "sekstiliun", + "septiliun", + "oktiliun", + "noniliun", + "desiliun", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.count("-") == 1: + _, num = text.split("-") + if num.isdigit() or num in _num_words: + return True + return False + + +def is_currency(text): + if text in LIST_CURRENCY: + return True + + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +LEX_ATTRS = {IS_CURRENCY: is_currency, LIKE_NUM: like_num} diff --git a/spacy/lang/id/punctuation.py b/spacy/lang/id/punctuation.py new file mode 100644 index 0000000..8303b8e --- /dev/null +++ b/spacy/lang/id/punctuation.py @@ -0,0 +1,60 @@ +from ..char_classes import ALPHA, _currency, _units, merge_chars, split_chars +from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES + +_units = ( + _units + "s bit Gbps Mbps mbps Kbps kbps ƒ ppi px " + "Hz kHz MHz GHz mAh " + "ratus rb ribu ribuan " + "juta jt jutaan mill?iar million bil[l]?iun bilyun billion " +) +_currency = _currency + r" USD Rp IDR RMB SGD S\$" +_months = ( + "Januari Februari Maret April Mei Juni Juli Agustus September " + "Oktober November Desember January February March May June " + "July August October December Jan Feb Mar Jun Jul Aug Sept " + "Oct Okt Nov Des " +) + + +UNITS = merge_chars(_units) +CURRENCY = merge_chars(_currency) +HTML_PREFIX = r"<(b|strong|i|em|p|span|div|br)\s?/>|]+)>" +HTML_SUFFIX = r"" +MONTHS = merge_chars(_months) +LIST_CURRENCY = split_chars(_currency) + +_prefixes = list(TOKENIZER_PREFIXES) +_prefixes.remove("#") # hashtag +_prefixes = _prefixes + LIST_CURRENCY + [HTML_PREFIX] + ["/", "—"] + +_suffixes = ( + TOKENIZER_SUFFIXES + + [r"\-[Nn]ya", "-[KkMm]u", "[—-]"] + + [ + # disabled: variable width currency variable + # r"(?<={c})(?:[0-9]+)".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9])%", + # disabled: variable width HTML_SUFFIX variable + # r"(?<=[0-9{a}]{h})(?:[\.,:-])".format(a=ALPHA, h=HTML_SUFFIX), + r"(?<=[0-9{a}])(?:{h})".format(a=ALPHA, h=HTML_SUFFIX), + ] +) + +_infixes = TOKENIZER_INFIXES + [ + r"(?<=[0-9])[\\/](?=[0-9%-])", + r"(?<=[0-9])%(?=[{a}0-9/])".format(a=ALPHA), + # disabled: variable width units variable + # r"(?<={u})[\/-](?=[0-9])".format(u=UNITS), + # disabled: variable width months variable + # r"(?<={m})[\/-](?=[0-9])".format(m=MONTHS), + r'(?<=[0-9)][.,])"(?=[0-9])', + r'(?<=[{a})][.,\'])["—](?=[{a}])'.format(a=ALPHA), + r"(?<=[{a}])-(?=[0-9])".format(a=ALPHA), + r"(?<=[0-9])-(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[\/-](?={c}|[{a}])".format(a=ALPHA, c=CURRENCY), +] + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/id/stop_words.py b/spacy/lang/id/stop_words.py new file mode 100644 index 0000000..b1bfaea --- /dev/null +++ b/spacy/lang/id/stop_words.py @@ -0,0 +1,118 @@ +STOP_WORDS = set( + """ +ada adalah adanya adapun agak agaknya agar akan akankah akhir akhiri akhirnya +aku akulah amat amatlah anda andalah antar antara antaranya apa apaan apabila +apakah apalagi apatah artinya asal asalkan atas atau ataukah ataupun awal +awalnya + +bagai bagaikan bagaimana bagaimanakah bagaimanapun bagi bagian bahkan bahwa +bahwasanya baik bakal bakalan balik banyak bapak baru bawah beberapa begini +beginian beginikah beginilah begitu begitukah begitulah begitupun bekerja +belakang belakangan belum belumlah benar benarkah benarlah berada berakhir +berakhirlah berakhirnya berapa berapakah berapalah berapapun berarti berawal +berbagai berdatangan beri berikan berikut berikutnya berjumlah berkali-kali +berkata berkehendak berkeinginan berkenaan berlainan berlalu berlangsung +berlebihan bermacam bermacam-macam bermaksud bermula bersama bersama-sama +bersiap bersiap-siap bertanya bertanya-tanya berturut berturut-turut bertutur +berujar berupa besar betul betulkah biasa biasanya bila bilakah bisa bisakah +boleh bolehkah bolehlah buat bukan bukankah bukanlah bukannya bulan bung + +cara caranya cukup cukupkah cukuplah cuma + +dahulu dalam dan dapat dari daripada datang dekat demi demikian demikianlah +dengan depan di dia diakhiri diakhirinya dialah diantara diantaranya diberi +diberikan diberikannya dibuat dibuatnya didapat didatangkan digunakan +diibaratkan diibaratkannya diingat diingatkan diinginkan dijawab dijelaskan +dijelaskannya dikarenakan dikatakan dikatakannya dikerjakan diketahui +diketahuinya dikira dilakukan dilalui dilihat dimaksud dimaksudkan +dimaksudkannya dimaksudnya diminta dimintai dimisalkan dimulai dimulailah +dimulainya dimungkinkan dini dipastikan diperbuat diperbuatnya dipergunakan +diperkirakan diperlihatkan diperlukan diperlukannya dipersoalkan dipertanyakan +dipunyai diri dirinya disampaikan disebut disebutkan disebutkannya disini +disinilah ditambahkan ditandaskan ditanya ditanyai ditanyakan ditegaskan +ditujukan ditunjuk ditunjuki ditunjukkan ditunjukkannya ditunjuknya dituturkan +dituturkannya diucapkan diucapkannya diungkapkan dong dua dulu + +empat enggak enggaknya entah entahlah + +guna gunakan + +hal hampir hanya hanyalah hari harus haruslah harusnya hendak hendaklah +hendaknya hingga + +ia ialah ibarat ibaratkan ibaratnya ibu ikut ingat ingat-ingat ingin inginkah +inginkan ini inikah inilah itu itukah itulah + +jadi jadilah jadinya jangan jangankan janganlah jauh jawab jawaban jawabnya +jelas jelaskan jelaslah jelasnya jika jikalau juga jumlah jumlahnya justru + +kala kalau kalaulah kalaupun kalian kami kamilah kamu kamulah kan kapan +kapankah kapanpun karena karenanya kasus kata katakan katakanlah katanya ke +keadaan kebetulan kecil kedua keduanya keinginan kelamaan kelihatan +kelihatannya kelima keluar kembali kemudian kemungkinan kemungkinannya kenapa +kepada kepadanya kesampaian keseluruhan keseluruhannya keterlaluan ketika +khususnya kini kinilah kira kira-kira kiranya kita kitalah kok kurang + +lagi lagian lah lain lainnya lalu lama lamanya lanjut lanjutnya lebih lewat +lima luar + +macam maka makanya makin malah malahan mampu mampukah mana manakala manalagi +masa masalah masalahnya masih masihkah masing masing-masing mau maupun +melainkan melakukan melalui melihat melihatnya memang memastikan memberi +memberikan membuat memerlukan memihak meminta memintakan memisalkan memperbuat +mempergunakan memperkirakan memperlihatkan mempersiapkan mempersoalkan +mempertanyakan mempunyai memulai memungkinkan menaiki menambahkan menandaskan +menanti menanti-nanti menantikan menanya menanyai menanyakan mendapat +mendapatkan mendatang mendatangi mendatangkan menegaskan mengakhiri mengapa +mengatakan mengatakannya mengenai mengerjakan mengetahui menggunakan +menghendaki mengibaratkan mengibaratkannya mengingat mengingatkan menginginkan +mengira mengucapkan mengucapkannya mengungkapkan menjadi menjawab menjelaskan +menuju menunjuk menunjuki menunjukkan menunjuknya menurut menuturkan +menyampaikan menyangkut menyatakan menyebutkan menyeluruh menyiapkan merasa +mereka merekalah merupakan meski meskipun meyakini meyakinkan minta mirip +misal misalkan misalnya mula mulai mulailah mulanya mungkin mungkinkah + +nah naik namun nanti nantinya nyaris nyatanya + +oleh olehnya + +pada padahal padanya pak paling panjang pantas para pasti pastilah penting +pentingnya per percuma perlu perlukah perlunya pernah persoalan pertama +pertama-tama pertanyaan pertanyakan pihak pihaknya pukul pula pun punya + +rasa rasanya rata rupanya + +saat saatnya saja sajalah saling sama sama-sama sambil sampai sampai-sampai +sampaikan sana sangat sangatlah satu saya sayalah se sebab sebabnya sebagai +sebagaimana sebagainya sebagian sebaik sebaik-baiknya sebaiknya sebaliknya +sebanyak sebegini sebegitu sebelum sebelumnya sebenarnya seberapa sebesar +sebetulnya sebisanya sebuah sebut sebutlah sebutnya secara secukupnya sedang +sedangkan sedemikian sedikit sedikitnya seenaknya segala segalanya segera +seharusnya sehingga seingat sejak sejauh sejenak sejumlah sekadar sekadarnya +sekali sekali-kali sekalian sekaligus sekalipun sekarang sekarang sekecil +seketika sekiranya sekitar sekitarnya sekurang-kurangnya sekurangnya sela +selain selaku selalu selama selama-lamanya selamanya selanjutnya seluruh +seluruhnya semacam semakin semampu semampunya semasa semasih semata semata-mata +semaunya sementara semisal semisalnya sempat semua semuanya semula sendiri +sendirian sendirinya seolah seolah-olah seorang sepanjang sepantasnya +sepantasnyalah seperlunya seperti sepertinya sepihak sering seringnya serta +serupa sesaat sesama sesampai sesegera sesekali seseorang sesuatu sesuatunya +sesudah sesudahnya setelah setempat setengah seterusnya setiap setiba setibanya +setidak-tidaknya setidaknya setinggi seusai sewaktu siap siapa siapakah +siapapun sini sinilah soal soalnya suatu sudah sudahkah sudahlah supaya + +tadi tadinya tahu tahun tak tambah tambahnya tampak tampaknya tandas tandasnya +tanpa tanya tanyakan tanyanya tapi tegas tegasnya telah tempat tengah tentang +tentu tentulah tentunya tepat terakhir terasa terbanyak terdahulu terdapat +terdiri terhadap terhadapnya teringat teringat-ingat terjadi terjadilah +terjadinya terkira terlalu terlebih terlihat termasuk ternyata tersampaikan +tersebut tersebutlah tertentu tertuju terus terutama tetap tetapi tiap tiba +tiba-tiba tidak tidakkah tidaklah tiga tinggi toh tunjuk turut tutur tuturnya + +ucap ucapnya ujar ujarnya umum umumnya ungkap ungkapnya untuk usah usai + +waduh wah wahai waktu waktunya walau walaupun wong + +yaitu yakin yakni yang +""".split() +) diff --git a/spacy/lang/id/syntax_iterators.py b/spacy/lang/id/syntax_iterators.py new file mode 100644 index 0000000..0277986 --- /dev/null +++ b/spacy/lang/id/syntax_iterators.py @@ -0,0 +1,41 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + # fmt: off + labels = ["nsubj", "nsubj:pass", "obj", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/id/tokenizer_exceptions.py b/spacy/lang/id/tokenizer_exceptions.py new file mode 100644 index 0000000..8e20626 --- /dev/null +++ b/spacy/lang/id/tokenizer_exceptions.py @@ -0,0 +1,220 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ._tokenizer_exceptions_list import ID_BASE_EXCEPTIONS + +# Daftar singkatan dan Akronim dari: +# https://id.wiktionary.org/wiki/Wiktionary:Daftar_singkatan_dan_akronim_bahasa_Indonesia#A + +_exc = {} + +for orth in ID_BASE_EXCEPTIONS: + _exc[orth] = [{ORTH: orth}] + orth_title = orth.title() + _exc[orth_title] = [{ORTH: orth_title}] + orth_caps = orth.upper() + _exc[orth_caps] = [{ORTH: orth_caps}] + orth_lower = orth.lower() + _exc[orth_lower] = [{ORTH: orth_lower}] + orth_first_upper = orth[0].upper() + orth[1:] + _exc[orth_first_upper] = [{ORTH: orth_first_upper}] + if "-" in orth: + orth_title = "-".join([part.title() for part in orth.split("-")]) + _exc[orth_title] = [{ORTH: orth_title}] + orth_caps = "-".join([part.upper() for part in orth.split("-")]) + _exc[orth_caps] = [{ORTH: orth_caps}] + +for exc_data in [ + {ORTH: "Jan.", NORM: "Januari"}, + {ORTH: "Feb.", NORM: "Februari"}, + {ORTH: "Mar.", NORM: "Maret"}, + {ORTH: "Apr.", NORM: "April"}, + {ORTH: "Jun.", NORM: "Juni"}, + {ORTH: "Jul.", NORM: "Juli"}, + {ORTH: "Agu.", NORM: "Agustus"}, + {ORTH: "Ags.", NORM: "Agustus"}, + {ORTH: "Sep.", NORM: "September"}, + {ORTH: "Okt.", NORM: "Oktober"}, + {ORTH: "Nov.", NORM: "November"}, + {ORTH: "Des.", NORM: "Desember"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +_other_exc = { + "do'a": [{ORTH: "do'a", NORM: "doa"}], + "jum'at": [{ORTH: "jum'at", NORM: "Jumat"}], + "Jum'at": [{ORTH: "Jum'at", NORM: "Jumat"}], + "la'nat": [{ORTH: "la'nat", NORM: "laknat"}], + "ma'af": [{ORTH: "ma'af", NORM: "maaf"}], + "mu'jizat": [{ORTH: "mu'jizat", NORM: "mukjizat"}], + "Mu'jizat": [{ORTH: "Mu'jizat", NORM: "mukjizat"}], + "ni'mat": [{ORTH: "ni'mat", NORM: "nikmat"}], + "raka'at": [{ORTH: "raka'at", NORM: "rakaat"}], + "ta'at": [{ORTH: "ta'at", NORM: "taat"}], +} + +_exc.update(_other_exc) + +for orth in [ + "A.AB.", + "A.Ma.", + "A.Md.", + "A.Md.Keb.", + "A.Md.Kep.", + "A.P.", + "B.A.", + "B.Ch.E.", + "B.Sc.", + "Dr.", + "Dra.", + "Drs.", + "Hj.", + "Ka.", + "Kp.", + "M.AB", + "M.Ag.", + "M.AP", + "M.Arl", + "M.A.R.S", + "M.Hum.", + "M.I.Kom.", + "M.Kes,", + "M.Kom.", + "M.M.", + "M.P.", + "M.Pd.", + "M.Psi.", + "M.Psi.T.", + "M.Sc.", + "M.SArl", + "M.Si.", + "M.Sn.", + "M.T.", + "M.Th.", + "No.", + "Pjs.", + "Plt.", + "R.A.", + "S.AB", + "S.AP", + "S.Adm", + "S.Ag.", + "S.Agr", + "S.Ant", + "S.Arl", + "S.Ars", + "S.A.R.S", + "S.Ds", + "S.E.", + "S.E.I.", + "S.Farm", + "S.Gz.", + "S.H.", + "S.Han", + "S.H.Int", + "S.Hum", + "S.Hut.", + "S.In.", + "S.IK.", + "S.I.Kom.", + "S.I.P", + "S.IP", + "S.P.", + "S.Pt", + "S.Psi", + "S.Ptk", + "S.Keb", + "S.Ked", + "S.Kep", + "S.KG", + "S.KH", + "S.Kel", + "S.K.M.", + "S.Kedg.", + "S.Kedh.", + "S.Kom.", + "S.KPM", + "S.Mb", + "S.Mat", + "S.Par", + "S.Pd.", + "S.Pd.I.", + "S.Pd.SD", + "S.Pol.", + "S.Psi.", + "S.S.", + "S.SArl.", + "S.Sn", + "S.Si.", + "S.Si.Teol.", + "S.SI.", + "S.ST.", + "S.ST.Han", + "S.STP", + "S.Sos.", + "S.Sy.", + "S.T.", + "S.T.Han", + "S.Th.", + "S.Th.IS.TI.", + "S.T.P.", + "S.TrK", + "S.Tekp.", + "S.Th.", + "Prof.", + "drg.", + "KH.", + "Ust.", + "Lc", + "Pdt.", + "S.H.H.", + "Rm.", + "Ps.", + "St.", + "M.A.", + "M.B.A", + "M.Eng.", + "M.Eng.Sc.", + "M.Pharm.", + "Dr. med", + "Dr.-Ing", + "Dr. rer. nat.", + "Dr. phil.", + "Dr. iur.", + "Dr. rer. oec", + "Dr. rer. pol.", + "R.Ng.", + "R.", + "R.M.", + "R.B.", + "R.P.", + "R.Ay.", + "Rr.", + "R.Ngt.", + "a.l.", + "a.n.", + "a.s.", + "b.d.", + "d.a.", + "d.l.", + "d/h", + "dkk.", + "dll.", + "dr.", + "drh.", + "ds.", + "dsb.", + "dst.", + "faks.", + "fax.", + "hlm.", + "i/o", + "n.b.", + "p.p.pjs.", + "s.d.", + "tel.", + "u.p.", +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/is/__init__.py b/spacy/lang/is/__init__.py new file mode 100644 index 0000000..af12600 --- /dev/null +++ b/spacy/lang/is/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class IcelandicDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Icelandic(Language): + lang = "is" + Defaults = IcelandicDefaults + + +__all__ = ["Icelandic"] diff --git a/spacy/lang/is/stop_words.py b/spacy/lang/is/stop_words.py new file mode 100644 index 0000000..917fb6d --- /dev/null +++ b/spacy/lang/is/stop_words.py @@ -0,0 +1,158 @@ +# Source: https://github.com/Xangis/extra-stopwords + +STOP_WORDS = set( + """ +afhverju +aftan +aftur +afþví +aldrei +allir +allt +alveg +annað +annars +bara +dag +eða +eftir +eiga +einhver +einhverjir +einhvers +eins +einu +eitthvað +ekkert +ekki +ennþá +eru +fara +fer +finna +fjöldi +fólk +framan +frá +frekar +fyrir +gegnum +geta +getur +gmg +gott +hann +hafa +hef +hefur +heyra +hér +hérna +hjá +hún +hvað +hvar +hver +hverjir +hverjum +hvernig +hvor +hvort +hægt +img +inn +kannski +koma +líka +lol +maður +mátt +mér +með +mega +meira +mig +mikið +minna +minni +missa +mjög +nei +niður +núna +oft +okkar +okkur +póst +póstur +rofl +saman +sem +sér +sig +sinni +síðan +sjá +smá +smátt +spurja +spyrja +staðar +stórt +svo +svona +sælir +sæll +taka +takk +til +tilvitnun +titlar +upp +var +vel +velkomin +velkominn +vera +verður +verið +vel +við +vil +vilja +vill +vita +væri +yfir +ykkar +það +þakka +þakkir +þannig +það +þar +þarf +þau +þeim +þeir +þeirra +þeirra +þegar +þess +þessa +þessi +þessu +þessum +þetta +þér +þið +þinn +þitt +þín +þráð +þráður +því +þær +ætti +""".split() +) diff --git a/spacy/lang/it/__init__.py b/spacy/lang/it/__init__.py new file mode 100644 index 0000000..14458d8 --- /dev/null +++ b/spacy/lang/it/__init__.py @@ -0,0 +1,50 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import ItalianLemmatizer +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class ItalianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + stop_words = STOP_WORDS + syntax_iterators = SYNTAX_ITERATORS + + +class Italian(Language): + lang = "it" + Defaults = ItalianDefaults + + +@Italian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "pos_lookup", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return ItalianLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Italian"] diff --git a/spacy/lang/it/examples.py b/spacy/lang/it/examples.py new file mode 100644 index 0000000..ae85738 --- /dev/null +++ b/spacy/lang/it/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.it.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple vuole comprare una startup del Regno Unito per un miliardo di dollari", + "Le automobili a guida autonoma spostano la responsabilità assicurativa verso i produttori", + "San Francisco prevede di bandire i robot di consegna porta a porta", + "Londra è una grande città del Regno Unito.", +] diff --git a/spacy/lang/it/lemmatizer.py b/spacy/lang/it/lemmatizer.py new file mode 100644 index 0000000..bf86916 --- /dev/null +++ b/spacy/lang/it/lemmatizer.py @@ -0,0 +1,132 @@ +from typing import Dict, List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class ItalianLemmatizer(Lemmatizer): + """This lemmatizer was adapted from the Polish one (version of April 2021). + It implements lookup lemmatization based on the morphological lexicon + morph-it (Baroni and Zanchetta). The table lemma_lookup with non-POS-aware + entries is used as a backup for words that aren't handled by morph-it.""" + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "pos_lookup": + required = [ + "lemma_lookup_num", + "lemma_lookup_det", + "lemma_lookup_adp", + "lemma_lookup_adj", + "lemma_lookup_noun", + "lemma_lookup_pron", + "lemma_lookup_verb", + "lemma_lookup_aux", + "lemma_lookup_adv", + "lemma_lookup_other", + "lemma_lookup", + ] + return (required, []) + else: + return super().get_lookups_config(mode) + + def pos_lookup_lemmatize(self, token: Token) -> List[str]: + string = token.text + univ_pos = token.pos_ + morphology = token.morph.to_dict() + lookup_pos = univ_pos.lower() + if univ_pos == "PROPN": + lookup_pos = "noun" + elif univ_pos == "PART": + lookup_pos = "pron" + lookup_table = self.lookups.get_table("lemma_lookup_" + lookup_pos, {}) + if univ_pos == "NOUN": + return self.lemmatize_noun(string, morphology, lookup_table) + else: + if univ_pos != "PROPN": + string = string.lower() + if univ_pos == "DET": + return self.lemmatize_det(string, morphology, lookup_table) + elif univ_pos == "PRON": + return self.lemmatize_pron(string, morphology, lookup_table) + elif univ_pos == "ADP": + return self.lemmatize_adp(string, morphology, lookup_table) + elif univ_pos == "ADJ": + return self.lemmatize_adj(string, morphology, lookup_table) + else: + lemma = lookup_table.get(string, "") + if not lemma: + lookup_table = self.lookups.get_table("lemma_lookup_other") + lemma = lookup_table.get(string, "") + if not lemma: + lookup_table = self.lookups.get_table( + "lemma_lookup" + ) # "legacy" lookup table + lemma = lookup_table.get(string, string.lower()) + return [lemma] + + def lemmatize_det( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + if string in [ + "l'", + "lo", + "la", + "i", + "gli", + "le", + ]: + return ["il"] + if string in ["un'", "un", "una"]: + return ["uno"] + return [lookup_table.get(string, string)] + + def lemmatize_pron( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + if string in [ + "l'", + "li", + "la", + "gli", + "le", + ]: + return ["lo"] + if string in ["un'", "un", "una"]: + return ["uno"] + lemma = lookup_table.get(string, string) + if lemma == "alcun": + lemma = "alcuno" + elif lemma == "qualcun": + lemma = "qualcuno" + return [lemma] + + def lemmatize_adp( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + if string == "d'": + return ["di"] + return [lookup_table.get(string, string)] + + def lemmatize_adj( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + lemma = lookup_table.get(string, string) + if lemma == "alcun": + lemma = "alcuno" + elif lemma == "qualcun": + lemma = "qualcuno" + return [lemma] + + def lemmatize_noun( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + # this method is case-sensitive, in order to work + # for incorrectly tagged proper names + if string != string.lower(): + if string.lower() in lookup_table: + return [lookup_table[string.lower()]] + elif string in lookup_table: + return [lookup_table[string]] + return [string.lower()] + return [lookup_table.get(string, string)] diff --git a/spacy/lang/it/punctuation.py b/spacy/lang/it/punctuation.py new file mode 100644 index 0000000..51318b2 --- /dev/null +++ b/spacy/lang/it/punctuation.py @@ -0,0 +1,34 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + +ELISION = "'’" + + +_prefixes = [r"'[0-9][0-9]", r"[0-9]+°"] + BASE_TOKENIZER_PREFIXES + + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])(?:{h})(?=[{al}])".format(a=ALPHA, h=HYPHENS, al=ALPHA_LOWER), + r"(?<=[{a}0-9])[:<>=\/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}][{el}])(?=[{a}0-9\"])".format(a=ALPHA, el=ELISION), + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/it/stop_words.py b/spacy/lang/it/stop_words.py new file mode 100644 index 0000000..42adc79 --- /dev/null +++ b/spacy/lang/it/stop_words.py @@ -0,0 +1,83 @@ +STOP_WORDS = set( + """ +a abbastanza abbia abbiamo abbiano abbiate accidenti ad adesso affinche agl +agli ahime ahimè ai al alcuna alcuni alcuno all alla alle allo allora altri +altrimenti altro altrove altrui anche ancora anni anno ansa anticipo assai +attesa attraverso avanti avemmo avendo avente aver avere averlo avesse +avessero avessi avessimo aveste avesti avete aveva avevamo avevano avevate +avevi avevo avrai avranno avrebbe avrebbero avrei avremmo avremo avreste +avresti avrete avrà avrò avuta avute avuti avuto + +basta bene benissimo brava bravo + +casa caso cento certa certe certi certo che chi chicchessia chiunque ci c' +ciascuna ciascuno cima cio cioe circa citta città co codesta codesti codesto +cogli coi col colei coll coloro colui come cominci comunque con concernente +conciliarsi conclusione consiglio contro cortesia cos cosa cosi così cui + +d' da dagl dagli dai dal dall dall' dalla dalle dallo dappertutto davanti degl degli +dei del dell dell' della delle dello dentro detto deve di dice dietro dire +dirimpetto diventa diventare diventato dopo dov dove dovra dovrà dovunque due +dunque durante + +e ebbe ebbero ebbi ecc ecco ed effettivamente egli ella entrambi eppure era +erano eravamo eravate eri ero esempio esse essendo esser essere essi ex è + +fa faccia facciamo facciano facciate faccio facemmo facendo facesse facessero +facessi facessimo faceste facesti faceva facevamo facevano facevate facevi +facevo fai fanno farai faranno fare farebbe farebbero farei faremmo faremo +fareste faresti farete farà farò fatto favore fece fecero feci fin finalmente +finche fine fino forse forza fosse fossero fossi fossimo foste fosti fra +frattempo fu fui fummo fuori furono futuro generale + +gia già giacche giorni giorno gli gl' gliela gliele glieli glielo gliene governo +grande grazie gruppo + +ha haha hai hanno ho + +ieri il improvviso in inc infatti inoltre insieme intanto intorno invece io + +l' la là lasciato lato lavoro le lei li lo lontano loro lui lungo luogo + +m' ma macche magari maggior mai male malgrado malissimo mancanza marche me +medesimo mediante meglio meno mentre mesi mezzo mi mia mie miei mila miliardi +milioni minimi ministro mio modo molti moltissimo molto momento mondo mosto + +nazionale ne negl negli nei nel nell nella nelle nello nemmeno neppure nessun nessun' +nessuna nessuno nient' niente no noi non nondimeno nonostante nonsia nostra nostre +nostri nostro novanta nove nulla nuovo + +od oggi ogni ognuna ognuno oltre oppure ora ore osi ossia ottanta otto + +paese parecchi parecchie parecchio parte partendo peccato peggio per perche +perché percio perciò perfino pero persino persone però piedi pieno piglia piu +piuttosto più po pochissimo poco poi poiche possa possedere posteriore posto +potrebbe preferibilmente presa press prima primo principalmente probabilmente +proprio puo può pure purtroppo + +qualche qualcosa qualcuna qualcuno quale quali qualunque quando quanta quante +quanti quanto quantunque quasi quattro quel quel' quella quelle quelli quello quest quest' +questa queste questi questo qui quindi + +realmente recente recentemente registrazione relativo riecco salvo + +s' sara sarà sarai saranno sarebbe sarebbero sarei saremmo saremo sareste +saresti sarete saro sarò scola scopo scorso se secondo seguente seguito sei +sembra sembrare sembrato sembri sempre senza sette si sia siamo siano siate +siete sig solito solo soltanto sono sopra sotto spesso srl sta stai stando +stanno starai staranno starebbe starebbero starei staremmo staremo stareste +staresti starete starà starò stata state stati stato stava stavamo stavano +stavate stavi stavo stemmo stessa stesse stessero stessi stessimo stesso +steste stesti stette stettero stetti stia stiamo stiano stiate sto su sua +subito successivamente successivo sue sugl sugli sui sul sull sulla sulle +sullo suo suoi + +t' tale tali talvolta tanto te tempo ti titolo tra tranne tre trenta +troppo trovato tu tua tue tuo tuoi tutta tuttavia tutte tutti tutto + +uguali ulteriore ultimo un un' una uno uomo + +v' va vale vari varia varie vario verso vi via vicino visto vita voi volta volte +vostra vostre vostri vostro +""".split() +) diff --git a/spacy/lang/it/syntax_iterators.py b/spacy/lang/it/syntax_iterators.py new file mode 100644 index 0000000..9246276 --- /dev/null +++ b/spacy/lang/it/syntax_iterators.py @@ -0,0 +1,86 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "nsubj", + "nsubj:pass", + "obj", + "obl", + "obl:agent", + "nmod", + "pcomp", + "appos", + "ROOT", + ] + post_modifiers = ["flat", "flat:name", "fixed", "compound"] + dets = ["det", "det:poss"] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = {doc.vocab.strings.add(label) for label in labels} + np_modifs = {doc.vocab.strings.add(modifier) for modifier in post_modifiers} + np_label = doc.vocab.strings.add("NP") + adj_label = doc.vocab.strings.add("amod") + det_labels = {doc.vocab.strings.add(det) for det in dets} + det_pos = doc.vocab.strings.add("DET") + adp_label = doc.vocab.strings.add("ADP") + conj = doc.vocab.strings.add("conj") + conj_pos = doc.vocab.strings.add("CCONJ") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + right_childs = list(word.rights) + right_child = right_childs[0] if right_childs else None + + if right_child: + if ( + right_child.dep == adj_label + ): # allow chain of adjectives by expanding to right + right_end = right_child.right_edge + elif ( + right_child.dep in det_labels and right_child.pos == det_pos + ): # cut relative pronouns here + right_end = right_child + elif right_child.dep in np_modifs: # Check if we can expand to right + right_end = word.right_edge + else: + right_end = word + else: + right_end = word + prev_end = right_end.i + + left_index = word.left_edge.i + left_index = ( + left_index + 1 if word.left_edge.pos == adp_label else left_index + ) + + yield left_index, right_end.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + + left_index = word.left_edge.i # eliminate left attached conjunction + left_index = ( + left_index + 1 if word.left_edge.pos == conj_pos else left_index + ) + yield left_index, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/it/tokenizer_exceptions.py b/spacy/lang/it/tokenizer_exceptions.py new file mode 100644 index 0000000..2e7a5a1 --- /dev/null +++ b/spacy/lang/it/tokenizer_exceptions.py @@ -0,0 +1,64 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = { + "all'art.": [{ORTH: "all'"}, {ORTH: "art."}], + "dall'art.": [{ORTH: "dall'"}, {ORTH: "art."}], + "dell'art.": [{ORTH: "dell'"}, {ORTH: "art."}], + "L'art.": [{ORTH: "L'"}, {ORTH: "art."}], + "l'art.": [{ORTH: "l'"}, {ORTH: "art."}], + "nell'art.": [{ORTH: "nell'"}, {ORTH: "art."}], + "po'": [{ORTH: "po'"}], + "sett..": [{ORTH: "sett."}, {ORTH: "."}], +} + +for orth in [ + "..", + "....", + "a.C.", + "al.", + "all-path", + "art.", + "Art.", + "artt.", + "att.", + "avv.", + "Avv.", + "by-pass", + "c.d.", + "c/c", + "C.so", + "centro-sinistra", + "check-up", + "Civ.", + "cm.", + "Cod.", + "col.", + "Cost.", + "d.C.", + 'de"', + "distr.", + "E'", + "ecc.", + "e-mail", + "e/o", + "etc.", + "Jr.", + "n°", + "nord-est", + "pag.", + "Proc.", + "prof.", + "sett.", + "s.p.a.", + "s.n.c", + "s.r.l", + "ss.", + "St.", + "tel.", + "week-end", +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ja/__init__.py b/spacy/lang/ja/__init__.py new file mode 100644 index 0000000..492478a --- /dev/null +++ b/spacy/lang/ja/__init__.py @@ -0,0 +1,341 @@ +import re +from collections import namedtuple +from pathlib import Path +from typing import Any, Callable, Dict, Optional, Union + +import srsly +from thinc.api import Model + +from ... import util +from ...errors import Errors +from ...language import BaseDefaults, Language +from ...pipeline import Morphologizer +from ...pipeline.morphologizer import DEFAULT_MORPH_MODEL +from ...scorer import Scorer +from ...symbols import POS +from ...tokens import Doc, MorphAnalysis +from ...training import validate_examples +from ...util import DummyTokenizer, load_config_from_str, registry +from ...vocab import Vocab +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tag_bigram_map import TAG_BIGRAM_MAP +from .tag_map import TAG_MAP +from .tag_orth_map import TAG_ORTH_MAP + +DEFAULT_CONFIG = """ +[nlp] + +[nlp.tokenizer] +@tokenizers = "spacy.ja.JapaneseTokenizer" +split_mode = null +""" + + +def create_tokenizer(split_mode: Optional[str] = None): + def japanese_tokenizer_factory(nlp): + return JapaneseTokenizer(nlp.vocab, split_mode=split_mode) + + return japanese_tokenizer_factory + + +class JapaneseTokenizer(DummyTokenizer): + def __init__(self, vocab: Vocab, split_mode: Optional[str] = None) -> None: + self.vocab = vocab + self.split_mode = split_mode + self.tokenizer = try_sudachi_import(self.split_mode) + # if we're using split mode A we don't need subtokens + self.need_subtokens = not (split_mode is None or split_mode == "A") + + def __reduce__(self): + return JapaneseTokenizer, (self.vocab, self.split_mode) + + def __call__(self, text: str) -> Doc: + # convert sudachipy.morpheme.Morpheme to DetailedToken and merge continuous spaces + sudachipy_tokens = self.tokenizer.tokenize(text) + dtokens = self._get_dtokens(sudachipy_tokens) + dtokens, spaces = get_dtokens_and_spaces(dtokens, text) + + # create Doc with tag bi-gram based part-of-speech identification rules + words, tags, inflections, lemmas, norms, readings, sub_tokens_list = ( + zip(*dtokens) if dtokens else [[]] * 7 + ) + sub_tokens_list = list(sub_tokens_list) + doc = Doc(self.vocab, words=words, spaces=spaces) + next_pos = None # for bi-gram rules + for idx, (token, dtoken) in enumerate(zip(doc, dtokens)): + token.tag_ = dtoken.tag + if next_pos: # already identified in previous iteration + token.pos = next_pos + next_pos = None + else: + token.pos, next_pos = resolve_pos( + token.orth_, + dtoken.tag, + tags[idx + 1] if idx + 1 < len(tags) else None, + ) + # if there's no lemma info (it's an unk) just use the surface + token.lemma_ = dtoken.lemma if dtoken.lemma else dtoken.surface + morph = {} + if dtoken.inf: + # it's normal for this to be empty for non-inflecting types + morph["Inflection"] = dtoken.inf + token.norm_ = dtoken.norm + if dtoken.reading: + # punctuation is its own reading, but we don't want values like + # "=" here + morph["Reading"] = re.sub("[=|]", "_", dtoken.reading) + token.morph = MorphAnalysis(self.vocab, morph) + if self.need_subtokens: + doc.user_data["sub_tokens"] = sub_tokens_list + return doc + + def _get_dtokens(self, sudachipy_tokens, need_sub_tokens: bool = True): + sub_tokens_list = ( + self._get_sub_tokens(sudachipy_tokens) if need_sub_tokens else None + ) + dtokens = [ + DetailedToken( + token.surface(), # orth + "-".join([xx for xx in token.part_of_speech()[:4] if xx != "*"]), # tag + ";".join([xx for xx in token.part_of_speech()[4:] if xx != "*"]), # inf + token.dictionary_form(), # lemma + token.normalized_form(), + token.reading_form(), + ( + sub_tokens_list[idx] if sub_tokens_list else None + ), # user_data['sub_tokens'] + ) + for idx, token in enumerate(sudachipy_tokens) + if len(token.surface()) > 0 + # remove empty tokens which can be produced with characters like … that + ] + # Sudachi normalizes internally and outputs each space char as a token. + # This is the preparation for get_dtokens_and_spaces() to merge the continuous space tokens + return [ + t + for idx, t in enumerate(dtokens) + if idx == 0 + or not t.surface.isspace() + or t.tag != "空白" + or not dtokens[idx - 1].surface.isspace() + or dtokens[idx - 1].tag != "空白" + ] + + def _get_sub_tokens(self, sudachipy_tokens): + # do nothing for default split mode + if not self.need_subtokens: + return None + + sub_tokens_list = [] # list of (list of list of DetailedToken | None) + for token in sudachipy_tokens: + sub_a = token.split(self.tokenizer.SplitMode.A) + if len(sub_a) == 1: # no sub tokens + sub_tokens_list.append(None) + elif self.split_mode == "B": + sub_tokens_list.append([self._get_dtokens(sub_a, False)]) + else: # "C" + sub_b = token.split(self.tokenizer.SplitMode.B) + if len(sub_a) == len(sub_b): + dtokens = self._get_dtokens(sub_a, False) + sub_tokens_list.append([dtokens, dtokens]) + else: + sub_tokens_list.append( + [ + self._get_dtokens(sub_a, False), + self._get_dtokens(sub_b, False), + ] + ) + return sub_tokens_list + + def score(self, examples): + validate_examples(examples, "JapaneseTokenizer.score") + return Scorer.score_tokenization(examples) + + def _get_config(self) -> Dict[str, Any]: + return {"split_mode": self.split_mode} + + def _set_config(self, config: Dict[str, Any] = {}) -> None: + self.split_mode = config.get("split_mode", None) + + def to_bytes(self, **kwargs) -> bytes: + serializers = {"cfg": lambda: srsly.json_dumps(self._get_config())} + return util.to_bytes(serializers, []) + + def from_bytes(self, data: bytes, **kwargs) -> "JapaneseTokenizer": + deserializers = {"cfg": lambda b: self._set_config(srsly.json_loads(b))} + util.from_bytes(data, deserializers, []) + self.tokenizer = try_sudachi_import(self.split_mode) + return self + + def to_disk(self, path: Union[str, Path], **kwargs) -> None: + path = util.ensure_path(path) + serializers = {"cfg": lambda p: srsly.write_json(p, self._get_config())} + util.to_disk(path, serializers, []) + + def from_disk(self, path: Union[str, Path], **kwargs) -> "JapaneseTokenizer": + path = util.ensure_path(path) + serializers = {"cfg": lambda p: self._set_config(srsly.read_json(p))} + util.from_disk(path, serializers, []) + self.tokenizer = try_sudachi_import(self.split_mode) + return self + + +class JapaneseDefaults(BaseDefaults): + config = load_config_from_str(DEFAULT_CONFIG) + stop_words = STOP_WORDS + syntax_iterators = SYNTAX_ITERATORS + writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} + + +class Japanese(Language): + lang = "ja" + Defaults = JapaneseDefaults + + +@Japanese.factory( + "morphologizer", + assigns=["token.morph", "token.pos"], + default_config={ + "model": DEFAULT_MORPH_MODEL, + "overwrite": True, + "extend": True, + "scorer": {"@scorers": "spacy.morphologizer_scorer.v1"}, + }, + default_score_weights={ + "pos_acc": 0.5, + "morph_micro_f": 0.5, + "morph_per_feat": None, + }, +) +def make_morphologizer( + nlp: Language, + model: Model, + name: str, + overwrite: bool, + extend: bool, + scorer: Optional[Callable], +): + return Morphologizer( + nlp.vocab, model, name, overwrite=overwrite, extend=extend, scorer=scorer + ) + + +# Hold the attributes we need with convenient names +DetailedToken = namedtuple( + "DetailedToken", ["surface", "tag", "inf", "lemma", "norm", "reading", "sub_tokens"] +) + + +def try_sudachi_import(split_mode="A"): + """SudachiPy is required for Japanese support, so check for it. + It it's not available blow up and explain how to fix it. + split_mode should be one of these values: "A", "B", "C", None->"A".""" + try: + from sudachipy import dictionary, tokenizer + + split_mode = { + None: tokenizer.Tokenizer.SplitMode.A, + "A": tokenizer.Tokenizer.SplitMode.A, + "B": tokenizer.Tokenizer.SplitMode.B, + "C": tokenizer.Tokenizer.SplitMode.C, + }[split_mode] + tok = dictionary.Dictionary().create(mode=split_mode) + return tok + except ImportError: + raise ImportError( + "Japanese support requires SudachiPy and SudachiDict-core " + "(https://github.com/WorksApplications/SudachiPy). " + "Install with `pip install sudachipy sudachidict_core` or " + "install spaCy with `pip install spacy[ja]`." + ) from None + + +def resolve_pos(orth, tag, next_tag): + """If necessary, add a field to the POS tag for UD mapping. + Under Universal Dependencies, sometimes the same Unidic POS tag can + be mapped differently depending on the literal token or its context + in the sentence. This function returns resolved POSs for both token + and next_token by tuple. + """ + + # Some tokens have their UD tag decided based on the POS of the following + # token. + + # apply orth based mapping + if tag in TAG_ORTH_MAP: + orth_map = TAG_ORTH_MAP[tag] + if orth in orth_map: + return orth_map[orth], None # current_pos, next_pos + + # apply tag bi-gram mapping + if next_tag: + tag_bigram = tag, next_tag + if tag_bigram in TAG_BIGRAM_MAP: + current_pos, next_pos = TAG_BIGRAM_MAP[tag_bigram] + if current_pos is None: # apply tag uni-gram mapping for current_pos + return ( + TAG_MAP[tag][POS], + next_pos, + ) # only next_pos is identified by tag bi-gram mapping + else: + return current_pos, next_pos + + # apply tag uni-gram mapping + return TAG_MAP[tag][POS], None + + +def get_dtokens_and_spaces(dtokens, text, gap_tag="空白"): + # Compare the content of tokens and text, first + words = [x.surface for x in dtokens] + if "".join("".join(words).split()) != "".join(text.split()): + raise ValueError(Errors.E194.format(text=text, words=words)) + + text_dtokens = [] + text_spaces = [] + text_pos = 0 + # handle empty and whitespace-only texts + if len(words) == 0: + return text_dtokens, text_spaces + elif len([word for word in words if not word.isspace()]) == 0: + assert text.isspace() + text_dtokens = [DetailedToken(text, gap_tag, "", text, text, None, None)] + text_spaces = [False] + return text_dtokens, text_spaces + + # align words and dtokens by referring text, and insert gap tokens for the space char spans + for i, (word, dtoken) in enumerate(zip(words, dtokens)): + # skip all space tokens + if word.isspace(): + continue + try: + word_start = text[text_pos:].index(word) + except ValueError: + raise ValueError(Errors.E194.format(text=text, words=words)) from None + + # space token + if word_start > 0: + w = text[text_pos : text_pos + word_start] + text_dtokens.append(DetailedToken(w, gap_tag, "", w, w, None, None)) + text_spaces.append(False) + text_pos += word_start + + # content word + text_dtokens.append(dtoken) + text_spaces.append(False) + text_pos += len(word) + # poll a space char after the word + if i + 1 < len(dtokens) and dtokens[i + 1].surface == " ": + text_spaces[-1] = True + text_pos += 1 + + # trailing space token + if text_pos < len(text): + w = text[text_pos:] + text_dtokens.append(DetailedToken(w, gap_tag, "", w, w, None, None)) + text_spaces.append(False) + + return text_dtokens, text_spaces + + +__all__ = ["Japanese"] diff --git a/spacy/lang/ja/examples.py b/spacy/lang/ja/examples.py new file mode 100644 index 0000000..a07711c --- /dev/null +++ b/spacy/lang/ja/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ja.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "アップルがイギリスの新興企業を１０億ドルで購入を検討", + "自動運転車の損害賠償責任、自動車メーカーに一定の負担を求める", + "歩道を走る自動配達ロボ、サンフランシスコ市が走行禁止を検討", + "ロンドンはイギリスの大都市です。", +] diff --git a/spacy/lang/ja/stop_words.py b/spacy/lang/ja/stop_words.py new file mode 100644 index 0000000..98560d7 --- /dev/null +++ b/spacy/lang/ja/stop_words.py @@ -0,0 +1,48 @@ +# This list was created by taking the top 2000 words from a Wikipedia dump and +# filtering out everything that wasn't hiragana. ー (one) was also added. +# Considered keeping some non-hiragana words but too many place names were +# present. +STOP_WORDS = set( + """ +あ あっ あまり あり ある あるいは あれ +い いい いう いく いずれ いっ いつ いる いわ +うち +え +お おい おけ および おら おり +か かけ かつ かつて かなり から が +き きっかけ +くる くん +こ こう ここ こと この これ ご ごと +さ さらに さん +し しか しかし しまう しまっ しよう +す すぐ すべて する ず +せ せい せる +そう そこ そして その それ それぞれ +た たい ただし たち ため たら たり だ だけ だっ +ち ちゃん +つ つい つけ つつ +て で でき できる です +と とき ところ とっ とも どう +な ない なお なかっ ながら なく なけれ なし なっ など なら なり なる +に にて +ぬ +ね +の のち のみ +は はじめ ば +ひと +ぶり +へ べき +ほか ほとんど ほど ほぼ +ま ます また まで まま +み +も もう もっ もと もの +や やっ +よ よう よく よっ より よる よれ +ら らしい られ られる +る +れ れる +を +ん +一 +""".split() +) diff --git a/spacy/lang/ja/syntax_iterators.py b/spacy/lang/ja/syntax_iterators.py new file mode 100644 index 0000000..3467008 --- /dev/null +++ b/spacy/lang/ja/syntax_iterators.py @@ -0,0 +1,41 @@ +from typing import Iterator, Set, Tuple, Union + +from ...symbols import NOUN, PRON, PROPN, VERB +from ...tokens import Doc, Span + +# TODO: this can probably be pruned a bit +# fmt: off +labels = ["nsubj", "nmod", "ddoclike", "nsubjpass", "pcomp", "pdoclike", "doclike", "obl", "dative", "appos", "attr", "ROOT"] +# fmt: on + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + doc = doclike.doc # Ensure works on both Doc and Span. + np_deps = [doc.vocab.strings.add(label) for label in labels] + doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + seen: Set[int] = set() + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.i in seen: + continue + if word.dep in np_deps: + unseen = [w.i for w in word.subtree if w.i not in seen] + if not unseen: + continue + # this takes care of particles etc. + seen.update(j.i for j in word.subtree) + # This avoids duplicating embedded clauses + seen.update(range(word.i + 1)) + # if the head of this is a verb, mark that and rights seen + # Don't do the subtree as that can hide other phrases + if word.head.pos == VERB: + seen.add(word.head.i) + seen.update(w.i for w in word.head.rights) + yield unseen[0], word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/ja/tag_bigram_map.py b/spacy/lang/ja/tag_bigram_map.py new file mode 100644 index 0000000..9d15fc5 --- /dev/null +++ b/spacy/lang/ja/tag_bigram_map.py @@ -0,0 +1,28 @@ +from ...symbols import ADJ, AUX, NOUN, PART, VERB + +# mapping from tag bi-gram to pos of previous token +TAG_BIGRAM_MAP = { + # This covers only small part of AUX. + ("形容詞-非自立可能", "助詞-終助詞"): (AUX, None), + ("名詞-普通名詞-形状詞可能", "助動詞"): (ADJ, None), + # ("副詞", "名詞-普通名詞-形状詞可能"): (None, ADJ), + # This covers acl, advcl, obl and root, but has side effect for compound. + ("名詞-普通名詞-サ変可能", "動詞-非自立可能"): (VERB, AUX), + # This covers almost all of the deps + ("名詞-普通名詞-サ変形状詞可能", "動詞-非自立可能"): (VERB, AUX), + ("名詞-普通名詞-副詞可能", "動詞-非自立可能"): (None, VERB), + ("副詞", "動詞-非自立可能"): (None, VERB), + ("形容詞-一般", "動詞-非自立可能"): (None, VERB), + ("形容詞-非自立可能", "動詞-非自立可能"): (None, VERB), + ("接頭辞", "動詞-非自立可能"): (None, VERB), + ("助詞-係助詞", "動詞-非自立可能"): (None, VERB), + ("助詞-副助詞", "動詞-非自立可能"): (None, VERB), + ("助詞-格助詞", "動詞-非自立可能"): (None, VERB), + ("補助記号-読点", "動詞-非自立可能"): (None, VERB), + ("形容詞-一般", "接尾辞-名詞的-一般"): (None, PART), + ("助詞-格助詞", "形状詞-助動詞語幹"): (None, NOUN), + ("連体詞", "形状詞-助動詞語幹"): (None, NOUN), + ("動詞-一般", "助詞-副助詞"): (None, PART), + ("動詞-非自立可能", "助詞-副助詞"): (None, PART), + ("助動詞", "助詞-副助詞"): (None, PART), +} diff --git a/spacy/lang/ja/tag_map.py b/spacy/lang/ja/tag_map.py new file mode 100644 index 0000000..527c836 --- /dev/null +++ b/spacy/lang/ja/tag_map.py @@ -0,0 +1,89 @@ +from ...symbols import ( + ADJ, + ADP, + ADV, + AUX, + CCONJ, + DET, + INTJ, + NOUN, + NUM, + PART, + POS, + PRON, + PROPN, + PUNCT, + SCONJ, + SPACE, + SYM, + VERB, +) + +TAG_MAP = { + # Explanation of Unidic tags: + # https://www.gavo.t.u-tokyo.ac.jp/~mine/japanese/nlp+slp/UNIDIC_manual.pdf + # Universal Dependencies Mapping: (Some of the entries in this mapping are updated to v2.6 in the list below) + # http://universaldependencies.org/ja/overview/morphology.html + # http://universaldependencies.org/ja/pos/all.html + "記号-一般": { + POS: NOUN + }, # this includes characters used to represent sounds like ドレミ + "記号-文字": { + POS: NOUN + }, # this is for Greek and Latin characters having some meanings, or used as symbols, as in math + "感動詞-フィラー": {POS: INTJ}, + "感動詞-一般": {POS: INTJ}, + "空白": {POS: SPACE}, + "形状詞-一般": {POS: ADJ}, + "形状詞-タリ": {POS: ADJ}, + "形状詞-助動詞語幹": {POS: AUX}, + "形容詞-一般": {POS: ADJ}, + "形容詞-非自立可能": {POS: ADJ}, # XXX ADJ if alone, AUX otherwise + "助詞-格助詞": {POS: ADP}, + "助詞-係助詞": {POS: ADP}, + "助詞-終助詞": {POS: PART}, + "助詞-準体助詞": {POS: SCONJ}, # の as in 走るのが速い + "助詞-接続助詞": {POS: SCONJ}, # verb ending て0 + "助詞-副助詞": {POS: ADP}, # ばかり, つつ after a verb + "助動詞": {POS: AUX}, + "接続詞": {POS: CCONJ}, # XXX: might need refinement + "接頭辞": {POS: NOUN}, + "接尾辞-形状詞的": {POS: PART}, # がち, チック + "接尾辞-形容詞的": {POS: AUX}, # -らしい + "接尾辞-動詞的": {POS: PART}, # -じみ + "接尾辞-名詞的-サ変可能": {POS: NOUN}, # XXX see 名詞,普通名詞,サ変可能,* + "接尾辞-名詞的-一般": {POS: NOUN}, + "接尾辞-名詞的-助数詞": {POS: NOUN}, + "接尾辞-名詞的-副詞可能": {POS: NOUN}, # -後, -過ぎ + "代名詞": {POS: PRON}, + "動詞-一般": {POS: VERB}, + "動詞-非自立可能": {POS: AUX}, # XXX VERB if alone, AUX otherwise + "副詞": {POS: ADV}, + "補助記号-ＡＡ-一般": {POS: SYM}, # text art + "補助記号-ＡＡ-顔文字": {POS: PUNCT}, # kaomoji + "補助記号-一般": {POS: SYM}, + "補助記号-括弧開": {POS: PUNCT}, # open bracket + "補助記号-括弧閉": {POS: PUNCT}, # close bracket + "補助記号-句点": {POS: PUNCT}, # period or other EOS marker + "補助記号-読点": {POS: PUNCT}, # comma + "名詞-固有名詞-一般": {POS: PROPN}, # general proper noun + "名詞-固有名詞-人名-一般": {POS: PROPN}, # person's name + "名詞-固有名詞-人名-姓": {POS: PROPN}, # surname + "名詞-固有名詞-人名-名": {POS: PROPN}, # first name + "名詞-固有名詞-地名-一般": {POS: PROPN}, # place name + "名詞-固有名詞-地名-国": {POS: PROPN}, # country name + "名詞-助動詞語幹": {POS: AUX}, + "名詞-数詞": {POS: NUM}, # includes Chinese numerals + "名詞-普通名詞-サ変可能": { + POS: NOUN + }, # XXX: sometimes VERB in UDv2; suru-verb noun + "名詞-普通名詞-サ変形状詞可能": {POS: NOUN}, + "名詞-普通名詞-一般": {POS: NOUN}, + "名詞-普通名詞-形状詞可能": {POS: NOUN}, # XXX: sometimes ADJ in UDv2 + "名詞-普通名詞-助数詞可能": {POS: NOUN}, # counter / unit + "名詞-普通名詞-副詞可能": {POS: NOUN}, + "連体詞": {POS: DET}, # XXX this has exceptions based on literal token + # GSD tags. These aren't in Unidic, but we need them for the GSD data. + "外国語": {POS: PROPN}, # Foreign words + "絵文字・記号等": {POS: SYM}, # emoji / kaomoji ^^; +} diff --git a/spacy/lang/ja/tag_orth_map.py b/spacy/lang/ja/tag_orth_map.py new file mode 100644 index 0000000..9d32cde --- /dev/null +++ b/spacy/lang/ja/tag_orth_map.py @@ -0,0 +1,22 @@ +from ...symbols import DET, PART, PRON, SPACE, X + +# mapping from tag bi-gram to pos of previous token +TAG_ORTH_MAP = { + "空白": {" ": SPACE, "　": X}, + "助詞-副助詞": {"たり": PART}, + "連体詞": { + "あの": DET, + "かの": DET, + "この": DET, + "その": DET, + "どの": DET, + "彼の": DET, + "此の": DET, + "其の": DET, + "ある": PRON, + "こんな": PRON, + "そんな": PRON, + "どんな": PRON, + "あらゆる": PRON, + }, +} diff --git a/spacy/lang/kmr/__init__.py b/spacy/lang/kmr/__init__.py new file mode 100644 index 0000000..eee9e69 --- /dev/null +++ b/spacy/lang/kmr/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class KurmanjiDefaults(BaseDefaults): + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Kurmanji(Language): + lang = "kmr" + Defaults = KurmanjiDefaults + + +__all__ = ["Kurmanji"] diff --git a/spacy/lang/kmr/examples.py b/spacy/lang/kmr/examples.py new file mode 100644 index 0000000..5eb3620 --- /dev/null +++ b/spacy/lang/kmr/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.kmr.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Berê mirovan her tim li geşedana pêşerojê ye", # People's gaze is always on the development of the future + "Kawa Nemir di 14 salan de Ulysses wergerand Kurmancî.", # Kawa Nemir translated Ulysses into Kurmanji in 14 years. + "Mem Ararat hunermendekî Kurd yê bi nav û deng e.", # Mem Ararat is a famous Kurdish artist + "Firat Cewerî 40 sal e pirtûkên Kurdî dinivîsîne.", # Firat Ceweri has been writing Kurdish books for 40 years + "Rojnamegerê ciwan nûçeyeke balkêş li ser rewşa aborî nivîsand", # The young journalist wrote an interesting news article about the economic situation + "Sektora çandiniyê beşeke giring a belavkirina gaza serayê li seranserê cîhanê pêk tîne", # The agricultural sector constitutes an important part of greenhouse gas emissions worldwide + "Xwendekarên jêhatî di pêşbaziya matematîkê de serkeftî bûn", # Talented students succeeded in the mathematics competition + "Ji ber ji tunebûnê bavê min xwişkeke min nedan xwendin ew ji min re bû derd û kulek.", # Because of poverty, my father didn't send my sister to school, which became a pain and sorrow for me +] diff --git a/spacy/lang/kmr/lex_attrs.py b/spacy/lang/kmr/lex_attrs.py new file mode 100644 index 0000000..6b80204 --- /dev/null +++ b/spacy/lang/kmr/lex_attrs.py @@ -0,0 +1,138 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "sifir", + "yek", + "du", + "sê", + "çar", + "pênc", + "şeş", + "heft", + "heşt", + "neh", + "deh", + "yazde", + "dazde", + "sêzde", + "çarde", + "pazde", + "şazde", + "hevde", + "hejde", + "nozde", + "bîst", + "sî", + "çil", + "pêncî", + "şêst", + "heftê", + "heştê", + "nod", + "sed", + "hezar", + "milyon", + "milyar", +] + +_ordinal_words = [ + "yekem", + "yekemîn", + "duyem", + "duyemîn", + "sêyem", + "sêyemîn", + "çarem", + "çaremîn", + "pêncem", + "pêncemîn", + "şeşem", + "şeşemîn", + "heftem", + "heftemîn", + "heştem", + "heştemîn", + "nehem", + "nehemîn", + "dehem", + "dehemîn", + "yazdehem", + "yazdehemîn", + "dazdehem", + "dazdehemîn", + "sêzdehem", + "sêzdehemîn", + "çardehem", + "çardehemîn", + "pazdehem", + "pazdehemîn", + "şanzdehem", + "şanzdehemîn", + "hevdehem", + "hevdehemîn", + "hejdehem", + "hejdehemîn", + "nozdehem", + "nozdehemîn", + "bîstem", + "bîstemîn", + "sîyem", + "sîyemîn", + "çilem", + "çilemîn", + "pêncîyem", + "pênciyemîn", + "şêstem", + "şêstemîn", + "heftêyem", + "heftêyemîn", + "heştêyem", + "heştêyemîn", + "notem", + "notemîn", + "sedem", + "sedemîn", + "hezarem", + "hezaremîn", + "milyonem", + "milyonemîn", + "milyarem", + "milyaremîn", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + + # Check ordinal number + if text_lower in _ordinal_words: + return True + + if is_digit(text_lower): + return True + + return False + + +def is_digit(text): + endings = ("em", "yem", "emîn", "yemîn") + for ending in endings: + to = len(ending) + if text.endswith(ending) and text[:-to].isdigit(): + return True + + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/kmr/stop_words.py b/spacy/lang/kmr/stop_words.py new file mode 100644 index 0000000..aee33c2 --- /dev/null +++ b/spacy/lang/kmr/stop_words.py @@ -0,0 +1,44 @@ +STOP_WORDS = set( + """ +û +li +bi +di +da +de +ji +ku +ew +ez +tu +em +hûn +ew +ev +min +te +wî +wê +me +we +wan +vê +vî +va +çi +kî +kê +çawa +çima +kengî +li ku +çend +çiqas +her +hin +gelek +hemû +kes +tişt +""".split() +) diff --git a/spacy/lang/kn/__init__.py b/spacy/lang/kn/__init__.py new file mode 100644 index 0000000..44d53f6 --- /dev/null +++ b/spacy/lang/kn/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class KannadaDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Kannada(Language): + lang = "kn" + Defaults = KannadaDefaults + + +__all__ = ["Kannada"] diff --git a/spacy/lang/kn/examples.py b/spacy/lang/kn/examples.py new file mode 100644 index 0000000..7cbb7fc --- /dev/null +++ b/spacy/lang/kn/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.en.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "ಆಪಲ್ ಒಂದು ಯು.ಕೆ. ಸ್ಟಾರ್ಟ್ಅಪ್ ಅನ್ನು ೧ ಶತಕೋಟಿ ಡಾಲರ್ಗಳಿಗೆ ಖರೀದಿಸಲು ನೋಡುತ್ತಿದೆ.", + "ಸ್ವಾಯತ್ತ ಕಾರುಗಳು ವಿಮಾ ಹೊಣೆಗಾರಿಕೆಯನ್ನು ತಯಾರಕರ ಕಡೆಗೆ ಬದಲಾಯಿಸುತ್ತವೆ.", + "ಕಾಲುದಾರಿ ವಿತರಣಾ ರೋಬೋಟ್‌ಗಳನ್ನು ನಿಷೇಧಿಸುವುದನ್ನು ಸ್ಯಾನ್ ಫ್ರಾನ್ಸಿಸ್ಕೊ ​​ಪರಿಗಣಿಸುತ್ತದೆ.", + "ಲಂಡನ್ ಯುನೈಟೆಡ್ ಕಿಂಗ್‌ಡಂನ ದೊಡ್ಡ ನಗರ.", + "ನೀನು ಎಲ್ಲಿದಿಯಾ?", + "ಫ್ರಾನ್ಸಾದ ಅಧ್ಯಕ್ಷರು ಯಾರು?", + "ಯುನೈಟೆಡ್ ಸ್ಟೇಟ್ಸ್ನ ರಾಜಧಾನಿ ಯಾವುದು?", + "ಬರಾಕ್ ಒಬಾಮ ಯಾವಾಗ ಜನಿಸಿದರು?", +] diff --git a/spacy/lang/kn/stop_words.py b/spacy/lang/kn/stop_words.py new file mode 100644 index 0000000..dba9740 --- /dev/null +++ b/spacy/lang/kn/stop_words.py @@ -0,0 +1,86 @@ +STOP_WORDS = set( + """ +ಹಲವು +ಮೂಲಕ +ಹಾಗೂ +ಅದು +ನೀಡಿದ್ದಾರೆ +ಯಾವ +ಎಂದರು +ಅವರು +ಈಗ +ಎಂಬ +ಹಾಗಾಗಿ +ಅಷ್ಟೇ +ನಾವು +ಇದೇ +ಹೇಳಿ +ತಮ್ಮ +ಹೀಗೆ +ನಮ್ಮ +ಬೇರೆ +ನೀಡಿದರು +ಮತ್ತೆ +ಇದು +ಈ +ನೀವು +ನಾನು +ಇತ್ತು +ಎಲ್ಲಾ +ಯಾವುದೇ +ನಡೆದ +ಅದನ್ನು +ಎಂದರೆ +ನೀಡಿದೆ +ಹೀಗಾಗಿ +ಜೊತೆಗೆ +ಇದರಿಂದ +ನನಗೆ +ಅಲ್ಲದೆ +ಎಷ್ಟು +ಇದರ +ಇಲ್ಲ +ಕಳೆದ +ತುಂಬಾ +ಈಗಾಗಲೇ +ಮಾಡಿ +ಅದಕ್ಕೆ +ಬಗ್ಗೆ +ಅವರ +ಇದನ್ನು +ಆ +ಇದೆ +ಹೆಚ್ಚು +ಇನ್ನು +ಎಲ್ಲ +ಇರುವ +ಅವರಿಗೆ +ನಿಮ್ಮ +ಏನು +ಕೂಡ +ಇಲ್ಲಿ +ನನ್ನನ್ನು +ಕೆಲವು +ಮಾತ್ರ +ಬಳಿಕ +ಅಂತ +ತನ್ನ +ಆಗ +ಅಥವಾ +ಅಲ್ಲ +ಕೇವಲ +ಆದರೆ +ಮತ್ತು +ಇನ್ನೂ +ಅದೇ +ಆಗಿ +ಅವರನ್ನು +ಹೇಳಿದ್ದಾರೆ +ನಡೆದಿದೆ +ಇದಕ್ಕೆ +ಎಂಬುದು +ಎಂದು +ನನ್ನ +ಮೇಲೆ +""".split() +) diff --git a/spacy/lang/ko/__init__.py b/spacy/lang/ko/__init__.py new file mode 100644 index 0000000..3231e19 --- /dev/null +++ b/spacy/lang/ko/__init__.py @@ -0,0 +1,125 @@ +from typing import Any, Dict, Iterator + +from ...language import BaseDefaults, Language +from ...scorer import Scorer +from ...symbols import POS, X +from ...tokens import Doc +from ...training import validate_examples +from ...util import DummyTokenizer, load_config_from_str, registry +from ...vocab import Vocab +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .tag_map import TAG_MAP + +DEFAULT_CONFIG = """ +[nlp] + +[nlp.tokenizer] +@tokenizers = "spacy.ko.KoreanTokenizer" +""" + + +def create_tokenizer(): + def korean_tokenizer_factory(nlp): + return KoreanTokenizer(nlp.vocab) + + return korean_tokenizer_factory + + +class KoreanTokenizer(DummyTokenizer): + def __init__(self, vocab: Vocab): + self.vocab = vocab + self._mecab = try_mecab_import() # type: ignore[func-returns-value] + self._mecab_tokenizer = None + + @property + def mecab_tokenizer(self): + # This is a property so that initializing a pipeline with blank:ko is + # possible without actually requiring mecab-ko, e.g. to run + # `spacy init vectors ko` for a pipeline that will have a different + # tokenizer in the end. The languages need to match for the vectors + # to be imported and there's no way to pass a custom config to + # `init vectors`. + if self._mecab_tokenizer is None: + self._mecab_tokenizer = self._mecab("-F%f[0],%f[7]") + return self._mecab_tokenizer + + def __reduce__(self): + return KoreanTokenizer, (self.vocab,) + + def __call__(self, text: str) -> Doc: + dtokens = list(self.detailed_tokens(text)) + surfaces = [dt["surface"] for dt in dtokens] + doc = Doc(self.vocab, words=surfaces, spaces=list(check_spaces(text, surfaces))) + for token, dtoken in zip(doc, dtokens): + first_tag, sep, eomi_tags = dtoken["tag"].partition("+") + token.tag_ = first_tag # stem(어간) or pre-final(선어말 어미) + if token.tag_ in TAG_MAP: + token.pos = TAG_MAP[token.tag_][POS] + else: + token.pos = X + token.lemma_ = dtoken["lemma"] + doc.user_data["full_tags"] = [dt["tag"] for dt in dtokens] + return doc + + def detailed_tokens(self, text: str) -> Iterator[Dict[str, Any]]: + # 품사 태그(POS)[0], 의미 부류(semantic class)[1], 종성 유무(jongseong)[2], 읽기(reading)[3], + # 타입(type)[4], 첫번째 품사(start pos)[5], 마지막 품사(end pos)[6], 표현(expression)[7], * + for node in self.mecab_tokenizer.parse(text, as_nodes=True): + if node.is_eos(): + break + surface = node.surface + feature = node.feature + tag, _, expr = feature.partition(",") + lemma, _, remainder = expr.partition("/") + if lemma == "*": + lemma = surface + yield {"surface": surface, "lemma": lemma, "tag": tag} + + def score(self, examples): + validate_examples(examples, "KoreanTokenizer.score") + return Scorer.score_tokenization(examples) + + +class KoreanDefaults(BaseDefaults): + config = load_config_from_str(DEFAULT_CONFIG) + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} + infixes = TOKENIZER_INFIXES + + +class Korean(Language): + lang = "ko" + Defaults = KoreanDefaults + + +def try_mecab_import() -> None: + try: + from natto import MeCab + + return MeCab + except ImportError: + raise ImportError( + 'The Korean tokenizer ("spacy.ko.KoreanTokenizer") requires ' + "[mecab-ko](https://bitbucket.org/eunjeon/mecab-ko/src/master/README.md), " + "[mecab-ko-dic](https://bitbucket.org/eunjeon/mecab-ko-dic), " + "and [natto-py](https://github.com/buruzaemon/natto-py)" + ) from None + + +def check_spaces(text, tokens): + prev_end = -1 + start = 0 + for token in tokens: + idx = text.find(token, start) + if prev_end > 0: + yield prev_end != idx + prev_end = idx + len(token) + start = prev_end + if start > 0: + yield False + + +__all__ = ["Korean"] diff --git a/spacy/lang/ko/examples.py b/spacy/lang/ko/examples.py new file mode 100644 index 0000000..edb755e --- /dev/null +++ b/spacy/lang/ko/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ko.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "애플이 영국의 스타트업을 10억 달러에 인수하는 것을 알아보고 있다.", + "자율주행 자동차의 손해 배상 책임이 제조 업체로 옮겨 가다", + "샌프란시스코 시가 자동 배달 로봇의 보도 주행 금지를 검토 중이라고 합니다.", + "런던은 영국의 수도이자 가장 큰 도시입니다.", +] diff --git a/spacy/lang/ko/lex_attrs.py b/spacy/lang/ko/lex_attrs.py new file mode 100644 index 0000000..2c49aa3 --- /dev/null +++ b/spacy/lang/ko/lex_attrs.py @@ -0,0 +1,63 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "영", + "공", + # Native Korean number system + "하나", + "둘", + "셋", + "넷", + "다섯", + "여섯", + "일곱", + "여덟", + "아홉", + "열", + "스물", + "서른", + "마흔", + "쉰", + "예순", + "일흔", + "여든", + "아흔", + # Sino-Korean number system + "일", + "이", + "삼", + "사", + "오", + "육", + "칠", + "팔", + "구", + "십", + "백", + "천", + "만", + "십만", + "백만", + "천만", + "일억", + "십억", + "백억", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if any(char.lower() in _num_words for char in text): + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ko/punctuation.py b/spacy/lang/ko/punctuation.py new file mode 100644 index 0000000..c3c32ea --- /dev/null +++ b/spacy/lang/ko/punctuation.py @@ -0,0 +1,11 @@ +from ..char_classes import LIST_QUOTES +from ..punctuation import TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES + +_infixes = ( + ["·", "ㆍ", r"\(", r"\)"] + + [r"(?<=[0-9])~(?=[0-9-])"] + + LIST_QUOTES + + BASE_TOKENIZER_INFIXES +) + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/ko/stop_words.py b/spacy/lang/ko/stop_words.py new file mode 100644 index 0000000..3eba9fc --- /dev/null +++ b/spacy/lang/ko/stop_words.py @@ -0,0 +1,67 @@ +STOP_WORDS = set( + """ +이 +있 +하 +것 +들 +그 +되 +수 +이 +보 +않 +없 +나 +주 +아니 +등 +같 +때 +년 +가 +한 +지 +오 +말 +일 +그렇 +위하 +때문 +그것 +두 +말하 +알 +그러나 +받 +못하 +일 +그런 +또 +더 +많 +그리고 +좋 +크 +시키 +그러 +하나 +살 +데 +안 +어떤 +번 +나 +다른 +어떻 +들 +이렇 +점 +싶 +말 +좀 +원 +잘 +놓 +""".split() +) diff --git a/spacy/lang/ko/tag_map.py b/spacy/lang/ko/tag_map.py new file mode 100644 index 0000000..85598c3 --- /dev/null +++ b/spacy/lang/ko/tag_map.py @@ -0,0 +1,72 @@ +from ...symbols import ( + ADJ, + ADP, + ADV, + AUX, + CONJ, + DET, + INTJ, + NOUN, + NUM, + POS, + PRON, + PROPN, + PUNCT, + SYM, + VERB, + X, +) + +# 은전한닢(mecab-ko-dic)의 품사 태그를 universal pos tag로 대응시킴 +# https://docs.google.com/spreadsheets/d/1-9blXKjtjeKZqsf4NzHeYJCrr49-nXeRF6D80udfcwY/edit#gid=589544265 +# https://universaldependencies.org/u/pos/ +TAG_MAP = { + # J.{1,2} 조사 + "JKS": {POS: ADP}, + "JKC": {POS: ADP}, + "JKG": {POS: ADP}, + "JKO": {POS: ADP}, + "JKB": {POS: ADP}, + "JKV": {POS: ADP}, + "JKQ": {POS: ADP}, + "JX": {POS: ADP}, # 보조사 + "JC": {POS: CONJ}, # 접속 조사 + "MAJ": {POS: CONJ}, # 접속 부사 + "MAG": {POS: ADV}, # 일반 부사 + "MM": {POS: DET}, # 관형사 + "XPN": {POS: X}, # 접두사 + # XS. 접미사 + "XSN": {POS: X}, + "XSV": {POS: X}, + "XSA": {POS: X}, + "XR": {POS: X}, # 어근 + # E.{1,2} 어미 + "EP": {POS: X}, + "EF": {POS: X}, + "EC": {POS: X}, + "ETN": {POS: X}, + "ETM": {POS: X}, + "IC": {POS: INTJ}, # 감탄사 + "VV": {POS: VERB}, # 동사 + "VA": {POS: ADJ}, # 형용사 + "VX": {POS: AUX}, # 보조 용언 + "VCP": {POS: ADP}, # 긍정 지정사(이다) + "VCN": {POS: ADJ}, # 부정 지정사(아니다) + "NNG": {POS: NOUN}, # 일반 명사(general noun) + "NNB": {POS: NOUN}, # 의존 명사 + "NNBC": {POS: NOUN}, # 의존 명사(단위: unit) + "NNP": {POS: PROPN}, # 고유 명사(proper noun) + "NP": {POS: PRON}, # 대명사 + "NR": {POS: NUM}, # 수사(numerals) + "SN": {POS: NUM}, # 숫자 + # S.{1,2} 부호 + # 문장 부호 + "SF": {POS: PUNCT}, # period or other EOS marker + "SE": {POS: PUNCT}, + "SC": {POS: PUNCT}, # comma, etc. + "SSO": {POS: PUNCT}, # open bracket + "SSC": {POS: PUNCT}, # close bracket + "SY": {POS: SYM}, # 기타 기호 + "SL": {POS: X}, # 외국어 + "SH": {POS: X}, # 한자 +} diff --git a/spacy/lang/ky/__init__.py b/spacy/lang/ky/__init__.py new file mode 100644 index 0000000..fafc0f0 --- /dev/null +++ b/spacy/lang/ky/__init__.py @@ -0,0 +1,20 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class KyrgyzDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Kyrgyz(Language): + lang = "ky" + Defaults = KyrgyzDefaults + + +__all__ = ["Kyrgyz"] diff --git a/spacy/lang/ky/examples.py b/spacy/lang/ky/examples.py new file mode 100644 index 0000000..ba77ea9 --- /dev/null +++ b/spacy/lang/ky/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.ky.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple Улуу Британия стартабын $1 миллиардга сатып алууну көздөөдө.", + "Автоном автомобилдерди камсыздоо жоопкерчилиги өндүрүүчүлөргө артылды.", + "Сан-Франциско тротуар менен жүрүүчү робот-курьерлерге тыю салууну караштырууда.", + "Лондон - Улуу Британияда жайгашкан ири шаар.", + "Кайдасың?", + "Франциянын президенти ким?", + "Америка Кошмо Штаттарынын борбор калаасы кайсы шаар?", + "Барак Обама качан төрөлгөн?", +] diff --git a/spacy/lang/ky/lex_attrs.py b/spacy/lang/ky/lex_attrs.py new file mode 100644 index 0000000..bdf9934 --- /dev/null +++ b/spacy/lang/ky/lex_attrs.py @@ -0,0 +1,48 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "нөл", + "ноль", + "бир", + "эки", + "үч", + "төрт", + "беш", + "алты", + "жети", + "сегиз", + "тогуз", + "он", + "жыйырма", + "отуз", + "кырк", + "элүү", + "алтымыш", + "жетмиш", + "сексен", + "токсон", + "жүз", + "миң", + "миллион", + "миллиард", + "триллион", + "триллиард", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ky/punctuation.py b/spacy/lang/ky/punctuation.py new file mode 100644 index 0000000..6d89da2 --- /dev/null +++ b/spacy/lang/ky/punctuation.py @@ -0,0 +1,28 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) + +_hyphens_no_dash = HYPHENS.replace("-", "").strip("|").replace("||", "") +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?/()]+(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}{q}])[:<>=](?=[{a}])".format(a=ALPHA, q=CONCAT_QUOTES), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=CONCAT_QUOTES), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=_hyphens_no_dash), + r"(?<=[0-9])-(?=[{a}])".format(a=ALPHA), + r"(?<=[0-9])-(?=[0-9])", + ] +) + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/ky/stop_words.py b/spacy/lang/ky/stop_words.py new file mode 100644 index 0000000..ea40bdf --- /dev/null +++ b/spacy/lang/ky/stop_words.py @@ -0,0 +1,42 @@ +STOP_WORDS = set( + """ +ага адам айтты айтымында айтып ал алар +алардын алган алуу алып анда андан аны +анын ар + +бар басма баш башка башкы башчысы берген +биз билдирген билдирди бир биринчи бирок +бишкек болгон болот болсо болуп боюнча +буга бул + +гана + +да дагы деген деди деп + +жана жатат жаткан жаңы же жогорку жок жол +жолу + +кабыл калган кандай карата каршы катары +келген керек кийин кол кылмыш кыргыз +күнү көп + +маалымат мамлекеттик мен менен миң +мурдагы мыйзам мындай мүмкүн + +ошол ошондой + +сүрөт сөз + +тарабынан турган тууралуу + +укук учурда + +чейин чек + +экенин эки эл эле эмес эми эч + +үч үчүн + +өз +""".split() +) diff --git a/spacy/lang/ky/tokenizer_exceptions.py b/spacy/lang/ky/tokenizer_exceptions.py new file mode 100644 index 0000000..c93e3da --- /dev/null +++ b/spacy/lang/ky/tokenizer_exceptions.py @@ -0,0 +1,53 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +_abbrev_exc = [ + # Weekdays abbreviations + {ORTH: "дүй", NORM: "дүйшөмбү"}, + {ORTH: "шей", NORM: "шейшемби"}, + {ORTH: "шар", NORM: "шаршемби"}, + {ORTH: "бей", NORM: "бейшемби"}, + {ORTH: "жум", NORM: "жума"}, + {ORTH: "ишм", NORM: "ишемби"}, + {ORTH: "жек", NORM: "жекшемби"}, + # Months abbreviations + {ORTH: "янв", NORM: "январь"}, + {ORTH: "фев", NORM: "февраль"}, + {ORTH: "мар", NORM: "март"}, + {ORTH: "апр", NORM: "апрель"}, + {ORTH: "июн", NORM: "июнь"}, + {ORTH: "июл", NORM: "июль"}, + {ORTH: "авг", NORM: "август"}, + {ORTH: "сен", NORM: "сентябрь"}, + {ORTH: "окт", NORM: "октябрь"}, + {ORTH: "ноя", NORM: "ноябрь"}, + {ORTH: "дек", NORM: "декабрь"}, + # Number abbreviations + {ORTH: "млрд", NORM: "миллиард"}, + {ORTH: "млн", NORM: "миллион"}, +] + +for abbr in _abbrev_exc: + for orth in (abbr[ORTH], abbr[ORTH].capitalize(), abbr[ORTH].upper()): + _exc[orth] = [{ORTH: orth, NORM: abbr[NORM]}] + _exc[orth + "."] = [{ORTH: orth + ".", NORM: abbr[NORM]}] + +for exc_data in [ # "etc." abbreviations + {ORTH: "ж.б.у.с.", NORM: "жана башка ушул сыяктуу"}, + {ORTH: "ж.б.", NORM: "жана башка"}, + {ORTH: "ж.", NORM: "жыл"}, + {ORTH: "б.з.ч.", NORM: "биздин заманга чейин"}, + {ORTH: "б.з.", NORM: "биздин заман"}, + {ORTH: "кк.", NORM: "кылымдар"}, + {ORTH: "жж.", NORM: "жылдар"}, + {ORTH: "к.", NORM: "кылым"}, + {ORTH: "көч.", NORM: "көчөсү"}, + {ORTH: "м-н", NORM: "менен"}, + {ORTH: "б-ча", NORM: "боюнча"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/la/__init__.py b/spacy/lang/la/__init__.py new file mode 100644 index 0000000..d77ae26 --- /dev/null +++ b/spacy/lang/la/__init__.py @@ -0,0 +1,20 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class LatinDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + + +class Latin(Language): + lang = "la" + Defaults = LatinDefaults + + +__all__ = ["Latin"] diff --git a/spacy/lang/la/examples.py b/spacy/lang/la/examples.py new file mode 100644 index 0000000..db85500 --- /dev/null +++ b/spacy/lang/la/examples.py @@ -0,0 +1,22 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.la.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +# > Caes. BG 1.1 +# > Cic. De Amic. 1 +# > V. Georg. 1.1-5 +# > Gen. 1:1 +# > Galileo, Sid. Nunc. +# > van Schurman, Opusc. arg. 1 + +sentences = [ + "Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae, aliam Aquitani, tertiam qui ipsorum lingua Celtae, nostra Galli appellantur.", + "Q. Mucius augur multa narrare de C. Laelio socero suo memoriter et iucunde solebat nec dubitare illum in omni sermone appellare sapientem.", + "Quid faciat laetas segetes, quo sidere terram uertere, Maecenas, ulmisque adiungere uitis conueniat, quae cura boum, qui cultus habendo sit pecori, apibus quanta experientia parcis, hinc canere incipiam", + "In principio creavit Deus caelum et terram.", + "Quo sumpto, intelligatur lunaris globus, cuius maximus circulus CAF, centrum vero E, dimetiens CF, qui ad Terre diametrum est ut duo ad septem.", + "Cuicunque natura indita sunt principia, seu potentiae principiorum omnium artium, ac scientiarum, ei conveniunt omnes artes ac scientiae.", +] diff --git a/spacy/lang/la/lex_attrs.py b/spacy/lang/la/lex_attrs.py new file mode 100644 index 0000000..fcb35de --- /dev/null +++ b/spacy/lang/la/lex_attrs.py @@ -0,0 +1,34 @@ +import re + +from ...attrs import LIKE_NUM + +# cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4 +roman_numerals_compile = re.compile( + r"(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$" +) + +_num_words = """unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem undecim duodecim tredecim quattuordecim quindecim sedecim septendecim duodeviginti undeviginti viginti triginta quadraginta quinquaginta sexaginta septuaginta octoginta nonaginta centum ducenti ducentae ducenta trecenti trecentae trecenta quadringenti quadringentae quadringenta quingenti quingentae quingenta sescenti sescentae sescenta septingenti septingentae septingenta octingenti octingentae octingenta nongenti nongentae nongenta mille +""".split() + +_num_words += [item.replace("v", "u") for item in _num_words] +_num_words = set(_num_words) + +_ordinal_words = """primus prima primum secundus secunda secundum tertius tertia tertium quartus quarta quartum quintus quinta quintum sextus sexta sextum septimus septima septimum octavus octava octavum nonus nona nonum decimus decima decimum undecimus undecima undecimum duodecimus duodecima duodecimum duodevicesimus duodevicesima duodevicesimum undevicesimus undevicesima undevicesimum vicesimus vicesima vicesimum tricesimus tricesima tricesimum quadragesimus quadragesima quadragesimum quinquagesimus quinquagesima quinquagesimum sexagesimus sexagesima sexagesimum septuagesimus septuagesima septuagesimum octogesimus octogesima octogesimum nonagesimus nonagesima nonagesimum centesimus centesima centesimum ducentesimus ducentesima ducentesimum trecentesimus trecentesima trecentesimum quadringentesimus quadringentesima quadringentesimum quingentesimus quingentesima quingentesimum sescentesimus sescentesima sescentesimum septingentesimus septingentesima septingentesimum octingentesimus octingentesima octingentesimum nongentesimus nongentesima nongentesimum millesimus millesima millesimum""".split() + +_ordinal_words += [item.replace("v", "u") for item in _ordinal_words] +_ordinal_words = set(_ordinal_words) + + +def like_num(text): + if text.isdigit(): + return True + if roman_numerals_compile.match(text): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/la/stop_words.py b/spacy/lang/la/stop_words.py new file mode 100644 index 0000000..8b590bb --- /dev/null +++ b/spacy/lang/la/stop_words.py @@ -0,0 +1,37 @@ +# Corrected Perseus list, cf. https://wiki.digitalclassicist.org/Stopwords_for_Greek_and_Latin + +STOP_WORDS = set( + """ +ab ac ad adhuc aliqui aliquis an ante apud at atque aut autem + +cum cur + +de deinde dum + +ego enim ergo es est et etiam etsi ex + +fio + +haud hic + +iam idem igitur ille in infra inter interim ipse is ita + +magis modo mox + +nam ne nec necque neque nisi non nos + +o ob + +per possum post pro + +quae quam quare qui quia quicumque quidem quilibet quis quisnam quisquam quisque quisquis quo quoniam + +sed si sic sive sub sui sum super suus + +tam tamen trans tu tum + +ubi uel uero + +vel vero +""".split() +) diff --git a/spacy/lang/la/syntax_iterators.py b/spacy/lang/la/syntax_iterators.py new file mode 100644 index 0000000..39b4fb3 --- /dev/null +++ b/spacy/lang/la/syntax_iterators.py @@ -0,0 +1,86 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import AUX, NOUN, PRON, PROPN, VERB +from ...tokens import Doc, Span + +# NB: Modified from da on suggestion from https://github.com/explosion/spaCy/issues/7457#issuecomment-800349751 [PJB] + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + def is_verb_token(tok): + return tok.pos in [VERB, AUX] + + def get_left_bound(root): + left_bound = root + for tok in reversed(list(root.lefts)): + if tok.dep in np_left_deps: + left_bound = tok + return left_bound + + def get_right_bound(doc, root): + right_bound = root + for tok in root.rights: + if tok.dep in np_right_deps: + right = get_right_bound(doc, tok) + if list( + filter( + lambda t: is_verb_token(t) or t.dep in stop_deps, + doc[root.i : right.i], + ) + ): + break + else: + right_bound = right + return right_bound + + def get_bounds(doc, root): + return get_left_bound(root), get_right_bound(doc, root) + + doc = doclike.doc # Ensure works on both Doc and Span. + + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + if not len(doc): + return + + left_labels = [ + "det", + "fixed", + "nmod:poss", + "amod", + "flat", + "goeswith", + "nummod", + "appos", + ] + right_labels = [ + "fixed", + "nmod:poss", + "amod", + "flat", + "goeswith", + "nummod", + "appos", + "nmod", + "det", + ] + stop_labels = ["punct"] + + np_label = doc.vocab.strings.add("NP") + np_left_deps = [doc.vocab.strings.add(label) for label in left_labels] + np_right_deps = [doc.vocab.strings.add(label) for label in right_labels] + stop_deps = [doc.vocab.strings.add(label) for label in stop_labels] + + prev_right = -1 + for token in doclike: + if token.pos in [PROPN, NOUN, PRON]: + left, right = get_bounds(doc, token) + if left.i <= prev_right: + continue + yield left.i, right.i + 1, np_label + prev_right = right.i + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/la/tokenizer_exceptions.py b/spacy/lang/la/tokenizer_exceptions.py new file mode 100644 index 0000000..c0b9811 --- /dev/null +++ b/spacy/lang/la/tokenizer_exceptions.py @@ -0,0 +1,25 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +## TODO: Look into systematically handling u/v +_exc = { + "mecum": [{ORTH: "me"}, {ORTH: "cum"}], + "tecum": [{ORTH: "te"}, {ORTH: "cum"}], + "nobiscum": [{ORTH: "nobis"}, {ORTH: "cum"}], + "vobiscum": [{ORTH: "vobis"}, {ORTH: "cum"}], + "uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}], +} + +_abbrev_exc = """A. A.D. Aa. Aaa. Acc. Agr. Ap. Apr. April. A.U.C. Aug. C. Caes. Caess. Cc. Cn. Coll. Cons. Conss. Cos. Coss. D. D.N. Dat. Dd. Dec. Decemb. Decembr. F. Feb. Febr. Februar. Ian. Id. Imp. Impp. Imppp. Iul. Iun. K. Kal. L. M'. M. Mai. Mam. Mar. Mart. Med. N. Nn. Nob. Non. Nov. Novemb. Oct. Octob. Opet. Ord. P. Paul. Pf. Pl. Plur. Post. Pp. Prid. Pro. Procos. Q. Quint. S. S.C. Scr. Sept. Septemb. Ser. Sert. Sex. Sext. St. Sta. Suff. T. Ti. Trib. V. Vol. Vop. Vv.""".split() + +_abbrev_exc += [item.lower() for item in _abbrev_exc] +_abbrev_exc += [item.upper() for item in _abbrev_exc] +_abbrev_exc += [item.replace("v", "u").replace("V", "U") for item in _abbrev_exc] + +_abbrev_exc += ["d.N."] + +for orth in set(_abbrev_exc): + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/lb/__init__.py b/spacy/lang/lb/__init__.py new file mode 100644 index 0000000..2386b43 --- /dev/null +++ b/spacy/lang/lb/__init__.py @@ -0,0 +1,20 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class LuxembourgishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Luxembourgish(Language): + lang = "lb" + Defaults = LuxembourgishDefaults + + +__all__ = ["Luxembourgish"] diff --git a/spacy/lang/lb/examples.py b/spacy/lang/lb/examples.py new file mode 100644 index 0000000..a7a1048 --- /dev/null +++ b/spacy/lang/lb/examples.py @@ -0,0 +1,15 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.lb.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "An der Zäit hunn sech den Nordwand an d’Sonn gestridden, wie vun hinnen zwee wuel méi staark wier, wéi e Wanderer, deen an ee waarme Mantel agepak war, iwwert de Wee koum.", + "Si goufen sech eens, dass deejéinege fir de Stäerkste gëlle sollt, deen de Wanderer forcéiere géif, säi Mantel auszedoen.", + "Den Nordwand huet mat aller Force geblosen, awer wat e méi geblosen huet, wat de Wanderer sech méi a säi Mantel agewéckelt huet.", + "Um Enn huet den Nordwand säi Kampf opginn.", + "Dunn huet d’Sonn d’Loft mat hire frëndleche Strale gewiermt, a schonn no kuerzer Zäit huet de Wanderer säi Mantel ausgedoen.", + "Do huet den Nordwand missen zouginn, dass d’Sonn vun hinnen zwee de Stäerkste wier.", +] diff --git a/spacy/lang/lb/lex_attrs.py b/spacy/lang/lb/lex_attrs.py new file mode 100644 index 0000000..1192313 --- /dev/null +++ b/spacy/lang/lb/lex_attrs.py @@ -0,0 +1,40 @@ +from ...attrs import LIKE_NUM + +_num_words = set( + """ +null eent zwee dräi véier fënnef sechs ziwen aacht néng zéng eelef zwielef dräizéng +véierzéng foffzéng siechzéng siwwenzéng uechtzeng uechzeng nonnzéng nongzéng zwanzeg drësseg véierzeg foffzeg sechzeg siechzeg siwenzeg achtzeg achzeg uechtzeg uechzeg nonnzeg +honnert dausend millioun milliard billioun billiard trillioun triliard +""".split() +) + +_ordinal_words = set( + """ +éischten zweeten drëtten véierten fënneften sechsten siwenten aachten néngten zéngten eeleften +zwieleften dräizéngten véierzéngten foffzéngten siechzéngten uechtzéngen uechzéngten nonnzéngten nongzéngten zwanzegsten +drëssegsten véierzegsten foffzegsten siechzegsten siwenzegsten uechzegsten nonnzegsten +honnertsten dausendsten milliounsten +milliardsten billiounsten billiardsten trilliounsten trilliardsten +""".split() +) + + +def like_num(text): + """ + check if text resembles a number + """ + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/lb/punctuation.py b/spacy/lang/lb/punctuation.py new file mode 100644 index 0000000..8bdbf97 --- /dev/null +++ b/spacy/lang/lb/punctuation.py @@ -0,0 +1,21 @@ +from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER, LIST_ELLIPSES, LIST_ICONS + +ELISION = " ' ’ ".strip().replace(" ", "") + +abbrev = ("d", "D") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=^[{ab}][{el}])(?=[{a}])".format(ab=abbrev, a=ALPHA, el=ELISION), + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[0-9])-(?=[0-9])", + ] +) + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/lb/stop_words.py b/spacy/lang/lb/stop_words.py new file mode 100644 index 0000000..8f22ea6 --- /dev/null +++ b/spacy/lang/lb/stop_words.py @@ -0,0 +1,211 @@ +STOP_WORDS = set( + """ +a +à +äis +är +ärt +äert +ären +all +allem +alles +alleguer +als +also +am +an +anerefalls +ass +aus +awer +bei +beim +bis +bis +d' +dach +datt +däin +där +dat +de +dee +den +deel +deem +deen +deene +déi +den +deng +denger +dem +der +dësem +di +dir +do +da +dann +domat +dozou +drop +du +duerch +duerno +e +ee +em +een +eent +ë +en +ënner +ëm +ech +eis +eise +eisen +eiser +eises +eisereen +esou +een +eng +enger +engem +entweder +et +eréischt +falls +fir +géint +géif +gëtt +gët +geet +gi +ginn +gouf +gouff +goung +hat +haten +hatt +hätt +hei +hu +huet +hun +hunn +hiren +hien +hin +hier +hir +jidderen +jiddereen +jiddwereen +jiddereng +jiddwerengen +jo +ins +iech +iwwer +kann +kee +keen +kënne +kënnt +kéng +kéngen +kéngem +koum +kuckt +mam +mat +ma +mä +mech +méi +mécht +meng +menger +mer +mir +muss +nach +nämmlech +nämmelech +näischt +nawell +nëmme +nëmmen +net +nees +nee +no +nu +nom +och +oder +ons +onsen +onser +onsereen +onst +om +op +ouni +säi +säin +schonn +schonns +si +sid +sie +se +sech +seng +senge +sengem +senger +selwecht +selwer +sinn +sollten +souguer +sou +soss +sot +'t +tëscht +u +un +um +virdrun +vu +vum +vun +wann +war +waren +was +wat +wëllt +weider +wéi +wéini +wéinst +wi +wollt +wou +wouhin +zanter +ze +zu +zum +zwar +""".split() +) diff --git a/spacy/lang/lb/tokenizer_exceptions.py b/spacy/lang/lb/tokenizer_exceptions.py new file mode 100644 index 0000000..844826e --- /dev/null +++ b/spacy/lang/lb/tokenizer_exceptions.py @@ -0,0 +1,52 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +# TODO +# treat other apostrophes within words as part of the word: [op d'mannst], [fir d'éischt] (= exceptions) + +_exc = {} + +# translate / delete what is not necessary +for exc_data in [ + {ORTH: "’t", NORM: "et"}, + {ORTH: "’T", NORM: "et"}, + {ORTH: "'t", NORM: "et"}, + {ORTH: "'T", NORM: "et"}, + {ORTH: "wgl.", NORM: "wannechgelift"}, + {ORTH: "M.", NORM: "Monsieur"}, + {ORTH: "Mme.", NORM: "Madame"}, + {ORTH: "Dr.", NORM: "Dokter"}, + {ORTH: "Tel.", NORM: "Telefon"}, + {ORTH: "asw.", NORM: "an sou weider"}, + {ORTH: "etc.", NORM: "et cetera"}, + {ORTH: "bzw.", NORM: "bezéiungsweis"}, + {ORTH: "Jan.", NORM: "Januar"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# to be extended +for orth in [ + "z.B.", + "Dipl.", + "Dr.", + "etc.", + "i.e.", + "o.k.", + "O.K.", + "p.a.", + "p.s.", + "P.S.", + "phil.", + "q.e.d.", + "R.I.P.", + "rer.", + "sen.", + "ë.a.", + "U.S.", + "U.S.A.", +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/lex_attrs.py b/spacy/lang/lex_attrs.py new file mode 100644 index 0000000..3ac2042 --- /dev/null +++ b/spacy/lang/lex_attrs.py @@ -0,0 +1,213 @@ +import re +import unicodedata +from typing import Set + +from .. import attrs +from .tokenizer_exceptions import URL_MATCH + +_like_email = re.compile(r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)").match +_tlds = set( + "com|org|edu|gov|net|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|" + "name|pro|tel|travel|xyz|icu|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|" + "ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|" + "cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|" + "ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|" + "gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|" + "je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|" + "lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|" + "na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|" + "pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|" + "ss|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|" + "ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw".split("|") +) + + +def is_punct(text: str) -> bool: + for char in text: + if not unicodedata.category(char).startswith("P"): + return False + return True + + +def is_ascii(text: str) -> bool: + for char in text: + if ord(char) >= 128: + return False + return True + + +def like_num(text: str) -> bool: + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + # can be overwritten by lang with list of number words + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + return False + + +def is_bracket(text: str) -> bool: + brackets = ("(", ")", "[", "]", "{", "}", "<", ">") + return text in brackets + + +def is_quote(text: str) -> bool: + # fmt: off + quotes = ('"', "'", "`", "«", "»", "‘", "’", "‚", "‛", "“", "”", "„", "‟", "‹", "›", "❮", "❯", "''", "``") + # fmt: on + return text in quotes + + +def is_left_punct(text: str) -> bool: + # fmt: off + left_punct = ("(", "[", "{", "<", '"', "'", "«", "‘", "‚", "‛", "“", "„", "‟", "‹", "❮", "``") + # fmt: on + return text in left_punct + + +def is_right_punct(text: str) -> bool: + right_punct = (")", "]", "}", ">", '"', "'", "»", "’", "”", "›", "❯", "''") + return text in right_punct + + +def is_currency(text: str) -> bool: + # can be overwritten by lang with list of currency words, e.g. dollar, euro + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +def like_email(text: str) -> bool: + return bool(_like_email(text)) + + +def like_url(text: str) -> bool: + # We're looking for things that function in text like URLs. So, valid URL + # or not, anything they say http:// is going to be good. + if text.startswith("http://") or text.startswith("https://"): + return True + elif text.startswith("www.") and len(text) >= 5: + return True + if text[0] == "." or text[-1] == ".": + return False + if "@" in text: + # prevent matches on e-mail addresses – check after splitting the text + # to still allow URLs containing an '@' character (see #1715) + return False + for i in range(len(text)): + if text[i] == ".": + break + else: + return False + tld = text.rsplit(".", 1)[1].split(":", 1)[0] + if tld.endswith("/"): + return True + if tld.isalpha() and tld in _tlds: + return True + if URL_MATCH(text): + return True + return False + + +def word_shape(text: str) -> str: + if len(text) >= 100: + return "LONG" + shape = [] + last = "" + shape_char = "" + seq = 0 + for char in text: + if char.isalpha(): + if char.isupper(): + shape_char = "X" + else: + shape_char = "x" + elif char.isdigit(): + shape_char = "d" + else: + shape_char = char + if shape_char == last: + seq += 1 + else: + seq = 0 + last = shape_char + if seq < 4: + shape.append(shape_char) + return "".join(shape) + + +def lower(string: str) -> str: + return string.lower() + + +def prefix(string: str) -> str: + return string[0] + + +def suffix(string: str) -> str: + return string[-3:] + + +def is_alpha(string: str) -> bool: + return string.isalpha() + + +def is_digit(string: str) -> bool: + return string.isdigit() + + +def is_lower(string: str) -> bool: + return string.islower() + + +def is_space(string: str) -> bool: + return string.isspace() + + +def is_title(string: str) -> bool: + return string.istitle() + + +def is_upper(string: str) -> bool: + return string.isupper() + + +def is_stop(string: str, stops: Set[str] = set()) -> bool: + return string.lower() in stops + + +def get_lang(text: str, lang: str = "") -> str: + # This function is partially applied so lang code can be passed in + # automatically while still allowing pickling + return lang + + +LEX_ATTRS = { + attrs.LOWER: lower, + attrs.NORM: lower, + attrs.PREFIX: prefix, + attrs.SUFFIX: suffix, + attrs.IS_ALPHA: is_alpha, + attrs.IS_DIGIT: is_digit, + attrs.IS_LOWER: is_lower, + attrs.IS_SPACE: is_space, + attrs.IS_TITLE: is_title, + attrs.IS_UPPER: is_upper, + attrs.IS_STOP: is_stop, + attrs.LIKE_EMAIL: like_email, + attrs.LIKE_NUM: like_num, + attrs.IS_PUNCT: is_punct, + attrs.IS_ASCII: is_ascii, + attrs.SHAPE: word_shape, + attrs.IS_BRACKET: is_bracket, + attrs.IS_QUOTE: is_quote, + attrs.IS_LEFT_PUNCT: is_left_punct, + attrs.IS_RIGHT_PUNCT: is_right_punct, + attrs.IS_CURRENCY: is_currency, + attrs.LIKE_URL: like_url, +} diff --git a/spacy/lang/lg/__init__.py b/spacy/lang/lg/__init__.py new file mode 100644 index 0000000..a876853 --- /dev/null +++ b/spacy/lang/lg/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS + + +class LugandaDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + infixes = TOKENIZER_INFIXES + stop_words = STOP_WORDS + + +class Luganda(Language): + lang = "lg" + Defaults = LugandaDefaults + + +__all__ = ["Luganda"] diff --git a/spacy/lang/lg/examples.py b/spacy/lang/lg/examples.py new file mode 100644 index 0000000..5450c55 --- /dev/null +++ b/spacy/lang/lg/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.lg.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Mpa ebyafaayo ku byalo Nakatu ne Nkajja", + "Okuyita Ttembo kitegeeza kugwa ddalu", + "Ekifumu kino kyali kya mulimu ki?", + "Ekkovu we liyise wayitibwa mukululo", + "Akola mulimu ki oguvaamu ssente?", + "Emisumaali egikomerera embaawo giyitibwa nninga", + "Abooluganda ab’emmamba ababiri", + "Ekisaawe ky'ebyenjigiriza kya mugaso nnyo", +] diff --git a/spacy/lang/lg/lex_attrs.py b/spacy/lang/lg/lex_attrs.py new file mode 100644 index 0000000..3c60e3d --- /dev/null +++ b/spacy/lang/lg/lex_attrs.py @@ -0,0 +1,95 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nnooti", # Zero + "zeero", # zero + "emu", # one + "bbiri", # two + "ssatu", # three + "nnya", # four + "ttaano", # five + "mukaaga", # six + "musanvu", # seven + "munaana", # eight + "mwenda", # nine + "kkumi", # ten + "kkumi n'emu", # eleven + "kkumi na bbiri", # twelve + "kkumi na ssatu", # thirteen + "kkumi na nnya", # forteen + "kkumi na ttaano", # fifteen + "kkumi na mukaaga", # sixteen + "kkumi na musanvu", # seventeen + "kkumi na munaana", # eighteen + "kkumi na mwenda", # nineteen + "amakumi abiri", # twenty + "amakumi asatu", # thirty + "amakumi ana", # forty + "amakumi ataano", # fifty + "nkaaga", # sixty + "nsanvu", # seventy + "kinaana", # eighty + "kyenda", # ninety + "kikumi", # hundred + "lukumi", # thousand + "kakadde", # million + "kawumbi", # billion + "kase", # trillion + "katabalika", # quadrillion + "keesedde", # gajillion + "kafukunya", # bazillion + "ekisooka", # first + "ekyokubiri", # second + "ekyokusatu", # third + "ekyokuna", # fourth + "ekyokutaano", # fifith + "ekyomukaaga", # sixth + "ekyomusanvu", # seventh + "eky'omunaana", # eighth + "ekyomwenda", # nineth + "ekyekkumi", # tenth + "ekyekkumi n'ekimu", # eleventh + "ekyekkumi n'ebibiri", # twelveth + "ekyekkumi n'ebisatu", # thirteenth + "ekyekkumi n'ebina", # fourteenth + "ekyekkumi n'ebitaano", # fifteenth + "ekyekkumi n'omukaaga", # sixteenth + "ekyekkumi n'omusanvu", # seventeenth + "ekyekkumi n'omunaana", # eigteenth + "ekyekkumi n'omwenda", # nineteenth + "ekyamakumi abiri", # twentieth + "ekyamakumi asatu", # thirtieth + "ekyamakumi ana", # fortieth + "ekyamakumi ataano", # fiftieth + "ekyenkaaga", # sixtieth + "ekyensanvu", # seventieth + "ekyekinaana", # eightieth + "ekyekyenda", # ninetieth + "ekyekikumi", # hundredth + "ekyolukumi", # thousandth + "ekyakakadde", # millionth + "ekyakawumbi", # billionth + "ekyakase", # trillionth + "ekyakatabalika", # quadrillionth + "ekyakeesedde", # gajillionth + "ekyakafukunya", # bazillionth +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/lg/punctuation.py b/spacy/lang/lg/punctuation.py new file mode 100644 index 0000000..775c6b0 --- /dev/null +++ b/spacy/lang/lg/punctuation.py @@ -0,0 +1,26 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/lg/stop_words.py b/spacy/lang/lg/stop_words.py new file mode 100644 index 0000000..7bad593 --- /dev/null +++ b/spacy/lang/lg/stop_words.py @@ -0,0 +1,19 @@ +STOP_WORDS = set( + """ +abadde abalala abamu abangi abava ajja ali alina ani anti ateekeddwa atewamu +atya awamu aweebwa ayinza ba baali babadde babalina bajja +bajjanewankubade bali balina bandi bangi bano bateekeddwa baweebwa bayina bebombi beera bibye +bimu bingi bino bo bokka bonna buli bulijjo bulungi bwabwe bwaffe bwayo bwe bwonna bya byabwe +byaffe byebimu byonna ddaa ddala ddi e ebimu ebiri ebweruobulungi ebyo edda ejja ekirala ekyo +endala engeri ennyo era erimu erina ffe ffenna ga gujja gumu gunno guno gwa gwe kaseera kati +kennyini ki kiki kikino kikye kikyo kino kirungi kki ku kubangabyombi kubangaolwokuba kudda +kuva kuwa kwegamba kyaffe kye kyekimuoyo kyekyo kyonna leero liryo lwa lwaki lyabwezaabwe +lyaffe lyange mbadde mingi mpozzi mu mulinaoyina munda mwegyabwe nolwekyo nabadde nabo nandiyagadde +nandiye nanti naye ne nedda neera nga nnyingi nnyini nnyinza nnyo nti nyinza nze oba ojja okudda +okugenda okuggyako okutuusa okuva okuwa oli olina oluvannyuma olwekyobuva omuli ono osobola otya +oyina oyo seetaaga si sinakindi singa talina tayina tebaali tebaalina tebayina terina tetulina +tetuteekeddwa tewali teyalina teyayina tolina tu tuyina tulina tuyina twafuna twetaaga wa wabula +wabweru wadde waggulunnina wakati waliwobangi waliyo wandi wange wano wansi weebwa yabadde yaffe +ye yenna yennyini yina yonna ziba zijja zonna +""".split() +) diff --git a/spacy/lang/lij/__init__.py b/spacy/lang/lij/__init__.py new file mode 100644 index 0000000..3b8e972 --- /dev/null +++ b/spacy/lang/lij/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class LigurianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + stop_words = STOP_WORDS + + +class Ligurian(Language): + lang = "lij" + Defaults = LigurianDefaults + + +__all__ = ["Ligurian"] diff --git a/spacy/lang/lij/examples.py b/spacy/lang/lij/examples.py new file mode 100644 index 0000000..ec336b0 --- /dev/null +++ b/spacy/lang/lij/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.lij.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Sciusciâ e sciorbî no se peu.", + "Graçie di çetroin, che me son arrivæ.", + "Vegnime apreuvo, che ve fasso pescâ di òmmi.", + "Bella pe sempre l'ægua inta conchetta quande unn'agoggia d'ægua a se â trapaña.", +] diff --git a/spacy/lang/lij/punctuation.py b/spacy/lang/lij/punctuation.py new file mode 100644 index 0000000..c5c150d --- /dev/null +++ b/spacy/lang/lij/punctuation.py @@ -0,0 +1,11 @@ +from ..char_classes import ALPHA +from ..punctuation import TOKENIZER_INFIXES + +ELISION = " ' ’ ".strip().replace(" ", "").replace("\n", "") + + +_infixes = TOKENIZER_INFIXES + [ + r"(?<=[{a}][{el}])(?=[{a}])".format(a=ALPHA, el=ELISION) +] + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/lij/stop_words.py b/spacy/lang/lij/stop_words.py new file mode 100644 index 0000000..1d6f09d --- /dev/null +++ b/spacy/lang/lij/stop_words.py @@ -0,0 +1,39 @@ +STOP_WORDS = set( + """ +a à â a-a a-e a-i a-o aiva aloa an ancheu ancon apreuvo ascì atra atre atri atro avanti avei + +bella belle belli bello ben + +ch' che chì chi ciù co-a co-e co-i co-o comm' comme con cösa coscì cöse + +d' da da-a da-e da-i da-o dapeu de delongo derê di do doe doî donde dòppo + +é e ê ea ean emmo en ëse + +fin fiña + +gh' ghe guæei + +i î in insemme int' inta inte inti into + +l' lê lì lô + +m' ma manco me megio meno mezo mi + +na n' ne ni ninte nisciun nisciuña no + +o ò ô oua + +parte pe pe-a pe-i pe-e pe-o perché pittin pö primma pròpio + +quæ quand' quande quarche quella quelle quelli quello + +s' sce scê sci sciâ sciô sciù se segge seu sò solo son sott' sta stæta stæte stæti stæto ste sti sto + +tanta tante tanti tanto te ti torna tra tròppo tutta tutte tutti tutto + +un uña unn' unna + +za zu +""".split() +) diff --git a/spacy/lang/lij/tokenizer_exceptions.py b/spacy/lang/lij/tokenizer_exceptions.py new file mode 100644 index 0000000..cf5a1af --- /dev/null +++ b/spacy/lang/lij/tokenizer_exceptions.py @@ -0,0 +1,49 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for raw in [ + "a-e", + "a-o", + "a-i", + "a-a", + "co-a", + "co-e", + "co-i", + "co-o", + "da-a", + "da-e", + "da-i", + "da-o", + "pe-a", + "pe-e", + "pe-i", + "pe-o", +]: + for orth in [raw, raw.capitalize()]: + _exc[orth] = [{ORTH: orth}] + +# Prefix + prepositions with à (e.g. "sott'a-o") + +for prep in [ + "a-a", + "a-e", + "a-o", + "a-i", +]: + for prefix in [ + "sott'", + "sott’", + "contr'", + "contr’", + "ch'", + "ch’", + "s'", + "s’", + ]: + for prefix_orth in [prefix, prefix.capitalize()]: + _exc[prefix_orth + prep] = [{ORTH: prefix_orth}, {ORTH: prep}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/lt/__init__.py b/spacy/lang/lt/__init__.py new file mode 100644 index 0000000..f3ea257 --- /dev/null +++ b/spacy/lang/lt/__init__.py @@ -0,0 +1,21 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class LithuanianDefaults(BaseDefaults): + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Lithuanian(Language): + lang = "lt" + Defaults = LithuanianDefaults + + +__all__ = ["Lithuanian"] diff --git a/spacy/lang/lt/examples.py b/spacy/lang/lt/examples.py new file mode 100644 index 0000000..57d6eb4 --- /dev/null +++ b/spacy/lang/lt/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.lt.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Jaunikis pirmąją vestuvinę naktį iškeitė į areštinės gultą", + "Bepiločiai automobiliai išnaikins vairavimo mokyklas, autoservisus ir eismo nelaimes", + "Vilniuje galvojama uždrausti naudoti skėčius", + "Londonas yra didelis miestas Jungtinėje Karalystėje", + "Kur tu?", + "Kas yra Prancūzijos prezidentas?", + "Kokia yra Jungtinių Amerikos Valstijų sostinė?", + "Kada gimė Dalia Grybauskaitė?", +] diff --git a/spacy/lang/lt/lex_attrs.py b/spacy/lang/lt/lex_attrs.py new file mode 100644 index 0000000..28894a5 --- /dev/null +++ b/spacy/lang/lt/lex_attrs.py @@ -0,0 +1,1150 @@ +from ...attrs import LIKE_NUM + +_num_words = { + "antra", + "antrai", + "antrais", + "antram", + "antrame", + "antras", + "antri", + "antriems", + "antro", + "antroje", + "antromis", + "antroms", + "antros", + "antrose", + "antru", + "antruose", + "antrus", + "antrą", + "antrų", + "aštunta", + "aštuntai", + "aštuntais", + "aštuntam", + "aštuntame", + "aštuntas", + "aštunti", + "aštuntiems", + "aštunto", + "aštuntoje", + "aštuntomis", + "aštuntoms", + "aštuntos", + "aštuntose", + "aštuntu", + "aštuntuose", + "aštuntus", + "aštuntą", + "aštuntų", + "aštuoneri", + "aštuoneriais", + "aštuonerias", + "aštuoneriems", + "aštuoneriomis", + "aštuonerioms", + "aštuonerios", + "aštuoneriose", + "aštuoneriuose", + "aštuonerius", + "aštuonerių", + "aštuonetai", + "aštuonetais", + "aštuonetams", + "aštuonetas", + "aštuonete", + "aštuoneto", + "aštuonetu", + "aštuonetui", + "aštuonetuose", + "aštuonetus", + "aštuonetą", + "aštuonetų", + "aštuoni", + "aštuoniais", + "aštuonias", + "aštuoniasdešimt", + "aštuoniasdešimta", + "aštuoniasdešimtai", + "aštuoniasdešimtais", + "aštuoniasdešimtam", + "aštuoniasdešimtame", + "aštuoniasdešimtas", + "aštuoniasdešimti", + "aštuoniasdešimtiems", + "aštuoniasdešimties", + "aštuoniasdešimtimi", + "aštuoniasdešimtimis", + "aštuoniasdešimtims", + "aštuoniasdešimtis", + "aštuoniasdešimto", + "aštuoniasdešimtoje", + "aštuoniasdešimtomis", + "aštuoniasdešimtoms", + "aštuoniasdešimtos", + "aštuoniasdešimtose", + "aštuoniasdešimtu", + "aštuoniasdešimtuose", + "aštuoniasdešimtus", + "aštuoniasdešimtyje", + "aštuoniasdešimtys", + "aštuoniasdešimtyse", + "aštuoniasdešimtą", + "aštuoniasdešimtį", + "aštuoniasdešimtų", + "aštuoniasdešimčia", + "aštuoniasdešimčiai", + "aštuoniasdešimčių", + "aštuoniems", + "aštuoniolika", + "aštuoniolikai", + "aštuoniolikoje", + "aštuoniolikos", + "aštuoniolikta", + "aštuonioliktai", + "aštuonioliktais", + "aštuonioliktam", + "aštuonioliktame", + "aštuonioliktas", + "aštuoniolikti", + "aštuonioliktiems", + "aštuoniolikto", + "aštuonioliktoje", + "aštuonioliktomis", + "aštuonioliktoms", + "aštuonioliktos", + "aštuonioliktose", + "aštuonioliktu", + "aštuonioliktuose", + "aštuonioliktus", + "aštuonioliktą", + "aštuonioliktų", + "aštuoniomis", + "aštuonioms", + "aštuonios", + "aštuoniose", + "aštuonis", + "aštuoniuose", + "aštuonių", + "bilijonai", + "bilijonais", + "bilijonams", + "bilijonas", + "bilijone", + "bilijono", + "bilijonu", + "bilijonui", + "bilijonuose", + "bilijonus", + "bilijoną", + "bilijonų", + "devinta", + "devintai", + "devintais", + "devintam", + "devintame", + "devintas", + "devinti", + "devintiems", + "devinto", + "devintoje", + "devintomis", + "devintoms", + "devintos", + "devintose", + "devintu", + "devintuose", + "devintus", + "devintą", + "devintų", + "devyneri", + "devyneriais", + "devynerias", + "devyneriems", + "devyneriomis", + "devynerioms", + "devynerios", + "devyneriose", + "devyneriuose", + "devynerius", + "devynerių", + "devynetas", + "devynete", + "devyneto", + "devynetu", + "devynetui", + "devynetą", + "devyni", + "devyniais", + "devynias", + "devyniasdešimt", + "devyniasdešimta", + "devyniasdešimtai", + "devyniasdešimtais", + "devyniasdešimtam", + "devyniasdešimtame", + "devyniasdešimtas", + "devyniasdešimti", + "devyniasdešimtiems", + "devyniasdešimties", + "devyniasdešimtimi", + "devyniasdešimtimis", + "devyniasdešimtims", + "devyniasdešimtis", + "devyniasdešimto", + "devyniasdešimtoje", + "devyniasdešimtomis", + "devyniasdešimtoms", + "devyniasdešimtos", + "devyniasdešimtose", + "devyniasdešimtu", + "devyniasdešimtuose", + "devyniasdešimtus", + "devyniasdešimtyje", + "devyniasdešimtys", + "devyniasdešimtyse", + "devyniasdešimtą", + "devyniasdešimtį", + "devyniasdešimtų", + "devyniasdešimčia", + "devyniasdešimčiai", + "devyniasdešimčių", + "devyniems", + "devyniolika", + "devyniolikai", + "devyniolikoje", + "devyniolikos", + "devyniolikta", + "devynioliktai", + "devynioliktais", + "devynioliktam", + "devynioliktame", + "devynioliktas", + "devyniolikti", + "devynioliktiems", + "devyniolikto", + "devynioliktoje", + "devynioliktomis", + "devynioliktoms", + "devynioliktos", + "devynioliktose", + "devynioliktu", + "devynioliktuose", + "devynioliktus", + "devynioliktą", + "devynioliktų", + "devyniomis", + "devynioms", + "devynios", + "devyniose", + "devynis", + "devyniuose", + "devynių", + "dešimt", + "dešimta", + "dešimtai", + "dešimtais", + "dešimtam", + "dešimtame", + "dešimtas", + "dešimti", + "dešimtiems", + "dešimties", + "dešimtimi", + "dešimtimis", + "dešimtims", + "dešimtis", + "dešimto", + "dešimtoje", + "dešimtomis", + "dešimtoms", + "dešimtos", + "dešimtose", + "dešimtu", + "dešimtuose", + "dešimtus", + "dešimtyje", + "dešimtys", + "dešimtyse", + "dešimtą", + "dešimtį", + "dešimtų", + "dešimčia", + "dešimčiai", + "dešimčių", + "du", + "dutūkstanta", + "dutūkstantai", + "dutūkstantais", + "dutūkstantam", + "dutūkstantame", + "dutūkstantas", + "dutūkstanti", + "dutūkstantiems", + "dutūkstanto", + "dutūkstantoje", + "dutūkstantomis", + "dutūkstantoms", + "dutūkstantos", + "dutūkstantose", + "dutūkstantu", + "dutūkstantuose", + "dutūkstantus", + "dutūkstantą", + "dutūkstantų", + "dušimta", + "dušimtai", + "dušimtais", + "dušimtam", + "dušimtame", + "dušimtas", + "dušimti", + "dušimtiems", + "dušimto", + "dušimtoje", + "dušimtomis", + "dušimtoms", + "dušimtos", + "dušimtose", + "dušimtu", + "dušimtuose", + "dušimtus", + "dušimtą", + "dušimtų", + "dvejais", + "dvejas", + "dvejetai", + "dvejetais", + "dvejetams", + "dvejetas", + "dvejete", + "dvejeto", + "dvejetu", + "dvejetui", + "dvejetuose", + "dvejetus", + "dvejetą", + "dvejetų", + "dveji", + "dvejiems", + "dvejomis", + "dvejoms", + "dvejos", + "dvejose", + "dvejuose", + "dvejus", + "dvejų", + "dvi", + "dvidešimt", + "dvidešimta", + "dvidešimtai", + "dvidešimtais", + "dvidešimtam", + "dvidešimtame", + "dvidešimtas", + "dvidešimti", + "dvidešimtiems", + "dvidešimties", + "dvidešimtimi", + "dvidešimtimis", + "dvidešimtims", + "dvidešimtis", + "dvidešimto", + "dvidešimtoje", + "dvidešimtomis", + "dvidešimtoms", + "dvidešimtos", + "dvidešimtose", + "dvidešimtu", + "dvidešimtuose", + "dvidešimtus", + "dvidešimtyje", + "dvidešimtys", + "dvidešimtyse", + "dvidešimtą", + "dvidešimtį", + "dvidešimtų", + "dvidešimčia", + "dvidešimčiai", + "dvidešimčių", + "dviejose", + "dviejuose", + "dviejų", + "dviem", + "dvylika", + "dvylikai", + "dvylikoje", + "dvylikos", + "dvylikta", + "dvyliktai", + "dvyliktais", + "dvyliktam", + "dvyliktame", + "dvyliktas", + "dvylikti", + "dvyliktiems", + "dvylikto", + "dvyliktoje", + "dvyliktomis", + "dvyliktoms", + "dvyliktos", + "dvyliktose", + "dvyliktu", + "dvyliktuose", + "dvyliktus", + "dvyliktą", + "dvyliktų", + "keturi", + "keturiais", + "keturias", + "keturiasdešimt", + "keturiasdešimta", + "keturiasdešimtai", + "keturiasdešimtais", + "keturiasdešimtam", + "keturiasdešimtame", + "keturiasdešimtas", + "keturiasdešimti", + "keturiasdešimtiems", + "keturiasdešimties", + "keturiasdešimtimi", + "keturiasdešimtimis", + "keturiasdešimtims", + "keturiasdešimtis", + "keturiasdešimto", + "keturiasdešimtoje", + "keturiasdešimtomis", + "keturiasdešimtoms", + "keturiasdešimtos", + "keturiasdešimtose", + "keturiasdešimtu", + "keturiasdešimtuose", + "keturiasdešimtus", + "keturiasdešimtyje", + "keturiasdešimtys", + "keturiasdešimtyse", + "keturiasdešimtą", + "keturiasdešimtį", + "keturiasdešimtų", + "keturiasdešimčia", + "keturiasdešimčiai", + "keturiasdešimčių", + "keturiems", + "keturiolika", + "keturiolikai", + "keturiolikoje", + "keturiolikos", + "keturiolikta", + "keturioliktai", + "keturioliktais", + "keturioliktam", + "keturioliktame", + "keturioliktas", + "keturiolikti", + "keturioliktiems", + "keturiolikto", + "keturioliktoje", + "keturioliktomis", + "keturioliktoms", + "keturioliktos", + "keturioliktose", + "keturioliktu", + "keturioliktuose", + "keturioliktus", + "keturioliktą", + "keturioliktų", + "keturiomis", + "keturioms", + "keturios", + "keturiose", + "keturis", + "keturiuose", + "keturių", + "ketveri", + "ketveriais", + "ketverias", + "ketveriems", + "ketveriomis", + "ketverioms", + "ketverios", + "ketveriose", + "ketveriuose", + "ketverius", + "ketverių", + "ketvertai", + "ketvertais", + "ketvertams", + "ketvertas", + "ketverte", + "ketverto", + "ketvertu", + "ketvertui", + "ketvertuose", + "ketvertus", + "ketvertą", + "ketvertų", + "ketvirta", + "ketvirtai", + "ketvirtais", + "ketvirtam", + "ketvirtame", + "ketvirtas", + "ketvirti", + "ketvirtiems", + "ketvirto", + "ketvirtoje", + "ketvirtomis", + "ketvirtoms", + "ketvirtos", + "ketvirtose", + "ketvirtu", + "ketvirtuose", + "ketvirtus", + "ketvirtą", + "ketvirtų", + "milijardai", + "milijardais", + "milijardams", + "milijardas", + "milijarde", + "milijardo", + "milijardu", + "milijardui", + "milijarduose", + "milijardus", + "milijardą", + "milijardų", + "milijonai", + "milijonais", + "milijonams", + "milijonas", + "milijone", + "milijono", + "milijonu", + "milijonui", + "milijonuose", + "milijonus", + "milijoną", + "milijonų", + "penkeri", + "penkeriais", + "penkerias", + "penkeriems", + "penkeriomis", + "penkerioms", + "penkerios", + "penkeriose", + "penkeriuose", + "penkerius", + "penkerių", + "penketai", + "penketais", + "penketams", + "penketas", + "penkete", + "penketo", + "penketu", + "penketui", + "penketuose", + "penketus", + "penketą", + "penketų", + "penki", + "penkiais", + "penkias", + "penkiasdešimt", + "penkiasdešimta", + "penkiasdešimtai", + "penkiasdešimtais", + "penkiasdešimtam", + "penkiasdešimtame", + "penkiasdešimtas", + "penkiasdešimti", + "penkiasdešimtiems", + "penkiasdešimties", + "penkiasdešimtimi", + "penkiasdešimtimis", + "penkiasdešimtims", + "penkiasdešimtis", + "penkiasdešimto", + "penkiasdešimtoje", + "penkiasdešimtomis", + "penkiasdešimtoms", + "penkiasdešimtos", + "penkiasdešimtose", + "penkiasdešimtu", + "penkiasdešimtuose", + "penkiasdešimtus", + "penkiasdešimtyje", + "penkiasdešimtys", + "penkiasdešimtyse", + "penkiasdešimtą", + "penkiasdešimtį", + "penkiasdešimtų", + "penkiasdešimčia", + "penkiasdešimčiai", + "penkiasdešimčių", + "penkiems", + "penkiolika", + "penkiolikai", + "penkiolikoje", + "penkiolikos", + "penkiolikta", + "penkioliktai", + "penkioliktais", + "penkioliktam", + "penkioliktame", + "penkioliktas", + "penkiolikti", + "penkioliktiems", + "penkiolikto", + "penkioliktoje", + "penkioliktomis", + "penkioliktoms", + "penkioliktos", + "penkioliktose", + "penkioliktu", + "penkioliktuose", + "penkioliktus", + "penkioliktą", + "penkioliktų", + "penkiomis", + "penkioms", + "penkios", + "penkiose", + "penkis", + "penkiuose", + "penkišimta", + "penkišimtai", + "penkišimtais", + "penkišimtam", + "penkišimtame", + "penkišimtas", + "penkišimti", + "penkišimtiems", + "penkišimto", + "penkišimtoje", + "penkišimtomis", + "penkišimtoms", + "penkišimtos", + "penkišimtose", + "penkišimtu", + "penkišimtuose", + "penkišimtus", + "penkišimtą", + "penkišimtų", + "penkių", + "penkta", + "penktai", + "penktais", + "penktam", + "penktame", + "penktas", + "penkti", + "penktiems", + "penkto", + "penktoje", + "penktomis", + "penktoms", + "penktos", + "penktose", + "penktu", + "penktuose", + "penktus", + "penktą", + "penktų", + "pirma", + "pirmai", + "pirmais", + "pirmaisiais", + "pirmajai", + "pirmajam", + "pirmajame", + "pirmam", + "pirmame", + "pirmas", + "pirmasis", + "pirmi", + "pirmieji", + "pirmiems", + "pirmiesiems", + "pirmo", + "pirmoje", + "pirmoji", + "pirmojo", + "pirmojoje", + "pirmomis", + "pirmoms", + "pirmos", + "pirmose", + "pirmosiomis", + "pirmosioms", + "pirmosios", + "pirmosiose", + "pirmu", + "pirmuoju", + "pirmuose", + "pirmuosiuose", + "pirmuosius", + "pirmus", + "pirmą", + "pirmąja", + "pirmąją", + "pirmąjį", + "pirmąsias", + "pirmų", + "pirmųjų", + "pusantro", + "pusantros", + "pusantrų", + "pusdevinto", + "pusdevintos", + "pusdevintų", + "pusdešimto", + "pusdešimtos", + "pusdešimtų", + "pusketvirto", + "pusketvirtos", + "pusketvirtų", + "puspenkto", + "puspenktos", + "puspenktų", + "pusseptinto", + "pusseptintos", + "pusseptintų", + "pustrečio", + "pustrečios", + "pustrečių", + "pusšešto", + "pusšeštos", + "pusšeštų", + "septinta", + "septintai", + "septintais", + "septintam", + "septintame", + "septintas", + "septinti", + "septintiems", + "septinto", + "septintoje", + "septintomis", + "septintoms", + "septintos", + "septintose", + "septintu", + "septintuose", + "septintus", + "septintą", + "septintų", + "septyneri", + "septyneriais", + "septynerias", + "septyneriems", + "septyneriomis", + "septynerioms", + "septynerios", + "septyneriose", + "septyneriuose", + "septynerius", + "septynerių", + "septynetas", + "septynete", + "septyneto", + "septynetu", + "septynetui", + "septynetą", + "septyni", + "septyniais", + "septynias", + "septyniasdešimt", + "septyniasdešimta", + "septyniasdešimtai", + "septyniasdešimtais", + "septyniasdešimtam", + "septyniasdešimtame", + "septyniasdešimtas", + "septyniasdešimti", + "septyniasdešimtiems", + "septyniasdešimties", + "septyniasdešimtimi", + "septyniasdešimtimis", + "septyniasdešimtims", + "septyniasdešimtis", + "septyniasdešimto", + "septyniasdešimtoje", + "septyniasdešimtomis", + "septyniasdešimtoms", + "septyniasdešimtos", + "septyniasdešimtose", + "septyniasdešimtu", + "septyniasdešimtuose", + "septyniasdešimtus", + "septyniasdešimtyje", + "septyniasdešimtys", + "septyniasdešimtyse", + "septyniasdešimtą", + "septyniasdešimtį", + "septyniasdešimtų", + "septyniasdešimčia", + "septyniasdešimčiai", + "septyniasdešimčių", + "septyniems", + "septyniolika", + "septyniolikai", + "septyniolikoje", + "septyniolikos", + "septyniolikta", + "septynioliktai", + "septynioliktais", + "septynioliktam", + "septynioliktame", + "septynioliktas", + "septyniolikti", + "septynioliktiems", + "septyniolikto", + "septynioliktoje", + "septynioliktomis", + "septynioliktoms", + "septynioliktos", + "septynioliktose", + "septynioliktu", + "septynioliktuose", + "septynioliktus", + "septynioliktą", + "septynioliktų", + "septyniomis", + "septynioms", + "septynios", + "septyniose", + "septynis", + "septyniuose", + "septynių", + "skaitvardi", + "skaitvardis", + "skaitvardyje", + "skaitvardį", + "skaitvardžiai", + "skaitvardžiais", + "skaitvardžiams", + "skaitvardžio", + "skaitvardžiu", + "skaitvardžiui", + "skaitvardžiuose", + "skaitvardžius", + "skaitvardžių", + "trejais", + "trejas", + "trejetai", + "trejetais", + "trejetams", + "trejetas", + "trejete", + "trejeto", + "trejetu", + "trejetui", + "trejetuose", + "trejetus", + "trejetą", + "trejetų", + "treji", + "trejiems", + "trejomis", + "trejoms", + "trejos", + "trejose", + "trejuose", + "trejus", + "trejų", + "treti", + "tretiems", + "trečia", + "trečiai", + "trečiais", + "trečiam", + "trečiame", + "trečias", + "trečio", + "trečioje", + "trečiomis", + "trečioms", + "trečios", + "trečiose", + "trečiu", + "trečiuose", + "trečius", + "trečią", + "trečių", + "trijose", + "trijuose", + "trijų", + "trilijonai", + "trilijonais", + "trilijonams", + "trilijonas", + "trilijone", + "trilijono", + "trilijonu", + "trilijonui", + "trilijonuose", + "trilijonus", + "trilijoną", + "trilijonų", + "trimis", + "trims", + "tris", + "trisdešimt", + "trisdešimta", + "trisdešimtai", + "trisdešimtais", + "trisdešimtam", + "trisdešimtame", + "trisdešimtas", + "trisdešimti", + "trisdešimtiems", + "trisdešimties", + "trisdešimtimi", + "trisdešimtimis", + "trisdešimtims", + "trisdešimtis", + "trisdešimto", + "trisdešimtoje", + "trisdešimtomis", + "trisdešimtoms", + "trisdešimtos", + "trisdešimtose", + "trisdešimtu", + "trisdešimtuose", + "trisdešimtus", + "trisdešimtyje", + "trisdešimtys", + "trisdešimtyse", + "trisdešimtą", + "trisdešimtį", + "trisdešimtų", + "trisdešimčia", + "trisdešimčiai", + "trisdešimčių", + "trylika", + "trylikai", + "trylikoje", + "trylikos", + "trylikta", + "tryliktai", + "tryliktais", + "tryliktam", + "tryliktame", + "tryliktas", + "trylikti", + "tryliktiems", + "trylikto", + "tryliktoje", + "tryliktomis", + "tryliktoms", + "tryliktos", + "tryliktose", + "tryliktu", + "tryliktuose", + "tryliktus", + "tryliktą", + "tryliktų", + "trys", + "tūkstanta", + "tūkstantai", + "tūkstantais", + "tūkstantam", + "tūkstantame", + "tūkstantas", + "tūkstanti", + "tūkstantiems", + "tūkstantis", + "tūkstanto", + "tūkstantoje", + "tūkstantomis", + "tūkstantoms", + "tūkstantos", + "tūkstantose", + "tūkstantu", + "tūkstantuose", + "tūkstantus", + "tūkstantyje", + "tūkstantą", + "tūkstantį", + "tūkstantų", + "tūkstančiai", + "tūkstančiais", + "tūkstančiams", + "tūkstančio", + "tūkstančiu", + "tūkstančiui", + "tūkstančiuose", + "tūkstančius", + "tūkstančių", + "vienam", + "viename", + "vienas", + "vieno", + "vienu", + "vienuolika", + "vienuolikai", + "vienuolikoje", + "vienuolikos", + "vienuolikta", + "vienuoliktais", + "vienuoliktam", + "vienuoliktame", + "vienuoliktas", + "vienuolikti", + "vienuoliktiems", + "vienuolikto", + "vienuoliktoje", + "vienuoliktomis", + "vienuoliktoms", + "vienuoliktos", + "vienuoliktose", + "vienuoliktu", + "vienuoliktuose", + "vienuoliktus", + "vienuoliktą", + "vienuoliktų", + "vienuoliką", + "vieną", + "šešeri", + "šešeriais", + "šešerias", + "šešeriems", + "šešeriomis", + "šešerioms", + "šešerios", + "šešeriose", + "šešeriuose", + "šešerius", + "šešerių", + "šešetai", + "šešetais", + "šešetams", + "šešetas", + "šešete", + "šešeto", + "šešetu", + "šešetui", + "šešetuose", + "šešetus", + "šešetą", + "šešetų", + "šeši", + "šešiais", + "šešias", + "šešiasdešimt", + "šešiasdešimta", + "šešiasdešimtai", + "šešiasdešimtais", + "šešiasdešimtam", + "šešiasdešimtame", + "šešiasdešimtas", + "šešiasdešimti", + "šešiasdešimtiems", + "šešiasdešimties", + "šešiasdešimtimi", + "šešiasdešimtimis", + "šešiasdešimtims", + "šešiasdešimtis", + "šešiasdešimto", + "šešiasdešimtoje", + "šešiasdešimtomis", + "šešiasdešimtoms", + "šešiasdešimtos", + "šešiasdešimtose", + "šešiasdešimtu", + "šešiasdešimtuose", + "šešiasdešimtus", + "šešiasdešimtyje", + "šešiasdešimtys", + "šešiasdešimtyse", + "šešiasdešimtą", + "šešiasdešimtį", + "šešiasdešimtų", + "šešiasdešimčia", + "šešiasdešimčiai", + "šešiasdešimčių", + "šešiems", + "šešiolika", + "šešiolikai", + "šešiolikoje", + "šešiolikos", + "šešiolikta", + "šešioliktai", + "šešioliktais", + "šešioliktam", + "šešioliktame", + "šešioliktas", + "šešiolikti", + "šešioliktiems", + "šešiolikto", + "šešioliktoje", + "šešioliktomis", + "šešioliktoms", + "šešioliktos", + "šešioliktose", + "šešioliktu", + "šešioliktuose", + "šešioliktus", + "šešioliktą", + "šešioliktų", + "šešiomis", + "šešioms", + "šešios", + "šešiose", + "šešis", + "šešiuose", + "šešių", + "šešta", + "šeštai", + "šeštais", + "šeštam", + "šeštame", + "šeštas", + "šešti", + "šeštiems", + "šešto", + "šeštoje", + "šeštomis", + "šeštoms", + "šeštos", + "šeštose", + "šeštu", + "šeštuose", + "šeštus", + "šeštą", + "šeštų", + "šimtai", + "šimtais", + "šimtams", + "šimtas", + "šimte", + "šimto", + "šimtu", + "šimtui", + "šimtuose", + "šimtus", + "šimtą", + "šimtų", +} + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/lt/punctuation.py b/spacy/lang/lt/punctuation.py new file mode 100644 index 0000000..deef248 --- /dev/null +++ b/spacy/lang/lt/punctuation.py @@ -0,0 +1,31 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) +from ..punctuation import TOKENIZER_SUFFIXES + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +_suffixes = [r"\."] + list(TOKENIZER_SUFFIXES) + + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/lt/stop_words.py b/spacy/lang/lt/stop_words.py new file mode 100644 index 0000000..8c11b3f --- /dev/null +++ b/spacy/lang/lt/stop_words.py @@ -0,0 +1,1316 @@ +STOP_WORDS = { + "a", + "abejais", + "abejas", + "abejetam", + "abejetame", + "abejetas", + "abejeto", + "abejetu", + "abejetą", + "abeji", + "abejiems", + "abejomis", + "abejoms", + "abejos", + "abejose", + "abejuose", + "abejus", + "abejų", + "abi", + "abidvi", + "abiejose", + "abiejuose", + "abiejų", + "abiem", + "abiems", + "abigaliai", + "abipus", + "abu", + "abudu", + "aha", + "ai", + "aiman", + "aj", + "ajajai", + "ak", + "aleliuja", + "aliai", + "alio", + "amen", + "ana", + "anai", + "anaiptol", + "anais", + "anaisiais", + "anajai", + "anajam", + "anajame", + "anam", + "aname", + "anapus", + "anas", + "anasai", + "anasis", + "ane", + "anei", + "anie", + "aniedvi", + "aniedviem", + "anieji", + "aniem", + "aniemdviem", + "aniems", + "aniesiems", + "ano", + "anodviem", + "anoj", + "anoje", + "anoji", + "anojo", + "anojoje", + "anokia", + "anokiai", + "anokiais", + "anokiam", + "anokiame", + "anokias", + "anokie", + "anokiems", + "anokio", + "anokioje", + "anokiomis", + "anokioms", + "anokios", + "anokiose", + "anokiu", + "anokiuose", + "anokius", + "anokią", + "anokių", + "anoks", + "anoksai", + "anokį", + "anom", + "anomdviem", + "anomis", + "anoms", + "anos", + "anose", + "anosiomis", + "anosioms", + "anosios", + "anosiose", + "anot", + "ant", + "antai", + "antrokia", + "antrokiai", + "antrokiais", + "antrokiam", + "antrokiame", + "antrokias", + "antrokie", + "antrokiems", + "antrokio", + "antrokioje", + "antrokiomis", + "antrokioms", + "antrokios", + "antrokiose", + "antrokiu", + "antrokiuose", + "antrokius", + "antrokią", + "antrokių", + "antroks", + "antrokį", + "anuo", + "anuodu", + "anuoju", + "anuos", + "anuose", + "anuosiuose", + "anuosius", + "aną", + "anąja", + "anąją", + "anąjį", + "anąsias", + "anąįį", + "anų", + "anųdviejų", + "anųjų", + "apie", + "aplink", + "ar", + "arba", + "argi", + "arti", + "at", + "aukščiau", + "ačiū", + "aš", + "bakst", + "bambt", + "bau", + "be", + "bei", + "beje", + "bemaž", + "bene", + "bent", + "berods", + "bet", + "betgi", + "beveik", + "bis", + "brakšt", + "braukšt", + "bravo", + "bumbt", + "būtent", + "cakt", + "capt", + "cha", + "cit", + "cvakt", + "dar", + "dargi", + "daugmaž", + "deja", + "dievaž", + "din", + "dirst", + "dribt", + "drykt", + "dunkst", + "dvejokia", + "dvejokiai", + "dvejokiais", + "dvejokiam", + "dvejokiame", + "dvejokias", + "dvejokie", + "dvejokiems", + "dvejokio", + "dvejokioje", + "dvejokiomis", + "dvejokioms", + "dvejokios", + "dvejokiose", + "dvejokiu", + "dvejokiuose", + "dvejokius", + "dvejokią", + "dvejokių", + "dvejoks", + "dvejokį", + "dzin", + "dėka", + "dėkui", + "dėl", + "dėlei", + "dėlto", + "e", + "ech", + "ei", + "ej", + "et", + "fe", + "gaila", + "gal", + "galbūt", + "galgi", + "gan", + "gana", + "gi", + "greta", + "idant", + "iki", + "ir", + "irgi", + "it", + "itin", + "iš", + "išilgai", + "išvis", + "ja", + "jai", + "jais", + "jaisiais", + "jajai", + "jajam", + "jajame", + "jam", + "jame", + "jas", + "jau", + "jei", + "jeigu", + "ji", + "jie", + "jiedu", + "jiedvi", + "jiedviem", + "jieji", + "jiemdviem", + "jiems", + "jiesiems", + "jinai", + "jis", + "jisai", + "jo", + "jodviem", + "jog", + "joje", + "joji", + "jojo", + "jojoje", + "jokia", + "jokiai", + "jokiais", + "jokiam", + "jokiame", + "jokias", + "jokie", + "jokiems", + "jokio", + "jokioje", + "jokiomis", + "jokioms", + "jokios", + "jokiose", + "jokiu", + "jokiuose", + "jokius", + "jokią", + "jokių", + "joks", + "jokį", + "jomdviem", + "jomis", + "joms", + "jos", + "jose", + "josiomis", + "josioms", + "josios", + "josiose", + "judu", + "judvi", + "judviejose", + "judviejuose", + "judviejų", + "judviem", + "juk", + "jumis", + "jums", + "jumyse", + "juo", + "juodu", + "juoju", + "juos", + "juose", + "juosiuose", + "juosius", + "jus", + "ją", + "jąja", + "jąją", + "jąsias", + "jį", + "jįjį", + "jūs", + "jūsiške", + "jūsiškei", + "jūsiškes", + "jūsiškiai", + "jūsiškiais", + "jūsiškiam", + "jūsiškiame", + "jūsiškiams", + "jūsiškio", + "jūsiškis", + "jūsiškiu", + "jūsiškiuose", + "jūsiškius", + "jūsiškių", + "jūsiškė", + "jūsiškėje", + "jūsiškėmis", + "jūsiškėms", + "jūsiškės", + "jūsiškėse", + "jūsiškę", + "jūsiškį", + "jūsų", + "jų", + "jųdviejų", + "jųjų", + "kad", + "kada", + "kadangi", + "kai", + "kaip", + "kaipgi", + "kapt", + "kas", + "katra", + "katrai", + "katrais", + "katram", + "katrame", + "katras", + "katrie", + "katriedvi", + "katriems", + "katro", + "katroje", + "katromis", + "katroms", + "katros", + "katrose", + "katruo", + "katruodu", + "katruos", + "katruose", + "katrą", + "katrų", + "kaukšt", + "kažin", + "kažkas", + "kažkatra", + "kažkatras", + "kažkokia", + "kažkokiai", + "kažkokiais", + "kažkokiam", + "kažkokiame", + "kažkokias", + "kažkokie", + "kažkokiems", + "kažkokio", + "kažkokioje", + "kažkokiomis", + "kažkokioms", + "kažkokios", + "kažkokiose", + "kažkokiu", + "kažkokiuose", + "kažkokius", + "kažkokią", + "kažkokių", + "kažkoks", + "kažkoksai", + "kažkokį", + "kažkuri", + "kažkuria", + "kažkuriai", + "kažkuriais", + "kažkuriam", + "kažkuriame", + "kažkurias", + "kažkurie", + "kažkuriems", + "kažkurio", + "kažkurioje", + "kažkuriomis", + "kažkurioms", + "kažkurios", + "kažkuriose", + "kažkuris", + "kažkuriuo", + "kažkuriuos", + "kažkuriuose", + "kažkurią", + "kažkurių", + "kažkurį", + "keleri", + "keleriais", + "kelerias", + "keleriems", + "keleriomis", + "kelerioms", + "kelerios", + "keleriose", + "keleriuose", + "kelerius", + "kelerių", + "keletas", + "kelete", + "keleto", + "keletu", + "keletui", + "keletą", + "keli", + "keliais", + "kelias", + "keliasdešimt", + "keliems", + "kelinta", + "kelintai", + "kelintaisiais", + "kelintajai", + "kelintajam", + "kelintajame", + "kelintam", + "kelintame", + "kelintas", + "kelintasis", + "kelintieji", + "kelintiesiems", + "kelinto", + "kelintoje", + "kelintoji", + "kelintojo", + "kelintojoje", + "kelintos", + "kelintosioms", + "kelintosios", + "kelintosiose", + "kelintu", + "kelintuoju", + "kelintuosiuose", + "kelintuosius", + "kelintą", + "kelintąją", + "kelintąjį", + "kelintąsias", + "kelintųjų", + "keliolika", + "keliolikai", + "keliolikoje", + "keliolikos", + "keliomis", + "kelioms", + "kelios", + "keliose", + "kelis", + "keliuose", + "kelių", + "kiaurai", + "kiek", + "kiekviena", + "kiekvienai", + "kiekvienais", + "kiekvienam", + "kiekviename", + "kiekvienas", + "kiekvieni", + "kiekvieniems", + "kiekvieno", + "kiekvienoje", + "kiekvienomis", + "kiekvienoms", + "kiekvienos", + "kiekvienose", + "kiekvienu", + "kiekvienuose", + "kiekvienus", + "kiekvieną", + "kiekvienų", + "kieno", + "kita", + "kitai", + "kitais", + "kitam", + "kitame", + "kitas", + "kiti", + "kitiems", + "kito", + "kitoje", + "kitokia", + "kitokiai", + "kitokiais", + "kitokiam", + "kitokiame", + "kitokias", + "kitokie", + "kitokiems", + "kitokio", + "kitokioje", + "kitokiomis", + "kitokioms", + "kitokios", + "kitokiose", + "kitokiu", + "kitokiuose", + "kitokius", + "kitokią", + "kitokių", + "kitoks", + "kitokį", + "kitomis", + "kitoms", + "kitos", + "kitose", + "kitu", + "kituose", + "kitus", + "kitą", + "kitų", + "kodėl", + "kokia", + "kokiai", + "kokiais", + "kokiam", + "kokiame", + "kokias", + "kokie", + "kokiem", + "kokiems", + "kokio", + "kokioje", + "kokiomis", + "kokioms", + "kokios", + "kokiose", + "kokiu", + "kokiuose", + "kokius", + "kokią", + "kokių", + "koks", + "koksai", + "kokį", + "kol", + "kolei", + "kone", + "kuomet", + "kur", + "kurgi", + "kuri", + "kuria", + "kuriai", + "kuriais", + "kuriam", + "kuriame", + "kurias", + "kurie", + "kuriedvi", + "kuriem", + "kuriems", + "kurio", + "kurioje", + "kuriomis", + "kurioms", + "kurion", + "kurios", + "kuriose", + "kuris", + "kuriuo", + "kuriuodu", + "kuriuos", + "kuriuose", + "kurią", + "kurių", + "kurį", + "labanakt", + "labanaktis", + "labas", + "lai", + "lig", + "ligi", + "link", + "lyg", + "man", + "mana", + "manai", + "manais", + "manaisiais", + "manajai", + "manajam", + "manajame", + "manam", + "maname", + "manas", + "manasai", + "manasis", + "mane", + "mani", + "manieji", + "maniems", + "maniesiems", + "manim", + "manimi", + "maniške", + "maniškei", + "maniškes", + "maniškiai", + "maniškiais", + "maniškiam", + "maniškiame", + "maniškiams", + "maniškio", + "maniškis", + "maniškiu", + "maniškiuose", + "maniškius", + "maniškių", + "maniškė", + "maniškėje", + "maniškėmis", + "maniškėms", + "maniškės", + "maniškėse", + "maniškę", + "maniškį", + "mano", + "manoje", + "manoji", + "manojo", + "manojoje", + "manomis", + "manoms", + "manos", + "manose", + "manosiomis", + "manosioms", + "manosios", + "manosiose", + "manu", + "manuoju", + "manuose", + "manuosiuose", + "manuosius", + "manus", + "many", + "manyje", + "maną", + "manąja", + "manąją", + "manąjį", + "manąsias", + "manęs", + "manų", + "manųjų", + "mat", + "maždaug", + "mažne", + "mes", + "mudu", + "mudvi", + "mudviejose", + "mudviejuose", + "mudviejų", + "mudviem", + "mudviems", + "mumis", + "mums", + "mumyse", + "mus", + "mūs", + "mūsiške", + "mūsiškei", + "mūsiškes", + "mūsiškiai", + "mūsiškiais", + "mūsiškiam", + "mūsiškiame", + "mūsiškiams", + "mūsiškio", + "mūsiškis", + "mūsiškiu", + "mūsiškiuose", + "mūsiškius", + "mūsiškių", + "mūsiškė", + "mūsiškėje", + "mūsiškėmis", + "mūsiškėms", + "mūsiškės", + "mūsiškėse", + "mūsiškę", + "mūsiškį", + "mūsų", + "na", + "nagi", + "ne", + "nebe", + "nebent", + "negi", + "negu", + "nei", + "nejau", + "nejaugi", + "nekaip", + "nelyginant", + "nes", + "net", + "netgi", + "netoli", + "neva", + "niekatra", + "niekatrai", + "niekatrais", + "niekatram", + "niekatrame", + "niekatras", + "niekatrie", + "niekatriems", + "niekatro", + "niekatroje", + "niekatromis", + "niekatroms", + "niekatros", + "niekatrose", + "niekatruo", + "niekatruos", + "niekatruose", + "niekatrą", + "niekatrų", + "nors", + "nuo", + "nė", + "nėmaž", + "o", + "ogi", + "oho", + "oi", + "oj", + "op", + "opa", + "paeiliui", + "pagal", + "pagaliau", + "pakeliui", + "pala", + "palaipsniui", + "palei", + "pas", + "pasak", + "paskos", + "paskui", + "paskum", + "pat", + "pati", + "patiem", + "patiems", + "paties", + "pats", + "patsai", + "patys", + "patį", + "paukšt", + "pačia", + "pačiai", + "pačiais", + "pačiam", + "pačiame", + "pačias", + "pačioje", + "pačiomis", + "pačioms", + "pačios", + "pačiose", + "pačiu", + "pačiuose", + "pačius", + "pačią", + "pačių", + "per", + "pernelyg", + "pirm", + "pirma", + "pirmiau", + "pliumpt", + "plius", + "po", + "pokšt", + "prie", + "prieš", + "priešais", + "pro", + "pusiau", + "pykšt", + "rasi", + "rodos", + "sau", + "sava", + "savai", + "savais", + "savaisiais", + "savajai", + "savajam", + "savajame", + "savam", + "savame", + "savas", + "savasai", + "savasis", + "save", + "savi", + "savieji", + "saviems", + "saviesiems", + "savim", + "savimi", + "saviške", + "saviškei", + "saviškes", + "saviškiai", + "saviškiais", + "saviškiam", + "saviškiame", + "saviškiams", + "saviškio", + "saviškis", + "saviškiu", + "saviškiuose", + "saviškius", + "saviškių", + "saviškė", + "saviškėje", + "saviškėmis", + "saviškėms", + "saviškės", + "saviškėse", + "saviškę", + "saviškį", + "savo", + "savoje", + "savoji", + "savojo", + "savojoje", + "savomis", + "savoms", + "savos", + "savose", + "savosiomis", + "savosioms", + "savosios", + "savosiose", + "savu", + "savuoju", + "savuose", + "savuosiuose", + "savuosius", + "savus", + "savyj", + "savyje", + "savą", + "savąja", + "savąją", + "savąjį", + "savąsias", + "savęs", + "savų", + "savųjų", + "skersai", + "skradžiai", + "stačiai", + "strikt", + "strykt", + "stukt", + "su", + "sudie", + "sudieu", + "sudiev", + "sulig", + "ta", + "tad", + "tai", + "taigi", + "taip", + "taipogi", + "tais", + "taisiais", + "tajai", + "tajam", + "tajame", + "tam", + "tame", + "tamsta", + "tamstai", + "tamstas", + "tamstoje", + "tamstomis", + "tamstoms", + "tamstos", + "tamstose", + "tamstą", + "tamstų", + "tarp", + "tarsi", + "tartum", + "tarytum", + "tas", + "tasai", + "tau", + "tava", + "tavai", + "tavais", + "tavaisiais", + "tavajai", + "tavajam", + "tavajame", + "tavam", + "tavame", + "tavas", + "tavasai", + "tavasis", + "tave", + "tavi", + "tavieji", + "taviems", + "taviesiems", + "tavim", + "tavimi", + "taviške", + "taviškei", + "taviškes", + "taviškiai", + "taviškiais", + "taviškiam", + "taviškiame", + "taviškiams", + "taviškio", + "taviškis", + "taviškiu", + "taviškiuose", + "taviškius", + "taviškių", + "taviškė", + "taviškėje", + "taviškėmis", + "taviškėms", + "taviškės", + "taviškėse", + "taviškę", + "taviškį", + "tavo", + "tavoje", + "tavoji", + "tavojo", + "tavojoje", + "tavomis", + "tavoms", + "tavos", + "tavose", + "tavosiomis", + "tavosioms", + "tavosios", + "tavosiose", + "tavu", + "tavuoju", + "tavuose", + "tavuosiuose", + "tavuosius", + "tavus", + "tavy", + "tavyje", + "tavą", + "tavąja", + "tavąją", + "tavąjį", + "tavąsias", + "tavęs", + "tavų", + "tavųjų", + "tačiau", + "te", + "tegu", + "tegul", + "tie", + "tiedu", + "tiedvi", + "tiedviejose", + "tiedviejuose", + "tiedviejų", + "tiedviem", + "tiedviems", + "tieji", + "tiem", + "tiemdviem", + "tiems", + "ties", + "tiesiems", + "tiesiog", + "tik", + "tikriausiai", + "tiktai", + "to", + "todviem", + "toj", + "toje", + "toji", + "tojo", + "tojoje", + "tokia", + "tokiai", + "tokiais", + "tokiam", + "tokiame", + "tokias", + "tokie", + "tokiems", + "tokio", + "tokioje", + "tokiomis", + "tokioms", + "tokios", + "tokiose", + "tokiu", + "tokiuose", + "tokius", + "tokią", + "tokių", + "toks", + "toksai", + "tokį", + "tol", + "tolei", + "toliau", + "tom", + "tomdviem", + "tomis", + "toms", + "tos", + "tose", + "tosiomis", + "tosioms", + "tosios", + "tosiose", + "trakšt", + "trinkt", + "tu", + "tuo", + "tuodu", + "tuoju", + "tuos", + "tuose", + "tuosiuose", + "tuosius", + "turbūt", + "tą", + "tąja", + "tąją", + "tąjį", + "tąsias", + "tąįį", + "tūla", + "tūlai", + "tūlais", + "tūlam", + "tūlame", + "tūlas", + "tūli", + "tūliems", + "tūlo", + "tūloje", + "tūlomis", + "tūloms", + "tūlos", + "tūlose", + "tūlu", + "tūluose", + "tūlus", + "tūlą", + "tūlų", + "tų", + "tųdviejų", + "tųjų", + "ui", + "už", + "užtat", + "užuot", + "užvis", + "va", + "vai", + "valio", + "vau", + "viduj", + "vidury", + "vien", + "vienas", + "vienokia", + "vienoks", + "vietoj", + "virš", + "viršuj", + "viršum", + "vis", + "visa", + "visas", + "visgi", + "visokia", + "visoks", + "vos", + "vėl", + "vėlgi", + "y", + "ypač", + "čikšt", + "činkšt", + "ėgi", + "į", + "įkypai", + "įstrižai", + "ša", + "šalia", + "šast", + "še", + "šekit", + "ši", + "šia", + "šiai", + "šiaipjau", + "šiais", + "šiaisiais", + "šiajai", + "šiajam", + "šiajame", + "šiam", + "šiame", + "šiapus", + "šias", + "šie", + "šiedu", + "šiedvi", + "šiedviejose", + "šiedviejuose", + "šiedviejų", + "šiedviem", + "šiedviems", + "šieji", + "šiemdviem", + "šiems", + "šiesiems", + "šio", + "šiodviem", + "šioje", + "šioji", + "šiojo", + "šiojoje", + "šiokia", + "šiokiai", + "šiokiais", + "šiokiam", + "šiokiame", + "šiokias", + "šiokie", + "šiokiems", + "šiokio", + "šiokioje", + "šiokiomis", + "šiokioms", + "šiokios", + "šiokiose", + "šiokiu", + "šiokiuose", + "šiokius", + "šiokią", + "šiokių", + "šioks", + "šioksai", + "šiokį", + "šiomdviem", + "šiomis", + "šioms", + "šios", + "šiose", + "šiosiomis", + "šiosioms", + "šiosios", + "šiosiose", + "šis", + "šisai", + "šit", + "šita", + "šitai", + "šitais", + "šitaisiais", + "šitajai", + "šitajam", + "šitajame", + "šitam", + "šitame", + "šitas", + "šitasai", + "šitie", + "šitiedvi", + "šitiedviem", + "šitieji", + "šitiem", + "šitiemdviem", + "šitiems", + "šitiesiems", + "šito", + "šitodviem", + "šitoj", + "šitoje", + "šitoji", + "šitojo", + "šitojoje", + "šitokia", + "šitokiai", + "šitokiais", + "šitokiam", + "šitokiame", + "šitokias", + "šitokie", + "šitokiems", + "šitokio", + "šitokioje", + "šitokiomis", + "šitokioms", + "šitokios", + "šitokiose", + "šitokiu", + "šitokiuose", + "šitokius", + "šitokią", + "šitokių", + "šitoks", + "šitoksai", + "šitokį", + "šitom", + "šitomdviem", + "šitomis", + "šitoms", + "šitos", + "šitose", + "šitosiomis", + "šitosioms", + "šitosios", + "šitosiose", + "šituo", + "šituodu", + "šituoju", + "šituos", + "šituose", + "šituosiuose", + "šituosius", + "šitą", + "šitąja", + "šitąją", + "šitąsias", + "šitų", + "šitųdviejų", + "šitųjų", + "šiuo", + "šiuodu", + "šiuoju", + "šiuos", + "šiuose", + "šiuosiuose", + "šiuosius", + "šią", + "šiąja", + "šiąją", + "šiąsias", + "šių", + "šiųdviejų", + "šiųjų", + "škac", + "škic", + "šlept", + "šmurkšt", + "štai", + "šį", + "šįjį", + "žemiau", + "žūtbūt", +} diff --git a/spacy/lang/lt/tokenizer_exceptions.py b/spacy/lang/lt/tokenizer_exceptions.py new file mode 100644 index 0000000..d39b86d --- /dev/null +++ b/spacy/lang/lt/tokenizer_exceptions.py @@ -0,0 +1,14 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for orth in ["n-tosios", "?!"]: + _exc[orth] = [{ORTH: orth}] + +mod_base_exceptions = { + exc: val for exc, val in BASE_EXCEPTIONS.items() if not exc.endswith(".") +} +del mod_base_exceptions["8)"] +TOKENIZER_EXCEPTIONS = update_exc(mod_base_exceptions, _exc) diff --git a/spacy/lang/lv/__init__.py b/spacy/lang/lv/__init__.py new file mode 100644 index 0000000..fdfca5e --- /dev/null +++ b/spacy/lang/lv/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class LatvianDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Latvian(Language): + lang = "lv" + Defaults = LatvianDefaults + + +__all__ = ["Latvian"] diff --git a/spacy/lang/lv/stop_words.py b/spacy/lang/lv/stop_words.py new file mode 100644 index 0000000..2685c24 --- /dev/null +++ b/spacy/lang/lv/stop_words.py @@ -0,0 +1,167 @@ +# Source: https://github.com/stopwords-iso/stopwords-lv + +STOP_WORDS = set( + """ +aiz +ap +apakš +apakšpus +ar +arī +augšpus +bet +bez +bija +biji +biju +bijām +bijāt +būs +būsi +būsiet +būsim +būt +būšu +caur +diemžēl +diezin +droši +dēļ +esam +esat +esi +esmu +gan +gar +iekam +iekams +iekām +iekāms +iekš +iekšpus +ik +ir +it +itin +iz +ja +jau +jeb +jebšu +jel +jo +jā +ka +kamēr +kaut +kolīdz +kopš +kā +kļuva +kļuvi +kļuvu +kļuvām +kļuvāt +kļūs +kļūsi +kļūsiet +kļūsim +kļūst +kļūstam +kļūstat +kļūsti +kļūstu +kļūt +kļūšu +labad +lai +lejpus +līdz +līdzko +ne +nebūt +nedz +nekā +nevis +nezin +no +nu +nē +otrpus +pa +par +pat +pie +pirms +pret +priekš +pār +pēc +starp +tad +tak +tapi +taps +tapsi +tapsiet +tapsim +tapt +tapāt +tapšu +taču +te +tiec +tiek +tiekam +tiekat +tieku +tik +tika +tikai +tiki +tikko +tiklab +tiklīdz +tiks +tiksiet +tiksim +tikt +tiku +tikvien +tikām +tikāt +tikšu +tomēr +topat +turpretim +turpretī +tā +tādēļ +tālab +tāpēc +un +uz +vai +var +varat +varēja +varēji +varēju +varējām +varējāt +varēs +varēsi +varēsiet +varēsim +varēt +varēšu +vien +virs +virspus +vis +viņpus +zem +ārpus +šaipus +""".split() +) diff --git a/spacy/lang/mk/__init__.py b/spacy/lang/mk/__init__.py new file mode 100644 index 0000000..4414487 --- /dev/null +++ b/spacy/lang/mk/__init__.py @@ -0,0 +1,57 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...attrs import LANG +from ...language import BaseDefaults, Language +from ...lookups import Lookups +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .lemmatizer import MacedonianLemmatizer +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class MacedonianDefaults(BaseDefaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters[LANG] = lambda text: "mk" + + # Optional: replace flags with custom functions, e.g. like_num() + lex_attr_getters.update(LEX_ATTRS) + + # Merge base exceptions and custom tokenizer exceptions + tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) + stop_words = STOP_WORDS + + +class Macedonian(Language): + lang = "mk" + Defaults = MacedonianDefaults + + +@Macedonian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return MacedonianLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Macedonian"] diff --git a/spacy/lang/mk/lemmatizer.py b/spacy/lang/mk/lemmatizer.py new file mode 100644 index 0000000..f5a5eca --- /dev/null +++ b/spacy/lang/mk/lemmatizer.py @@ -0,0 +1,58 @@ +from collections import OrderedDict +from typing import List + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class MacedonianLemmatizer(Lemmatizer): + def rule_lemmatize(self, token: Token) -> List[str]: + string = token.text + univ_pos = token.pos_.lower() + + if univ_pos in ("", "eol", "space"): + return [string.lower()] + + if string[-3:] == "јќи": + string = string[:-3] + univ_pos = "verb" + + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + if not any( + ( + index_table.get(univ_pos), + exc_table.get(univ_pos), + rules_table.get(univ_pos), + ) + ): + if univ_pos == "propn": + return [string] + else: + return [string.lower()] + + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, []) + + orig = string + string = string.lower() + forms = [] + + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + continue + if form in index or not form.isalpha(): + forms.append(form) + + forms = list(OrderedDict.fromkeys(forms)) + for form in exceptions.get(string, []): + if form not in forms: + forms.insert(0, form) + if not forms: + forms.append(orig) + + return forms diff --git a/spacy/lang/mk/lex_attrs.py b/spacy/lang/mk/lex_attrs.py new file mode 100644 index 0000000..1d8f631 --- /dev/null +++ b/spacy/lang/mk/lex_attrs.py @@ -0,0 +1,138 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "нула", + "еден", + "една", + "едно", + "два", + "две", + "три", + "четири", + "пет", + "шест", + "седум", + "осум", + "девет", + "десет", + "единаесет", + "дванаесет", + "тринаесет", + "четиринаесет", + "петнаесет", + "шеснаесет", + "седумнаесет", + "осумнаесет", + "деветнаесет", + "дваесет", + "триесет", + "четириесет", + "педесет", + "шеесет", + "седумдесет", + "осумдесет", + "деведесет", + "сто", + "двесте", + "триста", + "четиристотини", + "петстотини", + "шестотини", + "седумстотини", + "осумстотини", + "деветстотини", + "илјада", + "илјади", + "милион", + "милиони", + "милијарда", + "милијарди", + "билион", + "билиони", + "двајца", + "тројца", + "четворица", + "петмина", + "шестмина", + "седуммина", + "осуммина", + "деветмина", + "обата", + "обајцата", + "прв", + "втор", + "трет", + "четврт", + "седм", + "осм", + "двестоти", + "два-три", + "два-триесет", + "два-триесетмина", + "два-тринаесет", + "два-тројца", + "две-три", + "две-тристотини", + "пет-шеесет", + "пет-шеесетмина", + "пет-шеснаесетмина", + "пет-шест", + "пет-шестмина", + "пет-шестотини", + "петина", + "осмина", + "седум-осум", + "седум-осумдесет", + "седум-осуммина", + "седум-осумнаесет", + "седум-осумнаесетмина", + "три-четириесет", + "три-четиринаесет", + "шеесет", + "шеесетина", + "шеесетмина", + "шеснаесет", + "шеснаесетмина", + "шест-седум", + "шест-седумдесет", + "шест-седумнаесет", + "шест-седумстотини", + "шестоти", + "шестотини", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + + text_lower = text.lower() + if text_lower in _num_words: + return True + + if text_lower.endswith(("а", "о", "и")): + if text_lower[:-1] in _num_words: + return True + + if text_lower.endswith(("ти", "та", "то", "на")): + if text_lower[:-2] in _num_words: + return True + + if text_lower.endswith(("ата", "иот", "ите", "ина", "чки")): + if text_lower[:-3] in _num_words: + return True + + if text_lower.endswith(("мина", "тина")): + if text_lower[:-4] in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/mk/stop_words.py b/spacy/lang/mk/stop_words.py new file mode 100644 index 0000000..312a456 --- /dev/null +++ b/spacy/lang/mk/stop_words.py @@ -0,0 +1,815 @@ +STOP_WORDS = set( + """ +а +абре +aв +аи +ако +алало +ам +ама +аман +ами +амин +априли-ли-ли +ау +аух +ауч +ах +аха +аха-ха +аш +ашколсум +ашколсун +ај +ајде +ајс +аџаба +бавно +бам +бам-бум +бап +бар +баре +барем +бау +бау-бау +баш +бај +бе +беа +бев +бевме +бевте +без +безбели +бездруго +белки +беше +би +бидејќи +бим +бис +бла +блазе +богами +божем +боц +браво +бравос +бре +бреј +брзо +бришка +бррр +бу +бум +буф +буц +бујрум +ваа +вам +варај +варда +вас +вај +ве +велат +вели +версус +веќе +ви +виа +види +вие +вистина +витос +внатре +во +воз +вон +впрочем +врв +вред +време +врз +всушност +втор +галиба +ги +гитла +го +годе +годишник +горе +гра +гуц +гљу +да +даан +дава +дал +дали +дан +два +дваесет +дванаесет +двајца +две +двесте +движам +движат +движи +движиме +движите +движиш +де +деведесет +девет +деветнаесет +деветстотини +деветти +дека +дел +делми +демек +десет +десетина +десетти +деситици +дејгиди +дејди +ди +дилми +дин +дип +дно +до +доволно +додека +додуша +докај +доколку +доправено +доправи +досамоти +доста +држи +дрн +друг +друга +другата +други +другиот +другите +друго +другото +дум +дур +дури +е +евала +еве +евет +ега +егиди +еден +едикојси +единаесет +единствено +еднаш +едно +ексик +ела +елбете +елем +ели +ем +еми +ене +ете +еурека +ех +еј +жими +жити +за +завал +заврши +зад +задека +задоволна +задржи +заедно +зар +зарад +заради +заре +зарем +затоа +зашто +згора +зема +земе +земува +зер +значи +зошто +зуј +и +иако +из +извезен +изгледа +измеѓу +износ +или +или-или +илјада +илјади +им +има +имаа +имаат +имавме +имавте +имам +имаме +имате +имаш +имаше +име +имено +именува +имплицира +имплицираат +имплицирам +имплицираме +имплицирате +имплицираш +инаку +индицира +исечок +исклучен +исклучена +исклучени +исклучено +искористен +искористена +искористени +искористено +искористи +искрај +исти +исто +итака +итн +их +иха +ихуу +иш +ишала +иј +ка +каде +кажува +како +каков +камоли +кај +ква +ки +кит +кло +клум +кога +кого +кого-годе +кое +кои +количество +количина +колку +кому +кон +користена +користени +користено +користи +кот +котрр +кош-кош +кој +која +којзнае +којшто +кр-кр-кр +крај +крек +крз +крк +крц +куку +кукуригу +куш +ле +лебами +леле +лели +ли +лиду +луп +ма +макар +малку +марш +мат +мац +машала +ме +мене +место +меѓу +меѓувреме +меѓутоа +ми +мое +може +можеби +молам +моли +мор +мора +море +мори +мразец +му +муклец +мутлак +муц +мјау +на +навидум +навистина +над +надвор +назад +накај +накрај +нали +нам +наместо +наоколу +направено +направи +напред +нас +наспоред +наспрема +наспроти +насред +натаму +натема +начин +наш +наша +наше +наши +нај +најдоцна +најмалку +најмногу +не +неа +него +негов +негова +негови +негово +незе +нека +некаде +некако +некаков +некого +некое +некои +неколку +некому +некој +некојси +нели +немој +нему +неоти +нечиј +нешто +нејзе +нејзин +нејзини +нејзино +нејсе +ни +нив +нивен +нивна +нивни +нивно +ние +низ +никаде +никако +никогаш +никого +никому +никој +ним +нити +нито +ниту +ничиј +ништо +но +нѐ +о +обр +ова +ова-она +оваа +овај +овде +овега +овие +овој +од +одавде +оди +однесува +односно +одошто +околу +олеле +олкацок +он +она +онаа +онака +онаков +онде +они +оние +оно +оној +оп +освем +освен +осем +осми +осум +осумдесет +осумнаесет +осумстотитни +отаде +оти +откако +откај +откога +отколку +оттаму +оттука +оф +ох +ој +па +пак +папа +пардон +пате-ќуте +пати +пау +паче +пеесет +пеки +пет +петнаесет +петстотини +петти +пи +пи-пи +пис +плас +плус +по +побавно +поблиску +побрзо +побуни +повеќе +повторно +под +подалеку +подолу +подоцна +подруго +позади +поинаква +поинакви +поинакво +поинаков +поинаку +покаже +покажува +покрај +полно +помалку +помеѓу +понатаму +понекогаш +понекој +поради +поразличен +поразлична +поразлични +поразлично +поседува +после +последен +последна +последни +последно +поспоро +потег +потоа +пошироко +прави +празно +прв +пред +през +преку +претежно +претходен +претходна +претходни +претходник +претходно +при +присвои +притоа +причинува +пријатно +просто +против +прр +пст +пук +пусто +пуф +пуј +пфуј +пшт +ради +различен +различна +различни +различно +разни +разоружен +разредлив +рамките +рамнообразно +растревожено +растреперено +расчувствувано +ратоборно +рече +роден +с +сакан +сам +сама +сами +самите +само +самоти +свое +свои +свој +своја +се +себе +себеси +сега +седми +седум +седумдесет +седумнаесет +седумстотини +секаде +секаков +секи +секогаш +секого +секому +секој +секојдневно +сем +сенешто +сепак +сериозен +сериозна +сериозни +сериозно +сет +сечиј +сешто +си +сиктер +сиот +сип +сиреч +сите +сичко +скок +скоро +скрц +следбеник +следбеничка +следен +следователно +следствено +сме +со +соне +сопствен +сопствена +сопствени +сопствено +сосе +сосем +сполај +според +споро +спрема +спроти +спротив +сред +среде +среќно +срочен +сст +става +ставаат +ставам +ставаме +ставате +ставаш +стави +сте +сто +стоп +страна +сум +сума +супер +сус +сѐ +та +таа +така +таква +такви +таков +тамам +таму +тангар-мангар +тандар-мандар +тап +твое +те +тебе +тебека +тек +текот +ти +тие +тизе +тик-так +тики +тоа +тогаш +тој +трак +трака-трука +трас +треба +трет +три +триесет +тринаест +триста +труп +трупа +трус +ту +тука +туку +тукушто +туф +у +уа +убаво +уви +ужасно +уз +ура +уу +уф +уха +уш +уште +фазен +фала +фил +филан +фис +фиу +фиљан +фоб +фон +ха +ха-ха +хе +хеј +хеј +хи +хм +хо +цак +цап +целина +цело +цигу-лигу +циц +чекај +често +четврт +четири +четириесет +четиринаесет +четирстотини +чие +чии +чик +чик-чирик +чини +чиш +чиј +чија +чијшто +чкрап +чому +чук +чукш +чуму +чунки +шеесет +шеснаесет +шест +шести +шестотини +ширум +шлак +шлап +шлапа-шлупа +шлуп +шмрк +што +штогоде +штом +штотуку +штрак +штрап +штрап-штруп +шуќур +ѓиди +ѓоа +ѓоамити +ѕан +ѕе +ѕин +ја +јадец +јазе +јали +јас +јаска +јок +ќе +ќешки +ѝ +џагара-магара +џанам +џив-џив + """.split() +) diff --git a/spacy/lang/mk/tokenizer_exceptions.py b/spacy/lang/mk/tokenizer_exceptions.py new file mode 100644 index 0000000..40f2c1d --- /dev/null +++ b/spacy/lang/mk/tokenizer_exceptions.py @@ -0,0 +1,93 @@ +from ...symbols import NORM, ORTH + +_exc = {} + + +_abbr_exc = [ + {ORTH: "м", NORM: "метар"}, + {ORTH: "мм", NORM: "милиметар"}, + {ORTH: "цм", NORM: "центиметар"}, + {ORTH: "см", NORM: "сантиметар"}, + {ORTH: "дм", NORM: "дециметар"}, + {ORTH: "км", NORM: "километар"}, + {ORTH: "кг", NORM: "килограм"}, + {ORTH: "дкг", NORM: "декаграм"}, + {ORTH: "дг", NORM: "дециграм"}, + {ORTH: "мг", NORM: "милиграм"}, + {ORTH: "г", NORM: "грам"}, + {ORTH: "т", NORM: "тон"}, + {ORTH: "кл", NORM: "килолитар"}, + {ORTH: "хл", NORM: "хектолитар"}, + {ORTH: "дкл", NORM: "декалитар"}, + {ORTH: "л", NORM: "литар"}, + {ORTH: "дл", NORM: "децилитар"}, +] +for abbr in _abbr_exc: + _exc[abbr[ORTH]] = [abbr] + +_abbr_line_exc = [ + {ORTH: "д-р", NORM: "доктор"}, + {ORTH: "м-р", NORM: "магистер"}, + {ORTH: "г-ѓа", NORM: "госпоѓа"}, + {ORTH: "г-ца", NORM: "госпоѓица"}, + {ORTH: "г-дин", NORM: "господин"}, +] + +for abbr in _abbr_line_exc: + _exc[abbr[ORTH]] = [abbr] + +_abbr_dot_exc = [ + {ORTH: "в.", NORM: "век"}, + {ORTH: "в.д.", NORM: "вршител на должност"}, + {ORTH: "г.", NORM: "година"}, + {ORTH: "г.г.", NORM: "господин господин"}, + {ORTH: "м.р.", NORM: "машки род"}, + {ORTH: "год.", NORM: "женски род"}, + {ORTH: "с.р.", NORM: "среден род"}, + {ORTH: "н.е.", NORM: "наша ера"}, + {ORTH: "о.г.", NORM: "оваа година"}, + {ORTH: "о.м.", NORM: "овој месец"}, + {ORTH: "с.", NORM: "село"}, + {ORTH: "т.", NORM: "точка"}, + {ORTH: "т.е.", NORM: "то ест"}, + {ORTH: "т.н.", NORM: "таканаречен"}, + {ORTH: "бр.", NORM: "број"}, + {ORTH: "гр.", NORM: "град"}, + {ORTH: "др.", NORM: "другар"}, + {ORTH: "и др.", NORM: "и друго"}, + {ORTH: "и сл.", NORM: "и слично"}, + {ORTH: "кн.", NORM: "книга"}, + {ORTH: "мн.", NORM: "множина"}, + {ORTH: "на пр.", NORM: "на пример"}, + {ORTH: "св.", NORM: "свети"}, + {ORTH: "сп.", NORM: "списание"}, + {ORTH: "с.", NORM: "страница"}, + {ORTH: "стр.", NORM: "страница"}, + {ORTH: "чл.", NORM: "член"}, + {ORTH: "арх.", NORM: "архитект"}, + {ORTH: "бел.", NORM: "белешка"}, + {ORTH: "гимн.", NORM: "гимназија"}, + {ORTH: "ден.", NORM: "денар"}, + {ORTH: "ул.", NORM: "улица"}, + {ORTH: "инж.", NORM: "инженер"}, + {ORTH: "проф.", NORM: "професор"}, + {ORTH: "студ.", NORM: "студент"}, + {ORTH: "бот.", NORM: "ботаника"}, + {ORTH: "мат.", NORM: "математика"}, + {ORTH: "мед.", NORM: "медицина"}, + {ORTH: "прил.", NORM: "прилог"}, + {ORTH: "прид.", NORM: "придавка"}, + {ORTH: "сврз.", NORM: "сврзник"}, + {ORTH: "физ.", NORM: "физика"}, + {ORTH: "хем.", NORM: "хемија"}, + {ORTH: "пр. н.", NORM: "природни науки"}, + {ORTH: "истор.", NORM: "историја"}, + {ORTH: "геогр.", NORM: "географија"}, + {ORTH: "литер.", NORM: "литература"}, +] + +for abbr in _abbr_dot_exc: + _exc[abbr[ORTH]] = [abbr] + + +TOKENIZER_EXCEPTIONS = _exc diff --git a/spacy/lang/ml/__init__.py b/spacy/lang/ml/__init__.py new file mode 100644 index 0000000..0b17b8a --- /dev/null +++ b/spacy/lang/ml/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class MalayalamDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Malayalam(Language): + lang = "ml" + Defaults = MalayalamDefaults + + +__all__ = ["Malayalam"] diff --git a/spacy/lang/ml/examples.py b/spacy/lang/ml/examples.py new file mode 100644 index 0000000..d067b8b --- /dev/null +++ b/spacy/lang/ml/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ml.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "അനാവശ്യമായി കണ്ണിലും മൂക്കിലും വായിലും സ്പർശിക്കാതിരിക്കുക", + "പൊതുരംഗത്ത് മലയാള ഭാഷയുടെ സമഗ്രപുരോഗതി ലക്ഷ്യമാക്കി പ്രവർത്തിക്കുന്ന സംഘടനയായ മലയാളഐക്യവേദിയുടെ വിദ്യാർത്ഥിക്കൂട്ടായ്മയാണ് വിദ്യാർത്ഥി മലയാളവേദി", + "എന്താണ്‌ കവാടങ്ങൾ?", + "ചുരുക്കത്തിൽ വിക്കിപീഡിയയുടെ ഉള്ളടക്കത്തിലേക്കുള്ള പടിപ്പുരകളാണ്‌‌ കവാടങ്ങൾ. അവ ലളിതവും വായനക്കാരനെ ആകർഷിക്കുന്നതുമായിരിക്കും", + "പതിനൊന്നുപേർ വീതമുള്ള രണ്ടു ടീമുകൾ കളിക്കുന്ന സംഘകായിക വിനോദമാണു ക്രിക്കറ്റ്", +] diff --git a/spacy/lang/ml/lex_attrs.py b/spacy/lang/ml/lex_attrs.py new file mode 100644 index 0000000..33a144f --- /dev/null +++ b/spacy/lang/ml/lex_attrs.py @@ -0,0 +1,76 @@ +from ...attrs import LIKE_NUM + +# reference 2: https://www.omniglot.com/language/numbers/malayalam.htm + +_num_words = [ + "പൂജ്യം ", + "ഒന്ന് ", + "രണ്ട് ", + "മൂന്ന് ", + "നാല്‌ ", + "അഞ്ച് ", + "ആറ് ", + "ഏഴ് ", + "എട്ട് ", + "ഒന്‍പത് ", + "പത്ത് ", + "പതിനൊന്ന്", + "പന്ത്രണ്ട്", + "പതി മൂന്നു", + "പതിനാല്", + "പതിനഞ്ച്", + "പതിനാറ്", + "പതിനേഴ്", + "പതിനെട്ട്", + "പത്തൊമ്പതു", + "ഇരുപത്", + "ഇരുപത്തിഒന്ന്", + "ഇരുപത്തിരണ്ട്‌", + "ഇരുപത്തിമൂന്ന്", + "ഇരുപത്തിനാല്", + "ഇരുപത്തിഅഞ്ചു", + "ഇരുപത്തിആറ്", + "ഇരുപത്തിഏഴ്", + "ഇരുപത്തിഎട്ടു", + "ഇരുപത്തിഒന്‍പത്", + "മുപ്പത്", + "മുപ്പത്തിഒന്ന്", + "മുപ്പത്തിരണ്ട്", + "മുപ്പത്തിമൂന്ന്", + "മുപ്പത്തിനാല്", + "മുപ്പത്തിഅഞ്ചു", + "മുപ്പത്തിആറ്", + "മുപ്പത്തിഏഴ്", + "മുപ്പത്തിഎട്ട്", + "മുപ്പത്തിഒന്‍പതു", + "നാല്‍പത്‌ ", + "അന്‍പത് ", + "അറുപത് ", + "എഴുപത് ", + "എണ്‍പത് ", + "തൊണ്ണൂറ് ", + "നുറ് ", + "ആയിരം ", + "പത്തുലക്ഷം", +] + + +def like_num(text): + """ + Check if text resembles a number + """ + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ml/stop_words.py b/spacy/lang/ml/stop_words.py new file mode 100644 index 0000000..441e935 --- /dev/null +++ b/spacy/lang/ml/stop_words.py @@ -0,0 +1,13 @@ +STOP_WORDS = set( + """ +അത് +ഇത് +ആയിരുന്നു +ആകുന്നു +വരെ +അന്നേരം +അന്ന് +ഇന്ന് +ആണ് +""".split() +) diff --git a/spacy/lang/mr/__init__.py b/spacy/lang/mr/__init__.py new file mode 100644 index 0000000..f980efb --- /dev/null +++ b/spacy/lang/mr/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class MarathiDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Marathi(Language): + lang = "mr" + Defaults = MarathiDefaults + + +__all__ = ["Marathi"] diff --git a/spacy/lang/mr/stop_words.py b/spacy/lang/mr/stop_words.py new file mode 100644 index 0000000..9b0cee9 --- /dev/null +++ b/spacy/lang/mr/stop_words.py @@ -0,0 +1,192 @@ +# Source: https://github.com/stopwords-iso/stopwords-mr/blob/master/stopwords-mr.txt, https://github.com/6/stopwords-json/edit/master/dist/mr.json +STOP_WORDS = set( + """ +न +अतरी +तो +हें +तें +कां +आणि +जें +जे +मग +ते +मी +जो +परी +गा +हे +ऐसें +आतां +नाहीं +तेथ +हा +तया +असे +म्हणे +काय +कीं +जैसें +तंव +तूं +होय +जैसा +आहे +पैं +तैसा +जरी +म्हणोनि +एक +ऐसा +जी +ना +मज +एथ +या +जेथ +जया +तुज +तेणें +तैं +पां +असो +करी +ऐसी +येणें +जाहला +तेंचि +आघवें +होती +कांहीं +होऊनि +एकें +मातें +ठायीं +ये +सकळ +केलें +जेणें +जाण +जैसी +होये +जेवीं +एऱ्हवीं +मीचि +किरीटी +दिसे +देवा +हो +तरि +कीजे +तैसे +आपण +तिये +कर्म +नोहे +इये +पडे +माझें +तैसी +लागे +नाना +जंव +कीर +अधिक +अनेक +अशी +असलयाचे +असलेल्या +असा +असून +असे +आज +आणि +आता +आपल्या +आला +आली +आले +आहे +आहेत +एक +एका +कमी +करणयात +करून +का +काम +काय +काही +किवा +की +केला +केली +केले +कोटी +गेल्या +घेऊन +जात +झाला +झाली +झाले +झालेल्या +टा +तर +तरी +तसेच +ता +ती +तीन +ते +तो +त्या +त्याचा +त्याची +त्याच्या +त्याना +त्यानी +त्यामुळे +त्री +दिली +दोन +न +पण +पम +परयतन +पाटील +म +मात्र +माहिती +मी +मुबी +म्हणजे +म्हणाले +म्हणून +या +याचा +याची +याच्या +याना +यानी +येणार +येत +येथील +येथे +लाख +व +व्यकत +सर्व +सागित्ले +सुरू +हजार +हा +ही +हे +होणार +होत +होता +होती +होते +""".split() +) diff --git a/spacy/lang/ms/__init__.py b/spacy/lang/ms/__init__.py new file mode 100644 index 0000000..f53ebfc --- /dev/null +++ b/spacy/lang/ms/__init__.py @@ -0,0 +1,24 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class MalayDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + syntax_iterators = SYNTAX_ITERATORS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Malay(Language): + lang = "ms" + Defaults = MalayDefaults + + +__all__ = ["Malay"] diff --git a/spacy/lang/ms/_tokenizer_exceptions_list.py b/spacy/lang/ms/_tokenizer_exceptions_list.py new file mode 100644 index 0000000..fba1dd7 --- /dev/null +++ b/spacy/lang/ms/_tokenizer_exceptions_list.py @@ -0,0 +1,1943 @@ +# from https://prpm.dbp.gov.my/cari1?keyword= +# dbp https://en.wikipedia.org/wiki/Dewan_Bahasa_dan_Pustaka +MS_BASE_EXCEPTIONS = set( + """ +aba-aba +abah-abah +abar-abar +abrit-abritan +abu-abu +abuk-abuk +abun-abun +acak-acak +acak-acakan +acang-acang +aci-aci +aci-acian +aci-acinya +adang-adang +adap-adapan +adik-beradik +aduk-adukan +agak-agak +agar-agar +agut-agut +air-cooled +ajar-ajar +aji-aji +akal-akal +akhir-akhir +aki-aki +alah-mengalahi +alan-alan +alang-alang +alang-alangan +alap-alap +ali-ali +alih-alih +aling-aling +aling-alingan +alip-alipan +alon-alon +alu-alu +alu-aluan +alun-alun +alur-alur +ambah-ambah +ambai-ambai +ambil-mengambil +ambring-ambringan +ambu-ambu +ambung-ambung +amin-amin +ampai-ampai +amung-amung +anai-anai +anak-anak +anak-anakan +anak-beranak +ancak-ancak +ancang-ancang +andang-andang +angan-angan +anggar-anggar +angin-angin +angin-anginan +angkul-angkul +angkup-angkup +angkut-angkut +ani-ani +aning-aning +anjang-anjang +anjing-anjing +anjung-anjung +anjung-anjungan +antar-antar +ante-mortem +anting-anting +antung-antung +anyam-menganyam +apa-apa +api-api +apit-apit +aprit-apritan +arah-arah +arak-arakan +aram-aram +ari-ari +aru-aru +asa-asaan +asam-asaman +asuh-asuh +atas-mengatasi +ati-ati +audio-visual +avant-garde +awang-awang +awang-gemawang +ayak-ayak +ayam-ayam +ayam-ayaman +ayang-ayang +ayeng-ayengan +ayun-temayun +back-up +bahu-membahu +baik-baik +bajang-bajang +baji-baji +balai-balai +balam-balam +balas-membalas +baling-baling +balut-balut +bangun-bangun +bantal-bantal +barat-barat +barau-barau +bari-bari +barung-barung +basa-basi +bata-bata +batir-batir +bau-bauan +bayang-bayang +bedil-bedal +begana-begini +bekal-bekalan +belat-belit +belu-belai +benggal-benggil +bengkal-bengkil +bengkang-bengkok +bengkang-bengkong +berabad-abad +berabun-rabun +berada-ada +beragah-agah +beragak-agak +beragam-ragam +beraja-raja +berakit-rakit +beraku-akuan +beralun-alun +beramah-ramahan +beramah-tamah +beramai-ramai +berambai-ambai +berambal-ambalan +beramuk-amukan +berandai-andai +berandai-randai +berang-berang +berangan-angan +beranggap-anggapan +berangguk-angguk +berangin-angin +berangka-angka +berangka-angkaan +berangkai-rangkai +beranja-anja +berantai-rantai +berapi-api +berapung-apung +berarak-arakan +beras-beras +berasing-asingan +beratus-ratus +berawas-awas +berayal-ayalan +berayun-ayun +berbagai-bagai +berbahas-bahasan +berbalas-balasan +berbalik-balik +berbanjar-banjar +berbantah-bantah +berbanyak-banyak +berbarik-barik +berbasah-basah +berbatu-batu +berbayang-bayang +berbecak-becak +berbedil-bedilan +berbeka-beka +berbelakang-belakangan +berbelang-belang +berbeli-belian +berbelit-belit +berbelok-belok +berbenar-benar +berbencah-bencah +berbesar-besar +berbidai-bidai +berbiku-biku +berbilik-bilik +berbinar-binar +berbincang-bincang +berbingkah-bingkah +berbintang-bintang +berbintik-bintik +berbintil-bintil +berbisik-bisik +berbolak-balik +berbolong-bolong +berbondong-bondong +berbongkah-bongkah +berbuai-buai +berbual-bual +berbukit-bukit +berbulan-bulan +berbunga-bunga +berbuntut-buntut +berbunuh-bunuhan +berburu-buru +berburuk-buruk +berbutir-butir +bercabang-cabang +bercaci-cacian +bercakap-cakap +bercakar-cakaran +bercantik-cantik +bercari-cari +bercari-carian +bercarik-carik +bercepat-cepat +bercerai-berai +bercerai-cerai +bercetai-cetai +bercikun-cikun +bercinta-cintaan +bercita-cita +berciut-ciut +berconteng-conteng +bercoreng-coreng +bercoreng-moreng +bercuit-cuit +bercumbu-cumbu +bercumbu-cumbuan +bercura-bura +bercura-cura +berdada-dadaan +berdahulu-dahuluan +berdalam-dalam +berdebar-debar +berdecap-decap +berdedai-dedai +berdegap-degap +berdegar-degar +berdeham-deham +berdekah-dekah +berdekat-dekat +berdelat-delat +berdembun-dembun +berdempang-dempang +berdendam-dendaman +berdengkang-dengkang +berdentang-dentang +berdentum-dentum +berdentung-dentung +berdepak-depak +berdepan-depan +berderai-derai +berderak-derak +berderau-derau +berdering-dering +berderung-derung +berdesak-desakan +berdesing-desing +berdesus-desus +berdikit-dikit +berdingkit-dingkit +berdua-dua +berduri-duri +berduru-duru +berduyun-duyun +berebut-rebut +berebut-rebutan +beregang-regang +berek-berek +berembut-rembut +berempat-empat +berenak-enak +berenteng-renteng +beresah-resah +berfoya-foya +bergagah-gagahan +bergagap-gagap +bergalur-galur +berganda-ganda +berganti-ganti +bergarah-garah +bergaruk-garuk +bergegas-gegas +bergelang-gelang +bergelap-gelap +bergelas-gelasan +bergeleng-geleng +bergemal-gemal +bergembut-gembut +bergerek-gerek +bergesa-gesa +bergilir-gilir +bergolek-golek +bergores-gores +bergotong-royong +bergugus-gugus +bergulung-gulung +bergulut-gulut +bergumpal-gumpal +bergunung-gunung +berhadap-hadapan +berhamun-hamun +berhandai-handai +berhanyut-hanyut +berhari-hari +berhati-hati +berhilau-hilau +berhujan-hujan +beria-ia +beria-ria +beriak-riak +beribu-ribu +berigi-rigi +bering-bering +beringat-ingat +beringgit-ringgit +berintik-rintik +beriring-iring +beriring-iringan +berjabir-jabir +berjaga-jaga +berjagung-jagung +berjalan-jalan +berjalar-jalar +berjalin-jalin +berjalur-jalur +berjam-jam +berjauh-jauhan +berjejal-jejal +berjela-jela +berjenis-jenis +berjenjang-jenjang +berjilid-jilid +berjinak-jinak +berjingkat-jingkat +berjingkrak-jingkrak +berjongkok-jongkok +berjubel-jubel +berjujut-jujutan +berjulai-julai +berjumbai-jumbai +berjurai-jurai +berjurus-jurus +berjuta-juta +berkaca-kaca +berkait-kaitan +berkala-kala +berkali-kali +berkanjar-kanjar +berkaok-kaok +berkarung-karung +berkasih-kasihan +berkata-kata +berkatak-katak +berkecai-kecai +berkecek-kecek +berkecil-kecil +berkecil-kecilan +berkedip-kedip +berkejang-kejang +berkejap-kejap +berkejar-kejaran +berkelar-kelar +berkelip-kelip +berkelit-kelit +berkelok-kelok +berkelompok-kelompok +berkelun-kelun +berkembur-kembur +berkempul-kempul +berkena-kenaan +berkenal-kenalan +berkendur-kendur +berkeok-keok +berkepak-kepak +berkepal-kepal +berkeping-keping +berkepul-kepul +berkeras-kerasan +berkeritik-keritik +berkeruit-keruit +berkerut-kerut +berketak-ketak +berketak-ketik +berketi-keti +berketil-ketil +berketuk-ketak +berketul-ketul +berkial-kial +berkian-kian +berkias-kiasan +berkibar-kibar +berkilah-kilah +berkilat-kilat +berkilau-kilauan +berkilo-kilo +berkinja-kinja +berkipas-kipas +berkira-kira +berkirim-kiriman +berkobar-kobar +berkobok-kobok +berkocak-kocak +berkodi-kodi +berkolek-kolek +berkopah-kopah +berkotak-kotak +berkuat-kuatan +berkunang-kunang +berkurun-kurun +berkusau-kusau +berkusu-kusu +berkusut-kusut +berkuting-kuting +berkutu-kutuan +berlabun-labun +berlain-lainan +berlalai-lalai +berlama-lama +berlambai-lambai +berlambak-lambak +berlampang-lampang +berlapang-lapang +berlapis-lapis +berlapuk-lapuk +berlarah-larah +berlarat-larat +berlari-larian +berlarik-larik +berlarut-larut +berlawak-lawak +berlayap-layapan +berlebih-lebih +berlebih-lebihan +berlekas-lekas +berlena-lena +berlengah-lengah +berlenggek-lenggek +berlenggok-lenggok +berleret-leret +berliang-liuk +berliku-liku +berlimpah-limpah +berlimpap-limpap +berlimpit-limpit +berlinang-linang +berlindak-lindak +berlipat-lipat +berlompok-lompok +berloncat-loncatan +berlopak-lopak +berlubang-lubang +bermaaf-maafan +bermacam-macam +bermain-main +bermalas-malas +bermanik-manik +bermanis-manis +bermanja-manja +bermasak-masak +bermati-mati +bermegah-megah +bermemek-memek +bermesra-mesraan +bermewah-mewah +berminggu-minggu +berminta-minta +bermuda-muda +bermudah-mudah +bermuka-muka +bermula-mula +bermulut-mulut +bernafsi-nafsi +bernaka-naka +berniat-niat +berogak-ogak +beroleng-oleng +berolok-olok +beromong-omong +beronggok-onggok +berorang-orang +beroyal-royal +berpada-pada +berpahit-pahit +berpair-pair +berpal-pal +berpalu-palu +berpalu-paluan +berpalun-palun +berpandai-pandai +berpandang-pandangan +berpangkat-pangkat +berpanjang-panjang +berpasang-pasang +berpasang-pasangan +berpayah-payah +berpeluh-peluh +berpeluk-pelukan +berpenat-penat +berpencar-pencar +berpendar-pendar +berpenggal-penggal +berperai-perai +berpesai-pesai +berpesta-pesta +berpesuk-pesuk +berpetak-petak +berpeti-peti +berpihak-pihak +berpijar-pijar +berpikul-pikul +berpilih-pilih +berpilin-pilin +berpindah-pindah +berpintal-pintal +berpirau-pirau +berpisah-pisah +berpolah-polah +berpongah-pongah +berpontang-panting +berporah-porah +berpotong-potong +berpuak-puak +berpual-pual +berpugak-pugak +berpuluh-puluh +berpulun-pulun +berpuntal-puntal +berpura-pura +berpusar-pusar +berpusing-pusing +berpusu-pusu +berputar-putar +bersaf-saf +bersahut-sahutan +bersakit-sakit +bersalah-salahan +bersalam-salaman +bersalin-salin +bersama-sama +bersambut-sambutan +bersampan-sampan +bersantai-santai +bersapa-sapaan +bersarang-sarang +bersedan-sedan +bersedia-sedia +bersedu-sedu +bersekat-sekat +berselang-selang +berselang-seli +bersembur-semburan +bersempit-sempit +bersenang-senang +bersenang-senangkan +bersenda-senda +bersendi-sendi +bersepah-sepah +bersepi-sepi +berserak-serak +berseri-seri +bersesak-sesak +bersetai-setai +bersia-sia +bersiap-siap +bersiar-siar +bersilir-silir +bersimbur-simburan +bersinau-sinau +bersorak-sorai +bersuap-suapan +bersudah-sudah +bersuka-suka +bersuka-sukaan +bersuku-suku +bersumpah-sumpahan +bersungguh-sungguh +bersungut-sungut +bersunyi-sunyi +bersusah-susah +bersusuk-susuk +bersusuk-susukan +bersutan-sutan +bertabur-tabur +bertahu-tahu +bertahun-tahun +bertajuk-tajuk +bertakik-takik +bertala-tala +bertali-tali +bertalu-talu +bertambah-tambah +bertanda-tandaan +bertangis-tangisan +bertangkil-tangkil +bertanya-tanya +bertarik-tarikan +bertatai-tatai +bertatih-tatih +bertawan-tawan +bertawar-tawaran +bertebu-tebu +bertebu-tebukan +berteguh-teguh +berteguh-teguhan +berteka-teki +bertelau-telau +bertele-tele +bertempat-tempat +bertempuh-tempuh +bertenang-tenang +bertenggang-tenggangan +bertentu-tentu +bertepek-tepek +berterang-terang +berterang-terangan +bertikam-tikaman +bertimbal-timbalan +bertimbun-timbun +bertimpa-timpa +bertimpas-timpas +bertingkah-tingkah +bertingkat-tingkat +bertinjau-tinjauan +bertiras-tiras +bertitar-titar +bertoboh-toboh +bertolak-tolak +bertolak-tolakan +bertolong-tolongan +bertonjol-tonjol +bertua-tua +bertua-tuaan +bertual-tual +bertubi-tubi +bertukar-tukar +bertukar-tukaran +bertukas-tukas +bertumpak-tumpak +bertunda-tunda +bertunjuk-tunjukan +bertura-tura +berturut-turut +bertutur-tutur +beruas-ruas +berubah-ubah +berulang-alik +berulang-ulang +berumbai-rumbai +berundung-undung +berunggas-runggas +berungkur-ungkuran +beruntai-untai +beruntun-runtun +berunyai-unyai +berupa-rupa +berura-ura +beruris-uris +berurut-urutan +berwarna-warna +berwarna-warni +berwindu-windu +berwiru-wiru +beryang-yang +besar-besaran +betak-betak +beti-beti +betul-betul +biang-biang +biar-biar +biji-bijian +bila-bila +bilang-bilang +bincang-bincut +bini-binian +biri-biri +biru-biru +bisik-bisik +biti-biti +bolak-balik +bolang-baling +bongkar-bangkir +buah-buahan +buat-buatan +buaya-buaya +bubun-bubun +bugi-bugi +built-in +bukan-bukan +bulan-bulan +bulan-bulanan +bulang-bulang +bulat-bulat +buli-buli +bulu-bulu +buluh-buluh +bulus-bulus +bunga-bungaan +bunuh-membunuh +bunyi-bunyian +buru-buru +burung-burungan +bye-bye +cabik-cabik +caing-caing +calar-balar +cara-cara +carut-marut +cawi-cawi +cebar-cebur +celam-celum +celangak-celinguk +celas-celus +celedang-celedok +celengkak-celengkok +cemas-cemas +centang-perenang +cepat-cepat +cerai-berai +ceruk-menceruk +ceruk-meruk +check-up +chit-chat +cirit-birit +cita-cita +close-up +closed-circuit +cobak-cabik +cobar-cabir +cola-cala +compang-camping +congak-cangit +congkah-cangkih +congkah-mangkih +copak-capik +corak-carik +corat-coret +coreng-moreng +cuang-caing +cubung-cubung +culik-culik +cuma-cuma +cumi-cumi +cungap-cangip +cupu-cupu +dahulu-mendahului +dali-dali +dapur-dapur +dari-dari +daru-daru +datang-datang +datang-mendatangi +daun-daunan +dawai-dawai +dayang-dayang +degap-degap +dekak-dekak +dekat-dekat +dengar-dengaran +desas-desus +diam-diam +do-it-yourself +dokok-dokok +dolak-dalik +dorong-mendorong +drive-in +dua-dua +dua-duanya +duduk-duduk +dulang-dulang +ecek-ecek +embuh-embuhan +empek-empek +empok-empok +encal-encal +endap-endap +endut-endutan +engah-engah +enggan-enggan +engkah-engkah +entah-berentah +erang-erot +erong-erong +fast-food +fifty-fifty +flip-flop +follow-up +foya-foya +gaba-gaba +gabai-gabai +gada-gada +gading-gading +gado-gado +gajah-gajahan +gala-gala +gali-galian +galing-galing +galu-galu +gamit-gamitan +gampang-gampangan +ganal-ganal +ganda-berganda +gapah-gopoh +gara-gara +garah-garah +gatal-gatal +gawar-gawar +gaya-gayanya +gedebak-gedebuk +gelang-gelang +gelembung-gelembungan +geli-geli +geliang-geliut +geliat-geliut +gempul-gempul +gendang-gendang +genjang-genjot +gerabak-gerubuk +gerak-gerik +gerbas-gerbus +gerit-gerit +geruh-gerah +getak-getuk +geti-geti +gila-gila +gila-gilaan +gilang-gemilang +gilap-gemilap +gili-gili +giling-giling +ginang-ginang +girik-girik +giring-giring +go-kart +golak-galik +gonta-ganti +gotong-royong +gual-gail +gudu-gudu +gula-gula +gulang-gulang +guna-guna +guntang-guntang +gunung-ganang +gunung-gemunung +gunung-gunungan +habis-habis +habis-habisan +halai-balai +half-time +hampir-hampir +harap-harapan +harum-haruman +hati-hati +heavy-duty +hebat-hebatan +hidup-hidup +hiru-biru +hiruk-pikuk +hubaya-hubaya +hula-hula +huru-hara +ibar-ibar +icak-icak +igau-igauan +ikut-ikut +ikut-ikutan +ilam-ilam +imbang-imbangan +inang-inang +inca-binca +incang-incut +ingat-ingat +ingat-ingatan +ingau-ingauan +inggang-inggung +injak-injak +iras-iras +iring-iringan +iseng-iseng +jadi-jadian +jala-jala +jamah-jamahan +jambu-jambu +jangan-jangan +jarang-jarang +jari-jari +jaring-jaring +jarum-jarum +jauh-jauh +jawi-jawi +jebat-jebatan +jelur-jelir +jendal-jendul +jenggar-jenggur +jentik-jentik +jerah-jerih +jolong-jolong +jongkar-jangkir +juak-juak +juang-juang +julung-julung +jurai-jurai +kabu-kabu +kacang-kacang +kacang-kacangan +kacau-balau +kadang-kadang +kail-kail +kait-kait +kakek-kakek +kalau-kalau +kaleng-kalengan +kalut-malut +kambing-kambing +kanak-kanak +kapa-kapa +kapan-kapan +kapu-kapu +karang-karangan +karang-mengarang +kareseh-peseh +karut-marut +katang-katang +kawa-kawa +kayu-kayuan +keabu-abuan +keasyik-asyikan +kebarat-baratan +kebasah-basahan +kebat-kebit +kebata-bataan +kebelanda-belandaan +kebiru-biruan +kebudak-budakan +kecil-kecilan +kecil-mengecil +kecuh-kecah +kedek-kedek +kegadis-gadisan +kegelap-gelapan +kegila-gilaan +kegirang-girangan +kehijau-hijauan +kehitam-hitaman +kejaga-jagaan +kejingga-jinggaan +kekabur-kaburan +kekanak-kanakan +kekoboi-koboian +kekuning-kuningan +kelak-kelik +kelak-keluk +kelaki-lakian +kelang-kelok +kelap-kelip +kelek-kelek +kelek-kelekan +kelik-kelik +kelip-kelip +kelusuh-kelasah +kelut-melut +kemak-kemik +kemalu-maluan +kemanja-manjaan +kemarah-marahan +kemasam-masaman +kemati-matian +kemerah-merahan +kempang-kempis +kempas-kempis +kemuda-mudaan +kena-mengena +kenal-mengenal +kenang-kenangan +kencang-kencung +kendang-kendang +kendang-kendangan +kentung-kentung +kenyat-kenyit +kepandir-pandiran +kepang-kepot +keperak-perakan +kepilu-piluan +kepura-puraan +keputih-putihan +kerah-kerahan +kerancak-rancakan +kerang-kerangan +kerang-keroh +kerang-kerung +kerap-kerap +keras-mengerasi +kercap-kercip +kercap-kercup +keriang-keriut +kernyat-kernyut +kerong-kerong +keropas-kerapis +kertak-kertuk +keruntang-pungkang +kesap-kesip +kesenak-senakan +kesewenang-wenangan +kesia-siaan +kesik-kesik +kesipu-sipuan +kesu-kesi +kesuh-kesih +kesuk-kesik +ketergesa-gesaan +keti-keti +ketidur-tiduran +ketiga-tiganya +ketua-tuaan +ketuan-tuanan +keungu-unguan +kia-kia +kiak-kiak +kial-kial +kiang-kiut +kibang-kibut +kicang-kecoh +kicang-kicu +kida-kida +kilau-mengilau +kili-kili +kira-kira +kira-kiraan +kisi-kisi +kocah-kacih +kodok-kodok +kolang-kaling +koleh-koleh +kolong-kolong +koma-koma +komat-kamit +kontal-kantil +kontang-kanting +kosak-kasik +kotak-katik +kotak-kotak +kuat-kuat +kucar-kacir +kucing-kucing +kucing-kucingan +kuda-kuda +kuda-kudaan +kudap-kudap +kulah-kulah +kulak-kulak +kulik-kulik +kulum-kulum +kumat-kamit +kunang-kunang +kupat-kapit +kupu-kupu +kura-kura +kurang-kurang +kusat-mesat +kutat-kutet +kuti-kuti +labi-labi +labu-labu +lagi-lagi +laguh-lagah +laki-laki +lalu-lalang +lama-kelamaan +lama-lama +lamat-lamat +lambat-lambat +lancar-lancar +langak-longok +langit-langit +lanja-lanjaan +lapat-lapat +large-scale +lari-lari +lauk-pauk +lawah-lawah +lawak-lawak +lawi-lawi +layang-layang +layu-layuan +lebih-lebih +legak-legok +lekak-lekuk +lekap-lekup +lekas-lekas +lekuh-lekih +lekup-lekap +lenggak-lenggok +lenggok-lenggok +lengket-lengket +lentam-lentum +lentang-lentok +lentang-lentung +lepa-lepa +lerang-lerang +lereng-lereng +letah-letai +letup-letup +liang-liuk +lidah-lidah +line-up +liuk-liuk +liung-liung +lobi-lobi +lock-up +lopak-lapik +lopak-lopak +lumba-lumba +lumi-lumi +luntang-lantung +lupa-lupa +lupa-lupaan +main-mainan +makan-makanan +make-up +malai-malai +malam-malam +malar-malar +mali-mali +malu-malu +mana-mana +manik-manik +manis-manisan +mark-up +masing-masing +mata-mata +mati-matian +maya-maya +megap-megap +megrek-megrek +melak-melak +melambai-lambai +melambai-lambaikan +melambat-lambatkan +melaun-laun +melawak-lawak +melayap-layap +melayap-layapkan +melebih-lebihi +melebih-lebihkan +melejang-lejangkan +melengah-lengah +melihat-lihat +melimpah-limpah +melincah-lincah +meloncat-loncat +melonco-lonco +melonjak-lonjak +memacak-macak +memaki-maki +memaksa-maksa +memandai-mandai +memanggil-manggil +memanis-manis +memanjut-manjut +memasak-masak +memata-matai +mematah-matah +mematut-matut +memayah-mayahkan +membagi-bagikan +membalik-balik +membangkit-bangkit +membayang-bayangi +membayang-bayangkan +membelai-belai +membenar-benar +membenar-benari +memberai-beraikan +membesar-besarkan +membolak-balikkan +membuang-buang +membuat-buat +membunga-bungai +memburu-buru +memburu-burukan +memburuk-burukkan +memencak-mencak +memencar-mencar +memetak-metak +memetang-metangkan +memetir-metir +memikir-mikirkan +memilih-milih +meminang-minang +meminta-minta +memisah-misahkan +memontang-mantingkan +memperamat-amat +memperamat-amatkan +memperbagai-bagaikan +memperganda-gandakan +memperganduh-ganduhkan +mempermacam-macamkan +memperolok-olokkan +mempersama-samakan +mempertubi-tubi +mempertubi-tubikan +memperturut-turutkan +memuja-muja +memukang-mukang +memulun-mulun +memundi-mundi +memundi-mundikan +memuyu-muyu +menagak-nagak +menakut-nakuti +menanjur-nanjur +menanti-nanti +menari-nari +mencabik-cabik +mencabik-cabikkan +mencaing-caing +mencak-mencak +mencakup-cakup +mencapak-capak +mencari-cari +mencarik-carik +mencarut-carut +mencengis-cengis +mencepak-cepak +mencepuk-cepuk +mencerai-beraikan +mencetai-cetai +menciap-ciap +menciar-ciar +mencita-citakan +menciut-ciut +mencoang-coang +mencubit-cubit +mencuri-curi +mendecap-decap +mendengking-dengking +menderak-derakkan +menderau-derau +menderu-deru +mendesas-desuskan +mendesus-desus +mendewa-dewakan +mendudu-dudu +menebu-nebu +menegur-neguri +mengabung-ngabung +mengaci-acikan +mengada-ada +mengaduk-aduk +mengagak-agak +mengagak-agihkan +mengagut-agut +mengais-ngais +mengali-ali +mengalur-alur +mengamang-amang +mengamat-amati +mengambai-ambaikan +mengambang-ambang +mengancak-ancak +mengangan-angankan +mengangguk-angguk +mengangin-anginkan +mengangkat-angkat +mengap-mengap +mengapa-apai +mengapi-apikan +mengarah-arahi +mengata-ngatai +mengaum-aumkan +mengejan-ejan +mengelai-ngelai +mengelepik-ngelepik +mengelus-elus +mengembut-embut +mengenap-enapkan +mengenjak-enjak +mengepak-ngepak +mengepak-ngepakkan +menggaba-gabai +menggalur-galur +menggamak-gamak +menggapai-gapai +menggapai-gapaikan +menggelepar-gelepar +menggelepar-geleparkan +menggemak-gemak +menggerecak-gerecak +menggesa-gesakan +menggili-gili +menggorek-gorek +menggosok-gosok +mengguit-guit +menghalai-balaikan +menghinap-hinap +mengiang-ngiang +mengibas-ngibas +mengidam-idamkan +mengilah-ngilahkan +mengilai-ilai +mengilat-ngilatkan +mengilik-ngilik +mengimak-imak +mengiming-iming +menginjak-injak +mengipas-ngipas +mengira-ngira +mengira-ngirakan +mengiras-iras +mengiras-irasi +mengitar-ngitar +mengitik-ngitik +mengogok-ogok +mengolak-alikkan +mengoleng-oleng +mengongkang-ongkang +mengongkok-ongkok +mengonyah-anyih +mengotak-ngatikkan +mengoyak-ngoyakkan +mengoyak-oyak +menguar-nguarkan +menguar-uarkan +menguber-uber +mengubit-ubit +mengubrak-abrik +mengucar-ngacirkan +mengucek-ngucek +menguik-uik +menguis-uis +mengulit-ulit +menguman-uman +mengumbang-ambingkan +mengumpak-umpak +mengungkat-ungkat +mengungkit-ungkit +mengurik-urik +mengutak-ngatikkan +mengutik-ngutik +menimang-nimang +meningkat-ningkat +meniru-niru +meniup-niup +menjadi-jadi +menjengek-jengek +menjengit-jengit +menjilat-jilat +mentah-mentah +mentang-mentang +menunda-nunda +menusuk-nusuk +menyama-nyama +menyambar-nyambar +menyanjung-nyanjung +menyapu-nyapu +menyarat-nyarat +menyendi-nyendi +menyeret-nyeret +menyeru-nyerukan +menyia-nyiakan +menyungguh-nyungguhi +meraba-raba +merangkak-rangkak +merasa-rasai +meraung-raung +meraung-raungkan +merayau-rayau +merayu-rayu +mereka-reka +merelap-relap +meremah-remah +meremeh-temehkan +merempah-rempahi +merengek-rengek +merenik-renik +merenta-renta +merenyai-renyai +merintang-rintang +merintik-rintik +merobek-robek +meronta-ronta +merungus-rungus +merungut-rungut +mewarna-warnikan +meyakin-yakini +miju-miju +minta-minta +moga-moga +morat-marit +muda-mudi +mudah-mudahan +muka-muka +mula-mula +muluk-muluk +naga-naga +nanti-nantian +nasi-nasi +nasib-nasiban +nenek-nenek +nyolong-nyolong +ogah-ogahan +ogak-ogak +olak-alik +olak-olak +olang-aling +olang-alingan +oleh-oleh +olok-olok +olok-olokan +olong-olong +on-screen +onde-onde +one-to-one +oneng-oneng +ongkang-ongkang +ongol-ongol +onyah-anyih +orak-arik +orang-aring +orang-orangan +orok-orok +orong-orong +otak-otak +otak-otakan +padi-padian +pagi-pagi +palas-palas +paling-paling +palu-memalu +panas-panas +pandang-memandang +panji-panji +para-para +paru-paru +pasang-memasang +pasu-pasu +paya-paya +pecah-pecah +pelan-pelan +pengundang-undang +perang-perangan +perintang-rintang +perlahan-lahan +perlip-perlipan +pertama-tama +perundang-undangan +pesan-pesan +piat-piut +pick-up +pijak-pijak +pijar-pijar +pijat-pijat +pina-pina +pisang-pisang +play-off +pohon-pohonan +pokrol-pokrolan +polang-paling +poma-poma +pontang-panting +porak-parik +porak-peranda +potong-memotong +puji-pujian +pukang-pukang +pukul-memukul +pulang-pergi +pulut-pulut +pundi-pundi +punggung-memunggung +pura-pura +pusar-pusar +push-up +pusing-pusing +putus-putus +rada-rada +radio-frequency +ragu-ragu +rama-rama +rambu-rambu +rango-rango +rasa-rasanya +rata-rata +real-time +rebah-rebah +rebah-rebahan +redam-redam +reka-reka +reka-rekaan +remah-remah +remang-remang +rembah-rembih +remeh-temeh +rempah-rempah +repuh-repuh +riang-riang +ribu-ribu +rigi-rigi +robak-rabik +robat-rabit +role-play +roll-on +rombang-rambing +ruak-ruak +ruku-ruku +rumah-rumah +rumah-rumahan +rumput-rumputan +runding-merunding +runggu-rangga +runner-up +rupa-rupa +rupa-rupanya +saban-saban +sabung-menyabung +saing-menyaing +salah-salah +sama-sama +samar-samar +sambar-menyambar +sambung-bersambung +sambung-menyambung +sambut-menyambut +sampai-sampai +sandar-menyandar +sangat-sangat +sangkut-menyangkut +sapa-menyapa +sapu-sapu +sarit-sarit +satu-satu +satu-satunya +sayup-menyayup +sayup-sayup +sayur-mayur +sayur-sayuran +sci-fi +seakal-akal +seakan-akan +sealak-alak +sebaik-baiknya +sebelah-menyebelah +sebentar-sebentar +seberang-menyeberang +seboleh-bolehnya +sedalam-dalamnya +sedang-menyedang +sedap-sedapan +sedapat-dapatnya +sedikit-dikitnya +sedikit-sedikit +sedikit-sedikitnya +seelok-eloknya +segala-galanya +segan-menyegan +segan-menyegani +segan-segan +sehari-hari +sehari-harian +sejadi-jadinya +sekali-kali +sekali-sekali +sekira-kira +sekonyong-konyong +sekuasa-kuasanya +sekurang-kurangnya +sela-menyela +sela-sela +selama-lamanya +selambat-lambatnya +selang-seli +selang-seling +selar-belar +selat-latnya +selekas-lekasnya +selepas-lepas +self-esteem +self-help +sema-sema +semah-semah +semak-semak +semalam-malaman +semasa-masa +semata-mata +sembunyi-sembunyi +sembunyi-sembunyian +semena-mena +semenda-menyemenda +semengga-mengga +sementang-mentang +semu-semu +semut-semutan +sengal-sengal +sengau-sengauan +seolah-olah +sepala-pala +sepandai-pandai +sepetang-petangan +sepoi-sepoi +sepuas-puasnya +serang-menyerang +seraya-menyeraya +serba-serbi +serbah-serbih +serembah-serembih +sering-sering +serta-menyertai +serta-serta +sesal-menyesali +sesudah-sudah +sesudah-sudahnya +sesuka-suka +setempat-setempat +setengah-setengah +setidak-tidaknya +seupaya-upaya +seupaya-upayanya +sewaktu-waktu +sewenang-wenang +short-term +sia-sia +siang-siang +siapa-siapa +sibar-sibar +sibur-sibur +sida-sida +siku-siku +silah-silah +silang-menyilang +silir-semilir +sinar-seminar +sindir-menyindir +singgah-menyinggah +sorak-sorai +stand-by +stand-up +sudu-sudu +sudung-sudung +suka-suka +sulang-menyulang +sulur-suluran +sumpah-sumpah +sumpit-sumpit +sungguh-sungguh +sungut-sungut +suram-suram +surat-menyurat +suruh-suruhan +tabar-tabar +tabir-mabir +tabrak-tubruk +tabuh-tabuhan +tahu-menahu +tahu-tahu +takang-takik +take-off +takut-takut +takut-takutan +tali-bertali +tali-tali +tampak-tampak +tanam-menanam +tanam-tanaman +tanda-tanda +tangan-menangan +tangan-tangan +tanggung-menanggung +tapa-tapa +tapak-tapak +tari-menari +tari-tarian +tarik-menarik +tatah-tatah +tawak-tawak +tawang-tawang +tawar-menawar +tawar-tawar +tayum-temayum +tebu-tebu +tegak-tegak +teka-teki +temas-temas +tembak-menembak +temut-temut +tenggang-menenggang +teraba-raba +terambang-ambang +terang-terang +terang-terangan +teranggar-anggar +terangguk-angguk +teranggul-anggul +terangin-angin +terangkup-angkup +teranja-anja +terapung-apung +terayan-rayan +terayap-rayap +terbada-bada +terbahak-bahak +terbata-bata +terbatuk-batuk +terbayang-bayang +terbengkil-bengkil +terbirit-birit +terbuai-buai +terbuang-buang +terburu-buru +tercangak-cangak +tercengang-cengang +tercilap-cilap +tercongget-congget +tercungap-cungap +terdangka-dangka +terdengih-dengih +terekeh-ekeh +terembut-embut +terembut-rembut +terengah-engah +teresak-esak +tergagap-gagap +tergagau-gagau +tergaguk-gaguk +tergapai-gapai +tergegap-gegap +tergegas-gegas +tergelung-gelung +tergerenyeng-gerenyeng +tergesa-gesa +tergila-gila +tergontai-gontai +tergudik-gudik +terguling-guling +tergulut-gulut +terharak-harak +terharap-harap +terhengit-hengit +terhinggut-hinggut +terigau-igau +terincut-incut +teringa-inga +teringat-ingat +terinjak-injak +terjembak-jembak +terjerit-jerit +terkadang-kadang +terkakah-kakah +terkakak-kakak +terkanjar-kanjar +terkapah-kapah +terkapai-kapai +terkapung-kapung +terkatah-katah +terkatung-katung +terkecap-kecap +terkedek-kedek +terkedip-kedip +terkejar-kejar +terkekau-kekau +terkekeh-kekeh +terkekek-kekek +terkelinjat-kelinjat +terkelip-kelip +terkempul-kempul +terkemut-kemut +terkencar-kencar +terkepak-kepak +terkesot-kesot +terkesut-kesut +terkial-kial +terkincak-kincak +terkindap-kindap +terkinja-kinja +terkirai-kirai +terkitar-kitar +terkocoh-kocoh +terkokol-kokol +terkosel-kosel +terkoteng-koteng +terkumpal-kumpal +terlara-lara +terlayang-layang +terlebih-lebih +terlincah-lincah +terliuk-liuk +terlolong-lolong +terlongong-longong +termangu-mangu +termanja-manja +termata-mata +termengah-mengah +termimpi-mimpi +ternanti-nanti +terngiang-ngiang +teroleng-oleng +terpandang-pandang +terpecah-pecah +terpekik-pekik +terpereh-pereh +terpikau-pikau +terpinga-pinga +terpingkal-pingkal +terpontang-panting +terpusing-pusing +terputus-putus +tersanga-sanga +tersaruk-saruk +tersedan-sedan +tersedih-sedih +tersedu-sedu +tersendat-sendat +tersendeng-sendeng +tersengal-sengal +tersengguk-sengguk +tersengut-sengut +tersera-sera +terserak-serak +tersetai-setai +tersia-sia +tersipu-sipu +tersoja-soja +tersungkuk-sungkuk +tertagak-tagak +tertahan-tahan +tertatih-tatih +tertegun-tegun +tertekan-tekan +terteleng-teleng +terumbang-ambing +terumbang-umbang +terungkap-ungkap +terus-menerus +terus-terusan +think-tank +tiap-tiap +tiba-tiba +tidak-tidak +tidur-tidur +tie-dye +tiga-tiganya +tikam-menikam +tilik-menilik +timah-timah +timang-timangan +timbang-menimbang +timu-timu +tindih-bertindih +tinjau-meninjau +tip-off +tiru-tiruan +tiup-tiup +tokak-takik +tokok-menokok +tolak-menolak +tolong-menolong +top-level +trade-in +tua-tua +tuan-tuan +tuang-tuang +tuban-tuban +tukang-menukang +tukar-menukar +tulang-tulangan +tuli-tuli +tulis-menulis +tumbuh-tumbuhan +tune-up +tunggang-tunggit +tupai-tupai +turun-temurun +turut-menurut +turut-turutan +two-tone +uar-uar +ubel-ubel +ubun-ubun +ubur-ubur +uci-uci +udap-udapan +ugal-ugalan +uir-uir +ujar-ujar +ukir-mengukir +ula-ula +ulak-ulak +ulang-alik +ulang-aling +ulang-ulang +ulap-ulap +ular-ular +ular-ularan +ulung-ulung +umang-umang +umbang-ambing +umbi-umbian +umbul-umbul +umbut-umbut +uncang-uncit +undak-undakan +undang-undang +unduk-unduk +undung-undung +undur-undur +unggat-unggit +ungkit-ungkit +unting-unting +untung-untung +untung-untungan +upside-down +ura-ura +uran-uran +urat-urat +uring-uringan +urup-urup +urup-urupan +urus-urus +user-user +user-useran +utar-utar +voice-over +walk-out +wangi-wangian +wanti-wanti +wara-wara +warna-warni +water-cooled +world-class +yang-yang +""".split() +) diff --git a/spacy/lang/ms/examples.py b/spacy/lang/ms/examples.py new file mode 100644 index 0000000..236e0c0 --- /dev/null +++ b/spacy/lang/ms/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ms.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Malaysia ialah sebuah negara yang terletak di Asia Tenggara.", + "Berapa banyak pelajar yang akan menghadiri majlis perpisahan sekolah?", + "Pengeluaran makanan berasal dari beberapa lokasi termasuk Cameron Highlands, Johor Bahru, dan Kuching.", + "Syarikat XYZ telah menghasilkan 20,000 unit produk baharu dalam setahun terakhir", + "Kuala Lumpur merupakan ibu negara Malaysia.Kau berada di mana semalam?", + "Siapa yang akan memimpin projek itu?", + "Siapa perdana menteri Malaysia sekarang?", +] diff --git a/spacy/lang/ms/lex_attrs.py b/spacy/lang/ms/lex_attrs.py new file mode 100644 index 0000000..2088c99 --- /dev/null +++ b/spacy/lang/ms/lex_attrs.py @@ -0,0 +1,65 @@ +import unicodedata + +from ...attrs import IS_CURRENCY, LIKE_NUM +from .punctuation import LIST_CURRENCY + +_num_words = [ + "kosong", + "satu", + "dua", + "tiga", + "empat", + "lima", + "enam", + "tujuh", + "lapan", + "sembilan", + "sepuluh", + "sebelas", + "belas", + "puluh", + "ratus", + "ribu", + "juta", + "billion", + "trillion", + "kuadrilion", + "kuintilion", + "sekstilion", + "septilion", + "oktilion", + "nonilion", + "desilion", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.count("-") == 1: + _, num = text.split("-") + if num.isdigit() or num in _num_words: + return True + return False + + +def is_currency(text): + if text in LIST_CURRENCY: + return True + + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +LEX_ATTRS = {IS_CURRENCY: is_currency, LIKE_NUM: like_num} diff --git a/spacy/lang/ms/punctuation.py b/spacy/lang/ms/punctuation.py new file mode 100644 index 0000000..a8d6c2e --- /dev/null +++ b/spacy/lang/ms/punctuation.py @@ -0,0 +1,60 @@ +from ..char_classes import ALPHA, _currency, _units, merge_chars, split_chars +from ..punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES + +_units = ( + _units + "s bit Gbps Mbps mbps Kbps kbps ƒ ppi px " + "Hz kHz MHz GHz mAh " + "ratus rb ribu ribuan " + "juta jt jutaan mill?iar million bil[l]?iun bilyun billion " +) +_currency = _currency + r" USD RM MYR Rp IDR RMB SGD S\$" +_months = ( + "Januari Februari Mac April Mei Jun Julai Ogos September " + "Oktober November Disember Januari Februari Mac Mei Jun " + "Julai Ogos Oktober Disember Jan Feb Mac Jun Julai Ogos Sept " + "Okt Nov Dis" +) + + +UNITS = merge_chars(_units) +CURRENCY = merge_chars(_currency) +HTML_PREFIX = r"<(b|strong|i|em|p|span|div|br)\s?/>|]+)>" +HTML_SUFFIX = r"" +MONTHS = merge_chars(_months) +LIST_CURRENCY = split_chars(_currency) + +_prefixes = list(TOKENIZER_PREFIXES) +_prefixes.remove("#") # hashtag +_prefixes = _prefixes + LIST_CURRENCY + [HTML_PREFIX] + ["/", "—"] + +_suffixes = ( + TOKENIZER_SUFFIXES + + [r"\-[Nn]ya", "-[KkMm]u", "[—-]"] + + [ + # disabled: variable width currency variable + # r"(?<={c})(?:[0-9]+)".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9])%", + # disabled: variable width HTML_SUFFIX variable + # r"(?<=[0-9{a}]{h})(?:[\.,:-])".format(a=ALPHA, h=HTML_SUFFIX), + r"(?<=[0-9{a}])(?:{h})".format(a=ALPHA, h=HTML_SUFFIX), + ] +) + +_infixes = TOKENIZER_INFIXES + [ + r"(?<=[0-9])[\\/](?=[0-9%-])", + r"(?<=[0-9])%(?=[{a}0-9/])".format(a=ALPHA), + # disabled: variable width units variable + # r"(?<={u})[\/-](?=[0-9])".format(u=UNITS), + # disabled: variable width months variable + # r"(?<={m})[\/-](?=[0-9])".format(m=MONTHS), + r'(?<=[0-9)][.,])"(?=[0-9])', + r'(?<=[{a})][.,\'])["—](?=[{a}])'.format(a=ALPHA), + r"(?<=[{a}])-(?=[0-9])".format(a=ALPHA), + r"(?<=[0-9])-(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[\/-](?={c}|[{a}])".format(a=ALPHA, c=CURRENCY), +] + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/ms/stop_words.py b/spacy/lang/ms/stop_words.py new file mode 100644 index 0000000..b1bfaea --- /dev/null +++ b/spacy/lang/ms/stop_words.py @@ -0,0 +1,118 @@ +STOP_WORDS = set( + """ +ada adalah adanya adapun agak agaknya agar akan akankah akhir akhiri akhirnya +aku akulah amat amatlah anda andalah antar antara antaranya apa apaan apabila +apakah apalagi apatah artinya asal asalkan atas atau ataukah ataupun awal +awalnya + +bagai bagaikan bagaimana bagaimanakah bagaimanapun bagi bagian bahkan bahwa +bahwasanya baik bakal bakalan balik banyak bapak baru bawah beberapa begini +beginian beginikah beginilah begitu begitukah begitulah begitupun bekerja +belakang belakangan belum belumlah benar benarkah benarlah berada berakhir +berakhirlah berakhirnya berapa berapakah berapalah berapapun berarti berawal +berbagai berdatangan beri berikan berikut berikutnya berjumlah berkali-kali +berkata berkehendak berkeinginan berkenaan berlainan berlalu berlangsung +berlebihan bermacam bermacam-macam bermaksud bermula bersama bersama-sama +bersiap bersiap-siap bertanya bertanya-tanya berturut berturut-turut bertutur +berujar berupa besar betul betulkah biasa biasanya bila bilakah bisa bisakah +boleh bolehkah bolehlah buat bukan bukankah bukanlah bukannya bulan bung + +cara caranya cukup cukupkah cukuplah cuma + +dahulu dalam dan dapat dari daripada datang dekat demi demikian demikianlah +dengan depan di dia diakhiri diakhirinya dialah diantara diantaranya diberi +diberikan diberikannya dibuat dibuatnya didapat didatangkan digunakan +diibaratkan diibaratkannya diingat diingatkan diinginkan dijawab dijelaskan +dijelaskannya dikarenakan dikatakan dikatakannya dikerjakan diketahui +diketahuinya dikira dilakukan dilalui dilihat dimaksud dimaksudkan +dimaksudkannya dimaksudnya diminta dimintai dimisalkan dimulai dimulailah +dimulainya dimungkinkan dini dipastikan diperbuat diperbuatnya dipergunakan +diperkirakan diperlihatkan diperlukan diperlukannya dipersoalkan dipertanyakan +dipunyai diri dirinya disampaikan disebut disebutkan disebutkannya disini +disinilah ditambahkan ditandaskan ditanya ditanyai ditanyakan ditegaskan +ditujukan ditunjuk ditunjuki ditunjukkan ditunjukkannya ditunjuknya dituturkan +dituturkannya diucapkan diucapkannya diungkapkan dong dua dulu + +empat enggak enggaknya entah entahlah + +guna gunakan + +hal hampir hanya hanyalah hari harus haruslah harusnya hendak hendaklah +hendaknya hingga + +ia ialah ibarat ibaratkan ibaratnya ibu ikut ingat ingat-ingat ingin inginkah +inginkan ini inikah inilah itu itukah itulah + +jadi jadilah jadinya jangan jangankan janganlah jauh jawab jawaban jawabnya +jelas jelaskan jelaslah jelasnya jika jikalau juga jumlah jumlahnya justru + +kala kalau kalaulah kalaupun kalian kami kamilah kamu kamulah kan kapan +kapankah kapanpun karena karenanya kasus kata katakan katakanlah katanya ke +keadaan kebetulan kecil kedua keduanya keinginan kelamaan kelihatan +kelihatannya kelima keluar kembali kemudian kemungkinan kemungkinannya kenapa +kepada kepadanya kesampaian keseluruhan keseluruhannya keterlaluan ketika +khususnya kini kinilah kira kira-kira kiranya kita kitalah kok kurang + +lagi lagian lah lain lainnya lalu lama lamanya lanjut lanjutnya lebih lewat +lima luar + +macam maka makanya makin malah malahan mampu mampukah mana manakala manalagi +masa masalah masalahnya masih masihkah masing masing-masing mau maupun +melainkan melakukan melalui melihat melihatnya memang memastikan memberi +memberikan membuat memerlukan memihak meminta memintakan memisalkan memperbuat +mempergunakan memperkirakan memperlihatkan mempersiapkan mempersoalkan +mempertanyakan mempunyai memulai memungkinkan menaiki menambahkan menandaskan +menanti menanti-nanti menantikan menanya menanyai menanyakan mendapat +mendapatkan mendatang mendatangi mendatangkan menegaskan mengakhiri mengapa +mengatakan mengatakannya mengenai mengerjakan mengetahui menggunakan +menghendaki mengibaratkan mengibaratkannya mengingat mengingatkan menginginkan +mengira mengucapkan mengucapkannya mengungkapkan menjadi menjawab menjelaskan +menuju menunjuk menunjuki menunjukkan menunjuknya menurut menuturkan +menyampaikan menyangkut menyatakan menyebutkan menyeluruh menyiapkan merasa +mereka merekalah merupakan meski meskipun meyakini meyakinkan minta mirip +misal misalkan misalnya mula mulai mulailah mulanya mungkin mungkinkah + +nah naik namun nanti nantinya nyaris nyatanya + +oleh olehnya + +pada padahal padanya pak paling panjang pantas para pasti pastilah penting +pentingnya per percuma perlu perlukah perlunya pernah persoalan pertama +pertama-tama pertanyaan pertanyakan pihak pihaknya pukul pula pun punya + +rasa rasanya rata rupanya + +saat saatnya saja sajalah saling sama sama-sama sambil sampai sampai-sampai +sampaikan sana sangat sangatlah satu saya sayalah se sebab sebabnya sebagai +sebagaimana sebagainya sebagian sebaik sebaik-baiknya sebaiknya sebaliknya +sebanyak sebegini sebegitu sebelum sebelumnya sebenarnya seberapa sebesar +sebetulnya sebisanya sebuah sebut sebutlah sebutnya secara secukupnya sedang +sedangkan sedemikian sedikit sedikitnya seenaknya segala segalanya segera +seharusnya sehingga seingat sejak sejauh sejenak sejumlah sekadar sekadarnya +sekali sekali-kali sekalian sekaligus sekalipun sekarang sekarang sekecil +seketika sekiranya sekitar sekitarnya sekurang-kurangnya sekurangnya sela +selain selaku selalu selama selama-lamanya selamanya selanjutnya seluruh +seluruhnya semacam semakin semampu semampunya semasa semasih semata semata-mata +semaunya sementara semisal semisalnya sempat semua semuanya semula sendiri +sendirian sendirinya seolah seolah-olah seorang sepanjang sepantasnya +sepantasnyalah seperlunya seperti sepertinya sepihak sering seringnya serta +serupa sesaat sesama sesampai sesegera sesekali seseorang sesuatu sesuatunya +sesudah sesudahnya setelah setempat setengah seterusnya setiap setiba setibanya +setidak-tidaknya setidaknya setinggi seusai sewaktu siap siapa siapakah +siapapun sini sinilah soal soalnya suatu sudah sudahkah sudahlah supaya + +tadi tadinya tahu tahun tak tambah tambahnya tampak tampaknya tandas tandasnya +tanpa tanya tanyakan tanyanya tapi tegas tegasnya telah tempat tengah tentang +tentu tentulah tentunya tepat terakhir terasa terbanyak terdahulu terdapat +terdiri terhadap terhadapnya teringat teringat-ingat terjadi terjadilah +terjadinya terkira terlalu terlebih terlihat termasuk ternyata tersampaikan +tersebut tersebutlah tertentu tertuju terus terutama tetap tetapi tiap tiba +tiba-tiba tidak tidakkah tidaklah tiga tinggi toh tunjuk turut tutur tuturnya + +ucap ucapnya ujar ujarnya umum umumnya ungkap ungkapnya untuk usah usai + +waduh wah wahai waktu waktunya walau walaupun wong + +yaitu yakin yakni yang +""".split() +) diff --git a/spacy/lang/ms/syntax_iterators.py b/spacy/lang/ms/syntax_iterators.py new file mode 100644 index 0000000..0277986 --- /dev/null +++ b/spacy/lang/ms/syntax_iterators.py @@ -0,0 +1,41 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + # fmt: off + labels = ["nsubj", "nsubj:pass", "obj", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/ms/tokenizer_exceptions.py b/spacy/lang/ms/tokenizer_exceptions.py new file mode 100644 index 0000000..e8b53fe --- /dev/null +++ b/spacy/lang/ms/tokenizer_exceptions.py @@ -0,0 +1,1532 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ._tokenizer_exceptions_list import MS_BASE_EXCEPTIONS + +# Daftar singkatan dan Akronim dari: +# https://ms.wiktionary.org/wiki/Wiktionary:Senarai_akronim_dan_singkatan + +_exc = {} + +for orth in MS_BASE_EXCEPTIONS: + _exc[orth] = [{ORTH: orth}] + orth_title = orth.title() + _exc[orth_title] = [{ORTH: orth_title}] + orth_caps = orth.upper() + _exc[orth_caps] = [{ORTH: orth_caps}] + orth_lower = orth.lower() + _exc[orth_lower] = [{ORTH: orth_lower}] + orth_first_upper = orth[0].upper() + orth[1:] + _exc[orth_first_upper] = [{ORTH: orth_first_upper}] + if "-" in orth: + orth_title = "-".join([part.title() for part in orth.split("-")]) + _exc[orth_title] = [{ORTH: orth_title}] + orth_caps = "-".join([part.upper() for part in orth.split("-")]) + _exc[orth_caps] = [{ORTH: orth_caps}] + +for exc_data in [ + {ORTH: "Jan.", NORM: "Januari"}, + {ORTH: "Feb.", NORM: "Februari"}, + {ORTH: "Mac.", NORM: "Mac"}, + {ORTH: "Apr.", NORM: "April"}, + {ORTH: "Jun.", NORM: "Jun"}, + {ORTH: "Jul.", NORM: "Julai"}, + {ORTH: "Ogos.", NORM: "Ogos"}, + {ORTH: "Sep.", NORM: "September"}, + {ORTH: "Okt.", NORM: "Oktober"}, + {ORTH: "Nov.", NORM: "November"}, + {ORTH: "Dis.", NORM: "Disember"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +_other_exc = { + "do'a": [{ORTH: "do'a", NORM: "doa"}], + "jum'at": [{ORTH: "jum'at", NORM: "Jumat"}], + "Jum'at": [{ORTH: "Jum'at", NORM: "Jumat"}], + "la'nat": [{ORTH: "la'nat", NORM: "laknat"}], + "ma'af": [{ORTH: "ma'af", NORM: "maaf"}], + "mu'jizat": [{ORTH: "mu'jizat", NORM: "mukjizat"}], + "Mu'jizat": [{ORTH: "Mu'jizat", NORM: "mukjizat"}], + "ni'mat": [{ORTH: "ni'mat", NORM: "nikmat"}], + "raka'at": [{ORTH: "raka'at", NORM: "rakaat"}], + "ta'at": [{ORTH: "ta'at", NORM: "taat"}], +} + +_exc.update(_other_exc) + +for orth in [ + "1 Kor.", + "1 Ptr.", + "1 Raj.", + "1 Sam.", + "1 Taw.", + "1 Tes.", + "1 Tim.", + "1 Yoh.", + "1Ch.", + "1Co.", + "1Jo.", + "1Ki.", + "1Pe.", + "1Sa.", + "1Th.", + "1Ti.", + "2 Kor.", + "2 Ptr.", + "2 Raj.", + "2 Sam.", + "2 Taw.", + "2 Tes.", + "2 Tim.", + "2 Yoh.", + "2Ch.", + "2Co.", + "2Jo.", + "2Ki.", + "2Pe.", + "2Sa.", + "2Th.", + "2Ti.", + "3 Yoh.", + "3D", + "3F", + "3Jo.", + "3M", + "8MP", + "AA", + "AAAAAA", + "AB", + "Abd.", + "ABC", + "ABIM", + "ABM", + "ABMI", + "ABS", + "AC", + "Ac", + "ACAPLPL", + "Act.", + "AD", + "AD LIB", + "ADAM", + "ADB", + "ADD", + "ADIL", + "ADN", + "ADR", + "ADRI", + "ADSL", + "ADUN", + "AFAS", + "AFTA", + "Ag", + "AGMARIS", + "AH", + "AI", + "AIA", + "AIDS", + "AIJV", + "AIM", + "a/k", + "ak", + "AKN", + "Al", + "a/l", + "AM", + "Am", + "Am.", + "AMN", + "Amo.", + "AMPS", + "Ams.", + "AMWA", + "AN", + "a.n.", + "ANGKASA", + "ANM", + "ANSI", + "Ant.", + "AOL", + "AP", + "a/p", + "APD", + "APEC", + "API", + "APIK", + "APM", + "APN", + "APP", + "Apr.", + "APRI", + "Ar", + "Ar.", + "ark.", + "A.S.", + "As", + "a.s.", + "ASA", + "ASAS 50", + "ASB", + "ASCII", + "ASEAN", + "ASEAN+3", + "ASEM", + "a.s.f.", + "ASN", + "a.s.o.", + "ASP", + "Ast.", + "A.T.", + "At", + "ATM", + "a.t.r.", + "ATUR", + "Au", + "AURI", + "Aug.", + "AWOL", + "Ayb.", + "B", + "BA", + "Ba", + "BAC", + "BAFIA", + "BAM", + "BANANA", + "BAPP", + "BASF", + "BATA", + "BB", + "BBC", + "BBE", + "BBS", + "BC", + "BCG", + "BCIC", + "b.d.", + "BDSSHAM", + "Be", + "BEER", + "BERNAMA", + "Bh", + "b.h.", + "Bhd.", + "Bi", + "BIDS", + "Bil.", + "bil.", + "BIMP-EAGA", + "Bio.", + "BIOS", + "BITMB", + "BJ", + "Bk", + "b.k.", + "BKAL", + "bkn.", + "BKP", + "BL", + "BLR", + "BM", + "BMI", + "BMW", + "BN", + "BNM", + "BO", + "BOJ", + "BOO", + "BOP", + "BOT", + "BP", + "b.p.", + "BPA", + "BPAs", + "bpd.", + "BPIMB", + "BPM", + "BPO", + "BPPH", + "Br", + "Br.", + "BSA", + "B.Sc.", + "B.Sh.", + "b.s.j.", + "BSN", + "Bt.", + "bt.", + "BWT", + "BYOB", + "C", + "C.", + "C/E", + "Ca", + "CAAM", + "CAD", + "CAM", + "CATV", + "CBS", + "CBT", + "CC", + "CCD", + "CCM", + "CCR", + "cct-km", + "CCTV", + "CCU", + "CD", + "Cd", + "CD-ROM", + "CD-RW", + "CDRC", + "Ce", + "CEO", + "CEPT", + "Cetak", + "Cf", + "CFO", + "CFTC", + "CGC", + "CGI", + "CH", + "CIA", + "CIAST", + "CID", + "CIDB", + "CIQ", + "CKD", + "CL", + "Cl", + "c.l.", + "CLI", + "CLOB", + "CM", + "Cm", + "cm.", + "CMAG", + "CMI", + "CMP", + "CNN", + "Co", + "COD", + "Col.", + "COLA", + "COMDEX", + "CP", + "CPI", + "CPO", + "CPR", + "CPU", + "Cr", + "CRDF", + "Cs", + "CST", + "CT", + "CTIP", + "CTRM", + "Cu", + "CUEPACS", + "D-8", + "d/a", + "DAGS", + "Dan.", + "DANCED", + "DAP", + "DARA", + "Db", + "DBKL", + "DBP", + "DBR", + "DC", + "DDA", + "DDT", + "DEB", + "Dec.", + "Deu.", + "DFIs", + "dgn.", + "DHL", + "DIBML", + "DIN", + "Dis.", + "DJ", + "d.l.l.", + "dlm.", + "dng.", + "DNS", + "DO", + "DOA", + "DOE", + "DOF", + "DOSH", + "doz.", + "DPPS", + "Dr.", + "dr.", + "drp.", + "drpd.", + "Ds", + "d.sb.", + "d.st.", + "DSTN2", + "Dt.", + "DTAs", + "DTMF", + "DTP", + "DTV", + "DUBES", + "DUNHILL", + "DV8", + "DVD", + "DVE", + "DVS", + "dw.t.", + "Dy", + "DYMM", + "E", + "E-Commerce", + "E-Dagang", + "E&E", + "E-Faraid", + "E-Government", + "E-Kerajaan", + "E-Mail", + "E-Services", + "E-Village", + "E-Zine", + "EALAF", + "EBI", + "EBP", + "EC", + "ECAFE", + "Ecc.", + "ECI", + "ECM", + "ECOSOC", + "ECP", + "ECR", + "EDI", + "EE", + "EEC", + "Ef.", + "EG", + "Eko.", + "EKS", + "ELWS", + "ELX", + "EMI", + "EMUs", + "En.", + "EP", + "EPF", + "Eph.", + "EPP", + "EPS", + "EPU", + "ER", + "Er", + "ERL", + "ERT", + "Es", + "ESCAP", + "ESOS", + "ESP", + "EST", + "Est.", + "ET", + "ETA", + "ETACS", + "ETC", + "ETD", + "EU", + "Eu", + "EVIAN", + "Exim Bank", + "Exo.", + "Eze.", + "Ezr.", + "F", + "FAM", + "FAMA", + "FAO", + "FAQ", + "FAX", + "FBI", + "FC", + "FCA", + "FCC", + "FDI", + "FE", + "Fe", + "f.e.", + "Feb.", + "FELCRA", + "FELDA", + "FI", + "FIA 1993", + "FIAT", + "FIC", + "FIDA", + "FIFA", + "FIMA", + "Fiz.", + "Flm.", + "Flp.", + "FM", + "Fm", + "FMUTM", + "FO", + "FOA", + "FOB", + "FOC", + "FOMCA", + "FORD", + "Fr", + "FRIM", + "FRTI", + "FSMP", + "FTA", + "FTE", + "FTP", + "G", + "g.", + "G15", + "G77", + "Ga", + "GAC", + "GACM", + "Gal.", + "GAPENA", + "GATS", + "GATT", + "GB", + "Gbps.", + "Gd", + "GDP", + "Ge", + "GEC", + "Gen.", + "Geo.", + "Geog.", + "Gerakan", + "GH", + "GIF", + "GII", + "GIS", + "GITIC", + "GITN", + "GJ", + "GLCs", + "GM", + "GMBH", + "GMI", + "GMT", + "GNP", + "GNS", + "GOLD", + "GP", + "GPC", + "GPIM", + "GPMS", + "GPO", + "GPP", + "GPS", + "GRO", + "GRS", + "GSMC", + "GST", + "GTZ", + "GUI", + "GWh.", + "H", + "Ha", + "Hab.", + "Hag.", + "Hak.", + "ham", + "hb.", + "HCI", + "HDTV", + "He", + "Heb.", + "Hf", + "Hg", + "HI-FI", + "HIS", + "HIV", + "Hj.", + "HMS", + "Ho", + "Hos.", + "HP", + "HRDC", + "HRDF", + "HRMIS", + "Hs", + "Hut.", + "I", + "I/O", + "IA", + "IAA", + "IADPs", + "IB", + "i.b.", + "IBA", + "IBFIM", + "IBG", + "Ibr.", + "IBRD", + "IBS", + "IC", + "ICA", + "ICBM", + "ICFM", + "ICI", + "ICM", + "ICOR", + "ICP", + "ICT", + "ICU", + "ID", + "Id.", + "IDB", + "IDFR", + "IE", + "i.e.", + "IFSB", + "IGAs", + "IGS", + "IHP", + "IHPG", + "IIM", + "IINA", + "IKKL", + "IKP", + "IKPH", + "IKS", + "Im.", + "IMD", + "IMF", + "IMP2", + "IMR", + "IMS-GT", + "IMT-GT", + "In", + "in.", + "INFRA", + "INSEP", + "INSPEN", + "INTAN", + "IOFC", + "IOU", + "IP", + "IPA", + "IPBA", + "IPCs", + "IPEBP", + "IPI", + "IPKIM", + "IPKPM", + "IPO", + "IPP", + "IPPM", + "IPPPM", + "i.pt.", + "IPTAR", + "IPTNM", + "IQR", + "Ir", + "IRA", + "IRPA", + "IRS", + "i.s.", + "ISA", + "Isa.", + "ISDN", + "ISMM", + "ISO", + "ISP", + "ist.", + "IT", + "i.t.", + "ITA", + "ITAF", + "ITEX", + "ITK", + "ITM", + "ITO", + "ITRCo", + "ITTA", + "ITU", + "JAK", + "JAKIM", + "Jam.", + "Jan.", + "Jb.", + "JBIC", + "JD", + "JDA", + "Jdg.", + "Jer.", + "Jh.", + "JICA", + "JJ", + "Jk.", + "JKKK", + "jkps.", + "JKR", + "JMTI", + "JOA", + "Joe.", + "Joh.", + "Jon.", + "Jos.", + "JP", + "JPA", + "JPEG", + "JPH", + "JPJ", + "JPSHK", + "JPS", + "JPT", + "JRDA", + "JSM", + "JT", + "Jud.", + "Jul.", + "Jun.", + "JVC", + "Jw.", + "K", + "K-Economy", + "KADA", + "KBE", + "KBIA", + "KBPA", + "KBSM", + "KD", + "Kd.", + "KDI", + "KDN", + "KDNK", + "KE", + "KEAP", + "Kej.", + "Kel.", + "KEM", + "KEMLU", + "kep.", + "Kg.", + "kg.", + "KGB", + "KGK", + "KH", + "ki.", + "Kid.", + "KIK", + "KIKMTT", + "KIM", + "Kim.", + "Kis.", + "KIX", + "KKGSK", + "KKK", + "KKPPA", + "KL", + "Kl.", + "KLCI", + "KLIA", + "KLIBOR", + "KLIM", + "KLM", + "KLSE", + "KM", + "KMM", + "KNK", + "KO", + "Kol.", + "Kom.", + "Komp.", + "KOMSAS", + "KPAI", + "KPB", + "KPBA", + "KPC", + "kpd.", + "KPE", + "KPIs", + "KPPL", + "KPPMS", + "KPWM", + "Kr", + "KRM", + "KSTI", + "KT", + "KTA", + "KTABKL", + "KTM", + "KTMB", + "kV", + "kW", + "kWh", + "kWj", + "KWSP", + "LA", + "La", + "LABOR", + "Lam.", + "LAN", + "LAPD", + "LASER", + "LAX", + "lb.", + "LC", + "LCD", + "LCHRF", + "LCLY", + "LED", + "Lev.", + "LFPR", + "LFS", + "LFX", + "LGM", + "Li", + "LID", + "Lin.", + "LKN", + "LKPM", + "LKPP", + "LKTP", + "LKWJ", + "LLB", + "LLC", + "LLN", + "LLS", + "LMSM", + "LNG", + "LOA", + "LOBATA", + "LOFSA", + "LPG", + "LPIP", + "LPKI", + "LPKLPL", + "LPKN", + "LPN", + "LPP", + "LPPK", + "LPPM", + "LPPP", + "LPPTP", + "Lr", + "LRs", + "LRT", + "LS", + "LTAKL", + "LTD", + "LTK", + "Lu", + "LUAS", + "Luk.", + "lw.", + "lwn.", + "M\n", + "m", + "M&A", + "MAB", + "MACRES", + "MAD", + "MADA", + "MAGERAN", + "MAHA", + "MAHSURI", + "Mal.", + "MALINDO", + "MAMPU", + "Mar.", + "MARA", + "MARC", + "MARDI", + "MARLBORO", + "MAS", + "MASSA", + "MASSCORP", + "Mat.", + "MATRADE", + "MAVCAP", + "MB", + "MBA", + "MBBS", + "MBM", + "MBO", + "MBS", + "MBTU", + "MC", + "MCA", + "MCB", + "MCSL", + "MCSv5", + "MD", + "Md", + "MDB", + "MDC", + "MDG", + "MDV", + "MEASAT", + "MEATJ", + "MECIB", + "MEMO", + "MENLU", + "MEPS", + "MES", + "MESDAQ", + "METEOR", + "MFI", + "MFIs", + "MG", + "Mg", + "MGM", + "MGR", + "MGS", + "MHA", + "Mi.", + "MIA", + "MIB", + "MIC", + "Mic.", + "MICE", + "MIDA", + "MIDF", + "MIDI", + "MIG", + "MIGHT", + "MII", + "MIMOS", + "MINDEF", + "MINT", + "mis.", + "MIT", + "MITC", + "MITI", + "Ml.", + "MLNG", + "mlpd.", + "MM", + "mm", + "MMN", + "mmscfd.", + "MMU", + "MMX", + "Mn", + "Mn.", + "MNA", + "MNCs", + "MO", + "Mo", + "MOA", + "MOD", + "MODEM", + "MOE", + "MOH", + "MOSTE", + "MOSTI", + "MOU", + "MP", + "MPB", + "MPEG", + "MPOB", + "MPP", + "mppa.", + "MPPJ", + "MPS", + "MPTM", + "MR", + "m.r.", + "MRB", + "MRELB", + "Mrk.", + "MRRDB", + "MS", + "MS-DOS", + "MSC", + "MSG", + "MSM", + "Mt", + "MTC", + "MTCP", + "MTD", + "MTDC", + "MTPB", + "MTV", + "Muz.", + "MV", + "MW", + "MY", + "MyKe", + "Mzm.", + "N", + "N/A", + "Na", + "NAB", + "NACIWID", + "Nah.", + "NAP", + "NASA", + "NATO", + "NAV", + "NB", + "Nb", + "NBA", + "NBC", + "NCR", + "Nd", + "NDP", + "Ne", + "NEAC", + "NEC", + "NEF", + "Neh.", + "NEP", + "NEqO", + "NERP", + "NF", + "NFPEs", + "NG", + "NGOs", + "NGV", + "NHEF", + "NHHES", + "NHK", + "Ni", + "NIDC", + "NIH", + "NIP", + "NIPA", + "NIS", + "NISIR", + "NITA", + "NITC", + "NITP", + "NIV", + "NLAC", + "NMPBSP", + "NMU", + "No", + "No.", + "no.", + "NOSS", + "Nov.", + "Np", + "NPC", + "NPCS", + "NPL", + "NRCC", + "NRW", + "NS", + "Ns", + "NSB", + "NTA", + "NTHRDC", + "NTMP", + "NTSC", + "Num.", + "NUTF", + "NVP", + "NVTC", + "NWRC", + "O", + "Ob.", + "Oba.", + "OC", + "OCPD", + "Oct.", + "OD", + "ODA", + "OECD", + "OEM", + "Ogo.", + "OHQs", + "OIC", + "Okt.", + "OPEC", + "OPP", + "OPP3", + "OPR", + "OS", + "Os", + "OSA", + "OT", + "OUG", + "oz.", + "P", + "P&P", + "PA", + "Pa", + "PABK", + "PABX", + "PAK", + "PAKSI", + "PAL", + "PALL MALL", + "PAS", + "PATA", + "PAWS", + "Pb", + "PBA", + "PBB", + "PBM", + "PBP", + "PBSM", + "PBT", + "PC", + "PC(s)", + "PCB", + "PCIRITA", + "PCM", + "PCMCIA", + "PCN", + "PD", + "Pd", + "pd.", + "PDS", + "PE", + "PEKEMAS", + "PEMADAM", + "PENA", + "PENIS", + "PERDANA", + "PERKESO", + "PERKIM", + "PERNAS", + "PERTAMA", + "PERTIWI", + "PESAKA", + "PETA", + "PETRONAS", + "PGU", + "Ph.", + "PHD", + "Phi.", + "Phm.", + "PIK", + "PIKOM", + "PIN", + "PINTAS", + "PIPM", + "PISK", + "PITA", + "PIXEL", + "PJ", + "PJK", + "PJKB", + "PJP", + "PKBM", + "PKBTA", + "PKEN", + "Pkh.", + "PKKM", + "PKLPA", + "PKM", + "PKNS", + "PKPIM", + "PKPM", + "PKR", + "PKS", + "Pl.", + "p.l.", + "PLA", + "PLC", + "PLCHP", + "PLCs", + "PLI", + "PLT", + "PLUS", + "PLWS", + "PM", + "Pm", + "PMM", + "PMP", + "PMR", + "PMS", + "Pn.", + "PNAT", + "PNS", + "PO", + "Po", + "POCPA", + "POKEMON", + "Pol.", + "POP", + "PORIM", + "PORLA", + "PORTAFOAM", + "PP", + "PPA", + "PPBE", + "PPBK", + "ppd.", + "PPGM", + "PPI", + "PPK", + "PPL", + "PPM", + "PPP", + "PPPB", + "PPPLM", + "PPPM", + "PPR", + "PPRT", + "PPS", + "PPTM", + "PPU", + "PR", + "Pr", + "Pr.", + "prb.", + "PRI", + "PRO", + "Pro.", + "Prof.", + "PROSPER", + "PROSTAR", + "PROTON", + "PS", + "PSA", + "Psa.", + "PSCs", + "PSDC", + "PSDH", + "Psi.", + "PSKE", + "PSRM", + "PST", + "PT", + "Pt", + "PTD", + "PTP", + "Pu", + "PUNB", + "QA", + "QC", + "QCC", + "R&D", + "RA", + "Ra", + "RAM", + "RAPP", + "Rat.", + "Rb", + "RCA", + "RDA", + "RDAs", + "RDCs", + "RE", + "Re", + "REHDA", + "Rev.", + "Rf", + "Rg", + "RGB", + "Rh", + "RI", + "RIDA", + "RIP", + "RISDA", + "r.l.", + "RM", + "Rm.", + "RMKe-8", + "Rn", + "ROC", + "ROM", + "Rom.", + "RPG", + "RPS", + "RRI", + "RRIM", + "RRJP", + "RRP", + "RSGC", + "RSS", + "RSVP", + "Rt.", + "RTA", + "RTM", + "Ru", + "Rut.", + "RWCR", + "RX", + "S", + "S/N", + "S&T", + "S-VHS", + "SA", + "SAC", + "SADCs", + "SAGA", + "SALCRA", + "SALM", + "SALT", + "SAM", + "SAP", + "SARS", + "Sas.", + "s.a.w.", + "SB", + "Sb", + "Sb.", + "SBA", + "SBB", + "sbg.", + "SBK", + "SC", + "Sc", + "SCA", + "SCADA", + "SCANS", + "SCSI", + "SCuM", + "SDCs", + "Sdn. Bhd.", + "sdr.", + "SDRC", + "Se", + "SEATO", + "SEB", + "SECAM", + "SEDCs", + "SEFF", + "Sej.", + "SEMS", + "Sep.", + "Sept.", + "SESB", + "SESCo", + "s.f.", + "Sg", + "SGPCA", + "SGPPI", + "SGPPKRM", + "SGX", + "Si", + "Si.", + "SIA 1983", + "SIC", + "SIM", + "SING", + "SIRIM", + "SITTDEC", + "sj.", + "SKDTP", + "SKM", + "SKSM", + "SL", + "Sl.", + "sl.", + "SLMCH", + "SLR", + "SM", + "Sm", + "SMART", + "SMEs", + "SMEt", + "SMIs", + "SMIDEC", + "SMIDP", + "SMJK", + "SMR", + "SMS", + "SMT", + "SMTP", + "SN", + "Sn", + "SOB", + "SOCSO", + "SOHO", + "Son.", + "SOS", + "Sos.", + "SP", + "SPA", + "SPAM", + "SPCA", + "SPKR", + "SPLAM", + "SPM", + "SPNB", + "SPSP", + "t.", + "Ta", + "Tadb.", + "TAF", + "TAF-W", + "Tani", + "TAP", + "TAR", + "TARBI", + "TB", + "Tb", + "TBA", + "TBTP", + "Tc", + "TCPD", + "TDCs", + "Te", + "TEKUN", + "TELCO", + "TELEX", + "TEUs", + "TFP", + "TGV", + "TH", + "Th", + "THIS", + "Ti", + "TICAD", + "Tit.", + "TKA", + "Tks.", + "Tl", + "TLDM", + "TM", + "Tm", + "TMB", + "TMK", + "TNB", + "TNSB", + "TNT", + "TOEFL", + "TP", + "TPIM", + "TPK", + "TPPP", + "TPPT", + "TPSM", + "TPUB", + "TQM", + "Tr.", + "TRIPs", + "tsb.", + "tscf.", + "t.sh.", + "t.s.t.", + "TT", + "t.t.", + "TUDM", + "TV", + "TVSMR", + "TWAIN", + "TX", + "TYPHIrapid", + "U", + "Ubat", + "UDA", + "Udg.", + "UFO", + "UH", + "UIA", + "UiTM", + "UK", + "UKM", + "UL", + "Ul.", + "ULC", + "UM", + "UMNO", + "UMS", + "UN", + "UN/OSCAL", + "UNCLE", + "UNCTAD", + "UNDP", + "UNESCO", + "UNFCCC", + "UNFPA", + "UNHCR", + "UNICEF", + "UNIMAS", + "UNTAET", + "UPE", + "UPM", + "UPS", + "UPSR", + "URL", + "US", + "USAINS", + "USD", + "USM", + "USNO", + "USS", + "USSR", + "UTC", + "UTF", + "utk.", + "UTM", + "V", + "VAT", + "VCC", + "VCD", + "VCR", + "VD", + "VDSC", + "VGA", + "VHF", + "VHS", + "VIP", + "VMS", + "VO", + "VOA", + "VoIP", + "VR", + "VSOP", + "VW", + "W", + "W/O", + "WAP", + "WAY", + "WC", + "WDDM", + "WDM", + "WHO", + "Why.", + "WIM", + "WPG", + "WTO", + "WWF", + "WWW", + "WYSIWYG", + "Xe", + "XO", + "XXL", + "Y", + "Y2K", + "YAB", + "Yak.", + "YAM", + "YAS", + "YB", + "Yb", + "Yeh.", + "Yer.", + "Yes.", + "yg.", + "Yl.", + "YM", + "YMCA", + "Yoh.", + "Yos.", + "Y.Th.", + "YTM", + "Yud.", + "Yun.", + "Za.", + "Zec.", + "Zef.", + "Zep.", + "ZIP", + "Zn", + "Zr", +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/nb/__init__.py b/spacy/lang/nb/__init__.py new file mode 100644 index 0000000..ef4665c --- /dev/null +++ b/spacy/lang/nb/__init__.py @@ -0,0 +1,51 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ...pipeline import Lemmatizer +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class NorwegianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class Norwegian(Language): + lang = "nb" + Defaults = NorwegianDefaults + + +@Norwegian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return Lemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Norwegian"] diff --git a/spacy/lang/nb/examples.py b/spacy/lang/nb/examples.py new file mode 100644 index 0000000..242dab7 --- /dev/null +++ b/spacy/lang/nb/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.nb.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple vurderer å kjøpe britisk oppstartfirma for en milliard dollar.", + "Selvkjørende biler flytter forsikringsansvaret over på produsentene.", + "San Francisco vurderer å forby robotbud på fortauene.", + "London er en stor by i Storbritannia.", +] diff --git a/spacy/lang/nb/punctuation.py b/spacy/lang/nb/punctuation.py new file mode 100644 index 0000000..a1fdb87 --- /dev/null +++ b/spacy/lang/nb/punctuation.py @@ -0,0 +1,69 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) + +# Punctuation adapted from Danish +_quotes = CONCAT_QUOTES.replace("'", "") +_list_punct = [x for x in LIST_PUNCT if x != "#"] +_list_icons = [x for x in LIST_ICONS if x != "°"] +_list_icons = [x.replace("\\u00B0", "") for x in _list_icons] +_list_quotes = [x for x in LIST_QUOTES if x != "\\'"] + + +_prefixes = ( + ["§", "%", "=", "—", "–", r"\+(?![0-9])"] + + _list_punct + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + + +_infixes = ( + LIST_ELLIPSES + + _list_icons + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + _list_quotes + + _list_icons + + ["—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=_quotes, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] + + [r"(?<=[^sSxXzZ])'"] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/nb/stop_words.py b/spacy/lang/nb/stop_words.py new file mode 100644 index 0000000..d9ed414 --- /dev/null +++ b/spacy/lang/nb/stop_words.py @@ -0,0 +1,50 @@ +STOP_WORDS = set( + """ +alle allerede alt and andre annen annet at av + +bak bare bedre beste blant ble bli blir blitt bris by både + +da dag de del dem den denne der dermed det dette disse du + +eller en enn er et ett etter + +fem fikk fire fjor flere folk for fortsatt fra fram +funnet få får fått før først første + +gang gi gikk gjennom gjorde gjort gjør gjøre god godt grunn gå går + +ha hadde ham han hans har hele helt henne hennes her hun + +i ifølge igjen ikke ingen inn + +ja jeg + +kamp kampen kan kl klart kom komme kommer kontakt kort kroner kunne kveld + +la laget land landet langt leder ligger like litt løpet + +man mange med meg mellom men mener mennesker mens mer mot mye må mål måtte + +ned neste noe noen nok ny nye nå når + +og også om opp opplyser oss over + +personer plass poeng på + +runde rundt + +sa saken samme sammen samtidig satt se seg seks selv senere ser sett +siden sier sin sine siste sitt skal skriver skulle slik som sted stedet stor +store står svært så + +ta tatt tid tidligere til tilbake tillegg tok tror + +under ut uten utenfor + +vant var ved veldig vi videre viktig vil ville viser vår være vært + +å år + +ønsker +""".split() +) diff --git a/spacy/lang/nb/syntax_iterators.py b/spacy/lang/nb/syntax_iterators.py new file mode 100644 index 0000000..89a8f5e --- /dev/null +++ b/spacy/lang/nb/syntax_iterators.py @@ -0,0 +1,39 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + # fmt: off + labels = ["nsubj", "nsubj:pass", "obj", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/nb/tokenizer_exceptions.py b/spacy/lang/nb/tokenizer_exceptions.py new file mode 100644 index 0000000..9b99a1d --- /dev/null +++ b/spacy/lang/nb/tokenizer_exceptions.py @@ -0,0 +1,222 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +for exc_data in [ + {ORTH: "jan.", NORM: "januar"}, + {ORTH: "feb.", NORM: "februar"}, + {ORTH: "mar.", NORM: "mars"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "jun.", NORM: "juni"}, + {ORTH: "jul.", NORM: "juli"}, + {ORTH: "aug.", NORM: "august"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "nov.", NORM: "november"}, + {ORTH: "des.", NORM: "desember"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "Ap.", + "Aq.", + "Ca.", + "Chr.", + "Co.", + "Co.", + "Dr.", + "F.eks.", + "Fr.p.", + "Frp.", + "Grl.", + "Kr.", + "Kr.F.", + "Kr.F.s", + "Mr.", + "Mrs.", + "Pb.", + "Pr.", + "Sp.", + "Sp.", + "St.", + "a.m.", + "ad.", + "adm.dir.", + "andelsnr", + "b.c.", + "bl.a.", + "bla.", + "bm.", + "bnr.", + "bto.", + "c.c.", + "ca.", + "cand.mag.", + "co.", + "d.d.", + "d.m.", + "d.y.", + "dept.", + "dr.", + "dr.med.", + "dr.philos.", + "dr.psychol.", + "dvs.", + "e.Kr.", + "e.l.", + "eg.", + "ekskl.", + "el.", + "et.", + "etc.", + "etg.", + "ev.", + "evt.", + "f.", + "f.Kr.", + "f.eks.", + "f.o.m.", + "fhv.", + "fk.", + "foreg.", + "fork.", + "fv.", + "fvt.", + "g.", + "gl.", + "gno.", + "gnr.", + "grl.", + "gt.", + "h.r.adv.", + "hhv.", + "hoh.", + "hr.", + "ifb.", + "ifm.", + "iht.", + "inkl.", + "istf.", + "jf.", + "jr.", + "jun.", + "juris.", + "kfr.", + "kgl.", + "kgl.res.", + "kl.", + "komm.", + "kr.", + "kst.", + "lat.", + "lø.", + "m.a.o.", + "m.fl.", + "m.m.", + "m.v.", + "ma.", + "mag.art.", + "md.", + "mfl.", + "mht.", + "mill.", + "min.", + "mnd.", + "moh.", + "mrd.", + "muh.", + "mv.", + "mva.", + "n.å.", + "ndf.", + "no.", + "nov.", + "nr.", + "nto.", + "nyno.", + "o.a.", + "o.l.", + "off.", + "ofl.", + "okt.", + "on.", + "op.", + "org.", + "osv.", + "ovf.", + "p.", + "p.a.", + "p.g.a.", + "p.m.", + "p.t.", + "pga.", + "ph.d.", + "pkt.", + "pr.", + "pst.", + "pt.", + "red.anm.", + "ref.", + "res.", + "res.kap.", + "resp.", + "rv.", + "s.", + "s.d.", + "s.k.", + "s.k.", + "s.u.", + "s.å.", + "sen.", + "sep.", + "siviling.", + "sms.", + "snr.", + "spm.", + "sr.", + "sst.", + "st.", + "st.meld.", + "st.prp.", + "stip.", + "stk.", + "stud.", + "sv.", + "såk.", + "sø.", + "t.h.", + "t.o.m.", + "t.v.", + "temp.", + "ti.", + "tils.", + "tilsv.", + "tl;dr", + "tlf.", + "to.", + "ult.", + "utg.", + "v.", + "vedk.", + "vedr.", + "vg.", + "vgs.", + "vha.", + "vit.ass.", + "vn.", + "vol.", + "vs.", + "vsa.", + "©NTB", + "årg.", + "årh.", + "§§", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ne/__init__.py b/spacy/lang/ne/__init__.py new file mode 100644 index 0000000..5c9e687 --- /dev/null +++ b/spacy/lang/ne/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class NepaliDefaults(BaseDefaults): + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Nepali(Language): + lang = "ne" + Defaults = NepaliDefaults + + +__all__ = ["Nepali"] diff --git a/spacy/lang/ne/examples.py b/spacy/lang/ne/examples.py new file mode 100644 index 0000000..cc3b382 --- /dev/null +++ b/spacy/lang/ne/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ne.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "एप्पलले अमेरिकी स्टार्टअप १ अर्ब डलरमा किन्ने सोच्दै छ", + "स्वायत्त कारहरूले बीमा दायित्व निर्माताहरु तिर बदल्छन्", + "स्यान फ्रांसिस्कोले फुटपाथ वितरण रोबोटहरु प्रतिबंध गर्ने विचार गर्दै छ", + "लन्डन यूनाइटेड किंगडमको एक ठूलो शहर हो।", + "तिमी कहाँ छौ?", + "फ्रान्स को राष्ट्रपति को हो?", + "संयुक्त राज्यको राजधानी के हो?", + "बराक ओबामा कहिले कहिले जन्मेका हुन्?", +] diff --git a/spacy/lang/ne/lex_attrs.py b/spacy/lang/ne/lex_attrs.py new file mode 100644 index 0000000..91d5b0e --- /dev/null +++ b/spacy/lang/ne/lex_attrs.py @@ -0,0 +1,94 @@ +from ...attrs import LIKE_NUM, NORM +from ..norm_exceptions import BASE_NORMS + +# fmt: off +_stem_suffixes = [ + ["ा", "ि", "ी", "ु", "ू", "ृ", "े", "ै", "ो", "ौ"], + ["ँ", "ं", "्", "ः"], + ["लाई", "ले", "बाट", "को", "मा", "हरू"], + ["हरूलाई", "हरूले", "हरूबाट", "हरूको", "हरूमा"], + ["इलो", "िलो", "नु", "ाउनु", "ई", "इन", "इन्", "इनन्"], + ["एँ", "इँन्", "इस्", "इनस्", "यो", "एन", "यौं", "एनौं", "ए", "एनन्"], + ["छु", "छौँ", "छस्", "छौ", "छ", "छन्", "छेस्", "छे", "छ्यौ", "छिन्", "हुन्छ"], + ["दै", "दिन", "दिँन", "दैनस्", "दैन", "दैनौँ", "दैनौं", "दैनन्"], + ["हुन्न", "न्न", "न्न्स्", "न्नौं", "न्नौ", "न्न्न्", "िई"], + ["अ", "ओ", "ऊ", "अरी", "साथ", "वित्तिकै", "पूर्वक"], + ["याइ", "ाइ", "बार", "वार", "चाँहि"], + ["ने", "ेको", "ेकी", "ेका", "ेर", "दै", "तै", "िकन", "उ", "न", "नन्"] +] +# fmt: on + +# reference 1: https://en.wikipedia.org/wiki/Numbers_in_Nepali_language +# reference 2: https://www.imnepal.com/nepali-numbers/ +_num_words = [ + "शुन्य", + "एक", + "दुई", + "तीन", + "चार", + "पाँच", + "छ", + "सात", + "आठ", + "नौ", + "दश", + "एघार", + "बाह्र", + "तेह्र", + "चौध", + "पन्ध्र", + "सोह्र", + "सोह्र", + "सत्र", + "अठार", + "उन्नाइस", + "बीस", + "तीस", + "चालीस", + "पचास", + "साठी", + "सत्तरी", + "असी", + "नब्बे", + "सय", + "हजार", + "लाख", + "करोड", + "अर्ब", + "खर्ब", +] + + +def norm(string): + # normalise base exceptions, e.g. punctuation or currency symbols + if string in BASE_NORMS: + return BASE_NORMS[string] + # set stem word as norm, if available, adapted from: + # https://github.com/explosion/spaCy/blob/master/spacy/lang/hi/lex_attrs.py + # https://www.researchgate.net/publication/237261579_Structure_of_Nepali_Grammar + for suffix_group in reversed(_stem_suffixes): + length = len(suffix_group[0]) + if len(string) <= length: + break + for suffix in suffix_group: + if string.endswith(suffix): + return string[:-length] + return string + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(", ", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {NORM: norm, LIKE_NUM: like_num} diff --git a/spacy/lang/ne/stop_words.py b/spacy/lang/ne/stop_words.py new file mode 100644 index 0000000..8470297 --- /dev/null +++ b/spacy/lang/ne/stop_words.py @@ -0,0 +1,494 @@ +# Source: https://github.com/sanjaalcorps/NepaliStopWords/blob/master/NepaliStopWords.txt + +STOP_WORDS = set( + """ +अक्सर +अगाडि +अगाडी +अघि +अझै +अठार +अथवा +अनि +अनुसार +अन्तर्गत +अन्य +अन्यत्र +अन्यथा +अब +अरु +अरुलाई +अरू +अर्को +अर्थात +अर्थात् +अलग +अलि +अवस्था +अहिले +आए +आएका +आएको +आज +आजको +आठ +आत्म +आदि +आदिलाई +आफनो +आफू +आफूलाई +आफै +आफैँ +आफ्नै +आफ्नो +आयो +उ +उक्त +उदाहरण +उनको +उनलाई +उनले +उनि +उनी +उनीहरुको +उन्नाइस +उप +उसको +उसलाई +उसले +उहालाई +ऊ +एउटा +एउटै +एक +एकदम +एघार +ओठ +औ +औं +कता +कति +कतै +कम +कमसेकम +कसरि +कसरी +कसै +कसैको +कसैलाई +कसैले +कसैसँग +कस्तो +कहाँबाट +कहिलेकाहीं +का +काम +कारण +कि +किन +किनभने +कुन +कुनै +कुन्नी +कुरा +कृपया +के +केहि +केही +को +कोहि +कोहिपनि +कोही +कोहीपनि +क्रमशः +गए +गएको +गएर +गयौ +गरि +गरी +गरे +गरेका +गरेको +गरेर +गरौं +गर्छ +गर्छन् +गर्छु +गर्दा +गर्दै +गर्न +गर्नु +गर्नुपर्छ +गर्ने +गैर +घर +चार +चाले +चाहनुहुन्छ +चाहन्छु +चाहिं +चाहिए +चाहिंले +चाहीं +चाहेको +चाहेर +चोटी +चौथो +चौध +छ +छन +छन् +छु +छू +छैन +छैनन् +छौ +छौं +जता +जताततै +जना +जनाको +जनालाई +जनाले +जब +जबकि +जबकी +जसको +जसबाट +जसमा +जसरी +जसलाई +जसले +जस्ता +जस्तै +जस्तो +जस्तोसुकै +जहाँ +जान +जाने +जाहिर +जुन +जुनै +जे +जो +जोपनि +जोपनी +झैं +ठाउँमा +ठीक +ठूलो +त +तता +तत्काल +तथा +तथापि +तथापी +तदनुसार +तपाइ +तपाई +तपाईको +तब +तर +तर्फ +तल +तसरी +तापनि +तापनी +तिन +तिनि +तिनिहरुलाई +तिनी +तिनीहरु +तिनीहरुको +तिनीहरू +तिनीहरूको +तिनै +तिमी +तिर +तिरको +ती +तीन +तुरन्त +तुरुन्त +तुरुन्तै +तेश्रो +तेस्कारण +तेस्रो +तेह्र +तैपनि +तैपनी +त्यत्तिकै +त्यत्तिकैमा +त्यस +त्यसकारण +त्यसको +त्यसले +त्यसैले +त्यसो +त्यस्तै +त्यस्तो +त्यहाँ +त्यहिँ +त्यही +त्यहीँ +त्यहीं +त्यो +त्सपछि +त्सैले +थप +थरि +थरी +थाहा +थिए +थिएँ +थिएन +थियो +दर्ता +दश +दिए +दिएको +दिन +दिनुभएको +दिनुहुन्छ +दुइ +दुइवटा +दुई +देखि +देखिन्छ +देखियो +देखे +देखेको +देखेर +दोश्री +दोश्रो +दोस्रो +द्वारा +धन्न +धेरै +धौ +न +नगर्नु +नगर्नू +नजिकै +नत्र +नत्रभने +नभई +नभएको +नभनेर +नयाँ +नि +निकै +निम्ति +निम्न +निम्नानुसार +निर्दिष्ट +नै +नौ +पक्का +पक्कै +पछाडि +पछाडी +पछि +पछिल्लो +पछी +पटक +पनि +पन्ध्र +पर्छ +पर्थ्यो +पर्दैन +पर्ने +पर्नेमा +पर्याप्त +पहिले +पहिलो +पहिल्यै +पाँच +पांच +पाचौँ +पाँचौं +पिच्छे +पूर्व +पो +प्रति +प्रतेक +प्रत्यक +प्राय +प्लस +फरक +फेरि +फेरी +बढी +बताए +बने +बरु +बाट +बारे +बाहिर +बाहेक +बाह्र +बिच +बिचमा +बिरुद्ध +बिशेष +बिस +बीच +बीचमा +बीस +भए +भएँ +भएका +भएकालाई +भएको +भएन +भएर +भन +भने +भनेको +भनेर +भन् +भन्छन् +भन्छु +भन्दा +भन्दै +भन्नुभयो +भन्ने +भन्या +भयेन +भयो +भर +भरि +भरी +भा +भित्र +भित्री +भीत्र +म +मध्य +मध्ये +मलाई +मा +मात्र +मात्रै +माथि +माथी +मुख्य +मुनि +मुन्तिर +मेरो +मैले +यति +यथोचित +यदि +यद्ध्यपि +यद्यपि +यस +यसका +यसको +यसपछि +यसबाहेक +यसमा +यसरी +यसले +यसो +यस्तै +यस्तो +यहाँ +यहाँसम्म +यही +या +यी +यो +र +रही +रहेका +रहेको +रहेछ +राखे +राख्छ +राम्रो +रुपमा +रूप +रे +लगभग +लगायत +लाई +लाख +लागि +लागेको +ले +वटा +वरीपरी +वा +वाट +वापत +वास्तवमा +शायद +सक्छ +सक्ने +सँग +संग +सँगको +सँगसँगै +सँगै +संगै +सङ्ग +सङ्गको +सट्टा +सत्र +सधै +सबै +सबैको +सबैलाई +समय +समेत +सम्भव +सम्म +सय +सरह +सहित +सहितै +सही +साँच्चै +सात +साथ +साथै +सायद +सारा +सुनेको +सुनेर +सुरु +सुरुको +सुरुमै +सो +सोचेको +सोचेर +सोही +सोह्र +स्थित +स्पष्ट +हजार +हरे +हरेक +हामी +हामीले +हाम्रा +हाम्रो +हुँदैन +हुन +हुनत +हुनु +हुने +हुनेछ +हुन् +हुन्छ +हुन्थ्यो +हैन +हो +होइन +होकि +होला +""".split() +) diff --git a/spacy/lang/nl/__init__.py b/spacy/lang/nl/__init__.py new file mode 100644 index 0000000..213041a --- /dev/null +++ b/spacy/lang/nl/__init__.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from .lemmatizer import DutchLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class DutchDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class Dutch(Language): + lang = "nl" + Defaults = DutchDefaults + + +@Dutch.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return DutchLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Dutch"] diff --git a/spacy/lang/nl/examples.py b/spacy/lang/nl/examples.py new file mode 100644 index 0000000..3440f01 --- /dev/null +++ b/spacy/lang/nl/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.nl.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple overweegt om voor 1 miljard een U.K. startup te kopen", + "Autonome auto's verschuiven de verzekeringverantwoordelijkheid naar producenten", + "San Francisco overweegt robots op voetpaden te verbieden", + "Londen is een grote stad in het Verenigd Koninkrijk", +] diff --git a/spacy/lang/nl/lemmatizer.py b/spacy/lang/nl/lemmatizer.py new file mode 100644 index 0000000..4f6b2ef --- /dev/null +++ b/spacy/lang/nl/lemmatizer.py @@ -0,0 +1,122 @@ +from typing import List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class DutchLemmatizer(Lemmatizer): + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "rule": + required = ["lemma_lookup", "lemma_rules", "lemma_exc", "lemma_index"] + return (required, []) + else: + return super().get_lookups_config(mode) + + def lookup_lemmatize(self, token: Token) -> List[str]: + """Overrides parent method so that a lowercased version of the string + is used to search the lookup table. This is necessary because our + lookup table consists entirely of lowercase keys.""" + lookup_table = self.lookups.get_table("lemma_lookup", {}) + string = token.text.lower() + return [lookup_table.get(string, string)] + + # Note: CGN does not distinguish AUX verbs, so we treat AUX as VERB. + def rule_lemmatize(self, token: Token) -> List[str]: + # Difference 1: self.rules is assumed to be non-None, so no + # 'is None' check required. + # String lowercased from the get-go. All lemmatization results in + # lowercased strings. For most applications, this shouldn't pose + # any problems, and it keeps the exceptions indexes small. If this + # creates problems for proper nouns, we can introduce a check for + # univ_pos == "PROPN". + cache_key = (token.lower, token.pos) + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + univ_pos = token.pos_.lower() + if univ_pos in ("", "eol", "space"): + forms = [string.lower()] + self.cache[cache_key] = forms + return forms + + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, {}) + + string = string.lower() + if univ_pos not in ( + "noun", + "verb", + "aux", + "adj", + "adv", + "pron", + "det", + "adp", + "num", + ): + forms = [string] + self.cache[cache_key] = forms + return forms + lemma_index = index_table.get(univ_pos, {}) + # string is already lemma + if string in lemma_index: + forms = [string] + self.cache[cache_key] = forms + return forms + exc_table = self.lookups.get_table("lemma_exc", {}) + exceptions = exc_table.get(univ_pos, {}) + # string is irregular token contained in exceptions index. + try: + forms = [exceptions[string][0]] + self.cache[cache_key] = forms + return forms + except KeyError: + pass + # string corresponds to key in lookup table + lookup_table = self.lookups.get_table("lemma_lookup", {}) + looked_up_lemma = lookup_table.get(string) + if looked_up_lemma and looked_up_lemma in lemma_index: + forms = [looked_up_lemma] + self.cache[cache_key] = forms + return forms + rules_table = self.lookups.get_table("lemma_rules", {}) + oov_forms = [] + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + pass + elif form in index: + forms = [form] + self.cache[cache_key] = forms + return forms + else: + oov_forms.append(form) + forms = list(dict.fromkeys(oov_forms)) + # Back-off through remaining return value candidates. + if forms: + for form in forms: + if form in exceptions: + forms = [form] + self.cache[cache_key] = forms + return forms + if looked_up_lemma: + forms = [looked_up_lemma] + self.cache[cache_key] = forms + return forms + else: + self.cache[cache_key] = forms + return forms + elif looked_up_lemma: + forms = [looked_up_lemma] + self.cache[cache_key] = forms + return forms + else: + forms = [string] + self.cache[cache_key] = forms + return forms diff --git a/spacy/lang/nl/lex_attrs.py b/spacy/lang/nl/lex_attrs.py new file mode 100644 index 0000000..488224c --- /dev/null +++ b/spacy/lang/nl/lex_attrs.py @@ -0,0 +1,40 @@ +from ...attrs import LIKE_NUM + +_num_words = set( + """ +nul een één twee drie vier vijf zes zeven acht negen tien elf twaalf dertien +veertien twintig dertig veertig vijftig zestig zeventig tachtig negentig honderd +duizend miljoen miljard biljoen biljard triljoen triljard +""".split() +) + +_ordinal_words = set( + """ +eerste tweede derde vierde vijfde zesde zevende achtste negende tiende elfde +twaalfde dertiende veertiende twintigste dertigste veertigste vijftigste +zestigste zeventigste tachtigste negentigste honderdste duizendste miljoenste +miljardste biljoenste biljardste triljoenste triljardste +""".split() +) + + +def like_num(text): + # This only does the most basic check for whether a token is a digit + # or matches one of the number words. In order to handle numbers like + # "drieëntwintig", more work is required. + # See this discussion: https://github.com/explosion/spaCy/pull/1177 + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/nl/punctuation.py b/spacy/lang/nl/punctuation.py new file mode 100644 index 0000000..c9a4c9e --- /dev/null +++ b/spacy/lang/nl/punctuation.py @@ -0,0 +1,64 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + LIST_UNITS, + PUNCT, + merge_chars, +) +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + +_prefixes = [",,"] + BASE_TOKENIZER_PREFIXES + + +# Copied from `de` package. Main purpose is to ensure that hyphens are not +# split on. + +_quotes = CONCAT_QUOTES.replace("'", "") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{}])\.(?=[{}])".format(ALPHA_LOWER, ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + ] +) + + +_list_units = [u for u in LIST_UNITS if u != "%"] +_units = merge_chars(" ".join(_list_units)) + +_suffixes = ( + ["''"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + ["—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=_units), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/nl/stop_words.py b/spacy/lang/nl/stop_words.py new file mode 100644 index 0000000..cd4fdef --- /dev/null +++ b/spacy/lang/nl/stop_words.py @@ -0,0 +1,72 @@ +# The original stop words list (added in f46ffe3) was taken from +# http://www.damienvanholten.com/downloads/dutch-stop-words.txt +# and consisted of about 100 tokens. +# In order to achieve parity with some of the better-supported +# languages, e.g., English, French, and German, this original list has been +# extended with 200 additional tokens. The main source of inspiration was +# https://raw.githubusercontent.com/stopwords-iso/stopwords-nl/master/stopwords-nl.txt. +# However, quite a bit of manual editing has taken place as well. +# Tokens whose status as a stop word is not entirely clear were admitted or +# rejected by deferring to their counterparts in the stop words lists for English +# and French. Similarly, those lists were used to identify and fill in gaps so +# that -- in principle -- each token contained in the English stop words list +# should have a Dutch counterpart here. + + +STOP_WORDS = set( + """ +aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaande aangezien achter achterna +afgelopen aldus alhoewel anderzijds + +ben bij bijna bijvoorbeeld behalve beide beiden beneden bent bepaald beter betere betreffende binnen binnenin boven +bovenal bovendien bovenstaand buiten + +daar dan dat de der den deze die dit doch doen door dus daarheen daarin daarna daarnet daarom daarop des dezelfde dezen +dien dikwijls doet doorgaand doorgaans + +een eens en er echter enige eerder eerst eerste eersten effe eigen elk elke enkel enkele enz erdoor etc even eveneens +evenwel + +ff + +ge geen geweest gauw gedurende gegeven gehad geheel gekund geleden gelijk gemogen geven geweest gewoon gewoonweg +geworden gij + +haar had heb hebben heeft hem het hier hij hoe hun hadden hare hebt hele hen hierbeneden hierboven hierin hoewel hun + +iemand iets ik in is idd ieder ikke ikzelf indien inmiddels inz inzake + +ja je jou jouw jullie jezelf jij jijzelf jouwe juist + +kan kon kunnen klaar konden krachtens kunnen kunt + +lang later liet liever + +maar me meer men met mij mijn moet mag mede meer meesten mezelf mijzelf min minder misschien mocht mochten moest moesten +moet moeten mogelijk mogen + +na naar niet niets nog nu nabij nadat net nogal nooit nr nu + +of om omdat ons ook op over omhoog omlaag omstreeks omtrent omver onder ondertussen ongeveer onszelf onze ooit opdat +opnieuw opzij over overigens + +pas pp precies prof publ + +reeds rond rondom + +sedert sinds sindsdien slechts sommige spoedig steeds + +‘t 't te tegen toch toen tot tamelijk ten tenzij ter terwijl thans tijdens toe totdat tussen + +u uit uw uitgezonderd uwe uwen + +van veel voor vaak vanaf vandaan vanuit vanwege veeleer verder verre vervolgens vgl volgens vooraf vooral vooralsnog +voorbij voordat voordien voorheen voorop voort voorts vooruit vrij vroeg + +want waren was wat we wel werd wezen wie wij wil worden waar waarom wanneer want weer weg wegens weinig weinige weldra +welk welke welken werd werden wiens wier wilde wordt + +zal ze zei zelf zich zij zijn zo zonder zou zeer zeker zekere zelfde zelfs zichzelf zijnde zijne zo’n zoals zodra zouden + zoveel zowat zulk zulke zulks zullen zult +""".split() +) diff --git a/spacy/lang/nl/syntax_iterators.py b/spacy/lang/nl/syntax_iterators.py new file mode 100644 index 0000000..d7388a3 --- /dev/null +++ b/spacy/lang/nl/syntax_iterators.py @@ -0,0 +1,75 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on Doc and Span. + The definition is inspired by https://www.nltk.org/book/ch07.html + Consider : [Noun + determinant / adjective] and also [Pronoun] + """ + # fmt: off + # labels = ["nsubj", "nsubj:pass", "obj", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + + # Check for dependencies: POS, DEP + if not doc.has_annotation("POS"): + raise ValueError(Errors.E1019) + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + # See UD tags: https://universaldependencies.org/u/dep/index.html + # amod = adjectival modifier + # nmod:poss = possessive nominal modifier + # nummod = numeric modifier + # det = determiner + # det:poss = possessive determiner + noun_deps = [ + doc.vocab.strings[label] for label in ["amod", "nmod:poss", "det", "det:poss"] + ] + + # nsubj = nominal subject + # nsubj:pass = passive nominal subject + pronoun_deps = [doc.vocab.strings[label] for label in ["nsubj", "nsubj:pass"]] + + # Label NP for the Span to identify it as Noun-Phrase + span_label = doc.vocab.strings.add("NP") + + # Only NOUNS and PRONOUNS matter + end_span = -1 + for i, word in enumerate(filter(lambda x: x.pos in [PRON, NOUN], doclike)): + # For NOUNS + # Pick children from syntactic parse (only those with certain dependencies) + if word.pos == NOUN: + # Some debugging. It happens that VERBS are POS-TAGGED as NOUNS + # We check if the word has a "nsubj", if it's the case, we eliminate it + nsubjs = filter( + lambda x: x.dep == doc.vocab.strings["nsubj"], word.children + ) + next_word = next(nsubjs, None) + if next_word is not None: + # We found some nsubj, so we skip this word. Otherwise, consider it a normal NOUN + continue + + children = filter(lambda x: x.dep in noun_deps, word.children) + children_i = [c.i for c in children] + [word.i] + + start_span = min(children_i) + if start_span >= end_span: + end_span = max(children_i) + 1 + yield start_span, end_span, span_label + + # PRONOUNS only if it is the subject of a verb + elif word.pos == PRON: + if word.dep in pronoun_deps: + start_span = word.i + if start_span >= end_span: + end_span = word.i + 1 + yield start_span, end_span, span_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/nl/tokenizer_exceptions.py b/spacy/lang/nl/tokenizer_exceptions.py new file mode 100644 index 0000000..85ad49f --- /dev/null +++ b/spacy/lang/nl/tokenizer_exceptions.py @@ -0,0 +1,1607 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +# Extensive list of both common and uncommon dutch abbreviations copied from +# github.com/diasks2/pragmatic_segmenter, a Ruby library for rule-based +# sentence boundary detection (MIT, Copyright 2015 Kevin S. Dias). +# Source file: https://github.com/diasks2/pragmatic_segmenter/blob/master/lib/pragmatic_segmenter/languages/dutch.rb +# (Last commit: 4d1477b) + +# Main purpose of such an extensive list: considerably improved sentence +# segmentation. + +# Note: This list has been copied over largely as-is. Some of the abbreviations +# are extremely domain-specific. Tokenizer performance may benefit from some +# slight pruning, although no performance regression has been observed so far. + +abbrevs = [ + "a.2d.", + "a.a.", + "a.a.j.b.", + "a.f.t.", + "a.g.j.b.", + "a.h.v.", + "a.h.w.", + "a.hosp.", + "a.i.", + "a.j.b.", + "a.j.t.", + "a.m.", + "a.m.r.", + "a.p.m.", + "a.p.r.", + "a.p.t.", + "a.s.", + "a.t.d.f.", + "a.u.b.", + "a.v.a.", + "a.w.", + "aanbev.", + "aanbev.comm.", + "aant.", + "aanv.st.", + "aanw.", + "vnw.", + "aanw.vnw.", + "abd.", + "abm.", + "abs.", + "acc.act.", + "acc.bedr.m.", + "acc.bedr.t.", + "achterv.", + "act.dr.", + "act.dr.fam.", + "act.fisc.", + "act.soc.", + "adm.akk.", + "adm.besl.", + "adm.lex.", + "adm.onderr.", + "adm.ov.", + "adv.", + "adv.", + "gen.", + "adv.bl.", + "afd.", + "afl.", + "aggl.verord.", + "agr.", + "al.", + "alg.", + "alg.richts.", + "amén.", + "ann.dr.", + "ann.dr.lg.", + "ann.dr.sc.pol.", + "ann.ét.eur.", + "ann.fac.dr.lg.", + "ann.jur.créd.", + "ann.jur.créd.règl.coll.", + "ann.not.", + "ann.parl.", + "ann.prat.comm.", + "app.", + "arb.", + "aud.", + "arbbl.", + "arbh.", + "arbit.besl.", + "arbrb.", + "arr.", + "arr.cass.", + "arr.r.v.st.", + "arr.verbr.", + "arrondrb.", + "art.", + "artw.", + "aud.", + "b.", + "b.", + "b.&w.", + "b.a.", + "b.a.s.", + "b.b.o.", + "b.best.dep.", + "b.br.ex.", + "b.coll.fr.gem.comm.", + "b.coll.vl.gem.comm.", + "b.d.cult.r.", + "b.d.gem.ex.", + "b.d.gem.reg.", + "b.dep.", + "b.e.b.", + "b.f.r.", + "b.fr.gem.ex.", + "b.fr.gem.reg.", + "b.i.h.", + "b.inl.j.d.", + "b.inl.s.reg.", + "b.j.", + "b.l.", + "b.o.z.", + "b.prov.r.", + "b.r.h.", + "b.s.", + "b.sr.", + "b.stb.", + "b.t.i.r.", + "b.t.s.z.", + "b.t.w.rev.", + "b.v.", + "b.ver.coll.gem.gem.comm.", + "b.verg.r.b.", + "b.versl.", + "b.vl.ex.", + "b.voorl.reg.", + "b.w.", + "b.w.gew.ex.", + "b.z.d.g.", + "b.z.v.", + "bab.", + "bedr.org.", + "begins.", + "beheersov.", + "bekendm.comm.", + "bel.", + "bel.besch.", + "bel.w.p.", + "beleidsov.", + "belg.", + "grondw.", + "ber.", + "ber.w.", + "besch.", + "besl.", + "beslagr.", + "bestuurswet.", + "bet.", + "betr.", + "betr.", + "vnw.", + "bevest.", + "bew.", + "bijbl.", + "ind.", + "eig.", + "bijbl.n.bijdr.", + "bijl.", + "bijv.", + "bijw.", + "bijz.decr.", + "bin.b.", + "bkh.", + "bl.", + "blz.", + "bm.", + "bn.", + "rh.", + "bnw.", + "bouwr.", + "br.parl.", + "bs.", + "bull.", + "bull.adm.pénit.", + "bull.ass.", + "bull.b.m.m.", + "bull.bel.", + "bull.best.strafinr.", + "bull.bmm.", + "bull.c.b.n.", + "bull.c.n.c.", + "bull.cbn.", + "bull.centr.arb.", + "bull.cnc.", + "bull.contr.", + "bull.doc.min.fin.", + "bull.f.e.b.", + "bull.feb.", + "bull.fisc.fin.r.", + "bull.i.u.m.", + "bull.inf.ass.secr.soc.", + "bull.inf.i.e.c.", + "bull.inf.i.n.a.m.i.", + "bull.inf.i.r.e.", + "bull.inf.iec.", + "bull.inf.inami.", + "bull.inf.ire.", + "bull.inst.arb.", + "bull.ium.", + "bull.jur.imm.", + "bull.lég.b.", + "bull.off.", + "bull.trim.b.dr.comp.", + "bull.us.", + "bull.v.b.o.", + "bull.vbo.", + "bv.", + "bw.", + "bxh.", + "byz.", + "c.", + "c.a.", + "c.a.-a.", + "c.a.b.g.", + "c.c.", + "c.c.i.", + "c.c.s.", + "c.conc.jur.", + "c.d.e.", + "c.d.p.k.", + "c.e.", + "c.ex.", + "c.f.", + "c.h.a.", + "c.i.f.", + "c.i.f.i.c.", + "c.j.", + "c.l.", + "c.n.", + "c.o.d.", + "c.p.", + "c.pr.civ.", + "c.q.", + "c.r.", + "c.r.a.", + "c.s.", + "c.s.a.", + "c.s.q.n.", + "c.v.", + "c.v.a.", + "c.v.o.", + "ca.", + "cadeaust.", + "cah.const.", + "cah.dr.europ.", + "cah.dr.immo.", + "cah.dr.jud.", + "cal.", + "2d.", + "cal.", + "3e.", + "cal.", + "rprt.", + "cap.", + "carg.", + "cass.", + "cass.", + "verw.", + "cert.", + "cf.", + "ch.", + "chron.", + "chron.d.s.", + "chron.dr.not.", + "cie.", + "cie.", + "verz.schr.", + "cir.", + "circ.", + "circ.z.", + "cit.", + "cit.loc.", + "civ.", + "cl.et.b.", + "cmt.", + "co.", + "cognoss.v.", + "coll.", + "v.", + "b.", + "colp.w.", + "com.", + "com.", + "cas.", + "com.v.min.", + "comm.", + "comm.", + "v.", + "comm.bijz.ov.", + "comm.erf.", + "comm.fin.", + "comm.ger.", + "comm.handel.", + "comm.pers.", + "comm.pub.", + "comm.straf.", + "comm.v.", + "comm.venn.", + "comm.verz.", + "comm.voor.", + "comp.", + "compt.w.", + "computerr.", + "con.m.", + "concl.", + "concr.", + "conf.", + "confl.w.", + "confl.w.huwbetr.", + "cons.", + "conv.", + "coöp.", + "ver.", + "corr.", + "corr.bl.", + "cour.fisc.", + "cour.immo.", + "cridon.", + "crim.", + "cur.", + "cur.", + "crt.", + "curs.", + "d.", + "d.-g.", + "d.a.", + "d.a.v.", + "d.b.f.", + "d.c.", + "d.c.c.r.", + "d.d.", + "d.d.p.", + "d.e.t.", + "d.gem.r.", + "d.h.", + "d.h.z.", + "d.i.", + "d.i.t.", + "d.j.", + "d.l.r.", + "d.m.", + "d.m.v.", + "d.o.v.", + "d.parl.", + "d.w.z.", + "dact.", + "dat.", + "dbesch.", + "dbesl.", + "dec.", + "decr.", + "decr.d.", + "decr.fr.", + "decr.vl.", + "decr.w.", + "def.", + "dep.opv.", + "dep.rtl.", + "derg.", + "desp.", + "det.mag.", + "deurw.regl.", + "dez.", + "dgl.", + "dhr.", + "disp.", + "diss.", + "div.", + "div.act.", + "div.bel.", + "dl.", + "dln.", + "dnotz.", + "doc.", + "hist.", + "doc.jur.b.", + "doc.min.fin.", + "doc.parl.", + "doctr.", + "dpl.", + "dpl.besl.", + "dr.", + "dr.banc.fin.", + "dr.circ.", + "dr.inform.", + "dr.mr.", + "dr.pén.entr.", + "dr.q.m.", + "drs.", + "ds.", + "dtp.", + "dwz.", + "dyn.", + "e.", + "e.a.", + "e.b.", + "tek.mod.", + "e.c.", + "e.c.a.", + "e.d.", + "e.e.", + "e.e.a.", + "e.e.g.", + "e.g.", + "e.g.a.", + "e.h.a.", + "e.i.", + "e.j.", + "e.m.a.", + "e.n.a.c.", + "e.o.", + "e.p.c.", + "e.r.c.", + "e.r.f.", + "e.r.h.", + "e.r.o.", + "e.r.p.", + "e.r.v.", + "e.s.r.a.", + "e.s.t.", + "e.v.", + "e.v.a.", + "e.w.", + "e&o.e.", + "ec.pol.r.", + "econ.", + "ed.", + "ed(s).", + "eff.", + "eig.", + "eig.mag.", + "eil.", + "elektr.", + "enmb.", + "enz.", + "err.", + "etc.", + "etq.", + "eur.", + "parl.", + "eur.t.s.", + "ev.", + "evt.", + "ex.", + "ex.crim.", + "exec.", + "f.", + "f.a.o.", + "f.a.q.", + "f.a.s.", + "f.i.b.", + "f.j.f.", + "f.o.b.", + "f.o.r.", + "f.o.s.", + "f.o.t.", + "f.r.", + "f.supp.", + "f.suppl.", + "fa.", + "facs.", + "fasc.", + "fg.", + "fid.ber.", + "fig.", + "fin.verh.w.", + "fisc.", + "fisc.", + "tijdschr.", + "fisc.act.", + "fisc.koer.", + "fl.", + "form.", + "foro.", + "it.", + "fr.", + "fr.cult.r.", + "fr.gem.r.", + "fr.parl.", + "fra.", + "ft.", + "g.", + "g.a.", + "g.a.v.", + "g.a.w.v.", + "g.g.d.", + "g.m.t.", + "g.o.", + "g.omt.e.", + "g.p.", + "g.s.", + "g.v.", + "g.w.w.", + "geb.", + "gebr.", + "gebrs.", + "gec.", + "gec.decr.", + "ged.", + "ged.st.", + "gedipl.", + "gedr.st.", + "geh.", + "gem.", + "gem.", + "gem.", + "gem.gem.comm.", + "gem.st.", + "gem.stem.", + "gem.w.", + "gemeensch.optr.", + "gemeensch.standp.", + "gemeensch.strat.", + "gemeent.", + "gemeent.b.", + "gemeent.regl.", + "gemeent.verord.", + "geol.", + "geopp.", + "gepubl.", + "ger.deurw.", + "ger.w.", + "gerekw.", + "gereq.", + "gesch.", + "get.", + "getr.", + "gev.m.", + "gev.maatr.", + "gew.", + "ghert.", + "gir.eff.verk.", + "gk.", + "gr.", + "gramm.", + "grat.w.", + "grootb.w.", + "grs.", + "grvm.", + "grw.", + "gst.", + "gw.", + "h.a.", + "h.a.v.o.", + "h.b.o.", + "h.e.a.o.", + "h.e.g.a.", + "h.e.geb.", + "h.e.gestr.", + "h.l.", + "h.m.", + "h.o.", + "h.r.", + "h.t.l.", + "h.t.m.", + "h.w.geb.", + "hand.", + "handelsn.w.", + "handelspr.", + "handelsr.w.", + "handelsreg.w.", + "handv.", + "harv.l.rev.", + "hc.", + "herald.", + "hert.", + "herz.", + "hfdst.", + "hfst.", + "hgrw.", + "hhr.", + "hist.", + "hooggel.", + "hoogl.", + "hosp.", + "hpw.", + "hr.", + "hr.", + "ms.", + "hr.ms.", + "hregw.", + "hrg.", + "hst.", + "huis.just.", + "huisv.w.", + "huurbl.", + "hv.vn.", + "hw.", + "hyp.w.", + "i.b.s.", + "i.c.", + "i.c.m.h.", + "i.e.", + "i.f.", + "i.f.p.", + "i.g.v.", + "i.h.", + "i.h.a.", + "i.h.b.", + "i.l.pr.", + "i.o.", + "i.p.o.", + "i.p.r.", + "i.p.v.", + "i.pl.v.", + "i.r.d.i.", + "i.s.m.", + "i.t.t.", + "i.v.", + "i.v.m.", + "i.v.s.", + "i.w.tr.", + "i.z.", + "ib.", + "ibid.", + "icip-ing.cons.", + "iem.", + "inc.", + "indic.soc.", + "indiv.", + "inf.", + "inf.i.d.a.c.", + "inf.idac.", + "inf.r.i.z.i.v.", + "inf.riziv.", + "inf.soc.secr.", + "ing.", + "ing.", + "cons.", + "ing.cons.", + "inst.", + "int.", + "int.", + "rechtsh.", + "strafz.", + "interm.", + "intern.fisc.act.", + "intern.vervoerr.", + "inv.", + "inv.", + "f.", + "inv.w.", + "inv.wet.", + "invord.w.", + "inz.", + "ir.", + "irspr.", + "iwtr.", + "j.", + "j.-cl.", + "j.c.b.", + "j.c.e.", + "j.c.fl.", + "j.c.j.", + "j.c.p.", + "j.d.e.", + "j.d.f.", + "j.d.s.c.", + "j.dr.jeun.", + "j.j.d.", + "j.j.p.", + "j.j.pol.", + "j.l.", + "j.l.m.b.", + "j.l.o.", + "j.p.a.", + "j.r.s.", + "j.t.", + "j.t.d.e.", + "j.t.dr.eur.", + "j.t.o.", + "j.t.t.", + "jaarl.", + "jb.hand.", + "jb.kred.", + "jb.kred.c.s.", + "jb.l.r.b.", + "jb.lrb.", + "jb.markt.", + "jb.mens.", + "jb.t.r.d.", + "jb.trd.", + "jeugdrb.", + "jeugdwerkg.w.", + "jhr.", + "jg.", + "jis.", + "jl.", + "journ.jur.", + "journ.prat.dr.fisc.fin.", + "journ.proc.", + "jr.", + "jrg.", + "jur.", + "jur.comm.fl.", + "jur.dr.soc.b.l.n.", + "jur.f.p.e.", + "jur.fpe.", + "jur.niv.", + "jur.trav.brux.", + "jurambt.", + "jv.cass.", + "jv.h.r.j.", + "jv.hrj.", + "jw.", + "k.", + "k.", + "k.b.", + "k.g.", + "k.k.", + "k.m.b.o.", + "k.o.o.", + "k.v.k.", + "k.v.v.v.", + "kadasterw.", + "kaderb.", + "kador.", + "kbo-nr.", + "kg.", + "kh.", + "kiesw.", + "kind.bes.v.", + "kkr.", + "kon.", + "koopv.", + "kr.", + "krankz.w.", + "ksbel.", + "kt.", + "ktg.", + "ktr.", + "kvdm.", + "kw.r.", + "kymr.", + "kzr.", + "kzw.", + "l.", + "l.b.", + "l.b.o.", + "l.bas.", + "l.c.", + "l.gew.", + "l.j.", + "l.k.", + "l.l.", + "l.o.", + "l.p.", + "l.r.b.", + "l.u.v.i.", + "l.v.r.", + "l.v.w.", + "l.w.", + "l'exp.-compt.b..", + "l’exp.-compt.b.", + "landinr.w.", + "landscrt.", + "lat.", + "law.ed.", + "lett.", + "levensverz.", + "lgrs.", + "lidw.", + "limb.rechtsl.", + "lit.", + "litt.", + "liw.", + "liwet.", + "lk.", + "ll.", + "ll.(l.)l.r.", + "loonw.", + "losbl.", + "ltd.", + "luchtv.", + "luchtv.w.", + "m.", + "m.", + "not.", + "m.a.v.o.", + "m.a.w.", + "m.b.", + "m.b.o.", + "m.b.r.", + "m.b.t.", + "m.d.g.o.", + "m.e.a.o.", + "m.e.r.", + "m.h.", + "m.h.d.", + "m.i.v.", + "m.j.t.", + "m.k.", + "m.m.", + "m.m.a.", + "m.m.h.h.", + "m.m.v.", + "m.n.", + "m.not.fisc.", + "m.nt.", + "m.o.", + "m.r.", + "m.s.a.", + "m.u.p.", + "m.v.a.", + "m.v.h.n.", + "m.v.t.", + "m.z.", + "maatr.teboekgest.luchtv.", + "maced.", + "mand.", + "max.", + "mbl.not.", + "me.", + "med.", + "med.", + "v.b.o.", + "med.b.u.f.r.", + "med.bufr.", + "med.vbo.", + "meerv.", + "meetbr.w.", + "mej.", + "mevr.", + "mém.adm.", + "mgr.", + "mgrs.", + "mhd.", + "mi.verantw.", + "mil.", + "mil.bed.", + "mil.ger.", + "min.", + "min.", + "aanbev.", + "min.", + "circ.", + "min.", + "fin.", + "min.j.omz.", + "min.just.circ.", + "mitt.", + "mln.", + "mnd.", + "mod.", + "mon.", + "mouv.comm.", + "mr.", + "ms.", + "muz.", + "mv.", + "n.", + "chr.", + "n.a.", + "n.a.g.", + "n.a.v.", + "n.b.", + "n.c.", + "n.chr.", + "n.d.", + "n.d.r.", + "n.e.a.", + "n.g.", + "n.h.b.c.", + "n.j.", + "n.j.b.", + "n.j.w.", + "n.l.", + "n.m.", + "n.m.m.", + "n.n.", + "n.n.b.", + "n.n.g.", + "n.n.k.", + "n.o.m.", + "n.o.t.k.", + "n.rapp.", + "n.tijd.pol.", + "n.v.", + "n.v.d.r.", + "n.v.d.v.", + "n.v.o.b.", + "n.v.t.", + "nat.besch.w.", + "nat.omb.", + "nat.pers.", + "ned.", + "ned.cult.r.", + "neg.verkl.", + "nhd.", + "wisk.", + "njcm-bull.", + "nl.", + "nnd.", + "no.", + "not.fisc.m.", + "not.w.", + "not.wet.", + "nr.", + "nrs.", + "nste.", + "nt.", + "numism.", + "o.", + "o.a.", + "o.b.", + "o.c.", + "o.g.", + "o.g.v.", + "o.i.", + "o.i.d.", + "o.m.", + "o.o.", + "o.o.d.", + "o.o.v.", + "o.p.", + "o.r.", + "o.regl.", + "o.s.", + "o.t.s.", + "o.t.t.", + "o.t.t.t.", + "o.t.t.z.", + "o.tk.t.", + "o.v.t.", + "o.v.t.t.", + "o.v.tk.t.", + "o.v.v.", + "ob.", + "obsv.", + "octr.", + "octr.gem.regl.", + "octr.regl.", + "oe.", + "off.pol.", + "ofra.", + "ohd.", + "omb.", + "omnil.", + "omz.", + "on.ww.", + "onderr.", + "onfrank.", + "onteig.w.", + "ontw.", + "b.w.", + "onuitg.", + "onz.", + "oorl.w.", + "op.cit.", + "opin.pa.", + "opm.", + "or.", + "ord.br.", + "ord.gem.", + "ors.", + "orth.", + "os.", + "osm.", + "ov.", + "ov.w.i.", + "ov.w.ii.", + "ov.ww.", + "overg.w.", + "overw.", + "ovkst.", + "oz.", + "p.", + "p.a.", + "p.a.o.", + "p.b.o.", + "p.e.", + "p.g.", + "p.j.", + "p.m.", + "p.m.a.", + "p.o.", + "p.o.j.t.", + "p.p.", + "p.v.", + "p.v.s.", + "pachtw.", + "pag.", + "pan.", + "pand.b.", + "pand.pér.", + "parl.gesch.", + "parl.gesch.", + "inv.", + "parl.st.", + "part.arb.", + "pas.", + "pasin.", + "pat.", + "pb.c.", + "pb.l.", + "pct.", + "pens.", + "pensioenverz.", + "per.ber.i.b.r.", + "per.ber.ibr.", + "pers.", + "st.", + "pft.", + "pk.", + "pktg.", + "plv.", + "po.", + "pol.", + "pol.off.", + "pol.r.", + "pol.w.", + "postbankw.", + "postw.", + "pp.", + "pr.", + "preadv.", + "pres.", + "prf.", + "prft.", + "prg.", + "prijz.w.", + "proc.", + "procesregl.", + "prof.", + "prot.", + "prov.", + "prov.b.", + "prov.instr.h.m.g.", + "prov.regl.", + "prov.verord.", + "prov.w.", + "publ.", + "pun.", + "pw.", + "q.b.d.", + "q.e.d.", + "q.q.", + "q.r.", + "r.", + "r.a.b.g.", + "r.a.c.e.", + "r.a.j.b.", + "r.b.d.c.", + "r.b.d.i.", + "r.b.s.s.", + "r.c.", + "r.c.b.", + "r.c.d.c.", + "r.c.j.b.", + "r.c.s.j.", + "r.cass.", + "r.d.c.", + "r.d.i.", + "r.d.i.d.c.", + "r.d.j.b.", + "r.d.j.p.", + "r.d.p.c.", + "r.d.s.", + "r.d.t.i.", + "r.e.", + "r.f.s.v.p.", + "r.g.a.r.", + "r.g.c.f.", + "r.g.d.c.", + "r.g.f.", + "r.g.z.", + "r.h.a.", + "r.i.c.", + "r.i.d.a.", + "r.i.e.j.", + "r.i.n.", + "r.i.s.a.", + "r.j.d.a.", + "r.j.i.", + "r.k.", + "r.l.", + "r.l.g.b.", + "r.med.", + "r.med.rechtspr.", + "r.n.b.", + "r.o.", + "r.ov.", + "r.p.", + "r.p.d.b.", + "r.p.o.t.", + "r.p.r.j.", + "r.p.s.", + "r.r.d.", + "r.r.s.", + "r.s.", + "r.s.v.p.", + "r.stvb.", + "r.t.d.f.", + "r.t.d.h.", + "r.t.l.", + "r.trim.dr.eur.", + "r.v.a.", + "r.verkb.", + "r.w.", + "r.w.d.", + "rap.ann.c.a.", + "rap.ann.c.c.", + "rap.ann.c.e.", + "rap.ann.c.s.j.", + "rap.ann.ca.", + "rap.ann.cass.", + "rap.ann.cc.", + "rap.ann.ce.", + "rap.ann.csj.", + "rapp.", + "rb.", + "rb.kh.", + "rdn.", + "rdnr.", + "re.pers.", + "rec.", + "rec.c.i.j.", + "rec.c.j.c.e.", + "rec.cij.", + "rec.cjce.", + "rec.gén.enr.not.", + "rechtsk.t.", + "rechtspl.zeem.", + "rechtspr.arb.br.", + "rechtspr.b.f.e.", + "rechtspr.bfe.", + "rechtspr.soc.r.b.l.n.", + "recl.reg.", + "rect.", + "red.", + "reg.", + "reg.huiz.bew.", + "reg.w.", + "registr.w.", + "regl.", + "regl.", + "r.v.k.", + "regl.besl.", + "regl.onderr.", + "regl.r.t.", + "rep.", + "rép.fisc.", + "rép.not.", + "rep.r.j.", + "rep.rj.", + "req.", + "res.", + "resp.", + "rev.", + "rev.", + "comp.", + "rev.", + "trim.", + "civ.", + "rev.", + "trim.", + "comm.", + "rev.acc.trav.", + "rev.adm.", + "rev.b.compt.", + "rev.b.dr.const.", + "rev.b.dr.intern.", + "rev.b.séc.soc.", + "rev.banc.fin.", + "rev.comm.", + "rev.cons.prud.", + "rev.dr.b.", + "rev.dr.commun.", + "rev.dr.étr.", + "rev.dr.fam.", + "rev.dr.intern.comp.", + "rev.dr.mil.", + "rev.dr.min.", + "rev.dr.pén.", + "rev.dr.pén.mil.", + "rev.dr.rur.", + "rev.dr.u.l.b.", + "rev.dr.ulb.", + "rev.exp.", + "rev.faill.", + "rev.fisc.", + "rev.gd.", + "rev.hist.dr.", + "rev.i.p.c.", + "rev.ipc.", + "rev.not.b.", + "rev.prat.dr.comm.", + "rev.prat.not.b.", + "rev.prat.soc.", + "rev.rec.", + "rev.rw.", + "rev.trav.", + "rev.trim.d.h.", + "rev.trim.dr.fam.", + "rev.urb.", + "richtl.", + "riv.dir.int.", + "riv.dir.int.priv.proc.", + "rk.", + "rln.", + "roln.", + "rom.", + "rondz.", + "rov.", + "rtl.", + "rubr.", + "ruilv.wet.", + "rv.verdr.", + "rvkb.", + "s.", + "s.", + "s.a.", + "s.b.n.", + "s.ct.", + "s.d.", + "s.e.c.", + "s.e.et.o.", + "s.e.w.", + "s.exec.rept.", + "s.hrg.", + "s.j.b.", + "s.l.", + "s.l.e.a.", + "s.l.n.d.", + "s.p.a.", + "s.s.", + "s.t.", + "s.t.b.", + "s.v.", + "s.v.p.", + "samenw.", + "sc.", + "sch.", + "scheidsr.uitspr.", + "schepel.besl.", + "sec.", + "secr.comm.", + "secr.gen.", + "sect.soc.", + "sess.", + "cas.", + "sir.", + "soc.", + "best.", + "soc.", + "handv.", + "soc.", + "verz.", + "soc.act.", + "soc.best.", + "soc.kron.", + "soc.r.", + "soc.sw.", + "soc.weg.", + "sofi-nr.", + "somm.", + "somm.ann.", + "sp.c.c.", + "sr.", + "ss.", + "st.doc.b.c.n.a.r.", + "st.doc.bcnar.", + "st.vw.", + "stagever.", + "stas.", + "stat.", + "stb.", + "stbl.", + "stcrt.", + "stud.dipl.", + "su.", + "subs.", + "subst.", + "succ.w.", + "suppl.", + "sv.", + "sw.", + "t.", + "t.a.", + "t.a.a.", + "t.a.n.", + "t.a.p.", + "t.a.s.n.", + "t.a.v.", + "t.a.v.w.", + "t.aann.", + "t.acc.", + "t.agr.r.", + "t.app.", + "t.b.b.r.", + "t.b.h.", + "t.b.m.", + "t.b.o.", + "t.b.p.", + "t.b.r.", + "t.b.s.", + "t.b.v.", + "t.bankw.", + "t.belg.not.", + "t.desk.", + "t.e.m.", + "t.e.p.", + "t.f.r.", + "t.fam.", + "t.fin.r.", + "t.g.r.", + "t.g.t.", + "t.g.v.", + "t.gem.", + "t.gez.", + "t.huur.", + "t.i.n.", + "t.j.k.", + "t.l.l.", + "t.l.v.", + "t.m.", + "t.m.r.", + "t.m.w.", + "t.mil.r.", + "t.mil.strafr.", + "t.not.", + "t.o.", + "t.o.r.b.", + "t.o.v.", + "t.ontv.", + "t.p.r.", + "t.pol.", + "t.r.", + "t.r.g.", + "t.r.o.s.", + "t.r.v.", + "t.s.r.", + "t.strafr.", + "t.t.", + "t.u.", + "t.v.c.", + "t.v.g.", + "t.v.m.r.", + "t.v.o.", + "t.v.v.", + "t.v.v.d.b.", + "t.v.w.", + "t.verz.", + "t.vred.", + "t.vreemd.", + "t.w.", + "t.w.k.", + "t.w.v.", + "t.w.v.r.", + "t.wrr.", + "t.z.", + "t.z.t.", + "t.z.v.", + "taalk.", + "tar.burg.z.", + "td.", + "techn.", + "telecomm.", + "th.", + "toel.", + "toel.st.v.w.", + "toep.", + "toep.regl.", + "tom.", + "top.", + "trans.b.", + "transp.r.", + "trb.", + "trib.", + "trib.civ.", + "trib.gr.inst.", + "ts.", + "ts.", + "best.", + "ts.", + "verv.", + "turnh.rechtsl.", + "tvpol.", + "tvpr.", + "tvrechtsgesch.", + "tw.", + "u.", + "u.a.", + "u.a.r.", + "u.a.v.", + "u.c.", + "u.c.c.", + "u.g.", + "u.p.", + "u.s.", + "u.s.d.c.", + "uitdr.", + "uitl.w.", + "uitv.besch.div.b.", + "uitv.besl.", + "uitv.besl.", + "succ.w.", + "uitv.besl.bel.rv.", + "uitv.besl.l.b.", + "uitv.reg.", + "inv.w.", + "uitv.reg.bel.d.", + "uitv.reg.afd.verm.", + "uitv.reg.lb.", + "uitv.reg.succ.w.", + "univ.", + "univ.verkl.", + "v.", + "v.", + "chr.", + "v.a.", + "v.a.v.", + "v.c.", + "v.C.", + "v.Chr.", + "v.chr.", + "v.d.", + "v.h.", + "v.huw.verm.", + "v.i.", + "v.i.o.", + "v.j.", + "v.k.a.", + "v.m.", + "v.o.f.", + "v.o.n.", + "v.onderh.verpl.", + "v.p.", + "v.r.", + "v.s.o.", + "v.t.t.", + "v.t.t.t.", + "v.tk.t.", + "v.toep.r.vert.", + "v.v.b.", + "v.v.g.", + "v.v.t.", + "v.v.t.t.", + "v.v.tk.t.", + "v.w.b.", + "v.z.m.", + "vb.", + "vb.bo.", + "vbb.", + "vc.", + "vd.", + "veldw.", + "ver.k.", + "ver.verg.gem.", + "gem.comm.", + "verbr.", + "verd.", + "verdr.", + "verdr.v.", + "tek.mod.", + "verenw.", + "verg.", + "verg.fr.gem.", + "comm.", + "verkl.", + "verkl.herz.gw.", + "verl.", + "deelw.", + "vern.", + "verord.", + "vers.r.", + "versch.", + "versl.c.s.w.", + "versl.csw.", + "vert.", + "verw.", + "verz.", + "verz.w.", + "verz.wett.besl.", + "verz.wett.decr.besl.", + "vgl.", + "vid.", + "viss.w.", + "vl.parl.", + "vl.r.", + "vl.t.gez.", + "vl.w.reg.", + "vl.w.succ.", + "vlg.", + "vn.", + "vnl.", + "vnw.", + "vo.", + "vo.bl.", + "voegw.", + "vol.", + "volg.", + "volt.", + "deelw.", + "voorl.", + "voorz.", + "vord.w.", + "vorst.d.", + "vr.", + "vred.", + "vrg.", + "vnw.", + "vrijgrs.", + "vs.", + "vt.", + "vw.", + "vz.", + "vzngr.", + "vzr.", + "w.", + "w.a.", + "w.b.r.", + "w.c.h.", + "w.conf.huw.", + "w.conf.huwelijksb.", + "w.consum.kr.", + "w.f.r.", + "w.g.", + "w.gew.r.", + "w.ident.pl.", + "w.just.doc.", + "w.kh.", + "w.l.r.", + "w.l.v.", + "w.mil.straf.spr.", + "w.n.", + "w.not.ambt.", + "w.o.", + "w.o.d.huurcomm.", + "w.o.d.k.", + "w.openb.manif.", + "w.parl.", + "w.r.", + "w.reg.", + "w.succ.", + "w.u.b.", + "w.uitv.pl.verord.", + "w.v.", + "w.v.k.", + "w.v.m.s.", + "w.v.r.", + "w.v.w.", + "w.venn.", + "wac.", + "wd.", + "wetb.", + "n.v.h.", + "wgb.", + "winkelt.w.", + "wisk.", + "wka-verkl.", + "wnd.", + "won.w.", + "woningw.", + "woonr.w.", + "wrr.", + "wrr.ber.", + "wrsch.", + "ws.", + "wsch.", + "wsr.", + "wtvb.", + "ww.", + "x.d.", + "z.a.", + "z.g.", + "z.i.", + "z.j.", + "z.o.z.", + "z.p.", + "z.s.m.", + "zg.", + "zgn.", + "zn.", + "znw.", + "zr.", + "zr.", + "ms.", + "zr.ms.", + "'m", + "'n", + "'ns", + "'s", + "'t", +] + +_exc = {} +for orth in abbrevs: + _exc[orth] = [{ORTH: orth}] + uppered = orth.upper() + capsed = orth.capitalize() + for i in [uppered, capsed]: + _exc[i] = [{ORTH: i}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/nn/__init__.py b/spacy/lang/nn/__init__.py new file mode 100644 index 0000000..ebbf070 --- /dev/null +++ b/spacy/lang/nn/__init__.py @@ -0,0 +1,20 @@ +from ...language import BaseDefaults, Language +from ..nb import SYNTAX_ITERATORS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class NorwegianNynorskDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + syntax_iterators = SYNTAX_ITERATORS + + +class NorwegianNynorsk(Language): + lang = "nn" + Defaults = NorwegianNynorskDefaults + + +__all__ = ["NorwegianNynorsk"] diff --git a/spacy/lang/nn/examples.py b/spacy/lang/nn/examples.py new file mode 100644 index 0000000..ee03bf9 --- /dev/null +++ b/spacy/lang/nn/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.nn.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +# sentences taken from Omsetjingsminne frå Nynorsk pressekontor 2022 (https://www.nb.no/sprakbanken/en/resource-catalogue/oai-nb-no-sbr-80/) +sentences = [ + "Konseptet går ut på at alle tre omgangar tel, alle hopparar må stille i kvalifiseringa og poengsummen skal telje.", + "Det er ein meir enn i same periode i fjor.", + "Det har lava ned enorme snømengder i store delar av Europa den siste tida.", + "Akhtar Chaudhry er ikkje innstilt på Oslo-lista til SV, men utfordrar Heikki Holmås om førsteplassen.", +] diff --git a/spacy/lang/nn/punctuation.py b/spacy/lang/nn/punctuation.py new file mode 100644 index 0000000..7b50b58 --- /dev/null +++ b/spacy/lang/nn/punctuation.py @@ -0,0 +1,74 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) +from ..punctuation import TOKENIZER_SUFFIXES + +_quotes = CONCAT_QUOTES.replace("'", "") +_list_punct = [x for x in LIST_PUNCT if x != "#"] +_list_icons = [x for x in LIST_ICONS if x != "°"] +_list_icons = [x.replace("\\u00B0", "") for x in _list_icons] +_list_quotes = [x for x in LIST_QUOTES if x != "\\'"] + + +_prefixes = ( + ["§", "%", "=", "—", "–", r"\+(?![0-9])"] + + _list_punct + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + + +_infixes = ( + LIST_ELLIPSES + + _list_icons + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + _list_quotes + + _list_icons + + ["—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=_quotes, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] + + [r"(?<=[^sSxXzZ])'"] +) +_suffixes += [ + suffix + for suffix in TOKENIZER_SUFFIXES + if suffix not in ["'s", "'S", "’s", "’S", r"\'"] +] + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/nn/tokenizer_exceptions.py b/spacy/lang/nn/tokenizer_exceptions.py new file mode 100644 index 0000000..4bfcb26 --- /dev/null +++ b/spacy/lang/nn/tokenizer_exceptions.py @@ -0,0 +1,228 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +for exc_data in [ + {ORTH: "jan.", NORM: "januar"}, + {ORTH: "feb.", NORM: "februar"}, + {ORTH: "mar.", NORM: "mars"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "jun.", NORM: "juni"}, + # note: "jul." is in the simple list below without a NORM exception + {ORTH: "aug.", NORM: "august"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "nov.", NORM: "november"}, + {ORTH: "des.", NORM: "desember"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "Ap.", + "Aq.", + "Ca.", + "Chr.", + "Co.", + "Dr.", + "F.eks.", + "Fr.p.", + "Frp.", + "Grl.", + "Kr.", + "Kr.F.", + "Kr.F.s", + "Mr.", + "Mrs.", + "Pb.", + "Pr.", + "Sp.", + "St.", + "a.m.", + "ad.", + "adm.dir.", + "adr.", + "b.c.", + "bl.a.", + "bla.", + "bm.", + "bnr.", + "bto.", + "c.c.", + "ca.", + "cand.mag.", + "co.", + "d.d.", + "d.m.", + "d.y.", + "dept.", + "dr.", + "dr.med.", + "dr.philos.", + "dr.psychol.", + "dss.", + "dvs.", + "e.Kr.", + "e.l.", + "eg.", + "eig.", + "ekskl.", + "el.", + "et.", + "etc.", + "etg.", + "ev.", + "evt.", + "f.", + "f.Kr.", + "f.eks.", + "f.o.m.", + "fhv.", + "fk.", + "foreg.", + "fork.", + "fv.", + "fvt.", + "g.", + "gl.", + "gno.", + "gnr.", + "grl.", + "gt.", + "h.r.adv.", + "hhv.", + "hoh.", + "hr.", + "ifb.", + "ifm.", + "iht.", + "inkl.", + "istf.", + "jf.", + "jr.", + "jul.", + "juris.", + "kfr.", + "kgl.", + "kgl.res.", + "kl.", + "komm.", + "kr.", + "kst.", + "lat.", + "lø.", + "m.a.", + "m.a.o.", + "m.fl.", + "m.m.", + "m.v.", + "ma.", + "mag.art.", + "md.", + "mfl.", + "mht.", + "mill.", + "min.", + "mnd.", + "moh.", + "mrd.", + "muh.", + "mv.", + "mva.", + "n.å.", + "ndf.", + "nr.", + "nto.", + "nyno.", + "o.a.", + "o.l.", + "obl.", + "off.", + "ofl.", + "on.", + "op.", + "org.", + "osv.", + "ovf.", + "p.", + "p.a.", + "p.g.a.", + "p.m.", + "p.t.", + "pga.", + "ph.d.", + "pkt.", + "pr.", + "pst.", + "pt.", + "red.anm.", + "ref.", + "res.", + "res.kap.", + "resp.", + "rv.", + "s.", + "s.d.", + "s.k.", + "s.u.", + "s.å.", + "sen.", + "sep.", + "siviling.", + "sms.", + "snr.", + "spm.", + "sr.", + "sst.", + "st.", + "st.meld.", + "st.prp.", + "stip.", + "stk.", + "stud.", + "sv.", + "såk.", + "sø.", + "t.d.", + "t.h.", + "t.o.m.", + "t.v.", + "temp.", + "ti.", + "tils.", + "tilsv.", + "tl;dr", + "tlf.", + "to.", + "ult.", + "utg.", + "v.", + "vedk.", + "vedr.", + "vg.", + "vgs.", + "vha.", + "vit.ass.", + "vn.", + "vol.", + "vs.", + "vsa.", + "§§", + "©NTB", + "årg.", + "årh.", +]: + _exc[orth] = [{ORTH: orth}] + +# Dates +for h in range(1, 31 + 1): + for period in ["."]: + _exc[f"{h}{period}"] = [{ORTH: f"{h}."}] + +_custom_base_exc = {"i.": [{ORTH: "i", NORM: "i"}, {ORTH: "."}]} +_exc.update(_custom_base_exc) + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/norm_exceptions.py b/spacy/lang/norm_exceptions.py new file mode 100644 index 0000000..f35f613 --- /dev/null +++ b/spacy/lang/norm_exceptions.py @@ -0,0 +1,59 @@ +# These exceptions are used to add NORM values based on a token's ORTH value. +# Individual languages can also add their own exceptions and overwrite them - +# for example, British vs. American spelling in English. + +# Norms are only set if no alternative is provided in the tokenizer exceptions. +# Note that this does not change any other token attributes. Its main purpose +# is to normalise the word representations so that equivalent tokens receive +# similar representations. For example: $ and € are very different, but they're +# both currency symbols. By normalising currency symbols to $, all symbols are +# seen as similar, no matter how common they are in the training data. + + +BASE_NORMS = { + "'s": "'s", + "'S": "'s", + "’s": "'s", + "’S": "'s", + "’": "'", + "‘": "'", + "´": "'", + "`": "'", + "”": '"', + "“": '"', + "''": '"', + "``": '"', + "´´": '"', + "„": '"', + "»": '"', + "«": '"', + "‘‘": '"', + "’’": '"', + "？": "?", + "！": "!", + "，": ",", + "；": ";", + "：": ":", + "。": ".", + "।": ".", + "…": "...", + "—": "-", + "–": "-", + "--": "-", + "---": "-", + "——": "-", + "€": "$", + "£": "$", + "¥": "$", + "฿": "$", + "US$": "$", + "C$": "$", + "A$": "$", + "₺": "$", + "₹": "$", + "৳": "$", + "₩": "$", + "Mex$": "$", + "₣": "$", + "E£": "$", +} diff --git a/spacy/lang/pl/__init__.py b/spacy/lang/pl/__init__.py new file mode 100644 index 0000000..50a3a8e --- /dev/null +++ b/spacy/lang/pl/__init__.py @@ -0,0 +1,55 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .lemmatizer import PolishLemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS + +TOKENIZER_EXCEPTIONS = { + exc: val for exc, val in BASE_EXCEPTIONS.items() if not exc.endswith(".") +} + + +class PolishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Polish(Language): + lang = "pl" + Defaults = PolishDefaults + + +@Polish.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "pos_lookup", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return PolishLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Polish"] diff --git a/spacy/lang/pl/examples.py b/spacy/lang/pl/examples.py new file mode 100644 index 0000000..cb55ed0 --- /dev/null +++ b/spacy/lang/pl/examples.py @@ -0,0 +1,15 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.pl.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Poczuł przyjemną woń mocnej kawy.", + "Istnieje wiele dróg oddziaływania substancji psychoaktywnej na układ nerwowy.", + "Powitał mnie biało-czarny kot, płosząc siedzące na płocie trzy dorodne dudki.", + "Nowy abonament pod lupą Komisji Europejskiej", + "Czy w ciągu ostatnich 48 godzin spożyłeś leki zawierające paracetamol?", + "Kto ma ochotę zapoznać się z innymi niż w książkach przygodami Muminków i ich przyjaciół, temu polecam komiks Tove Jansson „Muminki i morze”.", +] diff --git a/spacy/lang/pl/lemmatizer.py b/spacy/lang/pl/lemmatizer.py new file mode 100644 index 0000000..d1d2a9c --- /dev/null +++ b/spacy/lang/pl/lemmatizer.py @@ -0,0 +1,86 @@ +from typing import Dict, List, Tuple + +from ...pipeline import Lemmatizer +from ...tokens import Token + + +class PolishLemmatizer(Lemmatizer): + # This lemmatizer implements lookup lemmatization based on the Morfeusz + # dictionary (morfeusz.sgjp.pl/en) by Institute of Computer Science PAS. + # It utilizes some prefix based improvements for verb and adjectives + # lemmatization, as well as case-sensitive lemmatization for nouns. + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + if mode == "pos_lookup": + # fmt: off + required = [ + "lemma_lookup_adj", "lemma_lookup_adp", "lemma_lookup_adv", + "lemma_lookup_aux", "lemma_lookup_noun", "lemma_lookup_num", + "lemma_lookup_part", "lemma_lookup_pron", "lemma_lookup_verb" + ] + # fmt: on + return (required, []) + else: + return super().get_lookups_config(mode) + + def pos_lookup_lemmatize(self, token: Token) -> List[str]: + string = token.text + univ_pos = token.pos_ + morphology = token.morph.to_dict() + lookup_pos = univ_pos.lower() + if univ_pos == "PROPN": + lookup_pos = "noun" + lookup_table = self.lookups.get_table("lemma_lookup_" + lookup_pos, {}) + if univ_pos == "NOUN": + return self.lemmatize_noun(string, morphology, lookup_table) + if univ_pos != "PROPN": + string = string.lower() + if univ_pos == "ADJ": + return self.lemmatize_adj(string, morphology, lookup_table) + elif univ_pos == "VERB": + return self.lemmatize_verb(string, morphology, lookup_table) + return [lookup_table.get(string, string.lower())] + + def lemmatize_adj( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + # this method utilizes different procedures for adjectives + # with 'nie' and 'naj' prefixes + if string[:3] == "nie": + search_string = string[3:] + if search_string[:3] == "naj": + naj_search_string = search_string[3:] + if naj_search_string in lookup_table: + return [lookup_table[naj_search_string]] + if search_string in lookup_table: + return [lookup_table[search_string]] + if string[:3] == "naj": + naj_search_string = string[3:] + if naj_search_string in lookup_table: + return [lookup_table[naj_search_string]] + return [lookup_table.get(string, string)] + + def lemmatize_verb( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + # this method utilizes a different procedure for verbs + # with 'nie' prefix + if string[:3] == "nie": + search_string = string[3:] + if search_string in lookup_table: + return [lookup_table[search_string]] + return [lookup_table.get(string, string)] + + def lemmatize_noun( + self, string: str, morphology: dict, lookup_table: Dict[str, str] + ) -> List[str]: + # this method is case-sensitive, in order to work + # for incorrectly tagged proper names + if string != string.lower(): + if string.lower() in lookup_table: + return [lookup_table[string.lower()]] + elif string in lookup_table: + return [lookup_table[string]] + return [string.lower()] + return [lookup_table.get(string, string)] diff --git a/spacy/lang/pl/lex_attrs.py b/spacy/lang/pl/lex_attrs.py new file mode 100644 index 0000000..398f52a --- /dev/null +++ b/spacy/lang/pl/lex_attrs.py @@ -0,0 +1,65 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "zero", + "jeden", + "dwa", + "trzy", + "cztery", + "pięć", + "sześć", + "siedem", + "osiem", + "dziewięć", + "dziesięć", + "jedenaście", + "dwanaście", + "trzynaście", + "czternaście", + "pietnaście", + "szesnaście", + "siedemnaście", + "osiemnaście", + "dziewiętnaście", + "dwadzieścia", + "trzydzieści", + "czterdzieści", + "pięćdziesiąt", + "szcześćdziesiąt", + "siedemdziesiąt", + "osiemdziesiąt", + "dziewięćdziesiąt", + "sto", + "dwieście", + "trzysta", + "czterysta", + "pięćset", + "sześćset", + "siedemset", + "osiemset", + "dziewięćset", + "tysiąc", + "milion", + "miliard", + "bilion", + "biliard", + "trylion", + "tryliard", + "kwadrylion", +] + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/pl/punctuation.py b/spacy/lang/pl/punctuation.py new file mode 100644 index 0000000..84ff239 --- /dev/null +++ b/spacy/lang/pl/punctuation.py @@ -0,0 +1,56 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_HYPHENS, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + +_quotes = CONCAT_QUOTES.replace("'", "") + +_prefixes = _prefixes = [ + r"(długo|krótko|jedno|dwu|trzy|cztero)-" +] + BASE_TOKENIZER_PREFIXES + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + LIST_HYPHENS + + [ + r"(?<=[0-9{al}])\.(?=[0-9{au}])".format(al=ALPHA, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[:<>=\/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=_quotes), + ] +) + +_suffixes = ( + ["''", "’’", r"\.", "…"] + + LIST_PUNCT + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}])\.".format(au=ALPHA_UPPER), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/pl/stop_words.py b/spacy/lang/pl/stop_words.py new file mode 100644 index 0000000..075aec3 --- /dev/null +++ b/spacy/lang/pl/stop_words.py @@ -0,0 +1,78 @@ +# sources: https://github.com/bieli/stopwords/blob/master/polish.stopwords.txt and https://github.com/stopwords-iso/stopwords-pl + +STOP_WORDS = set( + """ +a aby ach acz aczkolwiek aj albo ale alez +ależ ani az aż + +bardziej bardzo beda bede bedzie bez bo bowiem by +byc byl byla byli bylo byly bym bynajmniej być był +była było były będzie będą będę + +cala cali caly cała cały chce choć ci cie +ciebie cię co cokolwiek coraz cos coś czasami czasem czemu +czy czyli często + +daleko dla dlaczego dlatego do dobrze dokad dokąd +dosc dość duzo dużo dwa dwaj dwie dwoje dzis +dzisiaj dziś + +gdy gdyby gdyz gdyż gdzie gdziekolwiek gdzies gdzieś go +godz + +i ich ile im inna inne inny +innych iv ix iz iż + +ja jak jakas jakaś jakby jaki jakichs jakichś jakie +jakis jakiz jakiś jakiż jakkolwiek jako jakos jakoś je jeden +jedna jednak jednakze jednakże jedno jednym jedynie jego jej jemu +jesli jest jestem jeszcze jezeli jeśli jeżeli juz już ją + +kazdy każdy kiedy kierunku kilka kilku kims kimś kto +ktokolwiek ktora ktore ktorego ktorej ktory ktorych ktorym ktorzy ktos +ktoś która które którego której który których którym którzy ku + +lecz lub + +ma mają mam mamy mało mi miał miedzy +mimo między mna mnie mną moga mogą moi moim moj +moja moje moze mozliwe mozna może możliwe można mu musi +my mój + +na nad nam nami nas nasi nasz nasza nasze +naszego naszych natomiast natychmiast nawet nia nic nich nie niech +niego niej niemu nigdy nim nimi niz nią niż no + +o obok od ok około on ona one +oni ono oraz oto owszem + +pan pana pani po pod podczas pomimo ponad +poniewaz ponieważ powinien powinna powinni powinno poza prawie przeciez +przecież przed przede przedtem przez przy + +raz razie roku rowniez również + +sam sama sie się skad skąd soba sobie sobą +sposob sposób swoje są + +ta tak taka taki takich takie takze także tam +te tego tej tel temu ten teraz też to toba +tobie tobą totez toteż totobą trzeba tu tutaj twoi twoim +twoj twoja twoje twym twój ty tych tylko tym tys +tzw tę + +u + +vi vii viii + +w wam wami was wasi wasz wasza wasze we +według wie wiele wielu więc więcej wlasnie wszyscy wszystkich wszystkie +wszystkim wszystko wtedy wy właśnie wśród + +xi xii xiii xiv xv + +z za zaden zadna zadne zadnych zapewne zawsze zaś +ze zeby znow znowu znów zostal został + +żaden żadna żadne żadnych że żeby""".split() +) diff --git a/spacy/lang/pt/__init__.py b/spacy/lang/pt/__init__.py new file mode 100644 index 0000000..be4041f --- /dev/null +++ b/spacy/lang/pt/__init__.py @@ -0,0 +1,23 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class PortugueseDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + prefixes = TOKENIZER_PREFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class Portuguese(Language): + lang = "pt" + Defaults = PortugueseDefaults + + +__all__ = ["Portuguese"] diff --git a/spacy/lang/pt/examples.py b/spacy/lang/pt/examples.py new file mode 100644 index 0000000..42ae602 --- /dev/null +++ b/spacy/lang/pt/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.pt.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple está querendo comprar uma startup do Reino Unido por 100 milhões de dólares", + "Carros autônomos empurram a responsabilidade do seguro para os fabricantes." + "São Francisco considera banir os robôs de entrega que andam pelas calçadas", + "Londres é a maior cidade do Reino Unido", +] diff --git a/spacy/lang/pt/lex_attrs.py b/spacy/lang/pt/lex_attrs.py new file mode 100644 index 0000000..de6a67f --- /dev/null +++ b/spacy/lang/pt/lex_attrs.py @@ -0,0 +1,118 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "zero", + "um", + "dois", + "três", + "tres", + "quatro", + "cinco", + "seis", + "sete", + "oito", + "nove", + "dez", + "onze", + "doze", + "dúzia", + "dúzias", + "duzia", + "duzias", + "treze", + "catorze", + "quinze", + "dezasseis", + "dezassete", + "dezoito", + "dezanove", + "vinte", + "trinta", + "quarenta", + "cinquenta", + "sessenta", + "setenta", + "oitenta", + "noventa", + "cem", + "cento", + "duzentos", + "trezentos", + "quatrocentos", + "quinhentos", + "seicentos", + "setecentos", + "oitocentos", + "novecentos", + "mil", + "milhão", + "milhao", + "milhões", + "milhoes", + "bilhão", + "bilhao", + "bilhões", + "bilhoes", + "trilhão", + "trilhao", + "trilhões", + "trilhoes", + "quadrilhão", + "quadrilhao", + "quadrilhões", + "quadrilhoes", +] + + +_ordinal_words = [ + "primeiro", + "segundo", + "terceiro", + "quarto", + "quinto", + "sexto", + "sétimo", + "oitavo", + "nono", + "décimo", + "vigésimo", + "trigésimo", + "quadragésimo", + "quinquagésimo", + "sexagésimo", + "septuagésimo", + "octogésimo", + "nonagésimo", + "centésimo", + "ducentésimo", + "trecentésimo", + "quadringentésimo", + "quingentésimo", + "sexcentésimo", + "septingentésimo", + "octingentésimo", + "nongentésimo", + "milésimo", + "milionésimo", + "bilionésimo", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "").replace("º", "").replace("ª", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/pt/punctuation.py b/spacy/lang/pt/punctuation.py new file mode 100644 index 0000000..60bd50d --- /dev/null +++ b/spacy/lang/pt/punctuation.py @@ -0,0 +1,15 @@ +from ..punctuation import ( + TOKENIZER_INFIXES as BASE_TOKENIZER_INFIXES, + TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES, + TOKENIZER_SUFFIXES as BASE_TOKENIZER_SUFFIXES, +) + +_prefixes = [r"\w{1,3}\$"] + BASE_TOKENIZER_PREFIXES + +_suffixes = BASE_TOKENIZER_SUFFIXES + +_infixes = [r"(\w+-\w+(-\w+)*)"] + BASE_TOKENIZER_INFIXES + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/pt/stop_words.py b/spacy/lang/pt/stop_words.py new file mode 100644 index 0000000..ce3c86f --- /dev/null +++ b/spacy/lang/pt/stop_words.py @@ -0,0 +1,66 @@ +STOP_WORDS = set( + """ +a à às área acerca ademais adeus agora ainda algo algumas alguns ali além ambas ambos antes +ao aos apenas apoia apoio apontar após aquela aquelas aquele aqueles aqui aquilo +as assim através atrás até aí + +baixo bastante bem boa bom breve + +cada caminho catorze cedo cento certamente certeza cima cinco coisa com como +comprida comprido conhecida conhecido conselho contra contudo corrente cuja +cujo custa cá + +da daquela daquele dar das de debaixo demais dentro depois des desde dessa desse +desta deste deve devem deverá dez dezanove dezasseis dezassete dezoito diante +direita disso diz dizem dizer do dois dos doze duas dá dão + +e é és ela elas ele eles em embora enquanto entre então era essa essas esse esses esta +estado estar estará estas estava este estes esteve estive estivemos estiveram +estiveste estivestes estou está estás estão eu eventual exemplo + +falta fará favor faz fazeis fazem fazemos fazer fazes fazia faço fez fim final +foi fomos for fora foram forma foste fostes fui + +geral grande grandes grupo + +inclusive iniciar inicio ir irá isso isto + +já + +lado lhe ligado local logo longe lugar lá + +maior maioria maiorias mais mal mas me meio menor menos meses mesmo meu meus mil +minha minhas momento muito muitos máximo mês + +na nada naquela naquele nas nem nenhuma nessa nesse nesta neste no nos nossa +nossas nosso nossos nova novas nove novo novos num numa nunca nuns não nível nós +número números + +o obrigada obrigado oitava oitavo oito onde ontem onze ora os ou outra outras outros + +para parece parte partir pegar pela pelas pelo pelos perto pode podem poder poderá +podia pois ponto pontos por porquanto porque porquê portanto porém posição +possivelmente posso possível pouca pouco povo primeira primeiro próprio próxima +próximo puderam pôde põe põem + +quais qual qualquer quando quanto quarta quarto quatro que quem quer querem quero +questão quieta quieto quinta quinto quinze quê + +relação + +sabe saber se segunda segundo sei seis sem sempre ser seria sete seu seus sexta +sexto sim sistema sob sobre sois somente somos sou sua suas são sétima sétimo só + +tais tal talvez também tanta tanto tarde te tem temos tempo tendes tenho tens +tentar tentaram tente tentei ter terceira terceiro teu teus teve tipo tive +tivemos tiveram tiveste tivestes toda todas todo todos treze três tu tua tuas +tudo tão têm + +um uma umas uns usa usar último + +vai vais valor veja vem vens ver vez vezes vinda vindo vinte você vocês vos vossa +vossas vosso vossos vários vão vêm vós + +zero +""".split() +) diff --git a/spacy/lang/pt/syntax_iterators.py b/spacy/lang/pt/syntax_iterators.py new file mode 100644 index 0000000..11017aa --- /dev/null +++ b/spacy/lang/pt/syntax_iterators.py @@ -0,0 +1,85 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = [ + "nsubj", + "nsubj:pass", + "obj", + "obl", + "obl:agent", + "nmod", + "pcomp", + "appos", + "ROOT", + ] + post_modifiers = ["flat", "flat:name", "fixed", "compound"] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = {doc.vocab.strings.add(label) for label in labels} + np_modifs = {doc.vocab.strings.add(modifier) for modifier in post_modifiers} + np_label = doc.vocab.strings.add("NP") + adj_label = doc.vocab.strings.add("amod") + det_label = doc.vocab.strings.add("det") + det_pos = doc.vocab.strings.add("DET") + adp_label = doc.vocab.strings.add("ADP") + conj = doc.vocab.strings.add("conj") + conj_pos = doc.vocab.strings.add("CCONJ") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + right_childs = list(word.rights) + right_child = right_childs[0] if right_childs else None + + if right_child: + if ( + right_child.dep == adj_label + ): # allow chain of adjectives by expanding to right + right_end = right_child.right_edge + elif ( + right_child.dep == det_label and right_child.pos == det_pos + ): # cut relative pronouns here + right_end = right_child + elif right_child.dep in np_modifs: # Check if we can expand to right + right_end = word.right_edge + else: + right_end = word + else: + right_end = word + prev_end = right_end.i + + left_index = word.left_edge.i + left_index = ( + left_index + 1 if word.left_edge.pos == adp_label else left_index + ) + + yield left_index, right_end.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.i + + left_index = word.left_edge.i # eliminate left attached conjunction + left_index = ( + left_index + 1 if word.left_edge.pos == conj_pos else left_index + ) + yield left_index, word.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/pt/tokenizer_exceptions.py b/spacy/lang/pt/tokenizer_exceptions.py new file mode 100644 index 0000000..e369eda --- /dev/null +++ b/spacy/lang/pt/tokenizer_exceptions.py @@ -0,0 +1,54 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +for orth in [ + "Adm.", + "Art.", + "art.", + "Av.", + "av.", + "Cia.", + "dom.", + "Dr.", + "dr.", + "e.g.", + "E.g.", + "E.G.", + "e/ou", + "ed.", + "eng.", + "etc.", + "Fund.", + "Gen.", + "Gov.", + "i.e.", + "I.e.", + "I.E.", + "Inc.", + "Jr.", + "km/h", + "Ltd.", + "Mr.", + "p.m.", + "Ph.D.", + "Rep.", + "Rev.", + "S/A", + "Sen.", + "Sr.", + "sr.", + "Sra.", + "sra.", + "vs.", + "tel.", + "pág.", + "pag.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/punctuation.py b/spacy/lang/punctuation.py new file mode 100644 index 0000000..e4a6392 --- /dev/null +++ b/spacy/lang/punctuation.py @@ -0,0 +1,78 @@ +from .char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + COMBINING_DIACRITICS, + CONCAT_QUOTES, + CURRENCY, + HYPHENS, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) + +TOKENIZER_PREFIXES = ( + ["§", "%", "=", "—", "–", r"\+(?![0-9])"] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + + +TOKENIZER_SUFFIXES = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + ["'s", "'S", "’s", "’S", "—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_INFIXES = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +# Some languages e.g. written with the Cyrillic alphabet permit the use of diacritics +# to mark stressed syllables in words where stress is distinctive. Such languages +# should use the COMBINING_DIACRITICS... suffix and infix regex lists in +# place of the standard ones. +COMBINING_DIACRITICS_TOKENIZER_SUFFIXES = list(TOKENIZER_SUFFIXES) + [ + r"(?<=[{a}][{d}])\.".format(a=ALPHA, d=COMBINING_DIACRITICS), +] + +COMBINING_DIACRITICS_TOKENIZER_INFIXES = list(TOKENIZER_INFIXES) + [ + r"(?<=[{al}][{d}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES, d=COMBINING_DIACRITICS + ), + r"(?<=[{a}][{d}]),(?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS), + r"(?<=[{a}][{d}])(?:{h})(?=[{a}])".format( + a=ALPHA, d=COMBINING_DIACRITICS, h=HYPHENS + ), + r"(?<=[{a}][{d}])[:<>=/](?=[{a}])".format(a=ALPHA, d=COMBINING_DIACRITICS), +] diff --git a/spacy/lang/ro/__init__.py b/spacy/lang/ro/__init__.py new file mode 100644 index 0000000..441fefb --- /dev/null +++ b/spacy/lang/ro/__init__.py @@ -0,0 +1,26 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + +# Lemma data note: +# Original pairs downloaded from http://www.lexiconista.com/datasets/lemmatization/ +# Replaced characters using cedillas with the correct ones (ș and ț) + + +class RomanianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Romanian(Language): + lang = "ro" + Defaults = RomanianDefaults + + +__all__ = ["Romanian"] diff --git a/spacy/lang/ro/examples.py b/spacy/lang/ro/examples.py new file mode 100644 index 0000000..46b4c9a --- /dev/null +++ b/spacy/lang/ro/examples.py @@ -0,0 +1,18 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ro import Romanian +>>> from spacy.lang.ro.examples import sentences +>>> nlp = Romanian() +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple plănuiește să cumpere o companie britanică pentru un miliard de dolari", + "Municipalitatea din San Francisco ia în calcul interzicerea roboților curieri pe trotuar", + "Londra este un oraș mare în Regatul Unit", + "Unde ești?", + "Cine este președintele Franței?", + "Care este capitala Statelor Unite?", + "Când s-a născut Barack Obama?", +] diff --git a/spacy/lang/ro/lex_attrs.py b/spacy/lang/ro/lex_attrs.py new file mode 100644 index 0000000..736aa91 --- /dev/null +++ b/spacy/lang/ro/lex_attrs.py @@ -0,0 +1,42 @@ +from ...attrs import LIKE_NUM + +_num_words = set( + """ +zero unu doi două trei patru cinci șase șapte opt nouă zece +unsprezece doisprezece douăsprezece treisprezece patrusprezece cincisprezece șaisprezece șaptesprezece optsprezece nouăsprezece +douăzeci treizeci patruzeci cincizeci șaizeci șaptezeci optzeci nouăzeci +sută mie milion miliard bilion trilion cvadrilion catralion cvintilion sextilion septilion enșpemii +""".split() +) + +_ordinal_words = set( + """ +primul doilea treilea patrulea cincilea șaselea șaptelea optulea nouălea zecelea +prima doua treia patra cincia șasea șaptea opta noua zecea +unsprezecelea doisprezecelea treisprezecelea patrusprezecelea cincisprezecelea șaisprezecelea șaptesprezecelea optsprezecelea nouăsprezecelea +unsprezecea douăsprezecea treisprezecea patrusprezecea cincisprezecea șaisprezecea șaptesprezecea optsprezecea nouăsprezecea +douăzecilea treizecilea patruzecilea cincizecilea șaizecilea șaptezecilea optzecilea nouăzecilea sutălea +douăzecea treizecea patruzecea cincizecea șaizecea șaptezecea optzecea nouăzecea suta +miilea mielea mia milionulea milioana miliardulea miliardelea miliarda enșpemia +""".split() +) + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ro/punctuation.py b/spacy/lang/ro/punctuation.py new file mode 100644 index 0000000..7259f9a --- /dev/null +++ b/spacy/lang/ro/punctuation.py @@ -0,0 +1,170 @@ +import itertools + +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, +) + +_list_icons = [x for x in LIST_ICONS if x != "°"] +_list_icons = [x.replace("\\u00B0", "") for x in _list_icons] + + +_ro_variants = { + "Ă": ["Ă", "A"], + "Â": ["Â", "A"], + "Î": ["Î", "I"], + "Ș": ["Ș", "Ş", "S"], + "Ț": ["Ț", "Ţ", "T"], +} + + +def _make_ro_variants(tokens): + variants = [] + for token in tokens: + upper_token = token.upper() + upper_char_variants = [_ro_variants.get(c, [c]) for c in upper_token] + upper_variants = ["".join(x) for x in itertools.product(*upper_char_variants)] + for variant in upper_variants: + variants.extend([variant, variant.lower(), variant.title()]) + return sorted(list(set(variants))) + + +# UD_Romanian-RRT closed class prefixes +# POS: ADP|AUX|CCONJ|DET|NUM|PART|PRON|SCONJ +_ud_rrt_prefixes = [ + "a-", + "c-", + "ce-", + "cu-", + "d-", + "de-", + "dintr-", + "e-", + "făr-", + "i-", + "l-", + "le-", + "m-", + "mi-", + "n-", + "ne-", + "p-", + "pe-", + "prim-", + "printr-", + "s-", + "se-", + "te-", + "v-", + "într-", + "ș-", + "și-", + "ți-", +] +_ud_rrt_prefix_variants = _make_ro_variants(_ud_rrt_prefixes) + + +# UD_Romanian-RRT closed class suffixes without NUM +# POS: ADP|AUX|CCONJ|DET|PART|PRON|SCONJ +_ud_rrt_suffixes = [ + "-a", + "-aceasta", + "-ai", + "-al", + "-ale", + "-alta", + "-am", + "-ar", + "-astea", + "-atâta", + "-au", + "-aș", + "-ați", + "-i", + "-ilor", + "-l", + "-le", + "-lea", + "-mea", + "-meu", + "-mi", + "-mă", + "-n", + "-ndărătul", + "-ne", + "-o", + "-oi", + "-or", + "-s", + "-se", + "-si", + "-te", + "-ul", + "-ului", + "-un", + "-uri", + "-urile", + "-urilor", + "-veți", + "-vă", + "-ăștia", + "-și", + "-ți", +] +_ud_rrt_suffix_variants = _make_ro_variants(_ud_rrt_suffixes) + + +_prefixes = ( + ["§", "%", "=", "—", "–", r"\+(?![0-9])"] + + _ud_rrt_prefix_variants + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + + +_suffixes = ( + _ud_rrt_suffix_variants + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + _list_icons + + ["—", "–"] + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +_infixes = ( + LIST_ELLIPSES + + _list_icons + + [ + r"(?<=[0-9])[+\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=](?=[{a}])".format(a=ALPHA), + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/ro/stop_words.py b/spacy/lang/ro/stop_words.py new file mode 100644 index 0000000..d68a81c --- /dev/null +++ b/spacy/lang/ro/stop_words.py @@ -0,0 +1,499 @@ +# Source: https://github.com/stopwords-iso/stopwords-ro +STOP_WORDS = set( + """ +a +abia +acea +aceasta +această +aceea +aceeasi +aceeași +acei +aceia +acel +acela +acelasi +același +acele +acelea +acest +acesta +aceste +acestea +acestei +acestia +acestui +aceşti +aceştia +acolo +acord +acum +adica +adică +ai +aia +aibă +aici +aiurea +al +ala +alaturi +ale +alea +alt +alta +altceva +altcineva +alte +altfel +alti +altii +altul +alături +am +anume +apoi +apai +apăi +ar +are +as +asa +asemenea +asta +astazi +astea +astfel +astăzi +asupra +atare +atat +atata +atatea +atatia +ati +atit +atita +atitea +atitia +atunci +au +avea +avem +aveţi +aveți +avut +azi +aş +aşadar +aţi +aș +așadar +ați +b +ba +bine +bucur +bună +c +ca +cam +cand +capat +care +careia +carora +caruia +cat +catre +caut +ce +cea +ceea +cei +ceilalti +cel +cele +celor +ceva +chiar +ci +cinci +cind +cine +cineva +cit +cita +cite +citeva +citi +citiva +conform +contra +cu +cui +cum +cumva +curând +curînd +când +cât +câte +câtva +câţi +câți +cînd +cît +cîte +cîtva +cîţi +cîți +că +căci +cărei +căror +cărora +cărui +căruia +către +d +da +daca +dacă +dar +dat +datorită +dată +dau +de +deasupra +deci +decit +degraba +deja +deoarece +departe +desi +despre +deşi +deși +din +dinaintea +dincolo +dincoace +dintr +dintr- +dintre +doar +doi +doilea +două +drept +dupa +după +dă +deunaseara +deunăseară +deunazi +deunăzi +e +ea +ei +el +ele +era +eram +este +eu +exact +eşti +ești +f +face +fara +fata +fel +fi +fie +fiecare +fii +fim +fiu +fiţi +fiți +foarte +fost +frumos +fără +g +geaba +graţie +grație +h +i +ia +iar +ieri +ii +il +imi +in +inainte +inapoi +inca +incotro +incit +insa +intr +intre +isi +iti +j +k +l +la +le +li +lor +lui +lângă +lîngă +m +ma +mai +mare +macar +măcar +mata +matale +mea +mei +mele +mereu +meu +mi +mie +mine +mod +mult +multa +multe +multi +multă +mulţi +mulţumesc +mulți +mulțumesc +mâine +mîine +mă +n +na +ne +neincetat +neîncetat +nevoie +ni +nici +nicidecum +nicidecat +nicidecât +niciodata +niciodată +nicăieri +nimeni +nimeri +nimic +niste +nişte +niște +noastre +noastră +noi +noroc +nostri +nostru +nou +noua +nouă +noştri +noștri +nu +numai +o +odata +odată +odinioara +odinioară +opt +or +ori +oricare +orice +oricine +oricum +oricând +oricât +oricînd +oricît +oriunde +p +pai +păi +parca +parcă +patra +patru +patrulea +pe +pentru +peste +pic +pina +plus +poate +pot +prea +prima +primul +prin +printr- +printre +putini +puţin +puţina +puţină +până +pînă +r +rog +s +sa +sa-mi +sa-ti +sai +sale +sau +se +si +sint +sintem +spate +spre +sub +sunt +suntem +sunteţi +sunteți +sus +sută +sînt +sîntem +sînteţi +sînteți +să +săi +său +t +ta +tale +te +ti +timp +tine +toata +toate +toată +tocmai +tot +toti +totul +totusi +totuşi +totuși +toţi +toți +trei +treia +treilea +tu +tuturor +tăi +tău +u +ul +ului +un +una +unde +undeva +unei +uneia +unele +uneori +unii +unor +unora +unu +unui +unuia +unul +v +va +vai +vi +voastre +voastră +voi +vom +vor +vostru +vouă +voştri +voștri +vreme +vreo +vreun +vă +x +z +zece +zero +zi +zice +îi +îl +îmi +împotriva +în +înainte +înaintea +încotro +încât +încît +între +întrucât +întrucît +îţi +îți +ăla +ălea +ăsta +ăstea +ăştia +ăștia +şapte +şase +şi +ştiu +ţi +ţie +șapte +șase +și +știu +ți +ție +""".split() +) diff --git a/spacy/lang/ro/tokenizer_exceptions.py b/spacy/lang/ro/tokenizer_exceptions.py new file mode 100644 index 0000000..a397b27 --- /dev/null +++ b/spacy/lang/ro/tokenizer_exceptions.py @@ -0,0 +1,95 @@ +from ...symbols import ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .punctuation import _make_ro_variants + +_exc = {} + + +# Source: https://en.wiktionary.org/wiki/Category:Romanian_abbreviations +for orth in [ + "1-a", + "2-a", + "3-a", + "4-a", + "5-a", + "6-a", + "7-a", + "8-a", + "9-a", + "10-a", + "11-a", + "12-a", + "1-ul", + "2-lea", + "3-lea", + "4-lea", + "5-lea", + "6-lea", + "7-lea", + "8-lea", + "9-lea", + "10-lea", + "11-lea", + "12-lea", + "d-voastră", + "dvs.", + "ing.", + "dr.", + "Rom.", + "str.", + "nr.", + "etc.", + "d.p.d.v.", + "dpdv", + "șamd.", + "ș.a.m.d.", + # below: from UD_Romanian-RRT: + "A.c.", + "A.f.", + "A.r.", + "Al.", + "Art.", + "Aug.", + "Bd.", + "Dem.", + "Dr.", + "Fig.", + "Fr.", + "Gh.", + "Gr.", + "Lt.", + "Nr.", + "Obs.", + "Prof.", + "Sf.", + "a.m.", + "a.r.", + "alin.", + "art.", + "d-l", + "d-lui", + "d-nei", + "ex.", + "fig.", + "ian.", + "lit.", + "lt.", + "p.a.", + "p.m.", + "pct.", + "prep.", + "sf.", + "tel.", + "univ.", + "îngr.", + "într-adevăr", + "Șt.", + "ș.a.", +]: + # note: does not distinguish capitalized-only exceptions from others + for variant in _make_ro_variants([orth]): + _exc[variant] = [{ORTH: variant}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py new file mode 100644 index 0000000..880965b --- /dev/null +++ b/spacy/lang/ru/__init__.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ..punctuation import ( + COMBINING_DIACRITICS_TOKENIZER_INFIXES, + COMBINING_DIACRITICS_TOKENIZER_SUFFIXES, +) +from .lemmatizer import RussianLemmatizer +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class RussianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES + + +class Russian(Language): + lang = "ru" + Defaults = RussianDefaults + + +@Russian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "pymorphy3", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return RussianLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Russian"] diff --git a/spacy/lang/ru/examples.py b/spacy/lang/ru/examples.py new file mode 100644 index 0000000..9595d58 --- /dev/null +++ b/spacy/lang/ru/examples.py @@ -0,0 +1,39 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ru.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + # Translations from English: + "Apple рассматривает возможность покупки стартапа из Соединённого Королевства за $1 млрд", + "Беспилотные автомобили перекладывают страховую ответственность на производителя", + "В Сан-Франциско рассматривается возможность запрета роботов-курьеров, которые перемещаются по тротуару", + "Лондон — это большой город в Соединённом Королевстве", + # Native Russian sentences: + # Colloquial: + "Да, нет, наверное!", # Typical polite refusal + "Обратите внимание на необыкновенную красоту этого города-героя Москвы, столицы нашей Родины!", # From a tour guide speech + # Examples of Bookish Russian: + # Quote from "The Golden Calf" + "Рио-де-Жанейро — это моя мечта, и не смейте касаться её своими грязными лапами!", + # Quotes from "Ivan Vasilievich changes his occupation" + "Ты пошто боярыню обидел, смерд?!!", + "Оставь меня, старушка, я в печали!", + # Quotes from Dostoevsky: + "Уж коли я, такой же, как и ты, человек грешный, над тобой умилился и пожалел тебя, кольми паче бог", + "В мечтах я нередко, говорит, доходил до страстных помыслов о служении человечеству и может быть действительно пошел бы на крест за людей, если б это вдруг как-нибудь потребовалось, а между тем я двух дней не в состоянии прожить ни с кем в одной комнате, о чем знаю из опыта", + "Зато всегда так происходило, что чем более я ненавидел людей в частности, тем пламеннее становилась любовь моя к человечеству вообще", + # Quotes from Chekhov: + "Ненужные дела и разговоры всё об одном отхватывают на свою долю лучшую часть времени, лучшие силы, и в конце концов остается какая-то куцая, бескрылая жизнь, какая-то чепуха, и уйти и бежать нельзя, точно сидишь в сумасшедшем доме или в арестантских ротах!", + # Quotes from Turgenev: + "Нравится тебе женщина, старайся добиться толку; а нельзя — ну, не надо, отвернись — земля не клином сошлась", + "Узенькое местечко, которое я занимаю, до того крохотно в сравнении с остальным пространством, где меня нет и где дела до меня нет; и часть времени, которую мне удастся прожить, так ничтожна перед вечностью, где меня не было и не будет...", + # Quotes from newspapers: + # Komsomolskaya Pravda: + "На заседании президиума правительства Москвы принято решение присвоить статус инвестиционного приоритетного проекта города Москвы киностудии Союзмультфильм", + "Глава Минобороны Сергей Шойгу заявил, что обстановка на этом стратегическом направлении требует непрерывного совершенствования боевого состава войск", + # Argumenty i Facty: + "На реплику лже-Говина — дескать, он (Волков) будет лучшим революционером — Стамп с энтузиазмом ответил: Непременно!", +] diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py new file mode 100644 index 0000000..1e41220 --- /dev/null +++ b/spacy/lang/ru/lemmatizer.py @@ -0,0 +1,217 @@ +from typing import Callable, Dict, List, Optional, Tuple + +from thinc.api import Model + +from ...pipeline import Lemmatizer +from ...pipeline.lemmatizer import lemmatizer_score +from ...symbols import POS +from ...tokens import Token +from ...vocab import Vocab + +PUNCT_RULES = {"«": '"', "»": '"'} + + +class RussianLemmatizer(Lemmatizer): + def __init__( + self, + vocab: Vocab, + model: Optional[Model], + name: str = "lemmatizer", + *, + mode: str = "pymorphy3", + overwrite: bool = False, + scorer: Optional[Callable] = lemmatizer_score, + ) -> None: + if mode in {"pymorphy2", "pymorphy2_lookup"}: + try: + from pymorphy2 import MorphAnalyzer + except ImportError: + raise ImportError( + "The lemmatizer mode 'pymorphy2' requires the " + "pymorphy2 library and dictionaries. Install them with: " + "pip install pymorphy2" + "# for Ukrainian dictionaries:" + "pip install pymorphy2-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="ru") + elif mode in {"pymorphy3", "pymorphy3_lookup"}: + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3" + "# for Ukrainian dictionaries:" + "pip install pymorphy3-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="ru") + super().__init__( + vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + def _pymorphy_lemmatize(self, token: Token) -> List[str]: + string = token.text + univ_pos = token.pos_ + morphology = token.morph.to_dict() + if univ_pos == "PUNCT": + return [PUNCT_RULES.get(string, string)] + if univ_pos not in ("ADJ", "DET", "NOUN", "NUM", "PRON", "PROPN", "VERB"): + return self._pymorphy_lookup_lemmatize(token) + analyses = self._morph.parse(string) + filtered_analyses = [] + for analysis in analyses: + if not analysis.is_known: + # Skip suggested parse variant for unknown word for pymorphy + continue + analysis_pos, _ = oc2ud(str(analysis.tag)) + if ( + analysis_pos == univ_pos + or (analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN")) + or ((analysis_pos == "PRON") and (univ_pos == "DET")) + ): + filtered_analyses.append(analysis) + if not len(filtered_analyses): + return [string.lower()] + if morphology is None or (len(morphology) == 1 and POS in morphology): + return list( + dict.fromkeys([analysis.normal_form for analysis in filtered_analyses]) + ) + if univ_pos in ("ADJ", "DET", "NOUN", "PROPN"): + features_to_compare = ["Case", "Number", "Gender"] + elif univ_pos == "NUM": + features_to_compare = ["Case", "Gender"] + elif univ_pos == "PRON": + features_to_compare = ["Case", "Number", "Gender", "Person"] + else: # VERB + features_to_compare = [ + "Aspect", + "Gender", + "Mood", + "Number", + "Tense", + "VerbForm", + "Voice", + ] + analyses, filtered_analyses = filtered_analyses, [] + for analysis in analyses: + _, analysis_morph = oc2ud(str(analysis.tag)) + for feature in features_to_compare: + if ( + feature in morphology + and feature in analysis_morph + and morphology[feature].lower() != analysis_morph[feature].lower() + ): + break + else: + filtered_analyses.append(analysis) + if not len(filtered_analyses): + return [string.lower()] + return list( + dict.fromkeys([analysis.normal_form for analysis in filtered_analyses]) + ) + + def _pymorphy_lookup_lemmatize(self, token: Token) -> List[str]: + string = token.text + analyses = self._morph.parse(string) + # often multiple forms would derive from the same normal form + # thus check _unique_ normal forms + normal_forms = set([an.normal_form for an in analyses]) + if len(normal_forms) == 1: + return [next(iter(normal_forms))] + return [string] + + def pymorphy2_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lemmatize(token) + + def pymorphy2_lookup_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lookup_lemmatize(token) + + def pymorphy3_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lemmatize(token) + + def pymorphy3_lookup_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lookup_lemmatize(token) + + +def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: + gram_map = { + "_POS": { + "ADJF": "ADJ", + "ADJS": "ADJ", + "ADVB": "ADV", + "Apro": "DET", + "COMP": "ADJ", # Can also be an ADV - unchangeable + "CONJ": "CCONJ", # Can also be a SCONJ - both unchangeable ones + "GRND": "VERB", + "INFN": "VERB", + "INTJ": "INTJ", + "NOUN": "NOUN", + "NPRO": "PRON", + "NUMR": "NUM", + "NUMB": "NUM", + "PNCT": "PUNCT", + "PRCL": "PART", + "PREP": "ADP", + "PRTF": "VERB", + "PRTS": "VERB", + "VERB": "VERB", + }, + "Animacy": {"anim": "Anim", "inan": "Inan"}, + "Aspect": {"impf": "Imp", "perf": "Perf"}, + "Case": { + "ablt": "Ins", + "accs": "Acc", + "datv": "Dat", + "gen1": "Gen", + "gen2": "Gen", + "gent": "Gen", + "loc2": "Loc", + "loct": "Loc", + "nomn": "Nom", + "voct": "Voc", + }, + "Degree": {"COMP": "Cmp", "Supr": "Sup"}, + "Gender": {"femn": "Fem", "masc": "Masc", "neut": "Neut"}, + "Mood": {"impr": "Imp", "indc": "Ind"}, + "Number": {"plur": "Plur", "sing": "Sing"}, + "NumForm": {"NUMB": "Digit"}, + "Person": {"1per": "1", "2per": "2", "3per": "3", "excl": "2", "incl": "1"}, + "Tense": {"futr": "Fut", "past": "Past", "pres": "Pres"}, + "Variant": {"ADJS": "Brev", "PRTS": "Brev"}, + "VerbForm": { + "GRND": "Conv", + "INFN": "Inf", + "PRTF": "Part", + "PRTS": "Part", + "VERB": "Fin", + }, + "Voice": {"actv": "Act", "pssv": "Pass"}, + "Abbr": {"Abbr": "Yes"}, + } + pos = "X" + morphology = dict() + unmatched = set() + grams = oc_tag.replace(" ", ",").split(",") + for gram in grams: + match = False + for categ, gmap in sorted(gram_map.items()): + if gram in gmap: + match = True + if categ == "_POS": + pos = gmap[gram] + else: + morphology[categ] = gmap[gram] + if not match: + unmatched.add(gram) + while len(unmatched) > 0: + gram = unmatched.pop() + if gram in ("Name", "Patr", "Surn", "Geox", "Orgn"): + pos = "PROPN" + elif gram == "Auxt": + pos = "AUX" + elif gram == "Pltm": + morphology["Number"] = "Ptan" + return pos, morphology diff --git a/spacy/lang/ru/lex_attrs.py b/spacy/lang/ru/lex_attrs.py new file mode 100644 index 0000000..e0b35bd --- /dev/null +++ b/spacy/lang/ru/lex_attrs.py @@ -0,0 +1,228 @@ +from ...attrs import LIKE_NUM + +_num_words = list( + set( + """ +ноль ноля нолю нолём ноле нулевой нулевого нулевому нулевым нулевом нулевая нулевую нулевое нулевые нулевых нулевыми + +четверть четверти четвертью четвертей четвертям четвертями четвертях + +треть трети третью третей третям третями третях + +половина половины половине половину половиной половин половинам половинами половинах половиною + +один одного одному одним одном +первой первого первому первом первый первым первых +во-первых +единица единицы единице единицу единицей единиц единицам единицами единицах единицею + +два двумя двум двух двоих двое две +второго второму второй втором вторым вторых +двойка двойки двойке двойку двойкой двоек двойкам двойками двойках двойкою +во-вторых +оба обе обеим обеими обеих обоим обоими обоих + +полтора полторы полутора + +три третьего третьему третьем третьим третий тремя трем трех трое троих трёх +тройка тройки тройке тройку тройкою троек тройкам тройками тройках тройкой +троечка троечки троечке троечку троечкой троечек троечкам троечками троечках троечкой +трешка трешки трешке трешку трешкой трешек трешкам трешками трешках трешкою +трёшка трёшки трёшке трёшку трёшкой трёшек трёшкам трёшками трёшках трёшкою +трояк трояка трояку трояком трояке трояки трояков троякам трояками трояках +треха треху трехой +трёха трёху трёхой +втроем втроём + +четыре четвертого четвертому четвертом четвертый четвертым четверка четырьмя четырем четырех четверо четырёх четверым +четверых +вчетвером + +пять пятого пятому пятом пятый пятым пятью пяти пятеро пятерых пятерыми +впятером +пятерочка пятерочки пятерочке пятерочками пятерочкой пятерочку пятерочкой пятерочками +пятёрочка пятёрочки пятёрочке пятёрочками пятёрочкой пятёрочку пятёрочкой пятёрочками +пятерка пятерки пятерке пятерками пятеркой пятерку пятерками +пятёрка пятёрки пятёрке пятёрками пятёркой пятёрку пятёрками +пятёра пятёры пятёре пятёрами пятёрой пятёру пятёрами +пятера пятеры пятере пятерами пятерой пятеру пятерами +пятак пятаки пятаке пятаками пятаком пятаку пятаками + +шесть шестерка шестого шестому шестой шестом шестым шестью шести шестеро шестерых +вшестером + +семь семерка седьмого седьмому седьмой седьмом седьмым семью семи семеро седьмых +всемером + +восемь восьмерка восьмого восьмому восемью восьмой восьмом восьмым восеми восьмером восьми восьмью +восьмерых +ввосьмером + +девять девятого девятому девятка девятом девятый девятым девятью девяти девятером вдевятером девятерых +вдевятером + +десять десятого десятому десятка десятом десятый десятым десятью десяти десятером десятых +вдесятером + +одиннадцать одиннадцатого одиннадцатому одиннадцатом одиннадцатый одиннадцатым одиннадцатью одиннадцати +одиннадцатых + +двенадцать двенадцатого двенадцатому двенадцатом двенадцатый двенадцатым двенадцатью двенадцати +двенадцатых + +тринадцать тринадцатого тринадцатому тринадцатом тринадцатый тринадцатым тринадцатью тринадцати +тринадцатых + +четырнадцать четырнадцатого четырнадцатому четырнадцатом четырнадцатый четырнадцатым четырнадцатью четырнадцати +четырнадцатых + +пятнадцать пятнадцатого пятнадцатому пятнадцатом пятнадцатый пятнадцатым пятнадцатью пятнадцати +пятнадцатых +пятнарик пятнарику пятнариком пятнарики + +шестнадцать шестнадцатого шестнадцатому шестнадцатом шестнадцатый шестнадцатым шестнадцатью шестнадцати +шестнадцатых + +семнадцать семнадцатого семнадцатому семнадцатом семнадцатый семнадцатым семнадцатью семнадцати семнадцатых + +восемнадцать восемнадцатого восемнадцатому восемнадцатом восемнадцатый восемнадцатым восемнадцатью восемнадцати +восемнадцатых + +девятнадцать девятнадцатого девятнадцатому девятнадцатом девятнадцатый девятнадцатым девятнадцатью девятнадцати +девятнадцатых + +двадцать двадцатого двадцатому двадцатом двадцатый двадцатым двадцатью двадцати двадцатых + +четвертак четвертака четвертаке четвертаку четвертаки четвертаком четвертаками + +тридцать тридцатого тридцатому тридцатом тридцатый тридцатым тридцатью тридцати тридцатых +тридцадка тридцадку тридцадке тридцадки тридцадкой тридцадкою тридцадками + +тридевять тридевяти тридевятью + +сорок сорокового сороковому сороковом сороковым сороковой сороковых +сорокет сорокета сорокету сорокете сорокеты сорокетом сорокетами сорокетам + +пятьдесят пятьдесятого пятьдесятому пятьюдесятью пятьдесятом пятьдесятый пятьдесятым пятидесяти пятьдесятых +полтинник полтинника полтиннике полтиннику полтинники полтинником полтинниками полтинникам полтинниках +пятидесятка пятидесятке пятидесятку пятидесятки пятидесяткой пятидесятками пятидесяткам пятидесятках +полтос полтоса полтосе полтосу полтосы полтосом полтосами полтосам полтосах + +шестьдесят шестьдесятого шестьдесятому шестьюдесятью шестьдесятом шестьдесятый шестьдесятым шестидесятые шестидесяти +шестьдесятых + +семьдесят семьдесятого семьдесятому семьюдесятью семьдесятом семьдесятый семьдесятым семидесяти семьдесятых + +восемьдесят восемьдесятого восемьдесятому восемьюдесятью восемьдесятом восемьдесятый восемьдесятым восемидесяти +восьмидесяти восьмидесятых + +девяносто девяностого девяностому девяностом девяностый девяностым девяноста девяностых + +сто сотого сотому сотом сотен сотый сотым ста +стольник стольника стольнику стольнике стольники стольником стольниками +сотка сотки сотке соткой сотками соткам сотках +сотня сотни сотне сотней сотнями сотням сотнях + +двести двумястами двухсотого двухсотому двухсотом двухсотый двухсотым двумстам двухстах двухсот + +триста тремястами трехсотого трехсотому трехсотом трехсотый трехсотым тремстам трехстах трехсот + +четыреста четырехсотого четырехсотому четырьмястами четырехсотом четырехсотый четырехсотым четыремстам четырехстах +четырехсот + +пятьсот пятисотого пятисотому пятьюстами пятисотом пятисотый пятисотым пятистам пятистах пятисот +пятисотка пятисотки пятисотке пятисоткой пятисотками пятисоткам пятисоткою пятисотках +пятихатка пятихатки пятихатке пятихаткой пятихатками пятихаткам пятихаткою пятихатках +пятифан пятифаны пятифане пятифаном пятифанами пятифанах + +шестьсот шестисотого шестисотому шестьюстами шестисотом шестисотый шестисотым шестистам шестистах шестисот + +семьсот семисотого семисотому семьюстами семисотом семисотый семисотым семистам семистах семисот + +восемьсот восемисотого восемисотому восемисотом восемисотый восемисотым восьмистами восьмистам восьмистах восьмисот + +девятьсот девятисотого девятисотому девятьюстами девятисотом девятисотый девятисотым девятистам девятистах девятисот + +тысяча тысячного тысячному тысячном тысячный тысячным тысячам тысячах тысячей тысяч тысячи тыс +косарь косаря косару косарем косарями косарях косарям косарей + +десятитысячный десятитысячного десятитысячному десятитысячным десятитысячном десятитысячная десятитысячной +десятитысячную десятитысячною десятитысячное десятитысячные десятитысячных десятитысячными + +двадцатитысячный двадцатитысячного двадцатитысячному двадцатитысячным двадцатитысячном двадцатитысячная +двадцатитысячной двадцатитысячную двадцатитысячною двадцатитысячное двадцатитысячные двадцатитысячных +двадцатитысячными + +тридцатитысячный тридцатитысячного тридцатитысячному тридцатитысячным тридцатитысячном тридцатитысячная +тридцатитысячной тридцатитысячную тридцатитысячною тридцатитысячное тридцатитысячные тридцатитысячных +тридцатитысячными + +сорокатысячный сорокатысячного сорокатысячному сорокатысячным сорокатысячном сорокатысячная +сорокатысячной сорокатысячную сорокатысячною сорокатысячное сорокатысячные сорокатысячных +сорокатысячными + +пятидесятитысячный пятидесятитысячного пятидесятитысячному пятидесятитысячным пятидесятитысячном пятидесятитысячная +пятидесятитысячной пятидесятитысячную пятидесятитысячною пятидесятитысячное пятидесятитысячные пятидесятитысячных +пятидесятитысячными + +шестидесятитысячный шестидесятитысячного шестидесятитысячному шестидесятитысячным шестидесятитысячном шестидесятитысячная +шестидесятитысячной шестидесятитысячную шестидесятитысячною шестидесятитысячное шестидесятитысячные шестидесятитысячных +шестидесятитысячными + +семидесятитысячный семидесятитысячного семидесятитысячному семидесятитысячным семидесятитысячном семидесятитысячная +семидесятитысячной семидесятитысячную семидесятитысячною семидесятитысячное семидесятитысячные семидесятитысячных +семидесятитысячными + +восьмидесятитысячный восьмидесятитысячного восьмидесятитысячному восьмидесятитысячным восьмидесятитысячном восьмидесятитысячная +восьмидесятитысячной восьмидесятитысячную восьмидесятитысячною восьмидесятитысячное восьмидесятитысячные восьмидесятитысячных +восьмидесятитысячными + +стотысячный стотысячного стотысячному стотысячным стотысячном стотысячная стотысячной стотысячную стотысячное +стотысячные стотысячных стотысячными стотысячною + +миллион миллионного миллионов миллионному миллионном миллионный миллионным миллионом миллиона миллионе миллиону +миллионов +лям ляма лямы лямом лямами лямах лямов +млн + +десятимиллионная десятимиллионной десятимиллионными десятимиллионный десятимиллионным десятимиллионному +десятимиллионными десятимиллионную десятимиллионное десятимиллионные десятимиллионных десятимиллионною + +миллиард миллиардного миллиардному миллиардном миллиардный миллиардным миллиардом миллиарда миллиарде миллиарду +миллиардов +лярд лярда лярды лярдом лярдами лярдах лярдов +млрд + +триллион триллионного триллионному триллионном триллионный триллионным триллионом триллиона триллионе триллиону +триллионов трлн + +квадриллион квадриллионного квадриллионному квадриллионный квадриллионным квадриллионом квадриллиона квадриллионе +квадриллиону квадриллионов квадрлн + +квинтиллион квинтиллионного квинтиллионному квинтиллионный квинтиллионным квинтиллионом квинтиллиона квинтиллионе +квинтиллиону квинтиллионов квинтлн + +i ii iii iv v vi vii viii ix x xi xii xiii xiv xv xvi xvii xviii xix xx xxi xxii xxiii xxiv xxv xxvi xxvii xxvii xxix +""".split() + ) +) + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + if text.endswith("%"): + text = text[:-1] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ru/stop_words.py b/spacy/lang/ru/stop_words.py new file mode 100644 index 0000000..d6ea6b4 --- /dev/null +++ b/spacy/lang/ru/stop_words.py @@ -0,0 +1,111 @@ +STOP_WORDS = set( + """ +а авось ага агу аж ай али алло ау ах ая + +б будем будет будете будешь буду будут будучи будь будьте бы был была были было +быть бац без безусловно бишь благо благодаря ближайшие близко более больше +будто бывает бывала бывали бываю бывают бытует + +в вам вами вас весь во вот все всё всего всей всем всём всеми всему всех всею +всея всю вся вы ваш ваша ваше ваши вдали вдобавок вдруг ведь везде вернее +взаимно взаправду видно вишь включая вместо внакладе вначале вне вниз внизу +вновь вовсе возможно воистину вокруг вон вообще вопреки вперекор вплоть +вполне вправду вправе впрочем впрямь вресноту вроде вряд всегда всюду +всякий всякого всякой всячески вчеред + +г го где гораздо гав + +д да для до дабы давайте давно давным даже далее далеко дальше данная +данного данное данной данном данному данные данный данных дану данунах +даром де действительно довольно доколе доколь долго должен должна +должно должны должный дополнительно другая другие другим другими +других другое другой + +е его едим едят ее её ей ел ела ем ему емъ если ест есть ешь еще ещё ею едва +ежели еле + +ж же + +з за затем зато зачем здесь значит зря + +и из или им ими имъ их ибо иль имеет имел имела имело именно иметь иначе +иногда иным иными итак ишь + +й + +к как кем ко когда кого ком кому комья которая которого которое которой котором +которому которою которую которые который которым которыми которых кто ка кабы +каждая каждое каждые каждый кажется казалась казались казалось казался казаться +какая какие каким какими каков какого какой какому какою касательно кой коли +коль конечно короче кроме кстати ку куда + +л ли либо лишь любая любого любое любой любом любую любыми любых + +м меня мне мной мною мог моги могите могла могли могло могу могут мое моё моего +моей моем моём моему моею можем может можете можешь мои мой моим моими моих +мочь мою моя мы мало меж между менее меньше мимо многие много многого многое +многом многому можно мол му + +н на нам нами нас наса наш наша наше нашего нашей нашем нашему нашею наши нашим +нашими наших нашу не него нее неё ней нем нём нему нет нею ним ними них но +наверняка наверху навряд навыворот над надо назад наиболее наизворот +наизнанку наипаче накануне наконец наоборот наперед наперекор наподобие +например напротив напрямую насилу настоящая настоящее настоящие настоящий +насчет нате находиться начала начале неважно негде недавно недалеко незачем +некем некогда некому некоторая некоторые некоторый некоторых некто некуда +нельзя немногие немногим немного необходимо необходимости необходимые +необходимым неоткуда непрерывно нередко несколько нету неужели нечего +нечем нечему нечто нешто нибудь нигде ниже низко никак никакой никем +никогда никого никому никто никуда ниоткуда нипочем ничего ничем ничему +ничто ну нужная нужно нужного нужные нужный нужных ныне нынешнее нынешней +нынешних нынче + +о об один одна одни одним одними одних одно одного одной одном одному одною +одну он она оне они оно от оба общую обычно ого однажды однако ой около оный +оп опять особенно особо особую особые откуда отнелижа отнелиже отовсюду +отсюда оттого оттот оттуда отчего отчему ох очевидно очень ом + +п по при паче перед под подавно поди подобная подобно подобного подобные +подобный подобным подобных поелику пожалуй пожалуйста позже поистине +пока покамест поколе поколь покуда покудова помимо понеже поприще пор +пора посему поскольку после посреди посредством потом потому потомушта +похожем почему почти поэтому прежде притом причем про просто прочего +прочее прочему прочими проще прям пусть + +р ради разве ранее рано раньше рядом + +с сам сама сами самим самими самих само самого самом самому саму свое своё +своего своей своем своём своему своею свои свой своим своими своих свою своя +себе себя собой собою самая самое самой самый самых сверх свыше се сего сей +сейчас сие сих сквозь сколько скорее скоро следует слишком смогут сможет +сначала снова со собственно совсем сперва спокону спустя сразу среди сродни +стал стала стали стало стать суть сызнова + +та то ту ты ти так такая такие таким такими таких такого такое такой таком такому такою +такую те тебе тебя тем теми тех тобой тобою того той только том томах тому +тот тою также таки таков такова там твои твоим твоих твой твоя твоё +теперь тогда тоже тотчас точно туда тут тьфу тая + +у уже увы уж ура ух ую + +ф фу + +х ха хе хорошо хотел хотела хотелось хотеть хоть хотя хочешь хочу хуже + +ч чего чем чём чему что чтобы часто чаще чей через чтоб чуть чхать чьим +чьих чьё чё + +ш ша + +щ ща щас + +ы ых ые ый + +э эта эти этим этими этих это этого этой этом этому этот этою эту эдак эдакий +эй эка экий этак этакий эх + +ю + +я явно явных яко якобы якоже +""".split() +) diff --git a/spacy/lang/ru/tokenizer_exceptions.py b/spacy/lang/ru/tokenizer_exceptions.py new file mode 100644 index 0000000..0a8c476 --- /dev/null +++ b/spacy/lang/ru/tokenizer_exceptions.py @@ -0,0 +1,407 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +_abbrev_exc = [ + # Weekdays abbreviations + {ORTH: "пн", NORM: "понедельник"}, + {ORTH: "вт", NORM: "вторник"}, + {ORTH: "ср", NORM: "среда"}, + {ORTH: "чт", NORM: "четверг"}, + {ORTH: "чтв", NORM: "четверг"}, + {ORTH: "пт", NORM: "пятница"}, + {ORTH: "сб", NORM: "суббота"}, + {ORTH: "сбт", NORM: "суббота"}, + {ORTH: "вс", NORM: "воскресенье"}, + {ORTH: "вскр", NORM: "воскресенье"}, + {ORTH: "воскр", NORM: "воскресенье"}, + # Months abbreviations + {ORTH: "янв", NORM: "январь"}, + {ORTH: "фев", NORM: "февраль"}, + {ORTH: "февр", NORM: "февраль"}, + {ORTH: "мар", NORM: "март"}, + # {ORTH: "март", NORM: "март"}, + {ORTH: "мрт", NORM: "март"}, + {ORTH: "апр", NORM: "апрель"}, + # {ORTH: "май", NORM: "май"}, + {ORTH: "июн", NORM: "июнь"}, + # {ORTH: "июнь", NORM: "июнь"}, + {ORTH: "июл", NORM: "июль"}, + # {ORTH: "июль", NORM: "июль"}, + {ORTH: "авг", NORM: "август"}, + {ORTH: "сен", NORM: "сентябрь"}, + {ORTH: "сент", NORM: "сентябрь"}, + {ORTH: "окт", NORM: "октябрь"}, + {ORTH: "октб", NORM: "октябрь"}, + {ORTH: "ноя", NORM: "ноябрь"}, + {ORTH: "нояб", NORM: "ноябрь"}, + {ORTH: "нбр", NORM: "ноябрь"}, + {ORTH: "дек", NORM: "декабрь"}, +] + +for abbrev_desc in _abbrev_exc: + abbrev = abbrev_desc[ORTH] + for orth in (abbrev, abbrev.capitalize(), abbrev.upper()): + _exc[orth] = [{ORTH: orth, NORM: abbrev_desc[NORM]}] + _exc[orth + "."] = [{ORTH: orth + ".", NORM: abbrev_desc[NORM]}] + + +for abbr in [ + # Year slang abbreviations + {ORTH: "2к15", NORM: "2015"}, + {ORTH: "2к16", NORM: "2016"}, + {ORTH: "2к17", NORM: "2017"}, + {ORTH: "2к18", NORM: "2018"}, + {ORTH: "2к19", NORM: "2019"}, + {ORTH: "2к20", NORM: "2020"}, + {ORTH: "2к21", NORM: "2021"}, + {ORTH: "2к22", NORM: "2022"}, + {ORTH: "2к23", NORM: "2023"}, + {ORTH: "2к24", NORM: "2024"}, + {ORTH: "2к25", NORM: "2025"}, + {ORTH: "2к26", NORM: "2026"}, + {ORTH: "2к27", NORM: "2027"}, + {ORTH: "2к28", NORM: "2028"}, + {ORTH: "2к29", NORM: "2029"}, + {ORTH: "2к30", NORM: "2030"}, +]: + _exc[abbr[ORTH]] = [abbr] + +for abbr in [ + # Profession and academic titles abbreviations + {ORTH: "ак.", NORM: "академик"}, + {ORTH: "акад.", NORM: "академик"}, + {ORTH: "д-р архитектуры", NORM: "доктор архитектуры"}, + {ORTH: "д-р биол. наук", NORM: "доктор биологических наук"}, + {ORTH: "д-р ветеринар. наук", NORM: "доктор ветеринарных наук"}, + {ORTH: "д-р воен. наук", NORM: "доктор военных наук"}, + {ORTH: "д-р геогр. наук", NORM: "доктор географических наук"}, + {ORTH: "д-р геол.-минерал. наук", NORM: "доктор геолого-минералогических наук"}, + {ORTH: "д-р искусствоведения", NORM: "доктор искусствоведения"}, + {ORTH: "д-р ист. наук", NORM: "доктор исторических наук"}, + {ORTH: "д-р культурологии", NORM: "доктор культурологии"}, + {ORTH: "д-р мед. наук", NORM: "доктор медицинских наук"}, + {ORTH: "д-р пед. наук", NORM: "доктор педагогических наук"}, + {ORTH: "д-р полит. наук", NORM: "доктор политических наук"}, + {ORTH: "д-р психол. наук", NORM: "доктор психологических наук"}, + {ORTH: "д-р с.-х. наук", NORM: "доктор сельскохозяйственных наук"}, + {ORTH: "д-р социол. наук", NORM: "доктор социологических наук"}, + {ORTH: "д-р техн. наук", NORM: "доктор технических наук"}, + {ORTH: "д-р фармацевт. наук", NORM: "доктор фармацевтических наук"}, + {ORTH: "д-р физ.-мат. наук", NORM: "доктор физико-математических наук"}, + {ORTH: "д-р филол. наук", NORM: "доктор филологических наук"}, + {ORTH: "д-р филос. наук", NORM: "доктор философских наук"}, + {ORTH: "д-р хим. наук", NORM: "доктор химических наук"}, + {ORTH: "д-р экон. наук", NORM: "доктор экономических наук"}, + {ORTH: "д-р юрид. наук", NORM: "доктор юридических наук"}, + {ORTH: "д-р", NORM: "доктор"}, + {ORTH: "д.б.н.", NORM: "доктор биологических наук"}, + {ORTH: "д.г.-м.н.", NORM: "доктор геолого-минералогических наук"}, + {ORTH: "д.г.н.", NORM: "доктор географических наук"}, + {ORTH: "д.и.н.", NORM: "доктор исторических наук"}, + {ORTH: "д.иск.", NORM: "доктор искусствоведения"}, + {ORTH: "д.м.н.", NORM: "доктор медицинских наук"}, + {ORTH: "д.п.н.", NORM: "доктор психологических наук"}, + {ORTH: "д.пед.н.", NORM: "доктор педагогических наук"}, + {ORTH: "д.полит.н.", NORM: "доктор политических наук"}, + {ORTH: "д.с.-х.н.", NORM: "доктор сельскохозяйственных наук"}, + {ORTH: "д.социол.н.", NORM: "доктор социологических наук"}, + {ORTH: "д.т.н.", NORM: "доктор технических наук"}, + {ORTH: "д.т.н", NORM: "доктор технических наук"}, + {ORTH: "д.ф.-м.н.", NORM: "доктор физико-математических наук"}, + {ORTH: "д.ф.н.", NORM: "доктор филологических наук"}, + {ORTH: "д.филос.н.", NORM: "доктор философских наук"}, + {ORTH: "д.фил.н.", NORM: "доктор филологических наук"}, + {ORTH: "д.х.н.", NORM: "доктор химических наук"}, + {ORTH: "д.э.н.", NORM: "доктор экономических наук"}, + {ORTH: "д.э.н", NORM: "доктор экономических наук"}, + {ORTH: "д.ю.н.", NORM: "доктор юридических наук"}, + {ORTH: "доц.", NORM: "доцент"}, + {ORTH: "и.о.", NORM: "исполняющий обязанности"}, + {ORTH: "к.б.н.", NORM: "кандидат биологических наук"}, + {ORTH: "к.воен.н.", NORM: "кандидат военных наук"}, + {ORTH: "к.г.-м.н.", NORM: "кандидат геолого-минералогических наук"}, + {ORTH: "к.г.н.", NORM: "кандидат географических наук"}, + {ORTH: "к.геогр.н", NORM: "кандидат географических наук"}, + {ORTH: "к.геогр.наук", NORM: "кандидат географических наук"}, + {ORTH: "к.и.н.", NORM: "кандидат исторических наук"}, + {ORTH: "к.иск.", NORM: "кандидат искусствоведения"}, + {ORTH: "к.м.н.", NORM: "кандидат медицинских наук"}, + {ORTH: "к.п.н.", NORM: "кандидат психологических наук"}, + {ORTH: "к.псх.н.", NORM: "кандидат психологических наук"}, + {ORTH: "к.пед.н.", NORM: "кандидат педагогических наук"}, + {ORTH: "канд.пед.наук", NORM: "кандидат педагогических наук"}, + {ORTH: "к.полит.н.", NORM: "кандидат политических наук"}, + {ORTH: "к.с.-х.н.", NORM: "кандидат сельскохозяйственных наук"}, + {ORTH: "к.социол.н.", NORM: "кандидат социологических наук"}, + {ORTH: "к.с.н.", NORM: "кандидат социологических наук"}, + {ORTH: "к.т.н.", NORM: "кандидат технических наук"}, + {ORTH: "к.ф.-м.н.", NORM: "кандидат физико-математических наук"}, + {ORTH: "к.ф.н.", NORM: "кандидат филологических наук"}, + {ORTH: "к.фил.н.", NORM: "кандидат филологических наук"}, + {ORTH: "к.филол.н", NORM: "кандидат филологических наук"}, + {ORTH: "к.фарм.наук", NORM: "кандидат фармакологических наук"}, + {ORTH: "к.фарм.н.", NORM: "кандидат фармакологических наук"}, + {ORTH: "к.фарм.н", NORM: "кандидат фармакологических наук"}, + {ORTH: "к.филос.наук", NORM: "кандидат философских наук"}, + {ORTH: "к.филос.н.", NORM: "кандидат философских наук"}, + {ORTH: "к.филос.н", NORM: "кандидат философских наук"}, + {ORTH: "к.х.н.", NORM: "кандидат химических наук"}, + {ORTH: "к.х.н", NORM: "кандидат химических наук"}, + {ORTH: "к.э.н.", NORM: "кандидат экономических наук"}, + {ORTH: "к.э.н", NORM: "кандидат экономических наук"}, + {ORTH: "к.ю.н.", NORM: "кандидат юридических наук"}, + {ORTH: "к.ю.н", NORM: "кандидат юридических наук"}, + {ORTH: "канд. архитектуры", NORM: "кандидат архитектуры"}, + {ORTH: "канд. биол. наук", NORM: "кандидат биологических наук"}, + {ORTH: "канд. ветеринар. наук", NORM: "кандидат ветеринарных наук"}, + {ORTH: "канд. воен. наук", NORM: "кандидат военных наук"}, + {ORTH: "канд. геогр. наук", NORM: "кандидат географических наук"}, + {ORTH: "канд. геол.-минерал. наук", NORM: "кандидат геолого-минералогических наук"}, + {ORTH: "канд. искусствоведения", NORM: "кандидат искусствоведения"}, + {ORTH: "канд. ист. наук", NORM: "кандидат исторических наук"}, + {ORTH: "к.ист.н.", NORM: "кандидат исторических наук"}, + {ORTH: "канд. культурологии", NORM: "кандидат культурологии"}, + {ORTH: "канд. мед. наук", NORM: "кандидат медицинских наук"}, + {ORTH: "канд. пед. наук", NORM: "кандидат педагогических наук"}, + {ORTH: "канд. полит. наук", NORM: "кандидат политических наук"}, + {ORTH: "канд. психол. наук", NORM: "кандидат психологических наук"}, + {ORTH: "канд. с.-х. наук", NORM: "кандидат сельскохозяйственных наук"}, + {ORTH: "канд. социол. наук", NORM: "кандидат социологических наук"}, + {ORTH: "к.соц.наук", NORM: "кандидат социологических наук"}, + {ORTH: "к.соц.н.", NORM: "кандидат социологических наук"}, + {ORTH: "к.соц.н", NORM: "кандидат социологических наук"}, + {ORTH: "канд. техн. наук", NORM: "кандидат технических наук"}, + {ORTH: "канд. фармацевт. наук", NORM: "кандидат фармацевтических наук"}, + {ORTH: "канд. физ.-мат. наук", NORM: "кандидат физико-математических наук"}, + {ORTH: "канд. филол. наук", NORM: "кандидат филологических наук"}, + {ORTH: "канд. филос. наук", NORM: "кандидат философских наук"}, + {ORTH: "канд. хим. наук", NORM: "кандидат химических наук"}, + {ORTH: "канд. экон. наук", NORM: "кандидат экономических наук"}, + {ORTH: "канд. юрид. наук", NORM: "кандидат юридических наук"}, + {ORTH: "в.н.с.", NORM: "ведущий научный сотрудник"}, + {ORTH: "мл. науч. сотр.", NORM: "младший научный сотрудник"}, + {ORTH: "м.н.с.", NORM: "младший научный сотрудник"}, + {ORTH: "проф.", NORM: "профессор"}, + {ORTH: "профессор.кафедры", NORM: "профессор кафедры"}, + {ORTH: "ст. науч. сотр.", NORM: "старший научный сотрудник"}, + {ORTH: "чл.-к.", NORM: "член корреспондент"}, + {ORTH: "чл.-корр.", NORM: "член-корреспондент"}, + {ORTH: "чл.-кор.", NORM: "член-корреспондент"}, + {ORTH: "дир.", NORM: "директор"}, + {ORTH: "зам. дир.", NORM: "заместитель директора"}, + {ORTH: "зав. каф.", NORM: "заведующий кафедрой"}, + {ORTH: "зав.кафедрой", NORM: "заведующий кафедрой"}, + {ORTH: "зав. кафедрой", NORM: "заведующий кафедрой"}, + {ORTH: "асп.", NORM: "аспирант"}, + {ORTH: "гл. науч. сотр.", NORM: "главный научный сотрудник"}, + {ORTH: "вед. науч. сотр.", NORM: "ведущий научный сотрудник"}, + {ORTH: "науч. сотр.", NORM: "научный сотрудник"}, + {ORTH: "к.м.с.", NORM: "кандидат в мастера спорта"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +for abbr in [ + # Literary phrases abbreviations + {ORTH: "и т.д.", NORM: "и так далее"}, + {ORTH: "и т.п.", NORM: "и тому подобное"}, + {ORTH: "т.д.", NORM: "так далее"}, + {ORTH: "т.п.", NORM: "тому подобное"}, + {ORTH: "т.е.", NORM: "то есть"}, + {ORTH: "т.к.", NORM: "так как"}, + {ORTH: "в т.ч.", NORM: "в том числе"}, + {ORTH: "и пр.", NORM: "и прочие"}, + {ORTH: "и др.", NORM: "и другие"}, + {ORTH: "т.н.", NORM: "так называемый"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +for abbr in [ + # Appeal to a person abbreviations + {ORTH: "г-н", NORM: "господин"}, + {ORTH: "г-да", NORM: "господа"}, + {ORTH: "г-жа", NORM: "госпожа"}, + {ORTH: "тов.", NORM: "товарищ"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +for abbr in [ + # Time periods abbreviations + {ORTH: "до н.э.", NORM: "до нашей эры"}, + {ORTH: "по н.в.", NORM: "по настоящее время"}, + {ORTH: "в н.в.", NORM: "в настоящее время"}, + {ORTH: "наст.", NORM: "настоящий"}, + {ORTH: "наст. время", NORM: "настоящее время"}, + {ORTH: "г.г.", NORM: "годы"}, + {ORTH: "гг.", NORM: "годы"}, + {ORTH: "т.г.", NORM: "текущий год"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +for abbr in [ + # Address forming elements abbreviations + {ORTH: "респ.", NORM: "республика"}, + {ORTH: "обл.", NORM: "область"}, + {ORTH: "г.ф.з.", NORM: "город федерального значения"}, + {ORTH: "а.обл.", NORM: "автономная область"}, + {ORTH: "а.окр.", NORM: "автономный округ"}, + {ORTH: "м.р-н", NORM: "муниципальный район"}, + {ORTH: "г.о.", NORM: "городской округ"}, + {ORTH: "г.п.", NORM: "городское поселение"}, + {ORTH: "с.п.", NORM: "сельское поселение"}, + {ORTH: "вн.р-н", NORM: "внутригородской район"}, + {ORTH: "вн.тер.г.", NORM: "внутригородская территория города"}, + {ORTH: "пос.", NORM: "поселение"}, + {ORTH: "р-н", NORM: "район"}, + {ORTH: "с/с", NORM: "сельсовет"}, + {ORTH: "г.", NORM: "город"}, + {ORTH: "п.г.т.", NORM: "поселок городского типа"}, + {ORTH: "пгт.", NORM: "поселок городского типа"}, + {ORTH: "р.п.", NORM: "рабочий поселок"}, + {ORTH: "рп.", NORM: "рабочий поселок"}, + {ORTH: "кп.", NORM: "курортный поселок"}, + {ORTH: "гп.", NORM: "городской поселок"}, + {ORTH: "п.", NORM: "поселок"}, + {ORTH: "в-ки", NORM: "выселки"}, + {ORTH: "г-к", NORM: "городок"}, + {ORTH: "з-ка", NORM: "заимка"}, + {ORTH: "п-к", NORM: "починок"}, + {ORTH: "киш.", NORM: "кишлак"}, + {ORTH: "п. ст.", NORM: "поселок станция"}, + {ORTH: "п. ж/д ст.", NORM: "поселок при железнодорожной станции"}, + {ORTH: "ж/д бл-ст", NORM: "железнодорожный блокпост"}, + {ORTH: "ж/д б-ка", NORM: "железнодорожная будка"}, + {ORTH: "ж/д в-ка", NORM: "железнодорожная ветка"}, + {ORTH: "ж/д к-ма", NORM: "железнодорожная казарма"}, + {ORTH: "ж/д к-т", NORM: "железнодорожный комбинат"}, + {ORTH: "ж/д пл-ма", NORM: "железнодорожная платформа"}, + {ORTH: "ж/д пл-ка", NORM: "железнодорожная площадка"}, + {ORTH: "ж/д п.п.", NORM: "железнодорожный путевой пост"}, + {ORTH: "ж/д о.п.", NORM: "железнодорожный остановочный пункт"}, + {ORTH: "ж/д рзд.", NORM: "железнодорожный разъезд"}, + {ORTH: "ж/д ст.", NORM: "железнодорожная станция"}, + {ORTH: "м-ко", NORM: "местечко"}, + {ORTH: "д.", NORM: "деревня"}, + {ORTH: "с.", NORM: "село"}, + {ORTH: "сл.", NORM: "слобода"}, + {ORTH: "ст.", NORM: "станция"}, + {ORTH: "ст-ца", NORM: "станица"}, + {ORTH: "у.", NORM: "улус"}, + {ORTH: "х.", NORM: "хутор"}, + {ORTH: "рзд.", NORM: "разъезд"}, + {ORTH: "зим.", NORM: "зимовье"}, + {ORTH: "б-г", NORM: "берег"}, + {ORTH: "ж/р", NORM: "жилой район"}, + {ORTH: "кв-л", NORM: "квартал"}, + {ORTH: "мкр.", NORM: "микрорайон"}, + {ORTH: "ост-в", NORM: "остров"}, + {ORTH: "платф.", NORM: "платформа"}, + {ORTH: "п/р", NORM: "промышленный район"}, + {ORTH: "р-н", NORM: "район"}, + {ORTH: "тер.", NORM: "территория"}, + { + ORTH: "тер. СНО", + NORM: "территория садоводческих некоммерческих объединений граждан", + }, + { + ORTH: "тер. ОНО", + NORM: "территория огороднических некоммерческих объединений граждан", + }, + {ORTH: "тер. ДНО", NORM: "территория дачных некоммерческих объединений граждан"}, + {ORTH: "тер. СНТ", NORM: "территория садоводческих некоммерческих товариществ"}, + {ORTH: "тер. ОНТ", NORM: "территория огороднических некоммерческих товариществ"}, + {ORTH: "тер. ДНТ", NORM: "территория дачных некоммерческих товариществ"}, + {ORTH: "тер. СПК", NORM: "территория садоводческих потребительских кооперативов"}, + {ORTH: "тер. ОПК", NORM: "территория огороднических потребительских кооперативов"}, + {ORTH: "тер. ДПК", NORM: "территория дачных потребительских кооперативов"}, + {ORTH: "тер. СНП", NORM: "территория садоводческих некоммерческих партнерств"}, + {ORTH: "тер. ОНП", NORM: "территория огороднических некоммерческих партнерств"}, + {ORTH: "тер. ДНП", NORM: "территория дачных некоммерческих партнерств"}, + {ORTH: "тер. ТСН", NORM: "территория товарищества собственников недвижимости"}, + {ORTH: "тер. ГСК", NORM: "территория гаражно-строительного кооператива"}, + {ORTH: "ус.", NORM: "усадьба"}, + {ORTH: "тер.ф.х.", NORM: "территория фермерского хозяйства"}, + {ORTH: "ю.", NORM: "юрты"}, + {ORTH: "ал.", NORM: "аллея"}, + {ORTH: "б-р", NORM: "бульвар"}, + {ORTH: "взв.", NORM: "взвоз"}, + {ORTH: "взд.", NORM: "въезд"}, + {ORTH: "дор.", NORM: "дорога"}, + {ORTH: "ззд.", NORM: "заезд"}, + {ORTH: "км", NORM: "километр"}, + {ORTH: "к-цо", NORM: "кольцо"}, + {ORTH: "лн.", NORM: "линия"}, + {ORTH: "мгстр.", NORM: "магистраль"}, + {ORTH: "наб.", NORM: "набережная"}, + {ORTH: "пер-д", NORM: "переезд"}, + {ORTH: "пер.", NORM: "переулок"}, + {ORTH: "пл-ка", NORM: "площадка"}, + {ORTH: "пл.", NORM: "площадь"}, + {ORTH: "пр-д", NORM: "проезд"}, + {ORTH: "пр-к", NORM: "просек"}, + {ORTH: "пр-ка", NORM: "просека"}, + {ORTH: "пр-лок", NORM: "проселок"}, + {ORTH: "пр-кт", NORM: "проспект"}, + {ORTH: "проул.", NORM: "проулок"}, + {ORTH: "рзд.", NORM: "разъезд"}, + {ORTH: "ряд", NORM: "ряд(ы)"}, + {ORTH: "с-р", NORM: "сквер"}, + {ORTH: "с-к", NORM: "спуск"}, + {ORTH: "сзд.", NORM: "съезд"}, + {ORTH: "туп.", NORM: "тупик"}, + {ORTH: "ул.", NORM: "улица"}, + {ORTH: "ш.", NORM: "шоссе"}, + {ORTH: "влд.", NORM: "владение"}, + {ORTH: "г-ж", NORM: "гараж"}, + {ORTH: "д.", NORM: "дом"}, + {ORTH: "двлд.", NORM: "домовладение"}, + {ORTH: "зд.", NORM: "здание"}, + {ORTH: "з/у", NORM: "земельный участок"}, + {ORTH: "кв.", NORM: "квартира"}, + {ORTH: "ком.", NORM: "комната"}, + {ORTH: "подв.", NORM: "подвал"}, + {ORTH: "кот.", NORM: "котельная"}, + {ORTH: "п-б", NORM: "погреб"}, + {ORTH: "к.", NORM: "корпус"}, + {ORTH: "ОНС", NORM: "объект незавершенного строительства"}, + {ORTH: "оф.", NORM: "офис"}, + {ORTH: "пав.", NORM: "павильон"}, + {ORTH: "помещ.", NORM: "помещение"}, + {ORTH: "раб.уч.", NORM: "рабочий участок"}, + {ORTH: "скл.", NORM: "склад"}, + {ORTH: "coop.", NORM: "сооружение"}, + {ORTH: "стр.", NORM: "строение"}, + {ORTH: "торг.зал", NORM: "торговый зал"}, + {ORTH: "а/п", NORM: "аэропорт"}, + {ORTH: "им.", NORM: "имени"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +for abbr in [ + # Others abbreviations + {ORTH: "тыс.руб.", NORM: "тысяч рублей"}, + {ORTH: "тыс.", NORM: "тысяч"}, + {ORTH: "руб.", NORM: "рубль"}, + {ORTH: "долл.", NORM: "доллар"}, + {ORTH: "прим.", NORM: "примечание"}, + {ORTH: "прим.ред.", NORM: "примечание редакции"}, + {ORTH: "см. также", NORM: "смотри также"}, + {ORTH: "см.", NORM: "смотри"}, + {ORTH: "кв.м.", NORM: "квадратный метр"}, + {ORTH: "м2", NORM: "квадратный метр"}, + {ORTH: "б/у", NORM: "бывший в употреблении"}, + {ORTH: "сокр.", NORM: "сокращение"}, + {ORTH: "чел.", NORM: "человек"}, + {ORTH: "б.п.", NORM: "базисный пункт"}, +]: + _exc[abbr[ORTH]] = [abbr] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/sa/__init__.py b/spacy/lang/sa/__init__.py new file mode 100644 index 0000000..c7c0e98 --- /dev/null +++ b/spacy/lang/sa/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class SanskritDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Sanskrit(Language): + lang = "sa" + Defaults = SanskritDefaults + + +__all__ = ["Sanskrit"] diff --git a/spacy/lang/sa/examples.py b/spacy/lang/sa/examples.py new file mode 100644 index 0000000..6a0bc4e --- /dev/null +++ b/spacy/lang/sa/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sa.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "अभ्यावहति कल्याणं विविधं वाक् सुभाषिता ।", + "मनसि व्याकुले चक्षुः पश्यन्नपि न पश्यति ।", + "यस्य बुद्धिर्बलं तस्य निर्बुद्धेस्तु कुतो बलम्?", + "परो अपि हितवान् बन्धुः बन्धुः अपि अहितः परः ।", + "अहितः देहजः व्याधिः हितम् आरण्यं औषधम् ॥", +] diff --git a/spacy/lang/sa/lex_attrs.py b/spacy/lang/sa/lex_attrs.py new file mode 100644 index 0000000..bdceb7e --- /dev/null +++ b/spacy/lang/sa/lex_attrs.py @@ -0,0 +1,127 @@ +from ...attrs import LIKE_NUM + +# reference 1: https://en.wikibooks.org/wiki/Sanskrit/Numbers + +_num_words = [ + "एकः", + "द्वौ", + "त्रयः", + "चत्वारः", + "पञ्च", + "षट्", + "सप्त", + "अष्ट", + "नव", + "दश", + "एकादश", + "द्वादश", + "त्रयोदश", + "चतुर्दश", + "पञ्चदश", + "षोडश", + "सप्तदश", + "अष्टादश", + "एकान्नविंशति", + "विंशति", + "एकाविंशति", + "द्वाविंशति", + "त्रयोविंशति", + "चतुर्विंशति", + "पञ्चविंशति", + "षड्विंशति", + "सप्तविंशति", + "अष्टाविंशति", + "एकान्नत्रिंशत्", + "त्रिंशत्", + "एकत्रिंशत्", + "द्वात्रिंशत्", + "त्रयत्रिंशत्", + "चतुस्त्रिंशत्", + "पञ्चत्रिंशत्", + "षट्त्रिंशत्", + "सप्तत्रिंशत्", + "अष्टात्रिंशत्", + "एकोनचत्वारिंशत्", + "चत्वारिंशत्", + "एकचत्वारिंशत्", + "द्वाचत्वारिंशत्", + "त्रयश्चत्वारिंशत्", + "चतुश्चत्वारिंशत्", + "पञ्चचत्वारिंशत्", + "षट्चत्वारिंशत्", + "सप्तचत्वारिंशत्", + "अष्टाचत्वारिंशत्", + "एकोनपञ्चाशत्", + "पञ्चाशत्", + "एकपञ्चाशत्", + "द्विपञ्चाशत्", + "त्रिपञ्चाशत्", + "चतुःपञ्चाशत्", + "पञ्चपञ्चाशत्", + "षट्पञ्चाशत्", + "सप्तपञ्चाशत्", + "अष्टपञ्चाशत्", + "एकोनषष्ठिः", + "षष्ठिः", + "एकषष्ठिः", + "द्विषष्ठिः", + "त्रिषष्ठिः", + "चतुःषष्ठिः", + "पञ्चषष्ठिः", + "षट्षष्ठिः", + "सप्तषष्ठिः", + "अष्टषष्ठिः", + "एकोनसप्ततिः", + "सप्ततिः", + "एकसप्ततिः", + "द्विसप्ततिः", + "त्रिसप्ततिः", + "चतुःसप्ततिः", + "पञ्चसप्ततिः", + "षट्सप्ततिः", + "सप्तसप्ततिः", + "अष्टसप्ततिः", + "एकोनाशीतिः", + "अशीतिः", + "एकाशीतिः", + "द्वशीतिः", + "त्र्यशीतिः", + "चतुरशीतिः", + "पञ्चाशीतिः", + "षडशीतिः", + "सप्ताशीतिः", + "अष्टाशीतिः", + "एकोननवतिः", + "नवतिः", + "एकनवतिः", + "द्विनवतिः", + "त्रिनवतिः", + "चतुर्नवतिः", + "पञ्चनवतिः", + "षण्णवतिः", + "सप्तनवतिः", + "अष्टनवतिः", + "एकोनशतम्", + "शतम्", +] + + +def like_num(text): + """ + Check if text resembles a number + """ + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/sa/stop_words.py b/spacy/lang/sa/stop_words.py new file mode 100644 index 0000000..30302a1 --- /dev/null +++ b/spacy/lang/sa/stop_words.py @@ -0,0 +1,515 @@ +# Source: https://gist.github.com/Akhilesh28/fe8b8e180f64b72e64751bc31cb6d323 + +STOP_WORDS = set( + """ +अहम् +आवाम् +वयम् +माम् मा +आवाम् +अस्मान् नः +मया +आवाभ्याम् +अस्माभिस् +मह्यम् मे +आवाभ्याम् नौ +अस्मभ्यम् नः +मत् +आवाभ्याम् +अस्मत् +मम मे +आवयोः +अस्माकम् नः +मयि +आवयोः +अस्मासु +त्वम् +युवाम् +यूयम् +त्वाम् त्वा +युवाम् वाम् +युष्मान् वः +त्वया +युवाभ्याम् +युष्माभिः +तुभ्यम् ते +युवाभ्याम् वाम् +युष्मभ्यम् वः +त्वत् +युवाभ्याम् +युष्मत् +तव ते +युवयोः वाम् +युष्माकम् वः +त्वयि +युवयोः +युष्मासु +सः +तौ +ते +तम् +तौ +तान् +तेन +ताभ्याम् +तैः +तस्मै +ताभ्याम् +तेभ्यः +तस्मात् +ताभ्याम् +तेभ्यः +तस्य +तयोः +तेषाम् +तस्मिन् +तयोः +तेषु +सा +ते +ताः +ताम् +ते +ताः +तया +ताभ्याम् +ताभिः +तस्यै +ताभ्याम् +ताभ्यः +तस्याः +ताभ्याम् +ताभ्यः +तस्य +तयोः +तासाम् +तस्याम् +तयोः +तासु +तत् +ते +तानि +तत् +ते +तानि +तया +ताभ्याम् +ताभिः +तस्यै +ताभ्याम् +ताभ्यः +तस्याः +ताभ्याम् +ताभ्यः +तस्य +तयोः +तासाम् +तस्याम् +तयोः +तासु +अयम् +इमौ +इमे +इमम् +इमौ +इमान् +अनेन +आभ्याम् +एभिः +अस्मै +आभ्याम् +एभ्यः +अस्मात् +आभ्याम् +एभ्यः +अस्य +अनयोः +एषाम् +अस्मिन् +अनयोः +एषु +इयम् +इमे +इमाः +इमाम् +इमे +इमाः +अनया +आभ्याम् +आभिः +अस्यै +आभ्याम् +आभ्यः +अस्याः +आभ्याम् +आभ्यः +अस्याः +अनयोः +आसाम् +अस्याम् +अनयोः +आसु +इदम् +इमे +इमानि +इदम् +इमे +इमानि +अनेन +आभ्याम् +एभिः +अस्मै +आभ्याम् +एभ्यः +अस्मात् +आभ्याम् +एभ्यः +अस्य +अनयोः +एषाम् +अस्मिन् +अनयोः +एषु +एषः +एतौ +एते +एतम् एनम् +एतौ एनौ +एतान् एनान् +एतेन +एताभ्याम् +एतैः +एतस्मै +एताभ्याम् +एतेभ्यः +एतस्मात् +एताभ्याम् +एतेभ्यः +एतस्य +एतस्मिन् +एतेषाम् +एतस्मिन् +एतस्मिन् +एतेषु +एषा +एते +एताः +एताम् एनाम् +एते एने +एताः एनाः +एतया एनया +एताभ्याम् +एताभिः +एतस्यै +एताभ्याम् +एताभ्यः +एतस्याः +एताभ्याम् +एताभ्यः +एतस्याः +एतयोः एनयोः +एतासाम् +एतस्याम् +एतयोः एनयोः +एतासु +एतत् एतद् +एते +एतानि +एतत् एतद् एनत् एनद् +एते एने +एतानि एनानि +एतेन एनेन +एताभ्याम् +एतैः +एतस्मै +एताभ्याम् +एतेभ्यः +एतस्मात् +एताभ्याम् +एतेभ्यः +एतस्य +एतयोः एनयोः +एतेषाम् +एतस्मिन् +एतयोः एनयोः +एतेषु +असौ +अमू +अमी +अमूम् +अमू +अमून् +अमुना +अमूभ्याम् +अमीभिः +अमुष्मै +अमूभ्याम् +अमीभ्यः +अमुष्मात् +अमूभ्याम् +अमीभ्यः +अमुष्य +अमुयोः +अमीषाम् +अमुष्मिन् +अमुयोः +अमीषु +असौ +अमू +अमूः +अमूम् +अमू +अमूः +अमुया +अमूभ्याम् +अमूभिः +अमुष्यै +अमूभ्याम् +अमूभ्यः +अमुष्याः +अमूभ्याम् +अमूभ्यः +अमुष्याः +अमुयोः +अमूषाम् +अमुष्याम् +अमुयोः +अमूषु +अमु +अमुनी +अमूनि +अमु +अमुनी +अमूनि +अमुना +अमूभ्याम् +अमीभिः +अमुष्मै +अमूभ्याम् +अमीभ्यः +अमुष्मात् +अमूभ्याम् +अमीभ्यः +अमुष्य +अमुयोः +अमीषाम् +अमुष्मिन् +अमुयोः +अमीषु +कः +कौ +के +कम् +कौ +कान् +केन +काभ्याम् +कैः +कस्मै +काभ्याम् +केभ्य +कस्मात् +काभ्याम् +केभ्य +कस्य +कयोः +केषाम् +कस्मिन् +कयोः +केषु +का +के +काः +काम् +के +काः +कया +काभ्याम् +काभिः +कस्यै +काभ्याम् +काभ्यः +कस्याः +काभ्याम् +काभ्यः +कस्याः +कयोः +कासाम् +कस्याम् +कयोः +कासु +किम् +के +कानि +किम् +के +कानि +केन +काभ्याम् +कैः +कस्मै +काभ्याम् +केभ्य +कस्मात् +काभ्याम् +केभ्य +कस्य +कयोः +केषाम् +कस्मिन् +कयोः +केषु +भवान् +भवन्तौ +भवन्तः +भवन्तम् +भवन्तौ +भवतः +भवता +भवद्भ्याम् +भवद्भिः +भवते +भवद्भ्याम् +भवद्भ्यः +भवतः +भवद्भ्याम् +भवद्भ्यः +भवतः +भवतोः +भवताम् +भवति +भवतोः +भवत्सु +भवती +भवत्यौ +भवत्यः +भवतीम् +भवत्यौ +भवतीः +भवत्या +भवतीभ्याम् +भवतीभिः +भवत्यै +भवतीभ्याम् +भवतीभिः +भवत्याः +भवतीभ्याम् +भवतीभिः +भवत्याः +भवत्योः +भवतीनाम् +भवत्याम् +भवत्योः +भवतीषु +भवत् +भवती +भवन्ति +भवत् +भवती +भवन्ति +भवता +भवद्भ्याम् +भवद्भिः +भवते +भवद्भ्याम् +भवद्भ्यः +भवतः +भवद्भ्याम् +भवद्भ्यः +भवतः +भवतोः +भवताम् +भवति +भवतोः +भवत्सु +अये +अरे +अरेरे +अविधा +असाधुना +अस्तोभ +अहह +अहावस् +आम् +आर्यहलम् +आह +आहो +इस् +उम् +उवे +काम् +कुम् +चमत् +टसत् +दृन् +धिक् +पाट् +फत् +फाट् +फुडुत् +बत +बाल् +वट् +व्यवस्तोभति व्यवस्तुभ् +षाट् +स्तोभ +हुम्मा +हूम् +अति +अधि +अनु +अप +अपि +अभि +अव +आ +उद् +उप +नि +निर् +परा +परि +प्र +प्रति +वि +सम् +अथवा उत +अन्यथा +इव +च +चेत् यदि +तु परन्तु +यतः करणेन हि यतस् यदर्थम् यदर्थे यर्हि यथा यत्कारणम् येन ही हिन +यथा यतस् +यद्यपि +यात् अवधेस् यावति +येन प्रकारेण +स्थाने +अह +एव +एवम् +कच्चित् +कु +कुवित् +कूपत् +च +चण् +चेत् +तत्र +नकिम् +नह +नुनम् +नेत् +भूयस् +मकिम् +मकिर् +यत्र +युगपत् +वा +शश्वत् +सूपत् +ह +हन्त +हि +""".split() +) diff --git a/spacy/lang/si/__init__.py b/spacy/lang/si/__init__.py new file mode 100644 index 0000000..08d0937 --- /dev/null +++ b/spacy/lang/si/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class SinhalaDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Sinhala(Language): + lang = "si" + Defaults = SinhalaDefaults + + +__all__ = ["Sinhala"] diff --git a/spacy/lang/si/examples.py b/spacy/lang/si/examples.py new file mode 100644 index 0000000..8e0ffec --- /dev/null +++ b/spacy/lang/si/examples.py @@ -0,0 +1,18 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.si.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "මෙය වාක්‍යයකි.", + "ඔබ කවුද?", + "ගූගල් සමාගම ඩොලර් මිලියන 500 කට එම ආයතනය මිලදී ගන්නා ලදී.", + "කොළඹ ශ්‍රී ලංකාවේ ප්‍රධානතම නගරය යි.", + "ප්‍රංශයේ ජනාධිපති කවරෙක් ද?", + "මට බිස්කට් 1 ක් දෙන්න", + "ඔවුන් ලකුණු 59 කින් තරඟය පරාජයට පත් විය.", + "1 ත් 10 ත් අතර සංඛ්‍යාවක් කියන්න", + "ඔහු සහ ඇය නුවර හෝ කොළඹ පදිංචි කරුවන් නොවේ", +] diff --git a/spacy/lang/si/lex_attrs.py b/spacy/lang/si/lex_attrs.py new file mode 100644 index 0000000..aa06185 --- /dev/null +++ b/spacy/lang/si/lex_attrs.py @@ -0,0 +1,61 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "බින්දුව", + "බිංදුව", + "එක", + "දෙක", + "තුන", + "හතර", + "පහ", + "හය", + "හත", + "අට", + "නවය", + "නමය", + "දහය", + "එකොළහ", + "දොළහ", + "දහතුන", + "දහහතර", + "දාහතර", + "පහළව", + "පහළොව", + "දහසය", + "දහහත", + "දාහත", + "දහඅට", + "දහනවය", + "විස්ස", + "තිහ", + "හතළිහ", + "පනහ", + "හැට", + "හැත්තෑව", + "අසූව", + "අනූව", + "සියය", + "දහස", + "දාහ", + "ලක්ෂය", + "මිලියනය", + "කෝටිය", + "බිලියනය", + "ට්‍රිලියනය", +] + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/si/stop_words.py b/spacy/lang/si/stop_words.py new file mode 100644 index 0000000..7d29bc1 --- /dev/null +++ b/spacy/lang/si/stop_words.py @@ -0,0 +1,195 @@ +STOP_WORDS = set( + """ +සහ +සමග +සමඟ +අහා +ආහ් +ආ +ඕහෝ +අනේ +අඳෝ +අපොයි +අපෝ +අයියෝ +ආයි +ඌයි +චී +චිහ් +චික් +හෝ‍ +දෝ +දෝහෝ +මෙන් +සේ +වැනි +බඳු +වන් +අයුරු +අයුරින් +ලෙස +වැඩි +ශ්‍රී +හා +ය +නිසා +නිසාවෙන් +බවට +බව +බවෙන් +නම් +වැඩි +සිට +දී +මහා +මහ +පමණ +පමණින් +පමන +වන +විට +විටින් +මේ +මෙලෙස +මෙයින් +ඇති +ලෙස +සිදු +වශයෙන් +යන +සඳහා +මගින් +හෝ‍ +ඉතා +ඒ +එම +ද +අතර +විසින් +සමග +පිළිබඳව +පිළිබඳ +තුළ +බව +වැනි +මහ +මෙම +මෙහි +මේ +වෙත +වෙතින් +වෙතට +වෙනුවෙන් +වෙනුවට +වෙන +ගැන +නෑ +අනුව +නව +පිළිබඳ +විශේෂ +දැනට +එහෙන් +මෙහෙන් +එහේ +මෙහේ +ම +තවත් +තව +සහ +දක්වා +ට +ගේ +එ +ක +ක් +බවත් +බවද +මත +ඇතුලු +ඇතුළු +මෙසේ +වඩා +වඩාත්ම +නිති +නිතිත් +නිතොර +නිතර +ඉක්බිති +දැන් +යලි +පුන +ඉතින් +සිට +සිටන් +පටන් +තෙක් +දක්වා +සා +තාක් +තුවක් +පවා +ද +හෝ‍ +වත් +විනා +හැර +මිස +මුත් +කිම +කිම් +ඇයි +මන්ද +හෙවත් +නොහොත් +පතා +පාසා +ගානෙ +තව +ඉතා +බොහෝ +වහා +සෙද +සැනින් +හනික +එම්බා +එම්බල +බොල +නම් +වනාහි +කලී +ඉඳුරා +අන්න +ඔන්න +මෙන්න +උදෙසා +පිණිස +සඳහා +අරබයා +නිසා +එනිසා +එබැවින් +බැවින් +හෙයින් +සේක් +සේක +ගැන +අනුව +පරිදි +විට +තෙක් +මෙතෙක් +මේතාක් +තුරු +තුරා +තුරාවට +තුලින් +නමුත් +එනමුත් +වස් +මෙන් +ලෙස +පරිදි +එහෙත් +""".split() +) diff --git a/spacy/lang/sk/__init__.py b/spacy/lang/sk/__init__.py new file mode 100644 index 0000000..2ed7448 --- /dev/null +++ b/spacy/lang/sk/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class SlovakDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Slovak(Language): + lang = "sk" + Defaults = SlovakDefaults + + +__all__ = ["Slovak"] diff --git a/spacy/lang/sk/examples.py b/spacy/lang/sk/examples.py new file mode 100644 index 0000000..079d0d2 --- /dev/null +++ b/spacy/lang/sk/examples.py @@ -0,0 +1,22 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sk.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Ardevop, s.r.o. je malá startup firma na území SR.", + "Samojazdiace autá presúvajú poistnú zodpovednosť na výrobcov automobilov.", + "Košice sú na východe.", + "Bratislava je hlavné mesto Slovenskej republiky.", + "Kde si?", + "Kto je prezidentom Francúzska?", + "Aké je hlavné mesto Slovenska?", + "Kedy sa narodil Andrej Kiska?", + "Včera som dostal 100€ na ruku.", + "Dnes je nedeľa 26.1.2020.", + "Narodil sa 15.4.1998 v Ružomberku.", + "Niekto mi povedal, že 500 eur je veľa peňazí.", + "Podaj mi ruku!", +] diff --git a/spacy/lang/sk/lex_attrs.py b/spacy/lang/sk/lex_attrs.py new file mode 100644 index 0000000..0caf62e --- /dev/null +++ b/spacy/lang/sk/lex_attrs.py @@ -0,0 +1,59 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "nula", + "jeden", + "dva", + "tri", + "štyri", + "päť", + "šesť", + "sedem", + "osem", + "deväť", + "desať", + "jedenásť", + "dvanásť", + "trinásť", + "štrnásť", + "pätnásť", + "šestnásť", + "sedemnásť", + "osemnásť", + "devätnásť", + "dvadsať", + "tridsať", + "štyridsať", + "päťdesiat", + "šesťdesiat", + "sedemdesiat", + "osemdesiat", + "deväťdesiat", + "sto", + "tisíc", + "milión", + "miliarda", + "bilión", + "biliarda", + "trilión", + "triliarda", + "kvadrilión", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/sk/stop_words.py b/spacy/lang/sk/stop_words.py new file mode 100644 index 0000000..017e7be --- /dev/null +++ b/spacy/lang/sk/stop_words.py @@ -0,0 +1,424 @@ +# Source: https://github.com/Ardevop-sk/stopwords-sk + +STOP_WORDS = set( + """ +a +aby +aj +ak +akej +akejže +ako +akom +akomže +akou +akouže +akože +aká +akáže +aké +akého +akéhože +akému +akémuže +akéže +akú +akúže +aký +akých +akýchže +akým +akými +akýmiže +akýmže +akýže +ale +alebo +ani +asi +avšak +až +ba +bez +bezo +bol +bola +boli +bolo +bude +budem +budeme +budete +budeš +budú +buď +by +byť +cez +cezo +dnes +do +ešte +ho +hoci +i +iba +ich +im +inej +inom +iná +iné +iného +inému +iní +inú +iný +iných +iným +inými +ja +je +jeho +jej +jemu +ju +k +kam +kamže +každou +každá +každé +každého +každému +každí +každú +každý +každých +každým +každými +kde +kej +kejže +keď +keďže +kie +kieho +kiehože +kiemu +kiemuže +kieže +koho +kom +komu +kou +kouže +kto +ktorej +ktorou +ktorá +ktoré +ktorí +ktorú +ktorý +ktorých +ktorým +ktorými +ku +ká +káže +ké +kéže +kú +kúže +ký +kýho +kýhože +kým +kýmu +kýmuže +kýže +lebo +leda +ledaže +len +ma +majú +mal +mala +mali +mať +medzi +mi +mne +mnou +moja +moje +mojej +mojich +mojim +mojimi +mojou +moju +možno +mu +musia +musieť +musí +musím +musíme +musíte +musíš +my +má +mám +máme +máte +máš +môcť +môj +môjho +môže +môžem +môžeme +môžete +môžeš +môžu +mňa +na +nad +nado +najmä +nami +naša +naše +našej +naši +našich +našim +našimi +našou +ne +nech +neho +nej +nejakej +nejakom +nejakou +nejaká +nejaké +nejakého +nejakému +nejakú +nejaký +nejakých +nejakým +nejakými +nemu +než +nich +nie +niektorej +niektorom +niektorou +niektorá +niektoré +niektorého +niektorému +niektorú +niektorý +niektorých +niektorým +niektorými +nielen +niečo +nim +nimi +nič +ničoho +ničom +ničomu +ničím +no +nám +nás +náš +nášho +ním +o +od +odo +on +ona +oni +ono +ony +oň +oňho +po +pod +podo +podľa +pokiaľ +popod +popri +potom +poza +pre +pred +predo +preto +pretože +prečo +pri +práve +s +sa +seba +sebe +sebou +sem +si +sme +so +som +ste +svoj +svoja +svoje +svojho +svojich +svojim +svojimi +svojou +svoju +svojím +sú +ta +tak +takej +takejto +taká +takáto +také +takého +takéhoto +takému +takémuto +takéto +takí +takú +takúto +taký +takýto +takže +tam +teba +tebe +tebou +teda +tej +tejto +ten +tento +ti +tie +tieto +tiež +to +toho +tohoto +tohto +tom +tomto +tomu +tomuto +toto +tou +touto +tu +tvoj +tvoja +tvoje +tvojej +tvojho +tvoji +tvojich +tvojim +tvojimi +tvojím +ty +tá +táto +tí +títo +tú +túto +tých +tým +tými +týmto +u +už +v +vami +vaša +vaše +vašej +vaši +vašich +vašim +vaším +veď +viac +vo +vy +vám +vás +váš +vášho +však +všetci +všetka +všetko +všetky +všetok +z +za +začo +začože +zo +áno +čej +či +čia +čie +čieho +čiemu +čiu +čo +čoho +čom +čomu +čou +čože +čí +čím +čími +ďalšia +ďalšie +ďalšieho +ďalšiemu +ďalšiu +ďalšom +ďalšou +ďalší +ďalších +ďalším +ďalšími +ňom +ňou +ňu +že +""".split() +) diff --git a/spacy/lang/sl/__init__.py b/spacy/lang/sl/__init__.py new file mode 100644 index 0000000..cd3d70f --- /dev/null +++ b/spacy/lang/sl/__init__.py @@ -0,0 +1,22 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class SlovenianDefaults(BaseDefaults): + stop_words = STOP_WORDS + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + + +class Slovenian(Language): + lang = "sl" + Defaults = SlovenianDefaults + + +__all__ = ["Slovenian"] diff --git a/spacy/lang/sl/examples.py b/spacy/lang/sl/examples.py new file mode 100644 index 0000000..7984611 --- /dev/null +++ b/spacy/lang/sl/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sl.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple načrtuje nakup britanskega startupa za 1 bilijon dolarjev", + "France Prešeren je umrl 8. februarja 1849 v Kranju", + "Staro ljubljansko letališče Moste bo obnovila družba BTC", + "London je največje mesto v Združenem kraljestvu.", + "Kje se skrivaš?", + "Kdo je predsednik Francije?", + "Katero je glavno mesto Združenih držav Amerike?", + "Kdaj je bil rojen Milan Kučan?", +] diff --git a/spacy/lang/sl/lex_attrs.py b/spacy/lang/sl/lex_attrs.py new file mode 100644 index 0000000..3c14930 --- /dev/null +++ b/spacy/lang/sl/lex_attrs.py @@ -0,0 +1,144 @@ +import unicodedata + +from ...attrs import IS_CURRENCY, LIKE_NUM + +_num_words = set( + """ + nula ničla nič ena dva tri štiri pet šest sedem osem + devet deset enajst dvanajst trinajst štirinajst petnajst + šestnajst sedemnajst osemnajst devetnajst dvajset trideset štirideset + petdeset šestdest sedemdeset osemdeset devedeset sto tisoč + milijon bilijon trilijon kvadrilijon nešteto + + en eden enega enemu ennem enim enih enima enimi ene eni eno + dveh dvema dvem dvoje trije treh trem tremi troje štirje štirih štirim štirimi + petih petim petimi šestih šestim šestimi sedmih sedmim sedmimi osmih osmim osmimi + devetih devetim devetimi desetih desetim desetimi enajstih enajstim enajstimi + dvanajstih dvanajstim dvanajstimi trinajstih trinajstim trinajstimi + šestnajstih šestnajstim šestnajstimi petnajstih petnajstim petnajstimi + sedemnajstih sedemnajstim sedemnajstimi osemnajstih osemnajstim osemnajstimi + devetnajstih devetnajstim devetnajstimi dvajsetih dvajsetim dvajsetimi + """.split() +) + +_ordinal_words = set( + """ + prvi drugi tretji četrti peti šesti sedmi osmi + deveti deseti enajsti dvanajsti trinajsti štirinajsti + petnajsti šestnajsti sedemnajsti osemnajsti devetnajsti + dvajseti trideseti štirideseti petdeseti šestdeseti sedemdeseti + osemdeseti devetdeseti stoti tisoči milijonti bilijonti + trilijonti kvadrilijonti nešteti + + prva druga tretja četrta peta šesta sedma osma + deveta deseta enajsta dvanajsta trinajsta štirnajsta + petnajsta šestnajsta sedemnajsta osemnajsta devetnajsta + dvajseta trideseta štirideseta petdeseta šestdeseta sedemdeseta + osemdeseta devetdeseta stota tisoča milijonta bilijonta + trilijonta kvadrilijonta nešteta + + prvo drugo tretje četrto peto šestro sedmo osmo + deveto deseto enajsto dvanajsto trinajsto štirnajsto + petnajsto šestnajsto sedemnajsto osemnajsto devetnajsto + dvajseto trideseto štirideseto petdeseto šestdeseto sedemdeseto + osemdeseto devetdeseto stoto tisočo milijonto bilijonto + trilijonto kvadrilijonto nešteto + + prvega drugega tretjega četrtega petega šestega sedmega osmega + devega desetega enajstega dvanajstega trinajstega štirnajstega + petnajstega šestnajstega sedemnajstega osemnajstega devetnajstega + dvajsetega tridesetega štiridesetega petdesetega šestdesetega sedemdesetega + osemdesetega devetdesetega stotega tisočega milijontega bilijontega + trilijontega kvadrilijontega neštetega + + prvemu drugemu tretjemu četrtemu petemu šestemu sedmemu osmemu devetemu desetemu + enajstemu dvanajstemu trinajstemu štirnajstemu petnajstemu šestnajstemu sedemnajstemu + osemnajstemu devetnajstemu dvajsetemu tridesetemu štiridesetemu petdesetemu šestdesetemu + sedemdesetemu osemdesetemu devetdesetemu stotemu tisočemu milijontemu bilijontemu + trilijontemu kvadrilijontemu neštetemu + + prvem drugem tretjem četrtem petem šestem sedmem osmem devetem desetem + enajstem dvanajstem trinajstem štirnajstem petnajstem šestnajstem sedemnajstem + osemnajstem devetnajstem dvajsetem tridesetem štiridesetem petdesetem šestdesetem + sedemdesetem osemdesetem devetdesetem stotem tisočem milijontem bilijontem + trilijontem kvadrilijontem neštetem + + prvim drugim tretjim četrtim petim šestim sedtim osmim devetim desetim + enajstim dvanajstim trinajstim štirnajstim petnajstim šestnajstim sedemnajstim + osemnajstim devetnajstim dvajsetim tridesetim štiridesetim petdesetim šestdesetim + sedemdesetim osemdesetim devetdesetim stotim tisočim milijontim bilijontim + trilijontim kvadrilijontim neštetim + + prvih drugih tretjih četrthih petih šestih sedmih osmih deveth desetih + enajstih dvanajstih trinajstih štirnajstih petnajstih šestnajstih sedemnajstih + osemnajstih devetnajstih dvajsetih tridesetih štiridesetih petdesetih šestdesetih + sedemdesetih osemdesetih devetdesetih stotih tisočih milijontih bilijontih + trilijontih kvadrilijontih nešteth + + prvima drugima tretjima četrtima petima šestima sedmima osmima devetima desetima + enajstima dvanajstima trinajstima štirnajstima petnajstima šestnajstima sedemnajstima + osemnajstima devetnajstima dvajsetima tridesetima štiridesetima petdesetima šestdesetima + sedemdesetima osemdesetima devetdesetima stotima tisočima milijontima bilijontima + trilijontima kvadrilijontima neštetima + + prve druge četrte pete šeste sedme osme devete desete + enajste dvanajste trinajste štirnajste petnajste šestnajste sedemnajste + osemnajste devetnajste dvajsete tridesete štiridesete petdesete šestdesete + sedemdesete osemdesete devetdesete stote tisoče milijonte bilijonte + trilijonte kvadrilijonte neštete + + prvimi drugimi tretjimi četrtimi petimi šestimi sedtimi osmimi devetimi desetimi + enajstimi dvanajstimi trinajstimi štirnajstimi petnajstimi šestnajstimi sedemnajstimi + osemnajstimi devetnajstimi dvajsetimi tridesetimi štiridesetimi petdesetimi šestdesetimi + sedemdesetimi osemdesetimi devetdesetimi stotimi tisočimi milijontimi bilijontimi + trilijontimi kvadrilijontimi neštetimi + """.split() +) + +_currency_words = set( + """ + evro evra evru evrom evrov evroma evrih evrom evre evri evr eur + cent centa centu cenom centov centoma centih centom cente centi + dolar dolarja dolarji dolarju dolarjem dolarjev dolarjema dolarjih dolarje usd + tolar tolarja tolarji tolarju tolarjem tolarjev tolarjema tolarjih tolarje tol + dinar dinarja dinarji dinarju dinarjem dinarjev dinarjema dinarjih dinarje din + funt funta funti funtu funtom funtov funtoma funtih funte gpb + forint forinta forinti forintu forintom forintov forintoma forintih forinte + zlot zlota zloti zlotu zlotom zlotov zlotoma zlotih zlote + rupij rupija rupiji rupiju rupijem rupijev rupijema rupijih rupije + jen jena jeni jenu jenom jenov jenoma jenih jene + kuna kuni kune kuno kun kunama kunah kunam kunami + marka marki marke markama markah markami + """.split() +) + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + if text_lower in _ordinal_words: + return True + return False + + +def is_currency(text): + text_lower = text.lower() + if text in _currency_words: + return True + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +LEX_ATTRS = {LIKE_NUM: like_num, IS_CURRENCY: is_currency} diff --git a/spacy/lang/sl/punctuation.py b/spacy/lang/sl/punctuation.py new file mode 100644 index 0000000..3be83eb --- /dev/null +++ b/spacy/lang/sl/punctuation.py @@ -0,0 +1,83 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + +INCLUDE_SPECIAL = ["\\+", "\\/", "\\•", "\\¯", "\\=", "\\×"] + HYPHENS.split("|") + +_prefixes = INCLUDE_SPECIAL + BASE_TOKENIZER_PREFIXES + +_suffixes = ( + INCLUDE_SPECIAL + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + # split initials like J.K. Rowling + r"(?<=[A-Z]\.)(?:[A-Z].)", + ] +) + +# a list of all suffixes following a hyphen that are shouldn't split (eg. BTC-jev) +# source: Obeliks tokenizer - https://github.com/clarinsi/obeliks/blob/master/obeliks/res/TokRulesPart1.txt +CONCAT_QUOTES = CONCAT_QUOTES.replace("'", "") +HYPHENS_PERMITTED = ( + "((a)|(evemu)|(evskega)|(i)|(jevega)|(jevska)|(jevskimi)|(jinemu)|(oma)|(ovim)|" + "(ovski)|(e)|(evi)|(evskem)|(ih)|(jevem)|(jevske)|(jevsko)|(jini)|(ov)|(ovima)|" + "(ovskih)|(em)|(evih)|(evskemu)|(ja)|(jevemu)|(jevskega)|(ji)|(jinih)|(ova)|" + "(ovimi)|(ovskim)|(ema)|(evim)|(evski)|(je)|(jevi)|(jevskem)|(jih)|(jinim)|" + "(ove)|(ovo)|(ovskima)|(ev)|(evima)|(evskih)|(jem)|(jevih)|(jevskemu)|(jin)|" + "(jinima)|(ovega)|(ovska)|(ovskimi)|(eva)|(evimi)|(evskim)|(jema)|(jevim)|" + "(jevski)|(jina)|(jinimi)|(ovem)|(ovske)|(ovsko)|(eve)|(evo)|(evskima)|(jev)|" + "(jevima)|(jevskih)|(jine)|(jino)|(ovemu)|(ovskega)|(u)|(evega)|(evska)|" + "(evskimi)|(jeva)|(jevimi)|(jevskim)|(jinega)|(ju)|(ovi)|(ovskem)|(evem)|" + "(evske)|(evsko)|(jeve)|(jevo)|(jevskima)|(jinem)|(om)|(ovih)|(ovskemu)|" + "(ovec)|(ovca)|(ovcu)|(ovcem)|(ovcev)|(ovcema)|(ovcih)|(ovci)|(ovce)|(ovcimi)|" + "(evec)|(evca)|(evcu)|(evcem)|(evcev)|(evcema)|(evcih)|(evci)|(evce)|(evcimi)|" + "(jevec)|(jevca)|(jevcu)|(jevcem)|(jevcev)|(jevcema)|(jevcih)|(jevci)|(jevce)|" + "(jevcimi)|(ovka)|(ovke)|(ovki)|(ovko)|(ovk)|(ovkama)|(ovkah)|(ovkam)|(ovkami)|" + "(evka)|(evke)|(evki)|(evko)|(evk)|(evkama)|(evkah)|(evkam)|(evkami)|(jevka)|" + "(jevke)|(jevki)|(jevko)|(jevk)|(jevkama)|(jevkah)|(jevkam)|(jevkami)|(timi)|" + "(im)|(ima)|(a)|(imi)|(e)|(o)|(ega)|(ti)|(em)|(tih)|(emu)|(tim)|(i)|(tima)|" + "(ih)|(ta)|(te)|(to)|(tega)|(tem)|(temu))" +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?!{hp}$)(?=[{a}])".format( + a=ALPHA, h=HYPHENS, hp=HYPHENS_PERMITTED + ), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/sl/stop_words.py b/spacy/lang/sl/stop_words.py new file mode 100644 index 0000000..8491efc --- /dev/null +++ b/spacy/lang/sl/stop_words.py @@ -0,0 +1,84 @@ +# Source: https://github.com/stopwords-iso/stopwords-sl + +STOP_WORDS = set( + """ +a ali + +b bi bil bila bile bili bilo biti blizu bo bodo bojo bolj bom bomo +boste bova boš brez + +c cel cela celi celo + +č če često četrta četrtek četrti četrto čez čigav + +d da daleč dan danes datum deset deseta deseti deseto devet +deveta deveti deveto do dober dobra dobri dobro dokler dol dolg +dolga dolgi dovolj drug druga drugi drugo dva dve + +e eden en ena ene eni enkrat eno etc. + +f + +g g. ga ga. gor gospa gospod + +h halo + +i idr. ii iii in iv ix iz + +j jaz je ji jih jim jo jutri + +k kadarkoli kaj kajti kako kakor kamor kamorkoli kar karkoli +katerikoli kdaj kdo kdorkoli ker ki kje kjer kjerkoli +ko koder koderkoli koga komu kot kratek kratka kratke kratki + +l lahka lahke lahki lahko le lep lepa lepe lepi lepo leto + +m majhen majhna majhni malce malo manj me med medtem mene +mesec mi midva midve mnogo moj moja moje mora morajo moram +moramo morate moraš morem mu + +n na nad naj najina najino najmanj naju največ nam narobe +nas nato nazaj naš naša naše ne nedavno nedelja nek neka +nekaj nekatere nekateri nekatero nekdo neke nekega neki +nekje neko nekoga nekoč ni nikamor nikdar nikjer nikoli +nič nje njega njegov njegova njegovo njej njemu njen +njena njeno nji njih njihov njihova njihovo njiju njim +njo njun njuna njuno no nocoj npr. + +o ob oba obe oboje od odprt odprta odprti okoli on +onadva one oni onidve osem osma osmi osmo oz. + +p pa pet peta petek peti peto po pod pogosto poleg poln +polna polni polno ponavadi ponedeljek ponovno potem +povsod pozdravljen pozdravljeni prav prava prave pravi +pravo prazen prazna prazno prbl. precej pred prej preko +pri pribl. približno primer pripravljen pripravljena +pripravljeni proti prva prvi prvo + +r ravno redko res reč + +s saj sam sama same sami samo se sebe sebi sedaj sedem +sedma sedmi sedmo sem seveda si sicer skoraj skozi slab sm +so sobota spet sreda srednja srednji sta ste stran stvar sva + +š šest šesta šesti šesto štiri + +t ta tak taka take taki tako takoj tam te tebe tebi tega +težak težka težki težko ti tista tiste tisti tisto tj. +tja to toda torek tretja tretje tretji tri tu tudi tukaj +tvoj tvoja tvoje + +u + +v vaju vam vas vaš vaša vaše ve vedno velik velika veliki +veliko vendar ves več vi vidva vii viii visok visoka visoke +visoki vsa vsaj vsak vsaka vsakdo vsake vsaki vsakomur vse +vsega vsi vso včasih včeraj + +x + +z za zadaj zadnji zakaj zaprta zaprti zaprto zdaj zelo zunaj + +ž že +""".split() +) diff --git a/spacy/lang/sl/tokenizer_exceptions.py b/spacy/lang/sl/tokenizer_exceptions.py new file mode 100644 index 0000000..ec4ea9e --- /dev/null +++ b/spacy/lang/sl/tokenizer_exceptions.py @@ -0,0 +1,273 @@ +from typing import Dict, List + +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc: Dict[str, List[Dict]] = {} + +_other_exc = { + "t.i.": [{ORTH: "t.", NORM: "tako"}, {ORTH: "i.", NORM: "imenovano"}], + "t.j.": [{ORTH: "t.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "T.j.": [{ORTH: "T.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "d.o.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "o.", NORM: "omejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.O.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "O.", NORM: "omejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.n.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "n.", NORM: "neomejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.N.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "N.", NORM: "neomejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.d.": [{ORTH: "d.", NORM: "delniška"}, {ORTH: "d.", NORM: "družba"}], + "D.D.": [{ORTH: "D.", NORM: "delniška"}, {ORTH: "D.", NORM: "družba"}], + "s.p.": [{ORTH: "s.", NORM: "samostojni"}, {ORTH: "p.", NORM: "podjetnik"}], + "S.P.": [{ORTH: "S.", NORM: "samostojni"}, {ORTH: "P.", NORM: "podjetnik"}], + "l.r.": [{ORTH: "l.", NORM: "lastno"}, {ORTH: "r.", NORM: "ročno"}], + "le-te": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "te"}], + "Le-te": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "te"}], + "le-ti": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ti"}], + "Le-ti": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ti"}], + "le-to": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "to"}], + "Le-to": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "to"}], + "le-ta": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ta"}], + "Le-ta": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ta"}], + "le-tega": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "tega"}], + "Le-tega": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "tega"}], +} + +_exc.update(_other_exc) + + +for exc_data in [ + {ORTH: "adm.", NORM: "administracija"}, + {ORTH: "aer.", NORM: "aeronavtika"}, + {ORTH: "agr.", NORM: "agronomija"}, + {ORTH: "amer.", NORM: "ameriško"}, + {ORTH: "anat.", NORM: "anatomija"}, + {ORTH: "angl.", NORM: "angleški"}, + {ORTH: "ant.", NORM: "antonim"}, + {ORTH: "antr.", NORM: "antropologija"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "arab.", NORM: "arabsko"}, + {ORTH: "arheol.", NORM: "arheologija"}, + {ORTH: "arhit.", NORM: "arhitektura"}, + {ORTH: "avg.", NORM: "avgust"}, + {ORTH: "avstr.", NORM: "avstrijsko"}, + {ORTH: "avt.", NORM: "avtomobilizem"}, + {ORTH: "bibl.", NORM: "biblijsko"}, + {ORTH: "biokem.", NORM: "biokemija"}, + {ORTH: "biol.", NORM: "biologija"}, + {ORTH: "bolg.", NORM: "bolgarski"}, + {ORTH: "bot.", NORM: "botanika"}, + {ORTH: "cit.", NORM: "citat"}, + {ORTH: "daj.", NORM: "dajalnik"}, + {ORTH: "del.", NORM: "deležnik"}, + {ORTH: "ed.", NORM: "ednina"}, + {ORTH: "etn.", NORM: "etnografija"}, + {ORTH: "farm.", NORM: "farmacija"}, + {ORTH: "filat.", NORM: "filatelija"}, + {ORTH: "filoz.", NORM: "filozofija"}, + {ORTH: "fin.", NORM: "finančništvo"}, + {ORTH: "fiz.", NORM: "fizika"}, + {ORTH: "fot.", NORM: "fotografija"}, + {ORTH: "fr.", NORM: "francoski"}, + {ORTH: "friz.", NORM: "frizerstvo"}, + {ORTH: "gastr.", NORM: "gastronomija"}, + {ORTH: "geogr.", NORM: "geografija"}, + {ORTH: "geol.", NORM: "geologija"}, + {ORTH: "geom.", NORM: "geometrija"}, + {ORTH: "germ.", NORM: "germanski"}, + {ORTH: "gl.", NORM: "glej"}, + {ORTH: "glag.", NORM: "glagolski"}, + {ORTH: "glasb.", NORM: "glasba"}, + {ORTH: "gled.", NORM: "gledališče"}, + {ORTH: "gost.", NORM: "gostinstvo"}, + {ORTH: "gozd.", NORM: "gozdarstvo"}, + {ORTH: "gr.", NORM: "grški"}, + {ORTH: "grad.", NORM: "gradbeništvo"}, + {ORTH: "hebr.", NORM: "hebrejsko"}, + {ORTH: "hrv.", NORM: "hrvaško"}, + {ORTH: "ide.", NORM: "indoevropsko"}, + {ORTH: "igr.", NORM: "igre"}, + {ORTH: "im.", NORM: "imenovalnik"}, + {ORTH: "iron.", NORM: "ironično"}, + {ORTH: "it.", NORM: "italijanski"}, + {ORTH: "itd.", NORM: "in tako dalje"}, + {ORTH: "itn.", NORM: "in tako naprej"}, + {ORTH: "ipd.", NORM: "in podobno"}, + {ORTH: "jap.", NORM: "japonsko"}, + {ORTH: "jul.", NORM: "julij"}, + {ORTH: "jun.", NORM: "junij"}, + {ORTH: "kit.", NORM: "kitajsko"}, + {ORTH: "knj.", NORM: "knjižno"}, + {ORTH: "knjiž.", NORM: "knjižno"}, + {ORTH: "kor.", NORM: "koreografija"}, + {ORTH: "lat.", NORM: "latinski"}, + {ORTH: "les.", NORM: "lesna stroka"}, + {ORTH: "lingv.", NORM: "lingvistika"}, + {ORTH: "lit.", NORM: "literarni"}, + {ORTH: "ljubk.", NORM: "ljubkovalno"}, + {ORTH: "lov.", NORM: "lovstvo"}, + {ORTH: "m.", NORM: "moški"}, + {ORTH: "mak.", NORM: "makedonski"}, + {ORTH: "mar.", NORM: "marec"}, + {ORTH: "mat.", NORM: "matematika"}, + {ORTH: "med.", NORM: "medicina"}, + {ORTH: "meh.", NORM: "mehiško"}, + {ORTH: "mest.", NORM: "mestnik"}, + {ORTH: "mdr.", NORM: "med drugim"}, + {ORTH: "min.", NORM: "mineralogija"}, + {ORTH: "mitol.", NORM: "mitologija"}, + {ORTH: "mn.", NORM: "množina"}, + {ORTH: "mont.", NORM: "montanistika"}, + {ORTH: "muz.", NORM: "muzikologija"}, + {ORTH: "nam.", NORM: "namenilnik"}, + {ORTH: "nar.", NORM: "narečno"}, + {ORTH: "nav.", NORM: "navadno"}, + {ORTH: "nedol.", NORM: "nedoločnik"}, + {ORTH: "nedov.", NORM: "nedovršni"}, + {ORTH: "neprav.", NORM: "nepravilno"}, + {ORTH: "nepreh.", NORM: "neprehodno"}, + {ORTH: "neskl.", NORM: "nesklonljiv(o)"}, + {ORTH: "nestrok.", NORM: "nestrokovno"}, + {ORTH: "num.", NORM: "numizmatika"}, + {ORTH: "npr.", NORM: "na primer"}, + {ORTH: "obrt.", NORM: "obrtništvo"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "or.", NORM: "orodnik"}, + {ORTH: "os.", NORM: "oseba"}, + {ORTH: "otr.", NORM: "otroško"}, + {ORTH: "oz.", NORM: "oziroma"}, + {ORTH: "pal.", NORM: "paleontologija"}, + {ORTH: "papir.", NORM: "papirništvo"}, + {ORTH: "ped.", NORM: "pedagogika"}, + {ORTH: "pisar.", NORM: "pisarniško"}, + {ORTH: "pog.", NORM: "pogovorno"}, + {ORTH: "polit.", NORM: "politika"}, + {ORTH: "polj.", NORM: "poljsko"}, + {ORTH: "poljud.", NORM: "poljudno"}, + {ORTH: "preg.", NORM: "pregovor"}, + {ORTH: "preh.", NORM: "prehodno"}, + {ORTH: "pren.", NORM: "preneseno"}, + {ORTH: "prid.", NORM: "pridevnik"}, + {ORTH: "prim.", NORM: "primerjaj"}, + {ORTH: "prisl.", NORM: "prislov"}, + {ORTH: "psih.", NORM: "psihologija"}, + {ORTH: "psiht.", NORM: "psihiatrija"}, + {ORTH: "rad.", NORM: "radiotehnika"}, + {ORTH: "rač.", NORM: "računalništvo"}, + {ORTH: "rib.", NORM: "ribištvo"}, + {ORTH: "rod.", NORM: "rodilnik"}, + {ORTH: "rus.", NORM: "rusko"}, + {ORTH: "s.", NORM: "srednji"}, + {ORTH: "sam.", NORM: "samostalniški"}, + {ORTH: "sed.", NORM: "sedanjik"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "slabš.", NORM: "slabšalno"}, + {ORTH: "slovan.", NORM: "slovansko"}, + {ORTH: "slovaš.", NORM: "slovaško"}, + {ORTH: "srb.", NORM: "srbsko"}, + {ORTH: "star.", NORM: "starinsko"}, + {ORTH: "stil.", NORM: "stilno"}, + {ORTH: "sv.", NORM: "svet(i)"}, + {ORTH: "teh.", NORM: "tehnika"}, + {ORTH: "tisk.", NORM: "tiskarstvo"}, + {ORTH: "tj.", NORM: "to je"}, + {ORTH: "tož.", NORM: "tožilnik"}, + {ORTH: "trg.", NORM: "trgovina"}, + {ORTH: "ukr.", NORM: "ukrajinski"}, + {ORTH: "um.", NORM: "umetnost"}, + {ORTH: "vel.", NORM: "velelnik"}, + {ORTH: "vet.", NORM: "veterina"}, + {ORTH: "vez.", NORM: "veznik"}, + {ORTH: "vn.", NORM: "visokonemško"}, + {ORTH: "voj.", NORM: "vojska"}, + {ORTH: "vrtn.", NORM: "vrtnarstvo"}, + {ORTH: "vulg.", NORM: "vulgarno"}, + {ORTH: "vznes.", NORM: "vzneseno"}, + {ORTH: "zal.", NORM: "založništvo"}, + {ORTH: "zastar.", NORM: "zastarelo"}, + {ORTH: "zgod.", NORM: "zgodovina"}, + {ORTH: "zool.", NORM: "zoologija"}, + {ORTH: "čeb.", NORM: "čebelarstvo"}, + {ORTH: "češ.", NORM: "češki"}, + {ORTH: "člov.", NORM: "človeškost"}, + {ORTH: "šah.", NORM: "šahovski"}, + {ORTH: "šalj.", NORM: "šaljivo"}, + {ORTH: "šp.", NORM: "španski"}, + {ORTH: "špan.", NORM: "špansko"}, + {ORTH: "šport.", NORM: "športni"}, + {ORTH: "štev.", NORM: "števnik"}, + {ORTH: "šved.", NORM: "švedsko"}, + {ORTH: "švic.", NORM: "švicarsko"}, + {ORTH: "ž.", NORM: "ženski"}, + {ORTH: "žarg.", NORM: "žargonsko"}, + {ORTH: "žel.", NORM: "železnica"}, + {ORTH: "živ.", NORM: "živost"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +abbrv = """ +Co. Ch. DIPL. DR. Dr. Ev. Inc. Jr. Kr. Mag. M. MR. Mr. Mt. Murr. Npr. OZ. +Opr. Osn. Prim. Roj. ST. Sim. Sp. Sred. St. Sv. Škofl. Tel. UR. Zb. +a. aa. ab. abc. abit. abl. abs. abt. acc. accel. add. adj. adv. aet. afr. akad. al. alban. all. alleg. +alp. alt. alter. alžir. am. an. andr. ang. anh. anon. ans. antrop. apoc. app. approx. apt. ar. arc. arch. +arh. arr. as. asist. assist. assoc. asst. astr. attn. aug. avstral. az. b. bab. bal. bbl. bd. belg. bioinf. +biomed. bk. bl. bn. borg. bp. br. braz. brit. bros. broš. bt. bu. c. ca. cal. can. cand. cantab. cap. capt. +cat. cath. cc. cca. cd. cdr. cdre. cent. cerkv. cert. cf. cfr. ch. chap. chem. chr. chs. cic. circ. civ. cl. +cm. cmd. cnr. co. cod. col. coll. colo. com. comp. con. conc. cond. conn. cons. cont. coop. corr. cost. cp. +cpl. cr. crd. cres. cresc. ct. cu. d. dan. dat. davč. ddr. dec. ded. def. dem. dent. dept. dia. dip. dipl. +dir. disp. diss. div. do. doc. dok. dol. doo. dop. dott. dr. dram. druž. družb. drž. dt. duh. dur. dvr. dwt. e. +ea. ecc. eccl. eccles. econ. edn. egipt. egr. ekon. eksp. el. em. enc. eng. eo. ep. err. esp. esq. est. +et. etc. etnogr. etnol. ev. evfem. evr. ex. exc. excl. exp. expl. ext. exx. f. fa. facs. fak. faks. fas. +fasc. fco. fcp. feb. febr. fec. fed. fem. ff. fff. fid. fig. fil. film. fiziol. fiziot. flam. fm. fo. fol. folk. +frag. fran. franc. fsc. g. ga. gal. gdč. ge. gen. geod. geog. geotehnol. gg. gimn. glas. glav. gnr. go. gor. +gosp. gp. graf. gram. gren. grš. gs. h. hab. hf. hist. ho. hort. i. ia. ib. ibid. id. idr. idridr. ill. imen. +imp. impf. impr. in. inc. incl. ind. indus. inf. inform. ing. init. ins. int. inv. inšp. inštr. inž. is. islam. +ist. ital. iur. iz. izbr. izd. izg. izgr. izr. izv. j. jak. jam. jan. jav. je. jez. jr. jsl. jud. jug. +jugoslovan. jur. juž. jv. jz. k. kal. kan. kand. kat. kdo. kem. kip. kmet. kol. kom. komp. konf. kont. kost. kov. +kp. kpfw. kr. kraj. krat. kub. kult. kv. kval. l. la. lab. lb. ld. let. lib. lik. litt. lj. ljud. ll. loc. log. +loč. lt. ma. madž. mag. manag. manjš. masc. mass. mater. max. maxmax. mb. md. mech. medic. medij. medn. +mehč. mem. menedž. mes. mess. metal. meteor. meteorol. mex. mi. mikr. mil. minn. mio. misc. miss. mit. mk. +mkt. ml. mlad. mlle. mlr. mm. mme. množ. mo. moj. moš. možn. mr. mrd. mrs. ms. msc. msgr. mt. murr. mus. mut. +n. na. nad. nadalj. nadom. nagl. nakl. namer. nan. naniz. nasl. nat. navt. nač. ned. nem. nik. nizoz. nm. nn. +no. nom. norv. notr. nov. novogr. ns. o. ob. obd. obj. oblač. obl. oblik. obr. obraz. obs. obst. obt. obč. oc. +oct. od. odd. odg. odn. odst. odv. oec. off. ok. okla. okr. ont. oo. op. opis. opp. opr. orch. ord. ore. oreg. +org. orient. orig. ork. ort. oseb. osn. ot. ozir. ošk. p. pag. par. para. parc. parl. part. past. pat. pdk. +pen. perf. pert. perz. pesn. pet. pev. pf. pfc. ph. pharm. phil. pis. pl. po. pod. podr. podaljš. pogl. pogoj. pojm. +pok. pokr. pol. poljed. poljub. polu. pom. pomen. pon. ponov. pop. por. port. pos. posl. posn. pov. pp. ppl. pr. +praet. prav. pravopis. pravosl. preb. pred. predl. predm. predp. preds. pref. pregib. prel. prem. premen. prep. +pres. pret. prev. pribl. prih. pril. primerj. primor. prip. pripor. prir. prist. priv. proc. prof. prog. proiz. +prom. pron. prop. prot. protest. prov. ps. pss. pt. publ. pz. q. qld. qu. quad. que. r. racc. rastl. razgl. +razl. razv. rd. red. ref. reg. rel. relig. rep. repr. rer. resp. rest. ret. rev. revol. rež. rim. rist. rkp. rm. +roj. rom. romun. rp. rr. rt. rud. ruš. ry. sal. samogl. san. sc. scen. sci. scr. sdv. seg. sek. sen. sept. ser. +sev. sg. sgt. sh. sig. sigg. sign. sim. sin. sing. sinh. skand. skl. sklad. sklanj. sklep. skr. sl. slik. slov. +slovak. slovn. sn. so. sob. soc. sociol. sod. sopomen. sopr. sor. sov. sovj. sp. spec. spl. spr. spreg. sq. sr. +sre. sred. sredoz. srh. ss. ssp. st. sta. stan. stanstar. stcsl. ste. stim. stol. stom. str. stroj. strok. stsl. +stud. sup. supl. suppl. svet. sz. t. tab. tech. ted. tehn. tehnol. tek. teks. tekst. tel. temp. ten. teol. ter. +term. test. th. theol. tim. tip. tisočl. tit. tl. tol. tolmač. tom. tor. tov. tr. trad. traj. trans. tren. +trib. tril. trop. trp. trž. ts. tt. tu. tur. turiz. tvor. tvorb. tč. u. ul. umet. un. univ. up. upr. ur. urad. +us. ust. utr. v. va. val. var. varn. ven. ver. verb. vest. vezal. vic. vis. viv. viz. viš. vod. vok. vol. vpr. +vrst. vrstil. vs. vv. vzd. vzg. vzh. vzor. w. wed. wg. wk. x. y. z. zah. zaim. zak. zap. zasl. zavar. zač. zb. +združ. zg. zn. znan. znanstv. zoot. zun. zv. zvd. á. é. ć. č. čas. čet. čl. člen. čustv. đ. ľ. ł. ş. ŠT. š. šir. +škofl. škot. šol. št. števil. štud. ů. ű. žen. žival. +""".split() + +for orth in abbrv: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/sq/__init__.py b/spacy/lang/sq/__init__.py new file mode 100644 index 0000000..1c8a5ac --- /dev/null +++ b/spacy/lang/sq/__init__.py @@ -0,0 +1,14 @@ +from ...language import BaseDefaults, Language +from .stop_words import STOP_WORDS + + +class AlbanianDefaults(BaseDefaults): + stop_words = STOP_WORDS + + +class Albanian(Language): + lang = "sq" + Defaults = AlbanianDefaults + + +__all__ = ["Albanian"] diff --git a/spacy/lang/sq/examples.py b/spacy/lang/sq/examples.py new file mode 100644 index 0000000..61bf713 --- /dev/null +++ b/spacy/lang/sq/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sq.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple po shqyrton blerjen e nje shoqërie të U.K. për 1 miliard dollarë", + "Makinat autonome ndryshojnë përgjegjësinë e sigurimit ndaj prodhuesve", + "San Francisko konsideron ndalimin e robotëve të shpërndarjes", + "Londra është një qytet i madh në Mbretërinë e Bashkuar.", +] diff --git a/spacy/lang/sq/stop_words.py b/spacy/lang/sq/stop_words.py new file mode 100644 index 0000000..f2b1a4f --- /dev/null +++ b/spacy/lang/sq/stop_words.py @@ -0,0 +1,229 @@ +# Source: https://github.com/andrixh/index-albanian + +STOP_WORDS = set( + """ +a +afert +ai +ajo +andej +anes +aq +as +asaj +ashtu +ata +ate +atij +atje +ato +aty +atyre +b +be +behem +behet +bej +beje +bejne +ben +bene +bere +beri +bie +c +ca +cdo +cfare +cila +cilat +cilave +cilen +ciles +cilet +cili +cilin +cilit +deri +dhe +dic +dicka +dickaje +dike +dikujt +dikush +disa +do +dot +drejt +duke +dy +e +edhe +ende +eshte +etj +fare +gjate +gje +gjitha +gjithcka +gjithe +gjithnje +here +i +ia +ishin +ishte +iu +ja +jam +jane +jap +je +jemi +jo +ju +k +ka +kam +kane +kem +kemi +keq +kesaj +keshtu +kete +ketej +ketij +keto +ketu +ketyre +kishin +kishte +kjo +krejt +kryer +kryesisht +kryhet +ku +kudo +kundrejt +kur +kurre +kush +ky +la +le +lloj +m +ma +madhe +marr +marre +mban +mbi +me +menjehere +merr +merret +mes +mi +midis +mire +mjaft +mori +mos +mua +mund +na +ndaj +nder +ndermjet +ndersa +ndonje +ndryshe +ne +nen +neper +nepermjet +nese +nga +nje +njera +nuk +ose +pa +pak +papritur +para +pas +pasi +pasur +per +perbashket +perpara +po +por +prane +prapa +prej +pse +qe +qene +qenet +rralle +rreth +rri +s +sa +saj +sapo +se +secila +sepse +sh +shih +shume +si +sic +sikur +sipas +siper +sone +t +ta +tani +te +tej +tek +teper +tere +ti +tij +tilla +tille +tjera +tjeret +tjeter +tjetren +to +tone +ty +tyre +u +ua +une +vazhdimisht +vend +vet +veta +vete +vetem +veten +vetes +vjen +yne +zakonisht +""".split() +) diff --git a/spacy/lang/sr/__init__.py b/spacy/lang/sr/__init__.py new file mode 100644 index 0000000..5f121d7 --- /dev/null +++ b/spacy/lang/sr/__init__.py @@ -0,0 +1,21 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class SerbianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Serbian(Language): + lang = "sr" + Defaults = SerbianDefaults + + +__all__ = ["Serbian"] diff --git a/spacy/lang/sr/examples.py b/spacy/lang/sr/examples.py new file mode 100644 index 0000000..2d34d42 --- /dev/null +++ b/spacy/lang/sr/examples.py @@ -0,0 +1,20 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sr.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + # Translations from English + "Apple планира куповину америчког стартапа за $1 милијарду.", + "Беспилотни аутомобили пребацују одговорност осигурања на произвођаче.", + "Лондон је велики град у Уједињеном Краљевству.", + "Где си ти?", + "Ко је председник Француске?", + # Serbian common and slang + "Moj ћале је инжењер!", + "Новак Ђоковић је најбољи тенисер света.", + "У Пироту има добрих кафана!", + "Музеј Николе Тесле се налази у Београду.", +] diff --git a/spacy/lang/sr/lemma_lookup_licence.txt b/spacy/lang/sr/lemma_lookup_licence.txt new file mode 100644 index 0000000..c2d92a0 --- /dev/null +++ b/spacy/lang/sr/lemma_lookup_licence.txt @@ -0,0 +1,32 @@ +Copyright @InProceedings{ljubesic16-new, + author = {Nikola Ljubešić and Filip Klubička and Željko Agić and Ivo-Pavao Jazbec}, + title = {New Inflectional Lexicons and Training Corpora for Improved Morphosyntactic Annotation of Croatian and Serbian}, + booktitle = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC 2016)}, + year = {2016}, + date = {23-28}, + location = {Portorož, Slovenia}, + editor = {Nicoletta Calzolari (Conference Chair) and Khalid Choukri and Thierry Declerck and Sara Goggi and Marko Grobelnik and Bente Maegaard and Joseph Mariani and Helene Mazo and Asuncion Moreno and Jan Odijk and Stelios Piperidis}, + publisher = {European Language Resources Association (ELRA)}, + address = {Paris, France}, + isbn = {978-2-9517408-9-1} + } + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +The licence of Serbian lemmas was adopted from Serbian lexicon: + - sr.lexicon (https://github.com/clarinsi/reldi-tagger/blob/master/sr.lexicon) + +Changelog: + - Lexicon is translated into cyrilic + - Word order is sorted \ No newline at end of file diff --git a/spacy/lang/sr/lex_attrs.py b/spacy/lang/sr/lex_attrs.py new file mode 100644 index 0000000..696b9fd --- /dev/null +++ b/spacy/lang/sr/lex_attrs.py @@ -0,0 +1,65 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "нула", + "један", + "два", + "три", + "четири", + "пет", + "шест", + "седам", + "осам", + "девет", + "десет", + "једанаест", + "дванаест", + "тринаест", + "четрнаест", + "петнаест", + "шеснаест", + "седамнаест", + "осамнаест", + "деветнаест", + "двадесет", + "тридесет", + "четрдесет", + "педесет", + "шездесет", + "седамдесет", + "осамдесет", + "деведесет", + "сто", + "двеста", + "триста", + "четиристо", + "петсто", + "шестсто", + "седамсто", + "осамсто", + "деветсто", + "хиљаду", + "милион", + "милијарду", + "трилион", + "квадрилион", + "квинтилион", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/sr/punctuation.py b/spacy/lang/sr/punctuation.py new file mode 100644 index 0000000..cafb0f6 --- /dev/null +++ b/spacy/lang/sr/punctuation.py @@ -0,0 +1,45 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + CURRENCY, + LIST_ELLIPSES, + LIST_ICONS, + LIST_PUNCT, + LIST_QUOTES, + PUNCT, + UNITS, +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=[0-9])\+", + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{a}{e}{p}(?:{q})])\.".format( + a=ALPHA, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + ] +) + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/sr/stop_words.py b/spacy/lang/sr/stop_words.py new file mode 100644 index 0000000..5df5509 --- /dev/null +++ b/spacy/lang/sr/stop_words.py @@ -0,0 +1,393 @@ +STOP_WORDS = set( + """ +а +авај +ако +ал +али +арх +ау +ах +аха +ај +бар +би +била +били +било +бисмо +бисте +бих +бијасмо +бијасте +бијах +бијаху +бијаше +биће +близу +број +брр +буде +будимо +будите +буду +будући +бум +бућ +вам +вама +вас +ваша +ваше +вашим +вашима +ваљда +веома +вероватно +већ +већина +ви +видео +више +врло +врх +га +где +гиц +год +горе +гђекоје +да +дакле +дана +данас +дај +два +де +дедер +делимице +делимично +дем +до +добар +добити +довечер +докле +доле +донекле +досад +доскоро +дотад +дотле +дошао +доћи +другамо +другде +други +е +ево +ено +ето +ех +ехе +еј +желела +желеле +желели +желело +желех +желећи +жели +за +заиста +зар +затим +зато +захвалити +зашто +збиља +зимус +знати +зум +и +иде +из +изван +изволи +између +изнад +икада +икакав +икаква +икакве +икакви +икаквим +икаквима +икаквих +икакво +икаквог +икаквога +икаквом +икаквоме +икаквој +или +им +има +имам +имао +испод +их +ију +ићи +кад +када +кога +којекакав +којима +коју +кришом +лани +ли +мали +мањи +ме +мене +мени +ми +мимо +мисли +много +могу +мора +морао +мој +моја +моје +моји +моју +моћи +му +на +над +након +нам +нама +нас +наша +наше +нашег +наши +наћи +не +негде +нека +некад +неке +неког +неку +нема +немам +неко +неће +нећемо +нећете +нећеш +нећу +ни +никада +никога +никоје +никоји +никоју +нисам +ниси +нисте +нису +ништа +ниједан +но +о +ова +овако +овамо +овај +овде +ове +овим +овима +ово +овој +од +одмах +око +около +он +онај +оне +оним +онима +оном +оној +ону +осим +остали +отишао +па +пак +питати +по +поводом +под +подаље +пожељан +пожељна +поиздаље +поименце +понекад +попреко +поред +после +потаман +потрбушке +поуздано +почетак +поједини +правити +први +преко +према +прије +пут +пљус +радије +с +са +сав +сада +сам +само +сасвим +сва +сваки +сви +свим +свог +свом +свој +своја +своје +своју +сву +свугде +се +себе +себи +си +смети +смо +ствар +стварно +сте +су +сутра +та +таèно +тако +такође +тамо +твој +твоја +твоје +твоји +твоју +те +тебе +теби +ти +тима +то +томе +тој +ту +у +увек +увијек +уз +уза +узалуд +уздуж +узети +умало +унутра +употребити +упркос +учинио +учинити +хало +хвала +хеј +хм +хоп +хоће +хоћемо +хоћете +хоћеш +хоћу +хтедосте +хтедох +хтедоше +хтела +хтеле +хтели +хтео +хтејасмо +хтејасте +хтејаху +хура +често +чијем +чији +чијим +чијима +шиц +штагод +што +штогод +ја +је +један +једини +једна +једне +једни +једно +једном +јер +јесам +јеси +јесмо +јесу +јим +јој +ју +јуче +његова +његово +њезин +њезина +њезино +њему +њен +њим +њима +њихова +њихово +њој +њу +ће +ћемо +ћете +ћеш +ћу +""".split() +) diff --git a/spacy/lang/sr/tokenizer_exceptions.py b/spacy/lang/sr/tokenizer_exceptions.py new file mode 100755 index 0000000..b7db0aa --- /dev/null +++ b/spacy/lang/sr/tokenizer_exceptions.py @@ -0,0 +1,92 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +_abbrev_exc = [ + # Weekdays abbreviations + {ORTH: "пoн", NORM: "понедељак"}, + {ORTH: "уто", NORM: "уторак"}, + {ORTH: "сре", NORM: "среда"}, + {ORTH: "чет", NORM: "четвртак"}, + {ORTH: "пет", NORM: "петак"}, + {ORTH: "суб", NORM: "субота"}, + {ORTH: "нед", NORM: "недеља"}, + # Months abbreviations + {ORTH: "јан", NORM: "јануар"}, + {ORTH: "феб", NORM: "фебруар"}, + {ORTH: "мар", NORM: "март"}, + {ORTH: "апр", NORM: "април"}, + {ORTH: "јуни", NORM: "јун"}, + {ORTH: "јули", NORM: "јул"}, + {ORTH: "авг", NORM: "август"}, + {ORTH: "сеп", NORM: "септембар"}, + {ORTH: "септ", NORM: "септембар"}, + {ORTH: "окт", NORM: "октобар"}, + {ORTH: "нов", NORM: "новембар"}, + {ORTH: "дец", NORM: "децембар"}, +] + + +for abbrev_desc in _abbrev_exc: + abbrev = abbrev_desc[ORTH] + for orth in (abbrev, abbrev.capitalize(), abbrev.upper()): + _exc[orth] = [{ORTH: orth, NORM: abbrev_desc[NORM]}] + _exc[orth + "."] = [{ORTH: orth + ".", NORM: abbrev_desc[NORM]}] + + +# common abbreviations +_slang_exc = [ + # without dot + {ORTH: "др", NORM: "доктор"}, + {ORTH: "гдин", NORM: "господин"}, + {ORTH: "гђа", NORM: "госпођа"}, + {ORTH: "гђица", NORM: "госпођица"}, + {ORTH: "мр", NORM: "магистар"}, + {ORTH: "Бгд", NORM: "београд"}, + {ORTH: "цм", NORM: "центиметар"}, + {ORTH: "м", NORM: "метар"}, + {ORTH: "км", NORM: "километар"}, + {ORTH: "мг", NORM: "милиграм"}, + {ORTH: "кг", NORM: "килограм"}, + {ORTH: "дл", NORM: "децилитар"}, + {ORTH: "хл", NORM: "хектолитар"}, + # with dot + {ORTH: "ул.", NORM: "улица"}, + {ORTH: "бр.", NORM: "број"}, + {ORTH: "нпр.", NORM: "на пример"}, + {ORTH: "тзв.", NORM: "такозван"}, + {ORTH: "проф.", NORM: "професор"}, + {ORTH: "стр.", NORM: "страна"}, + {ORTH: "једн.", NORM: "једнина"}, + {ORTH: "мн.", NORM: "множина"}, + {ORTH: "уч.", NORM: "ученик"}, + {ORTH: "разр.", NORM: "разред"}, + {ORTH: "инж.", NORM: "инжењер"}, + {ORTH: "гимн.", NORM: "гимназија"}, + {ORTH: "год.", NORM: "година"}, + {ORTH: "мед.", NORM: "медицина"}, + {ORTH: "гимн.", NORM: "гимназија"}, + {ORTH: "акад.", NORM: "академик"}, + {ORTH: "доц.", NORM: "доцент"}, + {ORTH: "итд.", NORM: "и тако даље"}, + {ORTH: "и сл.", NORM: "и слично"}, + {ORTH: "н.е.", NORM: "нове ере"}, + {ORTH: "о.г.", NORM: "ове године"}, + {ORTH: "л.к.", NORM: "лична карта"}, + {ORTH: "в.д.", NORM: "вршилац дужности"}, + {ORTH: "стр.", NORM: "страна"}, + # with qoute + {ORTH: "ал'", NORM: "али"}, + {ORTH: "ил'", NORM: "или"}, + {ORTH: "је л'", NORM: "је ли"}, + {ORTH: "да л'", NORM: "да ли"}, + {ORTH: "држ'те", NORM: "држите"}, +] + +for slang_desc in _slang_exc: + _exc[slang_desc[ORTH]] = [slang_desc] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/sv/__init__.py b/spacy/lang/sv/__init__.py new file mode 100644 index 0000000..bb4ee17 --- /dev/null +++ b/spacy/lang/sv/__init__.py @@ -0,0 +1,52 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ...pipeline import Lemmatizer +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class SwedishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + syntax_iterators = SYNTAX_ITERATORS + stop_words = STOP_WORDS + + +class Swedish(Language): + lang = "sv" + Defaults = SwedishDefaults + + +@Swedish.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "rule", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return Lemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Swedish"] diff --git a/spacy/lang/sv/examples.py b/spacy/lang/sv/examples.py new file mode 100644 index 0000000..ffea6e4 --- /dev/null +++ b/spacy/lang/sv/examples.py @@ -0,0 +1,13 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.sv.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple överväger att köpa brittisk startup för 1 miljard dollar.", + "Självkörande bilar förskjuter försäkringsansvar mot tillverkare.", + "San Fransisco överväger förbud mot leveransrobotar på trottoarer.", + "London är en storstad i Storbritannien.", +] diff --git a/spacy/lang/sv/lex_attrs.py b/spacy/lang/sv/lex_attrs.py new file mode 100644 index 0000000..8eeafed --- /dev/null +++ b/spacy/lang/sv/lex_attrs.py @@ -0,0 +1,58 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "noll", + "en", + "ett", + "två", + "tre", + "fyra", + "fem", + "sex", + "sju", + "åtta", + "nio", + "tio", + "elva", + "tolv", + "tretton", + "fjorton", + "femton", + "sexton", + "sjutton", + "arton", + "nitton", + "tjugo", + "trettio", + "fyrtio", + "femtio", + "sextio", + "sjuttio", + "åttio", + "nittio", + "hundra", + "tusen", + "miljon", + "miljard", + "biljon", + "biljard", + "kvadriljon", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/sv/punctuation.py b/spacy/lang/sv/punctuation.py new file mode 100644 index 0000000..64f1da9 --- /dev/null +++ b/spacy/lang/sv/punctuation.py @@ -0,0 +1,38 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + LIST_ELLIPSES, + LIST_ICONS, +) +from ..punctuation import TOKENIZER_SUFFIXES + +_quotes = CONCAT_QUOTES.replace("'", "") + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])[<>=](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]):(?=[{a}])".format(a=ALPHA_UPPER), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[{a}])".format(a=ALPHA, q=_quotes), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])[<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9]):(?=[{a}])".format(a=ALPHA_UPPER), + ] +) + +_suffixes = [ + suffix + for suffix in TOKENIZER_SUFFIXES + if suffix not in ["'s", "'S", "’s", "’S", r"\'"] +] +_suffixes += [r"(?<=[^sSxXzZ])\'"] + + +TOKENIZER_INFIXES = _infixes +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/sv/stop_words.py b/spacy/lang/sv/stop_words.py new file mode 100644 index 0000000..2422b2a --- /dev/null +++ b/spacy/lang/sv/stop_words.py @@ -0,0 +1,66 @@ +STOP_WORDS = set( + """ +aderton adertonde adjö aldrig alla allas allt alltid alltså än andra andras +annan annat ännu artonde arton åtminstone att åtta åttio åttionde åttonde av +även + +båda bådas bakom bara bäst bättre behöva behövas behövde behövt beslut beslutat +beslutit bland blev bli blir blivit bort borta bra + +då dag dagar dagarna dagen där därför de del delen dem den deras dess det detta +dig din dina dit ditt dock du + +efter eftersom elfte eller elva en enkel enkelt enkla enligt er era ert ett +ettusen + +få fanns får fått fem femte femtio femtionde femton femtonde fick fin finnas +finns fjärde fjorton fjortonde fler flera flesta följande för före förlåt förra +första fram framför från fyra fyrtio fyrtionde + +gå gälla gäller gällt går gärna gått genast genom gick gjorde gjort god goda +godare godast gör göra gott + +ha hade haft han hans har här heller hellre helst helt henne hennes hit hög +höger högre högst hon honom hundra hundraen hundraett hur + +i ibland idag igår igen imorgon in inför inga ingen ingenting inget innan inne +inom inte inuti + +ja jag jämfört + +kan kanske knappast kom komma kommer kommit kr kunde kunna kunnat kvar + +länge längre långsam långsammare långsammast långsamt längst långt lätt lättare +lättast legat ligga ligger lika likställd likställda lilla lite liten litet + +man många måste med mellan men mer mera mest mig min mina mindre minst mitt +mittemot möjlig möjligen möjligt möjligtvis mot mycket + +någon någonting något några när nästa ned nederst nedersta nedre nej ner ni nio +nionde nittio nittionde nitton nittonde nödvändig nödvändiga nödvändigt +nödvändigtvis nog noll nr nu nummer + +och också ofta oftast olika olikt om oss + +över övermorgon överst övre + +på + +rakt rätt redan + +så sade säga säger sagt samma sämre sämst sedan senare senast sent sex sextio +sextionde sexton sextonde sig sin sina sist sista siste sitt sjätte sju sjunde +sjuttio sjuttionde sjutton sjuttonde ska skall skulle slutligen små smått snart +som stor stora större störst stort + +tack tidig tidigare tidigast tidigt till tills tillsammans tio tionde tjugo +tjugoen tjugoett tjugonde tjugotre tjugotvå tjungo tolfte tolv tre tredje +trettio trettionde tretton trettonde två tvåhundra + +under upp ur ursäkt ut utan utanför ute + +vad vänster vänstra var vår vara våra varför varifrån varit varken värre +varsågod vart vårt vem vems verkligen vi vid vidare viktig viktigare viktigast +viktigt vilka vilken vilket vill +""".split() +) diff --git a/spacy/lang/sv/syntax_iterators.py b/spacy/lang/sv/syntax_iterators.py new file mode 100644 index 0000000..09153a8 --- /dev/null +++ b/spacy/lang/sv/syntax_iterators.py @@ -0,0 +1,39 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """Detect base noun phrases from a dependency parse. Works on Doc and Span.""" + # fmt: off + labels = ["nsubj", "nsubj:pass", "dobj", "obj", "iobj", "ROOT", "appos", "nmod", "nmod:poss"] + # fmt: on + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add("conj") + np_label = doc.vocab.strings.add("NP") + prev_end = -1 + for i, word in enumerate(doclike): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.left_edge.i <= prev_end: + continue + if word.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + prev_end = word.right_edge.i + yield word.left_edge.i, word.right_edge.i + 1, np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/sv/tokenizer_exceptions.py b/spacy/lang/sv/tokenizer_exceptions.py new file mode 100644 index 0000000..8fd3afb --- /dev/null +++ b/spacy/lang/sv/tokenizer_exceptions.py @@ -0,0 +1,156 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + + +# Verbs + +for verb_data in [ + {ORTH: "driver"}, + {ORTH: "kör"}, + {ORTH: "hörr"}, + {ORTH: "fattar"}, + {ORTH: "hajar"}, + {ORTH: "lever"}, + {ORTH: "serr"}, + {ORTH: "fixar"}, +]: + verb_data_tc = dict(verb_data) + verb_data_tc[ORTH] = verb_data_tc[ORTH].title() + for data in [verb_data, verb_data_tc]: + _exc[data[ORTH] + "u"] = [data, {ORTH: "u", NORM: "du"}] + +# Abbreviations for weekdays "sön." (for "söndag" / "söner") +# are left out because they are ambiguous. The same is the case +# for abbreviations "jul." and "Jul." ("juli" / "jul"). +for exc_data in [ + {ORTH: "jan.", NORM: "januari"}, + {ORTH: "febr.", NORM: "februari"}, + {ORTH: "feb.", NORM: "februari"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "jun.", NORM: "juni"}, + {ORTH: "aug.", NORM: "augusti"}, + {ORTH: "sept.", NORM: "september"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "nov.", NORM: "november"}, + {ORTH: "dec.", NORM: "december"}, + {ORTH: "mån.", NORM: "måndag"}, + {ORTH: "tis.", NORM: "tisdag"}, + {ORTH: "ons.", NORM: "onsdag"}, + {ORTH: "tors.", NORM: "torsdag"}, + {ORTH: "fre.", NORM: "fredag"}, + {ORTH: "lör.", NORM: "lördag"}, + {ORTH: "Jan.", NORM: "Januari"}, + {ORTH: "Febr.", NORM: "Februari"}, + {ORTH: "Feb.", NORM: "Februari"}, + {ORTH: "Apr.", NORM: "April"}, + {ORTH: "Jun.", NORM: "Juni"}, + {ORTH: "Aug.", NORM: "Augusti"}, + {ORTH: "Sept.", NORM: "September"}, + {ORTH: "Sep.", NORM: "September"}, + {ORTH: "Okt.", NORM: "Oktober"}, + {ORTH: "Nov.", NORM: "November"}, + {ORTH: "Dec.", NORM: "December"}, + {ORTH: "Mån.", NORM: "Måndag"}, + {ORTH: "Tis.", NORM: "Tisdag"}, + {ORTH: "Ons.", NORM: "Onsdag"}, + {ORTH: "Tors.", NORM: "Torsdag"}, + {ORTH: "Fre.", NORM: "Fredag"}, + {ORTH: "Lör.", NORM: "Lördag"}, + {ORTH: "sthlm", NORM: "Stockholm"}, + {ORTH: "gbg", NORM: "Göteborg"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +# Specific case abbreviations only +for orth in ["AB", "Dr.", "H.M.", "H.K.H.", "m/s", "M/S", "Ph.d.", "S:t", "s:t"]: + _exc[orth] = [{ORTH: orth}] + + +ABBREVIATIONS = [ + "ang", + "anm", + "bl.a", + "d.v.s", + "doc", + "dvs", + "e.d", + "e.kr", + "el.", + "eng", + "etc", + "exkl", + "ev", + "f.", + "f.d", + "f.kr", + "f.n", + "f.ö", + "fid", + "fig", + "forts", + "fr.o.m", + "förf", + "inkl", + "iofs", + "jur.", + "kap", + "kl", + "kor.", + "kr", + "kungl", + "lat", + "m.a.o", + "m.fl", + "m.m", + "max", + "milj", + "min.", + "mos", + "mt", + "mvh", + "o.d", + "o.s.v", + "obs", + "osv", + "p.g.a", + "proc", + "prof", + "ref", + "resp", + "s.a.s", + "s.k", + "s.t", + "sid", + "t.ex", + "t.h", + "t.o.m", + "t.v", + "tel", + "ung.", + "vol", + "v.", + "äv", + "övers", +] + +# Add abbreviation for trailing punctuation too. If the abbreviation already has a trailing punctuation - skip it. +for abbr in ABBREVIATIONS: + if not abbr.endswith("."): + ABBREVIATIONS.append(abbr + ".") + +for orth in ABBREVIATIONS: + _exc[orth] = [{ORTH: orth}] + capitalized = orth.capitalize() + _exc[capitalized] = [{ORTH: capitalized}] + +# Sentences ending in "i." (as in "... peka i."), "m." (as in "...än 2000 m."), +# should be tokenized as two separate tokens. +for orth in ["i", "m"]: + _exc[orth + "."] = [{ORTH: orth, NORM: orth}, {ORTH: "."}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ta/__init__.py b/spacy/lang/ta/__init__.py new file mode 100644 index 0000000..7fd2937 --- /dev/null +++ b/spacy/lang/ta/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class TamilDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Tamil(Language): + lang = "ta" + Defaults = TamilDefaults + + +__all__ = ["Tamil"] diff --git a/spacy/lang/ta/examples.py b/spacy/lang/ta/examples.py new file mode 100644 index 0000000..522cd92 --- /dev/null +++ b/spacy/lang/ta/examples.py @@ -0,0 +1,23 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ta.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "கிறிஸ்துமஸ் மற்றும் இனிய புத்தாண்டு வாழ்த்துக்கள்", + "எனக்கு என் குழந்தைப் பருவம் நினைவிருக்கிறது", + "உங்கள் பெயர் என்ன?", + "ஏறத்தாழ இலங்கைத் தமிழரில் மூன்றிலொரு பங்கினர் இலங்கையை விட்டு வெளியேறிப் பிற நாடுகளில் வாழ்கின்றனர்", + "இந்த ஃபோனுடன் சுமார் ரூ.2,990 மதிப்புள்ள போட் ராக்கர்ஸ் நிறுவனத்தின் ஸ்போர்ட் புளூடூத் ஹெட்போன்ஸ் இலவசமாக வழங்கப்படவுள்ளது.", + "மட்டக்களப்பில் பல இடங்களில் வீட்டுத் திட்டங்களுக்கு இன்று அடிக்கல் நாட்டல்", + "ஐ போன்க்கு முகத்தை வைத்து அன்லாக் செய்யும் முறை மற்றும் விரலால் தொட்டு அன்லாக் செய்யும் முறையை வாட்ஸ் ஆப் நிறுவனம் இதற்கு முன் கண்டுபிடித்தது", + "இது ஒரு வாக்கியம்.", + "ஆப்பிள் நிறுவனம் யு.கே. தொடக்க நிறுவனத்தை ஒரு லட்சம் கோடிக்கு வாங்கப் பார்க்கிறது", + "தன்னாட்சி கார்கள் காப்பீட்டு பொறுப்பை உற்பத்தியாளரிடம் மாற்றுகின்றன", + "நடைபாதை விநியோக ரோபோக்களை தடை செய்வதை சான் பிரான்சிஸ்கோ கருதுகிறது", + "லண்டன் ஐக்கிய இராச்சியத்தில் ஒரு பெரிய நகரம்.", + "என்ன வேலை செய்கிறீர்கள்?", + "எந்த கல்லூரியில் படிக்கிறாய்?", +] diff --git a/spacy/lang/ta/lex_attrs.py b/spacy/lang/ta/lex_attrs.py new file mode 100644 index 0000000..d661255 --- /dev/null +++ b/spacy/lang/ta/lex_attrs.py @@ -0,0 +1,80 @@ +from ...attrs import LIKE_NUM + +_numeral_suffixes = {"பத்து": "பது", "ற்று": "று", "ரத்து": "ரம்", "சத்து": "சம்"} +_num_words = [ + "பூச்சியம்", + "ஒரு", + "ஒன்று", + "இரண்டு", + "மூன்று", + "நான்கு", + "ஐந்து", + "ஆறு", + "ஏழு", + "எட்டு", + "ஒன்பது", + "பத்து", + "பதினொன்று", + "பன்னிரண்டு", + "பதின்மூன்று", + "பதினான்கு", + "பதினைந்து", + "பதினாறு", + "பதினேழு", + "பதினெட்டு", + "பத்தொன்பது", + "இருபது", + "முப்பது", + "நாற்பது", + "ஐம்பது", + "அறுபது", + "எழுபது", + "எண்பது", + "தொண்ணூறு", + "நூறு", + "இருநூறு", + "முன்னூறு", + "நாநூறு", + "ஐநூறு", + "அறுநூறு", + "எழுநூறு", + "எண்ணூறு", + "தொள்ளாயிரம்", + "ஆயிரம்", + "ஒராயிரம்", + "லட்சம்", + "மில்லியன்", + "கோடி", + "பில்லியன்", + "டிரில்லியன்", +] + + +# 20-89 ,90-899,900-99999 and above have different suffixes +def suffix_filter(text): + # text without numeral suffixes + for num_suffix in _numeral_suffixes.keys(): + length = len(num_suffix) + if len(text) < length: + break + elif text.endswith(num_suffix): + return text[:-length] + _numeral_suffixes[num_suffix] + return text + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + elif suffix_filter(text) in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ta/stop_words.py b/spacy/lang/ta/stop_words.py new file mode 100644 index 0000000..abbff94 --- /dev/null +++ b/spacy/lang/ta/stop_words.py @@ -0,0 +1,131 @@ +# Stop words + +STOP_WORDS = set( + """ +ஒரு +என்று +மற்றும் +இந்த +இது +என்ற +கொண்டு +என்பது +பல +ஆகும் +அல்லது +அவர் +நான் +உள்ள +அந்த +இவர் +என +முதல் +என்ன +இருந்து +சில +என் +போன்ற +வேண்டும் +வந்து +இதன் +அது +அவன் +தான் +பலரும் +என்னும் +மேலும் +பின்னர் +கொண்ட +இருக்கும் +தனது +உள்ளது +போது +என்றும் +அதன் +தன் +பிறகு +அவர்கள் +வரை +அவள் +நீ +ஆகிய +இருந்தது +உள்ளன +வந்த +இருந்த +மிகவும் +இங்கு +மீது +ஓர் +இவை +இந்தக் +பற்றி +வரும் +வேறு +இரு +இதில் +போல் +இப்போது +அவரது +மட்டும் +இந்தப் +எனும் +மேல் +பின் +சேர்ந்த +ஆகியோர் +எனக்கு +இன்னும் +அந்தப் +அன்று +ஒரே +மிக +அங்கு +பல்வேறு +விட்டு +பெரும் +அதை +பற்றிய +உன் +அதிக +அந்தக் +பேர் +இதனால் +அவை +அதே +ஏன் +முறை +யார் +என்பதை +எல்லாம் +மட்டுமே +இங்கே +அங்கே +இடம் +இடத்தில் +அதில் +நாம் +அதற்கு +எனவே +பிற +சிறு +மற்ற +விட +எந்த +எனவும் +எனப்படும் +எனினும் +அடுத்த +இதனை +இதை +கொள்ள +இந்தத் +இதற்கு +அதனால் +தவிர +போல +வரையில் +சற்று +எனக் +""".split() +) diff --git a/spacy/lang/te/__init__.py b/spacy/lang/te/__init__.py new file mode 100644 index 0000000..611e974 --- /dev/null +++ b/spacy/lang/te/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class TeluguDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Telugu(Language): + lang = "te" + Defaults = TeluguDefaults + + +__all__ = ["Telugu"] diff --git a/spacy/lang/te/examples.py b/spacy/lang/te/examples.py new file mode 100644 index 0000000..4af8728 --- /dev/null +++ b/spacy/lang/te/examples.py @@ -0,0 +1,19 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.te import Telugu +>>> nlp = Telugu() +>>> from spacy.lang.te.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "ఆపిల్ 1 బిలియన్ డాలర్స్ కి యూ.కె. స్టార్ట్అప్ ని కొనాలని అనుకుంటుంది.", + "ఆటోనోమోస్ కార్లు భీమా బాధ్యతను తయారీదారులపైకి మళ్లిస్తాయి.", + "సాన్ ఫ్రాన్సిస్కో కాలిబాట డెలివరీ రోబోట్లను నిషేధించడానికి ఆలోచిస్తుంది.", + "లండన్ యునైటెడ్ కింగ్డమ్ లో పెద్ద సిటీ.", + "నువ్వు ఎక్కడ ఉన్నావ్?", + "ఫ్రాన్స్ అధ్యక్షుడు ఎవరు?", + "యునైటెడ్ స్టేట్స్ యొక్క రాజధాని ఏంటి?", + "బరాక్ ఒబామా ఎప్పుడు జన్మించారు?", +] diff --git a/spacy/lang/te/lex_attrs.py b/spacy/lang/te/lex_attrs.py new file mode 100644 index 0000000..ae11827 --- /dev/null +++ b/spacy/lang/te/lex_attrs.py @@ -0,0 +1,54 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "సున్నా", + "శూన్యం", + "ఒకటి", + "రెండు", + "మూడు", + "నాలుగు", + "ఐదు", + "ఆరు", + "ఏడు", + "ఎనిమిది", + "తొమ్మిది", + "పది", + "పదకొండు", + "పన్నెండు", + "పదమూడు", + "పద్నాలుగు", + "పదిహేను", + "పదహారు", + "పదిహేడు", + "పద్దెనిమిది", + "పందొమ్మిది", + "ఇరవై", + "ముప్పై", + "నలభై", + "యాభై", + "అరవై", + "డెబ్బై", + "ఎనభై", + "తొంబై", + "వంద", + "నూరు", + "వెయ్యి", + "లక్ష", + "కోటి", +] + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/te/stop_words.py b/spacy/lang/te/stop_words.py new file mode 100644 index 0000000..b18dab6 --- /dev/null +++ b/spacy/lang/te/stop_words.py @@ -0,0 +1,56 @@ +# Source: https://github.com/Xangis/extra-stopwords (MIT License) + +STOP_WORDS = set( + """ +అందరూ +అందుబాటులో +అడగండి +అడగడం +అడ్డంగా +అనుగుణంగా +అనుమతించు +అనుమతిస్తుంది +అయితే +ఇప్పటికే +ఉన్నారు +ఎక్కడైనా +ఎప్పుడు +ఎవరైనా +ఎవరో ఒకరు +ఏ +ఏదైనా +ఏమైనప్పటికి +ఏమైనప్పటికి +ఒక +ఒక ప్రక్కన +కనిపిస్తాయి +కాదు +కాదు +కూడా +గా +గురించి +చుట్టూ +చేయగలిగింది +తగిన +తర్వాత +తర్వాత +దాదాపు +దూరంగా +నిజంగా +పై +ప్రకారం +మధ్య +మధ్య +మరియు +మరొక +మళ్ళీ +మాత్రమే +మెచ్చుకో +వద్ద +వద్ద +వెంట +వేరుగా +వ్యతిరేకంగా +సంబంధం +""".split() +) diff --git a/spacy/lang/th/__init__.py b/spacy/lang/th/__init__.py new file mode 100644 index 0000000..551f50e --- /dev/null +++ b/spacy/lang/th/__init__.py @@ -0,0 +1,52 @@ +from ...language import BaseDefaults, Language +from ...tokens import Doc +from ...util import DummyTokenizer, load_config_from_str, registry +from ...vocab import Vocab +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + +DEFAULT_CONFIG = """ +[nlp] + +[nlp.tokenizer] +@tokenizers = "spacy.th.ThaiTokenizer" +""" + + +def create_thai_tokenizer(): + def thai_tokenizer_factory(nlp): + return ThaiTokenizer(nlp.vocab) + + return thai_tokenizer_factory + + +class ThaiTokenizer(DummyTokenizer): + def __init__(self, vocab: Vocab) -> None: + try: + from pythainlp.tokenize import word_tokenize + except ImportError: + raise ImportError( + "The Thai tokenizer requires the PyThaiNLP library: " + "https://github.com/PyThaiNLP/pythainlp" + ) from None + self.word_tokenize = word_tokenize + self.vocab = vocab + + def __call__(self, text: str) -> Doc: + words = list(self.word_tokenize(text)) + spaces = [False] * len(words) + return Doc(self.vocab, words=words, spaces=spaces) + + +class ThaiDefaults(BaseDefaults): + config = load_config_from_str(DEFAULT_CONFIG) + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Thai(Language): + lang = "th" + Defaults = ThaiDefaults + + +__all__ = ["Thai"] diff --git a/spacy/lang/th/lex_attrs.py b/spacy/lang/th/lex_attrs.py new file mode 100644 index 0000000..80f6ccb --- /dev/null +++ b/spacy/lang/th/lex_attrs.py @@ -0,0 +1,58 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "ศูนย์", + "หนึ่ง", + "สอง", + "สาม", + "สี่", + "ห้า", + "หก", + "เจ็ด", + "แปด", + "เก้า", + "สิบ", + "สิบเอ็ด", + "ยี่สิบ", + "ยี่สิบเอ็ด", + "สามสิบ", + "สามสิบเอ็ด", + "สี่สิบ", + "สี่สิบเอ็ด", + "ห้าสิบ", + "ห้าสิบเอ็ด", + "หกสิบเอ็ด", + "เจ็ดสิบ", + "เจ็ดสิบเอ็ด", + "แปดสิบ", + "แปดสิบเอ็ด", + "เก้าสิบ", + "เก้าสิบเอ็ด", + "ร้อย", + "พัน", + "ล้าน", + "พันล้าน", + "หมื่นล้าน", + "แสนล้าน", + "ล้านล้าน", + "ล้านล้านล้าน", + "ล้านล้านล้านล้าน", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/th/stop_words.py b/spacy/lang/th/stop_words.py new file mode 100644 index 0000000..2823281 --- /dev/null +++ b/spacy/lang/th/stop_words.py @@ -0,0 +1,75 @@ +STOP_WORDS = set( + """ +ทั้งนี้ ดัง ขอ รวม หลังจาก เป็น หลัง หรือ ๆ เกี่ยวกับ ซึ่งได้แก่ ด้วยเพราะ ด้วยว่า ด้วยเหตุเพราะ +ด้วยเหตุว่า สุดๆ เสร็จแล้ว เช่น เข้า ถ้า ถูก ถึง ต่างๆ ใคร เปิดเผย ครา รือ ตาม ใน ได้แก่ ได้แต่ +ได้ที่ ตลอดถึง นอกจากว่า นอกนั้น จริง อย่างดี ส่วน เพียงเพื่อ เดียว จัด ทั้งที ทั้งคน ทั้งตัว ไกลๆ +ถึงเมื่อใด คงจะ ถูกๆ เป็นที นับแต่ที่ นับแต่นั้น รับรอง ด้าน เป็นต้นมา ทุก กระทั่ง กระทำ จวบ ซึ่งก็ จะ +ครบครัน นับแต่ เยอะๆ เพียงไหน เปลี่ยนแปลง ไป่ ผ่านๆ เพื่อที่ รวมๆ กว้างขวาง เสียยิ่ง เปลี่ยน ผ่าน +ทรง ทว่า กันเถอะ เกี่ยวๆ ใดๆ ครั้งที่ ครั้งนั้น ครั้งนี้ ครั้งละ ครั้งหลัง ครั้งหลังสุด ร่วมกัน ร่วมด้วย ก็ตามที +ที่สุด ผิดๆ ยืนยง เยอะ ครั้งๆ ใครๆ นั่นเอง เสมือนว่า เสร็จ ตลอดศก ทั้งที่ ยืนยัน ด้วยที่ บัดนี้ +ด้วยประการฉะนี้ ซึ่งกัน ตลอดทั่วถึง ตลอดทั่วทั้ง ตลอดปี เป็นการ นั่นแหละ พร้อม เถิด ทั้ง สืบเนื่อง ตั้งแต่ +กลับ กล่าวคือ กลุ่มก้อน กลุ่มๆ ครั้งครา ส่ง รวดเร็ว เสร็จสิ้น เสีย เสียก่อน เสียจน อดีต ตั้ง เกิด อาจ +อีก ตลอดเวลา ภายหน้า ภายหลัง มอง มันๆ มองว่า มัก มักจะ มัน หาก คงอยู่ เป็นที่ เป็นที่สุด +เป็นเพราะเป็นเพราะว่า เกี่ยวกัน เพียงไร เป็นแต่เพียง กล่าว จนบัดนี้ เป็นอัน จน จนเมื่อ จนแม้ ใกล้ +ใหม่ๆ เป็นเพียง อย่างที่ ถูกต้อง ทั้งนั้น ทั้งนั้นด้วย กันดีกว่า กันดีไหม นั่นไง ตรงๆ แยะๆ เป็นต้น ใกล้ๆ +ซึ่งๆ ด้วยกัน ดังเคย เถอะ เสมือนกับ ไป คือ ขณะนี้ นอกจาก เพื่อที่จะ ขณะหนึ่ง ขวาง ครัน อยาก ไว้ +แบบ นอกจากนี้ เนื่องจาก เดียวกัน คง ให้มา อนึ่ง ก็แล้วแต่ ต้อง ข้าง เพื่อว่า จนแม้น ครั้งหนึ่ง อะไร ซึ่ง +เกินๆ ด้วยเหตุนั้น กันและกัน รับ ระหว่าง ครั้งไหน เสร็จกัน ถึงอย่างไร ขาด ข้าฯ เข้าใจ ครบ ครั้งใด +ครบถ้วน ระยะ ไม่ เกือบ เกือบจะ เกือบๆ แก่ แก อย่างโน้น ดังกับว่า จริงจัง เยอะแยะ นั่น ด้วย ถึงแม้ว่า +มาก ตลอดกาลนาน ตลอดระยะเวลา ตลอดจน ตลอดไป เป็นอันๆ เป็นอาทิ ก็ต่อเมื่อ สู่ เมื่อ เพื่อ ก็ กับ +ด้วยเหมือนกัน ด้วยเหตุนี้ ครั้งคราว ราย ร่วม เป็นอันมาก สูง รวมกัน รวมทั้ง ร่วมมือ เป็นเพียงว่า รวมถึง +ต่อ นะ กว้าง มา ครับ ตลอดทั้ง การ นั้นๆ น่า เป็นอันว่า เพราะ วัน จนขณะนี้ จนตลอด จนถึง ข้า อย่างใด +ไหนๆ ก่อนหน้านี้ ก่อนๆ สูงกว่า สูงส่ง สูงสุด สูงๆ เสียด้วย เสียนั่น เสียนี่ เสียนี่กระไร เสียนั่นเอง สุด +สําหรับ ว่า ลง ภายใต้ เพื่อให้ ภายนอก ภายใน เฉพาะ ซึ่งกันและกัน ง่าย ง่ายๆ ไง ถึงแม้จะ ถึงเมื่อไร +เกิน ก็ได้ คราใด คราที่ ตลอดวัน นับ ดังเก่า ดั่งเก่า หลาย หนึ่ง ถือว่า ก่อนหน้า นับตั้งแต่ จรด จริงๆ +จวน จวนเจียน ตลอดมา กลุ่ม กระนั้น ข้างๆ ตรง ข้าพเจ้า กว่า เกี่ยวเนื่อง ขึ้น ให้ไป ผล แต่ เอง เห็น +จึง ได้ ให้ โดย จริงๆจังๆ ดั่งกับว่า ทั้งนั้นเพราะ นอก นอกเหนือ น่ะ กันนะ ขณะเดียวกัน แยะ +นอกเหนือจาก น้อย ก่อน จวนจะ ข้างเคียง ก็ตามแต่ จรดกับ น้อยกว่า นั่นเป็น นักๆ ครั้งกระนั้น เลย ไกล +สิ้นกาลนาน ครั้ง รือว่า เก็บ อย่างเช่น บาง ดั่ง ดังกล่าว ดังกับ รึ รึว่า ออก แรก จง ยืนนาน ได้มา ตน +ตนเอง ได้รับ ระยะๆ กระผม กันไหม กันเอง กำลังจะ กำหนด กู กำลัง ความ แล้ว และ ต่าง อย่างน้อย +อย่างนั้น อย่างนี้ ก็คือ ก็แค่ ด้วยเหตุที่ ใหญ่ๆ ให้ดี ยัง เป็นเพื่อ ก็ตาม ผู้ ต่อกัน ถือ ซึ่งก็คือ ภายภาค +ภายภาคหน้า ก็ดี ก็จะ อยู่ เสียยิ่งนัก ใหม่ ขณะ เริ่ม เรา ขวางๆ เสียแล้ว ใคร่ ใคร่จะ ตนฯ ของ แห่ง +รวด ดั่งกับ ถึงเมื่อ น้อยๆ นับจากนั้น ตลอด ตลอดกาล เสร็จสมบูรณ์ เขียน กว้างๆ ยืนยาว ถึงแก่ ขณะใด +ขณะใดๆ ขณะที่ ขณะนั้น จนทั่ว ภาคฯ ภาย เป็นแต่ อย่าง พบ ภาค ให้แด่ เสียจนกระทั่ง เสียจนถึง +จนกระทั่ง จนกว่า ตลอดทั่ว เป็นๆ นอกจากนั้น ผิด ครั้งก่อน แก้ไข ขั้น กัน ช่วง จาก รวมด้วย เขา +ด้วยเช่นกัน นอกจากที่ เป็นต้นไป ข้างต้น ข้างบน ข้างล่าง ถึงจะ ถึงบัดนั้น ถึงแม้ มี ทาง เคย นับจากนี้ +อย่างเดียว เกี่ยวข้อง นี้ นํา นั้น ที่ ทําให้ ทํา ครานั้น ครานี้ คราหนึ่ง คราไหน คราว คราวก่อน คราวใด +คราวที่ คราวนั้น คราวนี้ คราวโน้น คราวละ คราวหน้า คราวหนึ่ง คราวหลัง คราวไหน คราวๆ คล้าย +คล้ายกัน คล้ายกันกับ คล้ายกับ คล้ายกับว่า คล้ายว่า ควร ค่อน ค่อนข้าง ค่อนข้างจะ ค่อยไปทาง ค่อนมาทาง ค +่อย ค่อยๆ คะ ค่ะ คำ คิด คิดว่า คุณ คุณๆ เคยๆ แค่ แค่จะ แค่นั้น แค่นี้ แค่เพียง แค่ว่า แค่ไหน จังๆ +จวบกับ จวบจน จ้ะ จ๊ะ จะได้ จัง จัดการ จัดงาน จัดแจง จัดตั้ง จัดทำ จัดหา จัดให้ จับ จ้า จ๋า จากนั้น +จากนี้ จากนี้ไป จำ จำเป็น จำพวก จึงจะ จึงเป็น จู่ๆ ฉะนั้น ฉะนี้ ฉัน เฉกเช่น เฉย เฉยๆ ไฉน ช่วงก่อน ช +่วงต่อไป ช่วงถัดไป ช่วงท้าย ช่วงที่ ช่วงนั้น ช่วงนี้ ช่วงระหว่าง ช่วงแรก ช่วงหน้า ช่วงหลัง ช่วงๆ ช่วย ช้า +ช้านาน ชาว ช้าๆ เช่นก่อน เช่นกัน เช่นเคย เช่นดัง เช่นดังก่อน เช่นดังเก่า เช่นดังที่ เช่นดังว่า +เช่นเดียวกัน เช่นเดียวกับ เช่นใด เช่นที่ เช่นที่เคย เช่นที่ว่า เช่นนั้น เช่นนั้นเอง เช่นนี้ เช่นเมื่อ เช่นไร +เชื่อ เชื่อถือ เชื่อมั่น เชื่อว่า ใช่ ใช้ ซะ ซะก่อน ซะจน ซะจนกระทั่ง ซะจนถึง ดั่งเคย ต่างก็ ต่างหาก +ตามด้วย ตามแต่ ตามที่ ตามๆ เต็มไปด้วย เต็มไปหมด เต็มๆ แต่ก็ แต่ก่อน แต่จะ แต่เดิม แต่ต้อง แต่ถ้า +แต่ทว่า แต่ที่ แต่นั้น แต่เพียง แต่เมื่อ แต่ไร แต่ละ แต่ว่า แต่ไหน แต่อย่างใด โต โตๆ ใต้ ถ้าจะ ถ้าหาก +ทั้งปวง ทั้งเป็น ทั้งมวล ทั้งสิ้น ทั้งหมด ทั้งหลาย ทั้งๆ ทัน ทันใดนั้น ทันที ทันทีทันใด ทั่ว ทำให้ ทำๆ ที ที่จริง +ที่ซึ่ง ทีเดียว ทีใด ที่ใด ที่ได้ ทีเถอะ ที่แท้ ที่แท้จริง ที่นั้น ที่นี้ ทีไร ทีละ ที่ละ ที่แล้ว ที่ว่า ที่แห่งนั้น ทีๆ ที่ๆ +ทุกคน ทุกครั้ง ทุกครา ทุกคราว ทุกชิ้น ทุกตัว ทุกทาง ทุกที ทุกที่ ทุกเมื่อ ทุกวัน ทุกวันนี้ ทุกสิ่ง ทุกหน ทุกแห่ง +ทุกอย่าง ทุกอัน ทุกๆ เท่า เท่ากัน เท่ากับ เท่าใด เท่าที่ เท่านั้น เท่านี้ แท้ แท้จริง เธอ นั้นไว นับแต่นี้ +นาง นางสาว น่าจะ นาน นานๆ นาย นำ นำพา นำมา นิด นิดหน่อย นิดๆ นี่ นี่ไง นี่นา นี่แน่ะ นี่แหละ นี้แหล่ +นี่เอง นี้เอง นู่น นู้น เน้น เนี่ย เนี่ยเอง ในช่วง ในที่ ในเมื่อ ในระหว่าง บน บอก บอกแล้ว บอกว่า บ่อย +บ่อยกว่า บ่อยครั้ง บ่อยๆ บัดดล บัดเดี๋ยวนี้ บัดนั้น บ้าง บางกว่า บางขณะ บางครั้ง บางครา บางคราว +บางที บางที่ บางแห่ง บางๆ ปฏิบัติ ประกอบ ประการ ประการฉะนี้ ประการใด ประการหนึ่ง ประมาณ +ประสบ ปรับ ปรากฏ ปรากฏว่า ปัจจุบัน ปิด เป็นด้วย เป็นดัง ผู้ใด เผื่อ เผื่อจะ เผื่อที่ เผื่อว่า ฝ่าย ฝ่ายใด +พบว่า พยายาม พร้อมกัน พร้อมกับ พร้อมด้วย พร้อมทั้ง พร้อมที่ พร้อมเพียง พวก พวกกัน พวกกู พวกแก +พวกเขา พวกคุณ พวกฉัน พวกท่าน พวกที่ พวกเธอ พวกนั้น พวกนี้ พวกนู้น พวกโน้น พวกมัน พวกมึง พอ พอกัน +พอควร พอจะ พอดี พอตัว พอที พอที่ พอเพียง พอแล้ว พอสม พอสมควร พอเหมาะ พอๆ พา พึง พึ่ง พื้นๆ พูด +เพราะฉะนั้น เพราะว่า เพิ่ง เพิ่งจะ เพิ่ม เพิ่มเติม เพียง เพียงแค่ เพียงใด เพียงแต่ เพียงพอ เพียงเพราะ +มากกว่า มากมาย มิ มิฉะนั้น มิใช่ มิได้ มีแต่ มึง มุ่ง มุ่งเน้น มุ่งหมาย เมื่อก่อน เมื่อครั้ง เมื่อครั้งก่อน +เมื่อคราวก่อน เมื่อคราวที่ เมื่อคราว เมื่อคืน เมื่อเช้า เมื่อใด เมื่อนั้น เมื่อนี้ เมื่อเย็น เมื่อวันวาน เมื่อวาน +แม้ แม้กระทั่ง แม้แต่ แม้นว่า แม้ว่า ไม่ค่อย ไม่ค่อยจะ ไม่ค่อยเป็น ไม่ใช่ ไม่เป็นไร ไม่ว่า ยก ยกให้ ยอม +ยอมรับ ย่อม ย่อย ยังคง ยังงั้น ยังงี้ ยังโง้น ยังไง ยังจะ ยังแต่ ยาก ยาว ยาวนาน ยิ่ง ยิ่งกว่า ยิ่งขึ้น +ยิ่งขึ้นไป ยิ่งจน ยิ่งจะ ยิ่งนัก ยิ่งเมื่อ ยิ่งแล้ว ยิ่งใหญ่ เร็ว เร็วๆ เราๆ เรียก เรียบ เรื่อย เรื่อยๆ ล้วน +ล้วนจน ล้วนแต่ ละ ล่าสุด เล็ก เล็กน้อย เล็กๆ เล่าว่า แล้วกัน แล้วแต่ แล้วเสร็จ วันใด วันนั้น วันนี้ วันไหน +สบาย สมัย สมัยก่อน สมัยนั้น สมัยนี้ สมัยโน้น ส่วนเกิน ส่วนด้อย ส่วนดี ส่วนใด ส่วนที่ ส่วนน้อย ส่วนนั้น ส +่วนมาก ส่วนใหญ่ สั้น สั้นๆ สามารถ สำคัญ สิ่ง สิ่งใด สิ่งนั้น สิ่งนี้ สิ่งไหน สิ้น แสดง แสดงว่า หน หนอ หนอย +หน่อย หมด หมดกัน หมดสิ้น หากแม้ หากแม้น หากแม้นว่า หากว่า หาความ หาใช่ หารือ เหตุ เหตุผล เหตุนั้น +เหตุนี้ เหตุไร เห็นแก่ เห็นควร เห็นจะ เห็นว่า เหลือ เหลือเกิน เหล่า เหล่านั้น เหล่านี้ แห่งใด แห่งนั้น +แห่งนี้ แห่งโน้น แห่งไหน แหละ ให้แก่ ใหญ่ ใหญ่โต อย่างมาก อย่างยิ่ง อย่างไรก็ อย่างไรก็ได้ อย่างไรเสีย +อย่างละ อย่างหนึ่ง อย่างๆ อัน อันจะ อันได้แก่ อันที่ อันที่จริง อันที่จะ อันเนื่องมาจาก อันละ อันๆ อาจจะ +อาจเป็น อาจเป็นด้วย อื่น อื่นๆ เอ็ง เอา ฯ ฯล ฯลฯ 555 กำ ขอโทษ เยี่ยม นี่คือ +""".split() +) diff --git a/spacy/lang/th/tokenizer_exceptions.py b/spacy/lang/th/tokenizer_exceptions.py new file mode 100644 index 0000000..954766d --- /dev/null +++ b/spacy/lang/th/tokenizer_exceptions.py @@ -0,0 +1,438 @@ +from ...symbols import ORTH + +_exc = { + # หน่วยงานรัฐ / government agency + "กกต.": [{ORTH: "กกต."}], + "กทท.": [{ORTH: "กทท."}], + "กทพ.": [{ORTH: "กทพ."}], + "กบข.": [{ORTH: "กบข."}], + "กบว.": [{ORTH: "กบว."}], + "กปน.": [{ORTH: "กปน."}], + "กปภ.": [{ORTH: "กปภ."}], + "กปส.": [{ORTH: "กปส."}], + "กผม.": [{ORTH: "กผม."}], + "กฟน.": [{ORTH: "กฟน."}], + "กฟผ.": [{ORTH: "กฟผ."}], + "กฟภ.": [{ORTH: "กฟภ."}], + "ก.ช.น.": [{ORTH: "ก.ช.น."}], + "กยศ.": [{ORTH: "กยศ."}], + "ก.ล.ต.": [{ORTH: "ก.ล.ต."}], + "กศ.บ.": [{ORTH: "กศ.บ."}], + "กศน.": [{ORTH: "กศน."}], + "กสท.": [{ORTH: "กสท."}], + "กอ.รมน.": [{ORTH: "กอ.รมน."}], + "กร.": [{ORTH: "กร."}], + "ขสมก.": [{ORTH: "ขสมก."}], + "คตง.": [{ORTH: "คตง."}], + "ครม.": [{ORTH: "ครม."}], + "คมช.": [{ORTH: "คมช."}], + "ตชด.": [{ORTH: "ตชด."}], + "ตม.": [{ORTH: "ตม."}], + "ตร.": [{ORTH: "ตร."}], + "ททท.": [{ORTH: "ททท."}], + "ททบ.": [{ORTH: "ททบ."}], + "ทบ.": [{ORTH: "ทบ."}], + "ทร.": [{ORTH: "ทร."}], + "ทอ.": [{ORTH: "ทอ."}], + "ทอท.": [{ORTH: "ทอท."}], + "ธ.ก.ส.": [{ORTH: "ธ.ก.ส."}], + "ธปท.": [{ORTH: "ธปท."}], + "ธอส.": [{ORTH: "ธอส."}], + "นย.": [{ORTH: "นย."}], + "ปตท.": [{ORTH: "ปตท."}], + "ป.ป.ช.": [{ORTH: "ป.ป.ช."}], + "ป.ป.ส.": [{ORTH: "ป.ป.ส."}], + "บพร.": [{ORTH: "บพร."}], + "บย.": [{ORTH: "บย."}], + "พสวท.": [{ORTH: "พสวท."}], + "มอก.": [{ORTH: "มอก."}], + "ยธ.": [{ORTH: "ยธ."}], + "รพช.": [{ORTH: "รพช."}], + "รฟท.": [{ORTH: "รฟท."}], + "รฟม.": [{ORTH: "รฟม."}], + "ศธ.": [{ORTH: "ศธ."}], + "ศนธ.": [{ORTH: "ศนธ."}], + "สกจ.": [{ORTH: "สกจ."}], + "สกท.": [{ORTH: "สกท."}], + "สกว.": [{ORTH: "สกว."}], + "สคบ.": [{ORTH: "สคบ."}], + "สจร.": [{ORTH: "สจร."}], + "สตง.": [{ORTH: "สตง."}], + "สทท.": [{ORTH: "สทท."}], + "สทร.": [{ORTH: "สทร."}], + "สธ": [{ORTH: "สธ"}], + "สนช.": [{ORTH: "สนช."}], + "สนนท.": [{ORTH: "สนนท."}], + "สปก.": [{ORTH: "สปก."}], + "สปช.": [{ORTH: "สปช."}], + "สปอ.": [{ORTH: "สปอ."}], + "สพช.": [{ORTH: "สพช."}], + "สยช.": [{ORTH: "สยช."}], + "สวช.": [{ORTH: "สวช."}], + "สวท.": [{ORTH: "สวท."}], + "สวทช.": [{ORTH: "สวทช."}], + "สคช.": [{ORTH: "สคช."}], + "สสว.": [{ORTH: "สสว."}], + "สสส.": [{ORTH: "สสส."}], + "สสวท.": [{ORTH: "สสวท."}], + "อตก.": [{ORTH: "อตก."}], + "อบจ.": [{ORTH: "อบจ."}], + "อบต.": [{ORTH: "อบต."}], + "อปพร.": [{ORTH: "อปพร."}], + "อย.": [{ORTH: "อย."}], + "อ.ส.ม.ท.": [{ORTH: "อ.ส.ม.ท."}], + # มหาวิทยาลัย / สถานศึกษา / university / college + "มทส.": [{ORTH: "มทส."}], + "มธ.": [{ORTH: "มธ."}], + "ม.อ.": [{ORTH: "ม.อ."}], + "มทร.": [{ORTH: "มทร."}], + "มมส.": [{ORTH: "มมส."}], + "วท.": [{ORTH: "วท."}], + "สตม.": [{ORTH: "สตม."}], + # ยศ / rank + "ดร.": [{ORTH: "ดร."}], + "ด.ต.": [{ORTH: "ด.ต."}], + "จ.ต.": [{ORTH: "จ.ต."}], + "จ.ท.": [{ORTH: "จ.ท."}], + "จ.ส.ต.": [{ORTH: "จ.ส.ต."}], + "จสต.": [{ORTH: "จสต."}], + "จ.ส.ท.": [{ORTH: "จ.ส.ท."}], + "จ.ส.อ.": [{ORTH: "จ.ส.อ."}], + "จ.อ.": [{ORTH: "จ.อ."}], + "ทพญ.": [{ORTH: "ทพญ."}], + "ทนพ.": [{ORTH: "ทนพ."}], + "นจอ.": [{ORTH: "นจอ."}], + "น.ช.": [{ORTH: "น.ช."}], + "น.ญ.": [{ORTH: "น.ญ."}], + "น.ต.": [{ORTH: "น.ต."}], + "น.ท.": [{ORTH: "น.ท."}], + "นตท.": [{ORTH: "นตท."}], + "นนส.": [{ORTH: "นนส."}], + "นนร.": [{ORTH: "นนร."}], + "นนอ.": [{ORTH: "นนอ."}], + "นพ.": [{ORTH: "นพ."}], + "นพท.": [{ORTH: "นพท."}], + "นรจ.": [{ORTH: "นรจ."}], + "นรต.": [{ORTH: "นรต."}], + "นศพ.": [{ORTH: "นศพ."}], + "นศท.": [{ORTH: "นศท."}], + "น.สพ.": [{ORTH: "น.สพ."}], + "น.อ.": [{ORTH: "น.อ."}], + "บช.ก.": [{ORTH: "บช.ก."}], + "บช.น.": [{ORTH: "บช.น."}], + "ผกก.": [{ORTH: "ผกก."}], + "ผกก.ภ.": [{ORTH: "ผกก.ภ."}], + "ผจก.": [{ORTH: "ผจก."}], + "ผช.": [{ORTH: "ผช."}], + "ผชก.": [{ORTH: "ผชก."}], + "ผช.ผอ.": [{ORTH: "ผช.ผอ."}], + "ผญบ.": [{ORTH: "ผญบ."}], + "ผบ.": [{ORTH: "ผบ."}], + "ผบก.": [{ORTH: "ผบก."}], + "ผบก.น.": [{ORTH: "ผบก.น."}], + "ผบก.ป.": [{ORTH: "ผบก.ป."}], + "ผบก.ปค.": [{ORTH: "ผบก.ปค."}], + "ผบก.ปม.": [{ORTH: "ผบก.ปม."}], + "ผบก.ภ.": [{ORTH: "ผบก.ภ."}], + "ผบช.": [{ORTH: "ผบช."}], + "ผบช.ก.": [{ORTH: "ผบช.ก."}], + "ผบช.ตชด.": [{ORTH: "ผบช.ตชด."}], + "ผบช.น.": [{ORTH: "ผบช.น."}], + "ผบช.ภ.": [{ORTH: "ผบช.ภ."}], + "ผบ.ทบ.": [{ORTH: "ผบ.ทบ."}], + "ผบ.ตร.": [{ORTH: "ผบ.ตร."}], + "ผบ.ทร.": [{ORTH: "ผบ.ทร."}], + "ผบ.ทอ.": [{ORTH: "ผบ.ทอ."}], + "ผบ.ทสส.": [{ORTH: "ผบ.ทสส."}], + "ผวจ.": [{ORTH: "ผวจ."}], + "ผู้ว่าฯ": [{ORTH: "ผู้ว่าฯ"}], + "พ.จ.ต.": [{ORTH: "พ.จ.ต."}], + "พ.จ.ท.": [{ORTH: "พ.จ.ท."}], + "พ.จ.อ.": [{ORTH: "พ.จ.อ."}], + "พญ.": [{ORTH: "พญ."}], + "ฯพณฯ": [{ORTH: "ฯพณฯ"}], + "พ.ต.": [{ORTH: "พ.ต."}], + "พ.ท.": [{ORTH: "พ.ท."}], + "พ.อ.": [{ORTH: "พ.อ."}], + "พ.ต.อ.พิเศษ": [{ORTH: "พ.ต.อ.พิเศษ"}], + "พลฯ": [{ORTH: "พลฯ"}], + "พล.๑ รอ.": [{ORTH: "พล.๑ รอ."}], + "พล.ต.": [{ORTH: "พล.ต."}], + "พล.ต.ต.": [{ORTH: "พล.ต.ต."}], + "พล.ต.ท.": [{ORTH: "พล.ต.ท."}], + "พล.ต.อ.": [{ORTH: "พล.ต.อ."}], + "พล.ท.": [{ORTH: "พล.ท."}], + "พล.ปตอ.": [{ORTH: "พล.ปตอ."}], + "พล.ม.": [{ORTH: "พล.ม."}], + "พล.ม.๒": [{ORTH: "พล.ม.๒"}], + "พล.ร.ต.": [{ORTH: "พล.ร.ต."}], + "พล.ร.ท.": [{ORTH: "พล.ร.ท."}], + "พล.ร.อ.": [{ORTH: "พล.ร.อ."}], + "พล.อ.": [{ORTH: "พล.อ."}], + "พล.อ.ต.": [{ORTH: "พล.อ.ต."}], + "พล.อ.ท.": [{ORTH: "พล.อ.ท."}], + "พล.อ.อ.": [{ORTH: "พล.อ.อ."}], + "พ.อ.พิเศษ": [{ORTH: "พ.อ.พิเศษ"}], + "พ.อ.ต.": [{ORTH: "พ.อ.ต."}], + "พ.อ.ท.": [{ORTH: "พ.อ.ท."}], + "พ.อ.อ.": [{ORTH: "พ.อ.อ."}], + "ภกญ.": [{ORTH: "ภกญ."}], + "ม.จ.": [{ORTH: "ม.จ."}], + "มท1": [{ORTH: "มท1"}], + "ม.ร.ว.": [{ORTH: "ม.ร.ว."}], + "มล.": [{ORTH: "มล."}], + "ร.ต.": [{ORTH: "ร.ต."}], + "ร.ต.ต.": [{ORTH: "ร.ต.ต."}], + "ร.ต.ท.": [{ORTH: "ร.ต.ท."}], + "ร.ต.อ.": [{ORTH: "ร.ต.อ."}], + "ร.ท.": [{ORTH: "ร.ท."}], + "รมช.": [{ORTH: "รมช."}], + "รมต.": [{ORTH: "รมต."}], + "รมว.": [{ORTH: "รมว."}], + "รศ.": [{ORTH: "รศ."}], + "ร.อ.": [{ORTH: "ร.อ."}], + "ศ.": [{ORTH: "ศ."}], + "ส.ต.": [{ORTH: "ส.ต."}], + "ส.ต.ต.": [{ORTH: "ส.ต.ต."}], + "ส.ต.ท.": [{ORTH: "ส.ต.ท."}], + "ส.ต.อ.": [{ORTH: "ส.ต.อ."}], + "ส.ท.": [{ORTH: "ส.ท."}], + "สพ.": [{ORTH: "สพ."}], + "สพ.ญ.": [{ORTH: "สพ.ญ."}], + "สพ.ช.": [{ORTH: "สพ.ช."}], + "ส.อ.": [{ORTH: "ส.อ."}], + "อจ.": [{ORTH: "อจ."}], + "อจญ.": [{ORTH: "อจญ."}], + # วุฒิ / bachelor degree + "ป.": [{ORTH: "ป."}], + "ป.กศ.": [{ORTH: "ป.กศ."}], + "ป.กศ.สูง": [{ORTH: "ป.กศ.สูง"}], + "ปวช.": [{ORTH: "ปวช."}], + "ปวท.": [{ORTH: "ปวท."}], + "ปวส.": [{ORTH: "ปวส."}], + "ปทส.": [{ORTH: "ปทส."}], + "กษ.บ.": [{ORTH: "กษ.บ."}], + "กษ.ม.": [{ORTH: "กษ.ม."}], + "กษ.ด.": [{ORTH: "กษ.ด."}], + "ค.บ.": [{ORTH: "ค.บ."}], + "คศ.บ.": [{ORTH: "คศ.บ."}], + "คศ.ม.": [{ORTH: "คศ.ม."}], + "คศ.ด.": [{ORTH: "คศ.ด."}], + "ค.อ.บ.": [{ORTH: "ค.อ.บ."}], + "ค.อ.ม.": [{ORTH: "ค.อ.ม."}], + "ค.อ.ด.": [{ORTH: "ค.อ.ด."}], + "ทก.บ.": [{ORTH: "ทก.บ."}], + "ทก.ม.": [{ORTH: "ทก.ม."}], + "ทก.ด.": [{ORTH: "ทก.ด."}], + "ท.บ.": [{ORTH: "ท.บ."}], + "ท.ม.": [{ORTH: "ท.ม."}], + "ท.ด.": [{ORTH: "ท.ด."}], + "น.บ.": [{ORTH: "น.บ."}], + "น.ม.": [{ORTH: "น.ม."}], + "น.ด.": [{ORTH: "น.ด."}], + "นศ.บ.": [{ORTH: "นศ.บ."}], + "นศ.ม.": [{ORTH: "นศ.ม."}], + "นศ.ด.": [{ORTH: "นศ.ด."}], + "บช.บ.": [{ORTH: "บช.บ."}], + "บช.ม.": [{ORTH: "บช.ม."}], + "บช.ด.": [{ORTH: "บช.ด."}], + "บธ.บ.": [{ORTH: "บธ.บ."}], + "บธ.ม.": [{ORTH: "บธ.ม."}], + "บธ.ด.": [{ORTH: "บธ.ด."}], + "พณ.บ.": [{ORTH: "พณ.บ."}], + "พณ.ม.": [{ORTH: "พณ.ม."}], + "พณ.ด.": [{ORTH: "พณ.ด."}], + "พ.บ.": [{ORTH: "พ.บ."}], + "พ.ม.": [{ORTH: "พ.ม."}], + "พ.ด.": [{ORTH: "พ.ด."}], + "พธ.บ.": [{ORTH: "พธ.บ."}], + "พธ.ม.": [{ORTH: "พธ.ม."}], + "พธ.ด.": [{ORTH: "พธ.ด."}], + "พบ.บ.": [{ORTH: "พบ.บ."}], + "พบ.ม.": [{ORTH: "พบ.ม."}], + "พบ.ด.": [{ORTH: "พบ.ด."}], + "พย.บ.": [{ORTH: "พย.บ."}], + "พย.ม.": [{ORTH: "พย.ม."}], + "พย.ด.": [{ORTH: "พย.ด."}], + "พศ.บ.": [{ORTH: "พศ.บ."}], + "พศ.ม.": [{ORTH: "พศ.ม."}], + "พศ.ด.": [{ORTH: "พศ.ด."}], + "ภ.บ.": [{ORTH: "ภ.บ."}], + "ภ.ม.": [{ORTH: "ภ.ม."}], + "ภ.ด.": [{ORTH: "ภ.ด."}], + "ภ.สถ.บ.": [{ORTH: "ภ.สถ.บ."}], + "รป.บ.": [{ORTH: "รป.บ."}], + "รป.ม.": [{ORTH: "รป.ม."}], + "วท.บ.": [{ORTH: "วท.บ."}], + "วท.ม.": [{ORTH: "วท.ม."}], + "วท.ด.": [{ORTH: "วท.ด."}], + "ศ.บ.": [{ORTH: "ศ.บ."}], + "ศศ.บ.": [{ORTH: "ศศ.บ."}], + "ศษ.บ.": [{ORTH: "ศษ.บ."}], + "ศส.บ.": [{ORTH: "ศส.บ."}], + "สถ.บ.": [{ORTH: "สถ.บ."}], + "สถ.ม.": [{ORTH: "สถ.ม."}], + "สถ.ด.": [{ORTH: "สถ.ด."}], + "สพ.บ.": [{ORTH: "สพ.บ."}], + "อ.บ.": [{ORTH: "อ.บ."}], + "อ.ม.": [{ORTH: "อ.ม."}], + "อ.ด.": [{ORTH: "อ.ด."}], + # ปี / เวลา / year / time + "ชม.": [{ORTH: "ชม."}], + "จ.ศ.": [{ORTH: "จ.ศ."}], + "ค.ศ.": [{ORTH: "ค.ศ."}], + "ฮ.ศ.": [{ORTH: "ฮ.ศ."}], + "ว.ด.ป.": [{ORTH: "ว.ด.ป."}], + # ระยะทาง / distance + "ฮม.": [{ORTH: "ฮม."}], + "ดคม.": [{ORTH: "ดคม."}], + "ดม.": [{ORTH: "ดม."}], + "มม.": [{ORTH: "มม."}], + "ซม.": [{ORTH: "ซม."}], + "กม.": [{ORTH: "กม."}], + # น้ำหนัก / weight + "น.น.": [{ORTH: "น.น."}], + "ฮก.": [{ORTH: "ฮก."}], + "ดคก.": [{ORTH: "ดคก."}], + "ดก.": [{ORTH: "ดก."}], + "ซก.": [{ORTH: "ซก."}], + "มก.": [{ORTH: "มก."}], + "ก.": [{ORTH: "ก."}], + "กก.": [{ORTH: "กก."}], + # ปริมาตร / volume + "ฮล.": [{ORTH: "ฮล."}], + "ดคล.": [{ORTH: "ดคล."}], + "ดล.": [{ORTH: "ดล."}], + "ซล.": [{ORTH: "ซล."}], + "ล.": [{ORTH: "ล."}], + "กล.": [{ORTH: "กล."}], + "ลบ.": [{ORTH: "ลบ."}], + # พื้นที่ / area + "ตร.ซม.": [{ORTH: "ตร.ซม."}], + "ตร.ม.": [{ORTH: "ตร.ม."}], + "ตร.ว.": [{ORTH: "ตร.ว."}], + "ตร.กม.": [{ORTH: "ตร.กม."}], + # เดือน / month + "ม.ค.": [{ORTH: "ม.ค."}], + "ก.พ.": [{ORTH: "ก.พ."}], + "มี.ค.": [{ORTH: "มี.ค."}], + "เม.ย.": [{ORTH: "เม.ย."}], + "พ.ค.": [{ORTH: "พ.ค."}], + "มิ.ย.": [{ORTH: "มิ.ย."}], + "ก.ค.": [{ORTH: "ก.ค."}], + "ส.ค.": [{ORTH: "ส.ค."}], + "ก.ย.": [{ORTH: "ก.ย."}], + "ต.ค.": [{ORTH: "ต.ค."}], + "พ.ย.": [{ORTH: "พ.ย."}], + "ธ.ค.": [{ORTH: "ธ.ค."}], + # เพศ / gender + "ช.": [{ORTH: "ช."}], + "ญ.": [{ORTH: "ญ."}], + "ด.ช.": [{ORTH: "ด.ช."}], + "ด.ญ.": [{ORTH: "ด.ญ."}], + # ที่อยู่ / address + "ถ.": [{ORTH: "ถ."}], + "ต.": [{ORTH: "ต."}], + "อ.": [{ORTH: "อ."}], + "จ.": [{ORTH: "จ."}], + # สรรพนาม / pronoun + "ข้าฯ": [{ORTH: "ข้าฯ"}], + "ทูลเกล้าฯ": [{ORTH: "ทูลเกล้าฯ"}], + "น้อมเกล้าฯ": [{ORTH: "น้อมเกล้าฯ"}], + "โปรดเกล้าฯ": [{ORTH: "โปรดเกล้าฯ"}], + # การเมือง / politic + "ขจก.": [{ORTH: "ขจก."}], + "ขบด.": [{ORTH: "ขบด."}], + "นปช.": [{ORTH: "นปช."}], + "ปชป.": [{ORTH: "ปชป."}], + "ผกค.": [{ORTH: "ผกค."}], + "พท.": [{ORTH: "พท."}], + "พ.ร.ก.": [{ORTH: "พ.ร.ก."}], + "พ.ร.ฎ.": [{ORTH: "พ.ร.ฎ."}], + "พ.ร.บ.": [{ORTH: "พ.ร.บ."}], + "รธน.": [{ORTH: "รธน."}], + "รบ.": [{ORTH: "รบ."}], + "รสช.": [{ORTH: "รสช."}], + "ส.ก.": [{ORTH: "ส.ก."}], + "สจ.": [{ORTH: "สจ."}], + "สว.": [{ORTH: "สว."}], + "ส.ส.": [{ORTH: "ส.ส."}], + # ทั่วไป / general + "ก.ข.ค.": [{ORTH: "ก.ข.ค."}], + "กทม.": [{ORTH: "กทม."}], + "กรุงเทพฯ": [{ORTH: "กรุงเทพฯ"}], + "ขรก.": [{ORTH: "ขรก."}], + "ขส": [{ORTH: "ขส."}], + "ค.ร.น.": [{ORTH: "ค.ร.น."}], + "ค.ร.ม.": [{ORTH: "ค.ร.ม."}], + "ง.ด.": [{ORTH: "ง.ด."}], + "งป.": [{ORTH: "งป."}], + "จก.": [{ORTH: "จก."}], + "จขกท.": [{ORTH: "จขกท."}], + "จนท.": [{ORTH: "จนท."}], + "จ.ป.ร.": [{ORTH: "จ.ป.ร."}], + "จ.ม.": [{ORTH: "จ.ม."}], + "จย.": [{ORTH: "จย."}], + "จยย.": [{ORTH: "จยย."}], + "ตจว.": [{ORTH: "ตจว."}], + "โทร.": [{ORTH: "โทร."}], + "ธ.": [{ORTH: "ธ."}], + "น.ร.": [{ORTH: "น.ร."}], + "น.ศ.": [{ORTH: "น.ศ."}], + "น.ส.": [{ORTH: "น.ส."}], + "น.ส.๓": [{ORTH: "น.ส.๓"}], + "น.ส.๓ ก.": [{ORTH: "น.ส.๓ ก"}], + "นสพ.": [{ORTH: "นสพ."}], + "บ.ก.": [{ORTH: "บ.ก."}], + "บจก.": [{ORTH: "บจก."}], + "บงล.": [{ORTH: "บงล."}], + "บบส.": [{ORTH: "บบส."}], + "บมจ.": [{ORTH: "บมจ."}], + "บลจ.": [{ORTH: "บลจ."}], + "บ/ช": [{ORTH: "บ/ช"}], + "บร.": [{ORTH: "บร."}], + "ปชช.": [{ORTH: "ปชช."}], + "ปณ.": [{ORTH: "ปณ."}], + "ปณก.": [{ORTH: "ปณก."}], + "ปณส.": [{ORTH: "ปณส."}], + "ปธ.": [{ORTH: "ปธ."}], + "ปธน.": [{ORTH: "ปธน."}], + "ปอ.": [{ORTH: "ปอ."}], + "ปอ.พ.": [{ORTH: "ปอ.พ."}], + "พ.ก.ง.": [{ORTH: "พ.ก.ง."}], + "พ.ก.ส.": [{ORTH: "พ.ก.ส."}], + "พขร.": [{ORTH: "พขร."}], + "ภ.ง.ด.": [{ORTH: "ภ.ง.ด."}], + "ภ.ง.ด.๙": [{ORTH: "ภ.ง.ด.๙"}], + "ภ.ป.ร.": [{ORTH: "ภ.ป.ร."}], + "ภ.พ.": [{ORTH: "ภ.พ."}], + "ร.": [{ORTH: "ร."}], + "ร.ง.": [{ORTH: "ร.ง."}], + "ร.ด.": [{ORTH: "ร.ด."}], + "รปภ.": [{ORTH: "รปภ."}], + "รพ.": [{ORTH: "รพ."}], + "ร.พ.": [{ORTH: "ร.พ."}], + "รร.": [{ORTH: "รร."}], + "รสก.": [{ORTH: "รสก."}], + "ส.ค.ส.": [{ORTH: "ส.ค.ส."}], + "สต.": [{ORTH: "สต."}], + "สน.": [{ORTH: "สน."}], + "สนข.": [{ORTH: "สนข."}], + "สนง.": [{ORTH: "สนง."}], + "สนญ.": [{ORTH: "สนญ."}], + "ส.ป.ช.": [{ORTH: "ส.ป.ช."}], + "สภ.": [{ORTH: "สภ."}], + "ส.ล.น.": [{ORTH: "ส.ล.น."}], + "สวญ.": [{ORTH: "สวญ."}], + "สวป.": [{ORTH: "สวป."}], + "สว.สส.": [{ORTH: "สว.สส."}], + "ส.ห.": [{ORTH: "ส.ห."}], + "สอ.": [{ORTH: "สอ."}], + "สอท.": [{ORTH: "สอท."}], + "เสธ.": [{ORTH: "เสธ."}], + "หจก.": [{ORTH: "หจก."}], + "ห.ร.ม.": [{ORTH: "ห.ร.ม."}], +} + + +TOKENIZER_EXCEPTIONS = _exc diff --git a/spacy/lang/ti/__init__.py b/spacy/lang/ti/__init__.py new file mode 100644 index 0000000..510999f --- /dev/null +++ b/spacy/lang/ti/__init__.py @@ -0,0 +1,26 @@ +from ...attrs import LANG +from ...language import BaseDefaults, Language +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class TigrinyaDefaults(BaseDefaults): + lex_attr_getters = dict(Language.Defaults.lex_attr_getters) + lex_attr_getters.update(LEX_ATTRS) + lex_attr_getters[LANG] = lambda text: "ti" + tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS) + stop_words = STOP_WORDS + suffixes = TOKENIZER_SUFFIXES + writing_system = {"direction": "ltr", "has_case": False, "has_letters": True} + + +class Tigrinya(Language): + lang = "ti" + Defaults = TigrinyaDefaults + + +__all__ = ["Tigrinya"] diff --git a/spacy/lang/ti/examples.py b/spacy/lang/ti/examples.py new file mode 100644 index 0000000..146ac34 --- /dev/null +++ b/spacy/lang/ti/examples.py @@ -0,0 +1,17 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.ti.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "አፕል ብዩኬ ትርከብ ንግድ ብ1 ቢሊዮን ዶላር ንምግዛዕ ሐሲባ።", + "ፈላማይ ክታበት ኮቪድ 19 ተጀሚሩ፤ሓዱሽ ተስፋ ሂቡ ኣሎ", + "ቻንስለር ጀርመን ኣንገላ መርከል ዝርግሓ ቫይረስ ኮሮና ንምክልካል ጽኑዕ እገዳ ክግበር ጸዊዓ", + "ለንደን ብዓዲ እንግሊዝ ትርከብ ዓባይ ከተማ እያ።", + "ናበይ አለኻ፧", + "ናይ ፈረንሳይ ፕሬዝዳንት መን እዩ፧", + "ናይ አሜሪካ ዋና ከተማ እንታይ እያ፧", + "ኦባማ መዓስ ተወሊዱ፧", +] diff --git a/spacy/lang/ti/lex_attrs.py b/spacy/lang/ti/lex_attrs.py new file mode 100644 index 0000000..da56af6 --- /dev/null +++ b/spacy/lang/ti/lex_attrs.py @@ -0,0 +1,73 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "ዜሮ", + "ሓደ", + "ክልተ", + "ሰለስተ", + "ኣርባዕተ", + "ሓሙሽተ", + "ሽድሽተ", + "ሸውዓተ", + "ሽሞንተ", + "ትሽዓተ", + "ዓሰርተ", + "ዕስራ", + "ሰላሳ", + "ኣርብዓ", + "ሓምሳ", + "ሱሳ", + "ሰብዓ", + "ሰማንያ", + "ቴስዓ", + "ሚእቲ", + "ሺሕ", + "ሚልዮን", + "ቢልዮን", + "ትሪልዮን", + "ኳድሪልዮን", + "ጋዚልዮን", + "ባዚልዮን", +] + +# Tigrinya ordinals above 10 are the same as _num_words but start with "መበል " +_ordinal_words = [ + "ቀዳማይ", + "ካልኣይ", + "ሳልሳይ", + "ራብዓይ", + "ሓምሻይ", + "ሻድሻይ", + "ሻውዓይ", + "ሻምናይ", + "ታሽዓይ", + "ዓስራይ", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + + text_lower = text.lower() + if text_lower in _num_words: + return True + + # Check ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith("ይ"): + if text_lower[:-2].isdigit(): + return True + + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ti/punctuation.py b/spacy/lang/ti/punctuation.py new file mode 100644 index 0000000..f29f30e --- /dev/null +++ b/spacy/lang/ti/punctuation.py @@ -0,0 +1,25 @@ +from ..char_classes import ( + ALPHA_UPPER, + CURRENCY, + LIST_ELLIPSES, + LIST_PUNCT, + LIST_QUOTES, + UNITS, +) + +_list_punct = LIST_PUNCT + "፡ ። ፣ ፤ ፥ ፦ ፧ ፠ ፨".strip().split() + +_suffixes = ( + _list_punct + + LIST_ELLIPSES + + LIST_QUOTES + + [ + r"(?<=[0-9])\+", + # Tigrinya is written from Left-To-Right + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + ] +) + +TOKENIZER_SUFFIXES = _suffixes diff --git a/spacy/lang/ti/stop_words.py b/spacy/lang/ti/stop_words.py new file mode 100644 index 0000000..9bd7122 --- /dev/null +++ b/spacy/lang/ti/stop_words.py @@ -0,0 +1,27 @@ +# Stop words from Tigrinya Wordcount: https://github.com/fgaim/Tigrinya-WordCount/blob/main/ti_stop_words.txt + +# Stop words +STOP_WORDS = set( + """ +'ምበር 'ሞ 'ቲ 'ታ 'ኳ 'ውን 'ዚ 'የ 'ዩ 'ያ 'ዮም 'ዮን +ልዕሊ ሒዙ ሒዛ ሕጂ መበል መን መንጎ መጠን ማለት ምስ ምባል +ምእንቲ ምኽንያቱ ምኽንያት ምዃኑ ምዃንና ምዃኖም +ስለ ስለዚ ስለዝበላ ሽዑ ቅድሚ በለ በቲ በዚ ብምባል ብተወሳኺ ብኸመይ +ብዘይ ብዘይካ ብዙሕ ብዛዕባ ብፍላይ ተባሂሉ ነበረ ነቲ ነታ ነቶም +ነዚ ነይሩ ነገራት ነገር ናብ ናብቲ ናትኩም ናትኪ ናትካ ናትክን +ናይ ናይቲ ንሕና ንሱ ንሳ ንሳቶም ንስኺ ንስኻ ንስኻትኩም ንስኻትክን ንዓይ +ኢለ ኢሉ ኢላ ኢልካ ኢሎም ኢና ኢኻ ኢዩ ኣለኹ +ኣለዉ ኣለዎ ኣሎ ኣብ ኣብቲ ኣብታ ኣብኡ ኣብዚ ኣነ ኣዝዩ ኣይኮነን ኣይኰነን +እምበር እሞ እተን እቲ እታ እቶም እንተ እንተሎ +ኣላ እንተኾነ እንታይ እንከሎ እኳ እዋን እውን እዚ እዛ እዞም +እየ እየን እዩ እያ እዮም +ከሎ ከመይ ከም ከምቲ ከምኡ ከምዘሎ +ከምዚ ከኣ ኩሉ ካልእ ካልኦት ካብ ካብቲ ካብቶም ክሳብ ክሳዕ ክብል +ክንደይ ክንዲ ክኸውን ኮይኑ ኰይኑ ኵሉ ኸም ኸኣ ወይ +ዋላ ዘለና ዘለዉ ዘለዋ ዘለዎ ዘለዎም ዘላ ዘሎ ዘይብሉ +ዝርከብ ዝበሃል ዝበለ ዝብል ዝተባህለ ዝተኻየደ ዝተፈላለየ ዝተፈላለዩ +ዝነበረ ዝነበረት ዝነበሩ ዝካየድ ዝኸውን ዝኽእል ዝኾነ ዝዀነ +የለን ይቕረብ ይብል ይኸውን ይኹን ይኽእል ደኣ ድሕሪ ድማ +ገለ ገሊጹ ገና ገይሩ ግና ግን ጥራይ +""".split() +) diff --git a/spacy/lang/ti/tokenizer_exceptions.py b/spacy/lang/ti/tokenizer_exceptions.py new file mode 100644 index 0000000..711e4b4 --- /dev/null +++ b/spacy/lang/ti/tokenizer_exceptions.py @@ -0,0 +1,21 @@ +from ...symbols import NORM, ORTH + +_exc = {} + + +for exc_data in [ + {ORTH: "ት/ቤት"}, + {ORTH: "ወ/ሮ", NORM: "ወይዘሮ"}, + {ORTH: "ወ/ሪ", NORM: "ወይዘሪት"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "ዓ.ም.", + "ኪ.ሜ.", +]: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = _exc diff --git a/spacy/lang/tl/__init__.py b/spacy/lang/tl/__init__.py new file mode 100644 index 0000000..6849810 --- /dev/null +++ b/spacy/lang/tl/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class TagalogDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Tagalog(Language): + lang = "tl" + Defaults = TagalogDefaults + + +__all__ = ["Tagalog"] diff --git a/spacy/lang/tl/lex_attrs.py b/spacy/lang/tl/lex_attrs.py new file mode 100644 index 0000000..8866453 --- /dev/null +++ b/spacy/lang/tl/lex_attrs.py @@ -0,0 +1,56 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "sero", + "isa", + "dalawa", + "tatlo", + "apat", + "lima", + "anim", + "pito", + "walo", + "siyam", + "sampu", + "labing-isa", + "labindalawa", + "labintatlo", + "labing-apat", + "labinlima", + "labing-anim", + "labimpito", + "labing-walo", + "labinsiyam", + "dalawampu", + "tatlumpu", + "apatnapu", + "limampu", + "animnapu", + "pitumpu", + "walumpu", + "siyamnapu", + "daan", + "libo", + "milyon", + "bilyon", + "trilyon", + "quadrilyon", + "gajilyon", + "bazilyon", +] + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/tl/stop_words.py b/spacy/lang/tl/stop_words.py new file mode 100644 index 0000000..2560cda --- /dev/null +++ b/spacy/lang/tl/stop_words.py @@ -0,0 +1,151 @@ +STOP_WORDS = set( + """ +akin +aking +ako +alin +am +amin +aming +ang +ano +anumang +apat +at +atin +ating +ay +bababa +bago +bakit +bawat +bilang +dahil +dalawa +dapat +din +dito +doon +gagawin +gayunman +ginagawa +ginawa +ginawang +gumawa +gusto +habang +hanggang +hindi +huwag +iba +ibaba +ibabaw +ibig +ikaw +ilagay +ilalim +ilan +inyong +isa +isang +itaas +ito +iyo +iyon +iyong +ka +kahit +kailangan +kailanman +kami +kanila +kanilang +kanino +kanya +kanyang +kapag +kapwa +karamihan +katiyakan +katulad +kaya +kaysa +ko +kong +kulang +kumuha +kung +laban +lahat +lamang +likod +lima +maaari +maaaring +maging +mahusay +makita +marami +marapat +masyado +may +mayroon +mga +minsan +mismo +mula +muli +na +nabanggit +naging +nagkaroon +nais +nakita +namin +napaka +narito +nasaan +ng +ngayon +ni +nila +nilang +nito +niya +niyang +noon +o +pa +paano +pababa +paggawa +pagitan +pagkakaroon +pagkatapos +palabas +pamamagitan +panahon +pangalawa +para +paraan +pareho +pataas +pero +pumunta +pumupunta +sa +saan +sabi +sabihin +sarili +sila +sino +siya +tatlo +tayo +tulad +tungkol +una +walang +""".split() +) diff --git a/spacy/lang/tl/tokenizer_exceptions.py b/spacy/lang/tl/tokenizer_exceptions.py new file mode 100644 index 0000000..b10c904 --- /dev/null +++ b/spacy/lang/tl/tokenizer_exceptions.py @@ -0,0 +1,18 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = { + "tayo'y": [{ORTH: "tayo"}, {ORTH: "'y", NORM: "ay"}], + "isa'y": [{ORTH: "isa"}, {ORTH: "'y", NORM: "ay"}], + "baya'y": [{ORTH: "baya"}, {ORTH: "'y", NORM: "ay"}], + "sa'yo": [{ORTH: "sa"}, {ORTH: "'yo", NORM: "iyo"}], + "ano'ng": [{ORTH: "ano"}, {ORTH: "'ng", NORM: "ang"}], + "siya'y": [{ORTH: "siya"}, {ORTH: "'y", NORM: "ay"}], + "nawa'y": [{ORTH: "nawa"}, {ORTH: "'y", NORM: "ay"}], + "papa'no": [{ORTH: "papa'no", NORM: "papaano"}], + "'di": [{ORTH: "'di", NORM: "hindi"}], +} + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/tn/__init__.py b/spacy/lang/tn/__init__.py new file mode 100644 index 0000000..4cb8a16 --- /dev/null +++ b/spacy/lang/tn/__init__.py @@ -0,0 +1,18 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS + + +class SetswanaDefaults(BaseDefaults): + infixes = TOKENIZER_INFIXES + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Setswana(Language): + lang = "tn" + Defaults = SetswanaDefaults + + +__all__ = ["Setswana"] diff --git a/spacy/lang/tn/examples.py b/spacy/lang/tn/examples.py new file mode 100644 index 0000000..fb6d96f --- /dev/null +++ b/spacy/lang/tn/examples.py @@ -0,0 +1,14 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.tn.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple e nyaka go reka JSE ka tlhwatlhwa ta R1 billion", + "Johannesburg ke toropo e kgolo mo Afrika Borwa.", + "O ko kae?", + "ke mang presidente ya Afrika Borwa?", + "ke eng toropo kgolo ya Afrika Borwa?", + "Nelson Mandela o belegwe leng?", +] diff --git a/spacy/lang/tn/lex_attrs.py b/spacy/lang/tn/lex_attrs.py new file mode 100644 index 0000000..c136d0a --- /dev/null +++ b/spacy/lang/tn/lex_attrs.py @@ -0,0 +1,107 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "lefela", + "nngwe", + "pedi", + "tharo", + "nne", + "tlhano", + "thataro", + "supa", + "robedi", + "robongwe", + "lesome", + "lesomenngwe", + "lesomepedi", + "sometharo", + "somenne", + "sometlhano", + "somethataro", + "somesupa", + "somerobedi", + "somerobongwe", + "someamabedi", + "someamararo", + "someamane", + "someamatlhano", + "someamarataro", + "someamasupa", + "someamarobedi", + "someamarobongwe", + "lekgolo", + "sekete", + "milione", + "bilione", + "terilione", + "kwatirilione", + "gajillione", + "bazillione", +] + + +_ordinal_words = [ + "ntlha", + "bobedi", + "boraro", + "bone", + "botlhano", + "borataro", + "bosupa", + "borobedi ", + "borobongwe", + "bolesome", + "bolesomengwe", + "bolesomepedi", + "bolesometharo", + "bolesomenne", + "bolesometlhano", + "bolesomethataro", + "bolesomesupa", + "bolesomerobedi", + "bolesomerobongwe", + "somamabedi", + "someamararo", + "someamane", + "someamatlhano", + "someamarataro", + "someamasupa", + "someamarobedi", + "someamarobongwe", + "lekgolo", + "sekete", + "milione", + "bilione", + "terilione", + "kwatirilione", + "gajillione", + "bazillione", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + + text_lower = text.lower() + if text_lower in _num_words: + return True + + # CHeck ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith("th"): + if text_lower[:-2].isdigit(): + return True + + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/tn/punctuation.py b/spacy/lang/tn/punctuation.py new file mode 100644 index 0000000..54d76fb --- /dev/null +++ b/spacy/lang/tn/punctuation.py @@ -0,0 +1,26 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/tn/stop_words.py b/spacy/lang/tn/stop_words.py new file mode 100644 index 0000000..f614771 --- /dev/null +++ b/spacy/lang/tn/stop_words.py @@ -0,0 +1,20 @@ +# Stop words +STOP_WORDS = set( + """ +ke gareng ga selekanyo tlhwatlhwa yo mongwe se +sengwe fa go le jalo gongwe ba na mo tikologong +jaaka kwa morago nna gonne ka sa pele nako teng +tlase fela ntle magareng tsona feta bobedi kgabaganya +moo gape kgatlhanong botlhe tsotlhe bokana e esi +setseng mororo dinako golo kgolo nnye wena gago +o ntse ntle tla goreng gangwe mang yotlhe gore +eo yona tseraganyo eng ne sentle re rona thata +godimo fitlha pedi masomamabedi lesomepedi mmogo +tharo tseo boraro tseno yone jaanong bobona bona +lesome tsaya tsamaiso nngwe masomethataro thataro +tsa mmatota tota sale thoko supa dira tshwanetse di mmalwa masisi +bonala e tshwanang bogolo tsenya tsweetswee karolo +sepe tlhalosa dirwa robedi robongwe lesomenngwe gaisa +tlhano lesometlhano botlalo lekgolo +""".split() +) diff --git a/spacy/lang/tokenizer_exceptions.py b/spacy/lang/tokenizer_exceptions.py new file mode 100644 index 0000000..b7d91d8 --- /dev/null +++ b/spacy/lang/tokenizer_exceptions.py @@ -0,0 +1,257 @@ +import re + +from ..symbols import NORM, ORTH +from .char_classes import ALPHA_LOWER + +# URL validation regex courtesy of: https://mathiasbynens.be/demo/url-regex +# and https://gist.github.com/dperini/729294 (Diego Perini, MIT License) +# A few mods to this regex to account for use cases represented in test_urls +URL_PATTERN = ( + # fmt: off + r"^" + # protocol identifier (mods: make optional and expand schemes) + # (see: https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml) + r"(?:(?:[\w\+\-\.]{2,})://)?" + # mailto:user or user:pass authentication + r"(?:\S+(?::\S*)?@)?" + r"(?:" + # IP address exclusion + # private & local networks + r"(?!(?:10|127)(?:\.\d{1,3}){3})" + r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})" + r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})" + # IP address dotted notation octets + # excludes loopback network 0.0.0.0 + # excludes reserved space >= 224.0.0.0 + # excludes network & broadcast addresses + # (first & last IP address of each class) + # MH: Do we really need this? Seems excessive, and seems to have caused + # Issue #957 + r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}" + r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))" + r"|" + # host & domain names + # mods: match is case-sensitive, so include [A-Z] + r"(?:" # noqa: E131 + r"(?:" # noqa: E131 + r"[A-Za-z0-9\u00a1-\uffff]" # noqa: E131 + r"[A-Za-z0-9\u00a1-\uffff_-]{0,62}" + r")?" + r"[A-Za-z0-9\u00a1-\uffff]\." + r")+" + # TLD identifier + # mods: use ALPHA_LOWER instead of a wider range so that this doesn't match + # strings like "lower.Upper", which can be split on "." by infixes in some + # languages + r"(?:[" + ALPHA_LOWER + "]{2,63})" + r")" + # port number + r"(?::\d{2,5})?" + # resource path + r"(?:[/?#]\S*)?" + r"$" + # fmt: on +).strip() + +URL_MATCH = re.compile("(?u)" + URL_PATTERN).match + + +BASE_EXCEPTIONS = {} + + +for exc_data in [ + {ORTH: " "}, + {ORTH: "\t"}, + {ORTH: "\\t"}, + {ORTH: "\n"}, + {ORTH: "\\n"}, + {ORTH: "\u2014"}, + {ORTH: "\u00a0", NORM: " "}, +]: + BASE_EXCEPTIONS[exc_data[ORTH]] = [exc_data] + + +for orth in [ + "'", + '\\")', + "", + "''", + "C++", + "a.", + "b.", + "c.", + "d.", + "e.", + "f.", + "g.", + "h.", + "i.", + "j.", + "k.", + "l.", + "m.", + "n.", + "o.", + "p.", + "q.", + "r.", + "s.", + "t.", + "u.", + "v.", + "w.", + "x.", + "y.", + "z.", + "ä.", + "ö.", + "ü.", +]: + BASE_EXCEPTIONS[orth] = [{ORTH: orth}] + + +emoticons = set( + r""" +:) +:-) +:)) +:-)) +:))) +:-))) +(: +(-: +=) +(= +:] +:-] +[: +[-: +[= +=] +:o) +(o: +:} +:-} +8) +8-) +(-8 +;) +;-) +(; +(-; +:( +:-( +:(( +:-(( +:((( +:-((( +): +)-: +=( +>:( +:') +:'-) +:'( +:'-( +:/ +:-/ +=/ +=| +:| +:-| +]= +=[ +:1 +:P +:-P +:p +:-p +:O +:-O +:o +:-o +:0 +:-0 +:() +>:o +:* +:-* +:3 +:-3 +=3 +:> +:-> +:X +:-X +:x +:-x +:D +:-D +;D +;-D +=D +xD +XD +xDD +XDD +8D +8-D + +^_^ +^__^ +^___^ +>.< +>.> +<.< +._. +;_; +-_- +-__- +v.v +V.V +v_v +V_V +o_o +o_O +O_o +O_O +0_o +o_0 +0_0 +o.O +O.o +O.O +o.o +0.0 +o.0 +0.o +@_@ +<3 +<33 +<333 +_<) +(*_*) +(¬_¬) +ಠ_ಠ +ಠ︵ಠ +(ಠ_ಠ) +¯\(ツ)/¯ +(╯°□°）╯︵┻━┻ +><(((*> +""".split() +) + + +for orth in emoticons: + BASE_EXCEPTIONS[orth] = [{ORTH: orth}] + + +# Moved from a suffix setting due to #9155 removing prefixes from consideration +# for lookbehinds +for u in "cfkCFK": + BASE_EXCEPTIONS[f"°{u}."] = [{ORTH: "°"}, {ORTH: f"{u}"}, {ORTH: "."}] diff --git a/spacy/lang/tr/__init__.py b/spacy/lang/tr/__init__.py new file mode 100644 index 0000000..9aa7521 --- /dev/null +++ b/spacy/lang/tr/__init__.py @@ -0,0 +1,21 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .syntax_iterators import SYNTAX_ITERATORS +from .tokenizer_exceptions import TOKEN_MATCH, TOKENIZER_EXCEPTIONS + + +class TurkishDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + token_match = TOKEN_MATCH + syntax_iterators = SYNTAX_ITERATORS + + +class Turkish(Language): + lang = "tr" + Defaults = TurkishDefaults + + +__all__ = ["Turkish"] diff --git a/spacy/lang/tr/examples.py b/spacy/lang/tr/examples.py new file mode 100644 index 0000000..c96e540 --- /dev/null +++ b/spacy/lang/tr/examples.py @@ -0,0 +1,20 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.tr.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Neredesin?", + "Neredesiniz?", + "Bu bir cümledir.", + "Sürücüsüz araçlar sigorta yükümlülüğünü üreticilere kaydırıyor.", + "San Francisco kaldırımda kurye robotları yasaklayabilir." + "Londra İngiltere'nin başkentidir.", + "Türkiye'nin başkenti neresi?", + "Bakanlar Kurulu 180 günlük eylem planını açıkladı.", + "Merkez Bankası, beklentiler doğrultusunda faizlerde değişikliğe gitmedi.", + "Cemal Sureya kimdir?", + "Bunlari Biliyor muydunuz?", + "Altinoluk Turkiye haritasinin neresinde yer alir?", +] diff --git a/spacy/lang/tr/lex_attrs.py b/spacy/lang/tr/lex_attrs.py new file mode 100644 index 0000000..2189932 --- /dev/null +++ b/spacy/lang/tr/lex_attrs.py @@ -0,0 +1,88 @@ +from ...attrs import LIKE_NUM + +# Thirteen, fifteen etc. are written separate: on üç + +_num_words = [ + "bir", + "iki", + "üç", + "dört", + "beş", + "altı", + "yedi", + "sekiz", + "dokuz", + "on", + "yirmi", + "otuz", + "kırk", + "elli", + "altmış", + "yetmiş", + "seksen", + "doksan", + "yüz", + "bin", + "milyon", + "milyar", + "trilyon", + "katrilyon", + "kentilyon", +] + + +_ordinal_words = [ + "birinci", + "ikinci", + "üçüncü", + "dördüncü", + "beşinci", + "altıncı", + "yedinci", + "sekizinci", + "dokuzuncu", + "onuncu", + "yirminci", + "otuzuncu", + "kırkıncı", + "ellinci", + "altmışıncı", + "yetmişinci", + "sekseninci", + "doksanıncı", + "yüzüncü", + "bininci", + "milyonuncu", + "milyarıncı", + "trilyonuncu", + "katrilyonuncu", + "kentilyonuncu", +] + +_ordinal_endings = ("inci", "ıncı", "nci", "ncı", "uncu", "üncü") + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + # Check cardinal number + if text_lower in _num_words: + return True + # Check ordinal number + if text_lower in _ordinal_words: + return True + if text_lower.endswith(_ordinal_endings): + if text_lower[:-3].isdigit() or text_lower[:-4].isdigit(): + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/tr/stop_words.py b/spacy/lang/tr/stop_words.py new file mode 100644 index 0000000..85dcff6 --- /dev/null +++ b/spacy/lang/tr/stop_words.py @@ -0,0 +1,557 @@ +# Source: https://github.com/stopwords-iso/stopwords-tr +STOP_WORDS = set( + """ +acaba +acep +adamakıllı +adeta +ait +ama +amma +anca +ancak +arada +artık +aslında +aynen +ayrıca +az +açıkça +açıkçası +bana +bari +bazen +bazı +bazısı +bazısına +bazısında +bazısından +bazısını +bazısının +başkası +başkasına +başkasında +başkasından +başkasını +başkasının +başka +belki +ben +bende +benden +beni +benim +beri +beriki +berikinin +berikiyi +berisi +bilcümle +bile +binaen +binaenaleyh +biraz +birazdan +birbiri +birbirine +birbirini +birbirinin +birbirinde +birbirinden +birden +birdenbire +biri +birine +birini +birinin +birinde +birinden +birice +birileri +birilerinde +birilerinden +birilerine +birilerini +birilerinin +birisi +birisine +birisini +birisinin +birisinde +birisinden +birkaç +birkaçı +birkaçına +birkaçını +birkaçının +birkaçında +birkaçından +birkez +birlikte +birçok +birçoğu +birçoğuna +birçoğunda +birçoğundan +birçoğunu +birçoğunun +birşey +birşeyi +bitevi +biteviye +bittabi +biz +bizatihi +bizce +bizcileyin +bizden +bize +bizi +bizim +bizimki +bizzat +boşuna +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +buracıkta +burada +buradan +burası +burasına +burasını +burasının +burasında +burasından +böyle +böylece +böylecene +böylelikle +böylemesine +böylesine +büsbütün +bütün +cuk +cümlesi +cümlesine +cümlesini +cümlesinin +cümlesinden +cümlemize +cümlemizi +cümlemizden +çabuk +çabukça +çeşitli +çok +çokları +çoklarınca +çokluk +çoklukla +çokça +çoğu +çoğun +çoğunca +çoğunda +çoğundan +çoğunlukla +çoğunu +çoğunun +çünkü +da +daha +dahası +dahi +dahil +dahilen +daima +dair +dayanarak +de +defa +dek +demin +demincek +deminden +denli +derakap +derhal +derken +değil +değin +diye +diğer +diğeri +diğerine +diğerini +diğerinden +dolayı +dolayısıyla +doğru +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +elbet +elbette +emme +en +enikonu +epey +epeyce +epeyi +esasen +esnasında +etmesi +etraflı +etraflıca +etti +ettiği +ettiğini +evleviyetle +evvel +evvela +evvelce +evvelden +evvelemirde +evveli +eğer +fakat +filanca +filancanın +gah +gayet +gayetle +gayri +gayrı +gelgelelim +gene +gerek +gerçi +geçende +geçenlerde +gibi +gibilerden +gibisinden +gine +göre +gırla +hakeza +halbuki +halen +halihazırda +haliyle +handiyse +hangi +hangisi +hangisine +hangisine +hangisinde +hangisinden +hani +hariç +hasebiyle +hasılı +hatta +hele +hem +henüz +hep +hepsi +hepsini +hepsinin +hepsinde +hepsinden +her +herhangi +herkes +herkesi +herkesin +herkesten +hiç +hiçbir +hiçbiri +hiçbirine +hiçbirini +hiçbirinin +hiçbirinde +hiçbirinden +hoş +hulasaten +iken +ila +ile +ilen +ilgili +ilk +illa +illaki +imdi +indinde +inen +insermi +ise +ister +itibaren +itibariyle +itibarıyla +iyi +iyice +iyicene +için +iş +işte +kadar +kaffesi +kah +kala +kanımca +karşın +kaynak +kaçı +kaçına +kaçında +kaçından +kaçını +kaçının +kelli +kendi +kendilerinde +kendilerinden +kendilerine +kendilerini +kendilerinin +kendini +kendisi +kendisinde +kendisinden +kendisine +kendisini +kendisinin +kere +kez +keza +kezalik +keşke +ki +kim +kimden +kime +kimi +kiminin +kimisi +kimisinde +kimisinden +kimisine +kimisinin +kimse +kimsecik +kimsecikler +külliyen +kısaca +kısacası +lakin +leh +lütfen +maada +madem +mademki +mamafih +mebni +međer +meğer +meğerki +meğerse +mu +mü +mı +mi +nasıl +nasılsa +nazaran +naşi +ne +neden +nedeniyle +nedenle +nedenler +nedenlerden +nedense +nerde +nerden +nerdeyse +nere +nerede +nereden +neredeyse +neresi +nereye +netekim +neye +neyi +neyse +nice +nihayet +nihayetinde +nitekim +niye +niçin +o +olan +olarak +oldu +olduklarını +oldukça +olduğu +olduğunu +olmak +olması +olsa +olsun +olup +olur +olursa +oluyor +ona +onca +onculayın +onda +ondan +onlar +onlara +onlardan +onları +onların +onu +onun +ora +oracık +oracıkta +orada +oradan +oranca +oranla +oraya +oysa +oysaki +öbür +öbürkü +öbürü +öbüründe +öbüründen +öbürüne +öbürünü +önce +önceden +önceleri +öncelikle +öteki +ötekisi +öyle +öylece +öylelikle +öylemesine +öz +pek +pekala +peki +pekçe +peyderpey +rağmen +sadece +sahi +sahiden +sana +sanki +sen +senden +seni +senin +siz +sizden +sizi +sizin +sonra +sonradan +sonraları +sonunda +şayet +şey +şeyden +şeyi +şeyler +şu +şuna +şuncacık +şunda +şundan +şunlar +şunları +şunların +şunu +şunun +şura +şuracık +şuracıkta +şurası +şöyle +şimdi +tabii +tam +tamam +tamamen +tamamıyla +tarafından +tek +tüm +üzere +var +vardı +vasıtasıyla +ve +velev +velhasıl +velhasılıkelam +veya +veyahut +ya +yahut +yakinen +yakında +yakından +yakınlarda +yalnız +yalnızca +yani +yapacak +yapmak +yaptı +yaptıkları +yaptığı +yaptığını +yapılan +yapılması +yapıyor +yeniden +yenilerde +yerine +yine +yok +yoksa +yoluyla +yüzünden +zarfında +zaten +zati +zira +""".split() +) diff --git a/spacy/lang/tr/syntax_iterators.py b/spacy/lang/tr/syntax_iterators.py new file mode 100644 index 0000000..ed58842 --- /dev/null +++ b/spacy/lang/tr/syntax_iterators.py @@ -0,0 +1,58 @@ +from typing import Iterator, Tuple, Union + +from ...errors import Errors +from ...symbols import NOUN, PRON, PROPN +from ...tokens import Doc, Span + + +def noun_chunks(doclike: Union[Doc, Span]) -> Iterator[Tuple[int, int, int]]: + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + # Please see documentation for Turkish NP structure + labels = [ + "nsubj", + "iobj", + "obj", + "obl", + "appos", + "orphan", + "dislocated", + "ROOT", + ] + doc = doclike.doc # Ensure works on both Doc and Span. + if not doc.has_annotation("DEP"): + raise ValueError(Errors.E029) + + np_deps = [doc.vocab.strings.add(label) for label in labels] + conj = doc.vocab.strings.add("conj") + flat = doc.vocab.strings.add("flat") + np_label = doc.vocab.strings.add("NP") + + def extend_right(w): # Playing a trick for flat + rindex = w.i + 1 + for rdep in doc[w.i].rights: # Extend the span to right if there is a flat + if rdep.dep == flat and rdep.pos in (NOUN, PROPN): + rindex = rdep.i + 1 + else: + break + return rindex + + prev_end = len(doc) + 1 + for i, word in reversed(list(enumerate(doclike))): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.i >= prev_end: + continue + if word.dep in np_deps: + prev_end = word.left_edge.i + yield word.left_edge.i, extend_right(word), np_label + elif word.dep == conj: + cc_token = word.left_edge + prev_end = cc_token.i + # Shave off cc tokens from the NP + yield cc_token.right_edge.i + 1, extend_right(word), np_label + + +SYNTAX_ITERATORS = {"noun_chunks": noun_chunks} diff --git a/spacy/lang/tr/tokenizer_exceptions.py b/spacy/lang/tr/tokenizer_exceptions.py new file mode 100644 index 0000000..d095a3d --- /dev/null +++ b/spacy/lang/tr/tokenizer_exceptions.py @@ -0,0 +1,190 @@ +import re + +from ...symbols import NORM, ORTH +from ..punctuation import ALPHA, ALPHA_LOWER + +_exc = {} + + +_abbr_period_exc = [ + {ORTH: "A.B.D.", NORM: "Amerika"}, + {ORTH: "Alb.", NORM: "albay"}, + {ORTH: "Ank.", NORM: "Ankara"}, + {ORTH: "Ar.Gör."}, + {ORTH: "Arş.Gör."}, + {ORTH: "Asb.", NORM: "astsubay"}, + {ORTH: "Astsb.", NORM: "astsubay"}, + {ORTH: "As.İz."}, + {ORTH: "as.iz."}, + {ORTH: "Atğm", NORM: "asteğmen"}, + {ORTH: "Av.", NORM: "avukat"}, + {ORTH: "Apt.", NORM: "apartmanı"}, + {ORTH: "apt.", NORM: "apartmanı"}, + {ORTH: "Bçvş.", NORM: "başçavuş"}, + {ORTH: "bçvş.", NORM: "başçavuş"}, + {ORTH: "bk.", NORM: "bakınız"}, + {ORTH: "bknz.", NORM: "bakınız"}, + {ORTH: "Bnb.", NORM: "binbaşı"}, + {ORTH: "bnb.", NORM: "binbaşı"}, + {ORTH: "Böl.", NORM: "bölümü"}, + {ORTH: "böl.", NORM: "bölümü"}, + {ORTH: "Bşk.", NORM: "başkanlığı"}, + {ORTH: "bşk.", NORM: "başkanlığı"}, + {ORTH: "Bştbp.", NORM: "baştabip"}, + {ORTH: "bştbp.", NORM: "baştabip"}, + {ORTH: "Bul.", NORM: "bulvarı"}, + {ORTH: "bul.", NORM: "bulvarı"}, + {ORTH: "Cad.", NORM: "caddesi"}, + {ORTH: "cad.", NORM: "caddesi"}, + {ORTH: "çev.", NORM: "çeviren"}, + {ORTH: "Çvş.", NORM: "çavuş"}, + {ORTH: "çvş.", NORM: "çavuş"}, + {ORTH: "dak.", NORM: "dakika"}, + {ORTH: "dk.", NORM: "dakika"}, + {ORTH: "Doç.", NORM: "doçent"}, + {ORTH: "doğ."}, + {ORTH: "Dr.", NORM: "doktor"}, + {ORTH: "dr.", NORM: "doktor"}, + {ORTH: "drl.", NORM: "derleyen"}, + {ORTH: "Dz.", NORM: "deniz"}, + {ORTH: "Dz.K.K.lığı"}, + {ORTH: "Dz.Kuv."}, + {ORTH: "Dz.Kuv.K."}, + {ORTH: "dzl.", NORM: "düzenleyen"}, + {ORTH: "Ecz.", NORM: "eczanesi"}, + {ORTH: "ecz.", NORM: "eczanesi"}, + {ORTH: "ekon.", NORM: "ekonomi"}, + {ORTH: "Fak.", NORM: "fakültesi"}, + {ORTH: "Gn.", NORM: "genel"}, + {ORTH: "Gnkur.", NORM: "Genelkurmay"}, + {ORTH: "Gn.Kur.", NORM: "Genelkurmay"}, + {ORTH: "gr.", NORM: "gram"}, + {ORTH: "Hst.", NORM: "hastanesi"}, + {ORTH: "hst.", NORM: "hastanesi"}, + {ORTH: "Hs.Uzm."}, + {ORTH: "huk.", NORM: "hukuk"}, + {ORTH: "Hv.", NORM: "hava"}, + {ORTH: "Hv.K.K.lığı"}, + {ORTH: "Hv.Kuv."}, + {ORTH: "Hv.Kuv.K."}, + {ORTH: "Hz.", NORM: "hazreti"}, + {ORTH: "Hz.Öz."}, + {ORTH: "İng.", NORM: "ingilizce"}, + {ORTH: "İst.", NORM: "İstanbul"}, + {ORTH: "Jeol.", NORM: "jeoloji"}, + {ORTH: "jeol.", NORM: "jeoloji"}, + {ORTH: "Korg.", NORM: "korgeneral"}, + {ORTH: "Kur.", NORM: "kurmay"}, + {ORTH: "Kur.Bşk."}, + {ORTH: "Kuv.", NORM: "kuvvetleri"}, + {ORTH: "Ltd.", NORM: "limited"}, + {ORTH: "ltd.", NORM: "limited"}, + {ORTH: "Mah.", NORM: "mahallesi"}, + {ORTH: "mah.", NORM: "mahallesi"}, + {ORTH: "max.", NORM: "maksimum"}, + {ORTH: "min.", NORM: "minimum"}, + {ORTH: "Müh.", NORM: "mühendisliği"}, + {ORTH: "müh.", NORM: "mühendisliği"}, + {ORTH: "M.Ö."}, + {ORTH: "M.S."}, + {ORTH: "Onb.", NORM: "onbaşı"}, + {ORTH: "Ord.", NORM: "ordinaryüs"}, + {ORTH: "Org.", NORM: "orgeneral"}, + {ORTH: "Ped.", NORM: "pedagoji"}, + {ORTH: "Prof.", NORM: "profesör"}, + {ORTH: "prof.", NORM: "profesör"}, + {ORTH: "Sb.", NORM: "subay"}, + {ORTH: "Sn.", NORM: "sayın"}, + {ORTH: "sn.", NORM: "saniye"}, + {ORTH: "Sok.", NORM: "sokak"}, + {ORTH: "sok.", NORM: "sokak"}, + {ORTH: "Şb.", NORM: "şube"}, + {ORTH: "şb.", NORM: "şube"}, + {ORTH: "Şti.", NORM: "şirketi"}, + {ORTH: "şti.", NORM: "şirketi"}, + {ORTH: "Tbp.", NORM: "tabip"}, + {ORTH: "tbp.", NORM: "tabip"}, + {ORTH: "T.C."}, + {ORTH: "Tel.", NORM: "telefon"}, + {ORTH: "tel.", NORM: "telefon"}, + {ORTH: "telg.", NORM: "telgraf"}, + {ORTH: "Tğm.", NORM: "teğmen"}, + {ORTH: "tğm.", NORM: "teğmen"}, + {ORTH: "tic.", NORM: "ticaret"}, + {ORTH: "Tug.", NORM: "tugay"}, + {ORTH: "Tuğg.", NORM: "tuğgeneral"}, + {ORTH: "Tümg.", NORM: "tümgeneral"}, + {ORTH: "Uzm.", NORM: "uzman"}, + {ORTH: "Üçvş.", NORM: "üstçavuş"}, + {ORTH: "Üni.", NORM: "üniversitesi"}, + {ORTH: "Ütğm.", NORM: "üsteğmen"}, + {ORTH: "vb."}, + {ORTH: "vs.", NORM: "vesaire"}, + {ORTH: "Yard.", NORM: "yardımcı"}, + {ORTH: "Yar.", NORM: "yardımcı"}, + {ORTH: "Yd.Sb."}, + {ORTH: "Yard.Doç."}, + {ORTH: "Yar.Doç."}, + {ORTH: "Yb.", NORM: "yarbay"}, + {ORTH: "Yrd.", NORM: "yardımcı"}, + {ORTH: "Yrd.Doç."}, + {ORTH: "Y.Müh."}, + {ORTH: "Y.Mim."}, + {ORTH: "yy.", NORM: "yüzyıl"}, +] + +for abbr in _abbr_period_exc: + _exc[abbr[ORTH]] = [abbr] + +_abbr_exc = [ + {ORTH: "AB", NORM: "Avrupa Birliği"}, + {ORTH: "ABD", NORM: "Amerika"}, + {ORTH: "ABS", NORM: "fren"}, + {ORTH: "AOÇ"}, + {ORTH: "ASKİ"}, + {ORTH: "Bağ-kur", NORM: "Bağkur"}, + {ORTH: "BDDK"}, + {ORTH: "BJK", NORM: "Beşiktaş"}, + {ORTH: "ESA", NORM: "Avrupa uzay ajansı"}, + {ORTH: "FB", NORM: "Fenerbahçe"}, + {ORTH: "GATA"}, + {ORTH: "GS", NORM: "Galatasaray"}, + {ORTH: "İSKİ"}, + {ORTH: "KBB"}, + {ORTH: "RTÜK", NORM: "radyo ve televizyon üst kurulu"}, + {ORTH: "TBMM"}, + {ORTH: "TC"}, + {ORTH: "TÜİK", NORM: "Türkiye istatistik kurumu"}, + {ORTH: "YÖK"}, +] + +for abbr in _abbr_exc: + _exc[abbr[ORTH]] = [abbr] + + +_num = r"[+-]?\d+([,.]\d+)*" +_ord_num = r"(\d+\.)" +_date = r"(((\d{1,2}[./-]){2})?(\d{4})|(\d{1,2}[./]\d{1,2}(\.)?))" +_dash_num = r"(([{al}\d]+/\d+)|(\d+/[{al}]))".format(al=ALPHA) +_roman_num = "M{0,3}(?:C[MD]|D?C{0,3})(?:X[CL]|L?X{0,3})(?:I[XV]|V?I{0,3})" +_roman_ord = r"({rn})\.".format(rn=_roman_num) +_time_exp = r"\d+(:\d+)*" + +_inflections = r"'[{al}]+".format(al=ALPHA_LOWER) +_abbrev_inflected = r"[{a}]+\.'[{al}]+".format(a=ALPHA, al=ALPHA_LOWER) + +_nums = r"(({d})|({dn})|({te})|({on})|({n})|({ro})|({rn}))({inf})?".format( + d=_date, + dn=_dash_num, + te=_time_exp, + on=_ord_num, + n=_num, + ro=_roman_ord, + rn=_roman_num, + inf=_inflections, +) + +TOKENIZER_EXCEPTIONS = _exc +TOKEN_MATCH = re.compile( + r"^({abbr})|({n})$".format(n=_nums, abbr=_abbrev_inflected) +).match diff --git a/spacy/lang/tt/__init__.py b/spacy/lang/tt/__init__.py new file mode 100644 index 0000000..ce04d09 --- /dev/null +++ b/spacy/lang/tt/__init__.py @@ -0,0 +1,20 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class TatarDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + infixes = TOKENIZER_INFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Tatar(Language): + lang = "tt" + Defaults = TatarDefaults + + +__all__ = ["Tatar"] diff --git a/spacy/lang/tt/examples.py b/spacy/lang/tt/examples.py new file mode 100644 index 0000000..723fcdd --- /dev/null +++ b/spacy/lang/tt/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.tt.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Apple Бөекбритания стартабын $1 миллиард өчен сатып алыун исәпли.", + "Автоном автомобильләр иминият җаваплылыкны җитештерүчеләргә күчерә.", + "Сан-Франциско тротуар буенча йөри торган робот-курьерларны тыю мөмкинлеген карый.", + "Лондон - Бөекбританиядә урнашкан зур шәһәр.", + "Син кайда?", + "Францияда кем президент?", + "Америка Кушма Штатларының башкаласы нинди шәһәр?", + "Барак Обама кайчан туган?", +] diff --git a/spacy/lang/tt/lex_attrs.py b/spacy/lang/tt/lex_attrs.py new file mode 100644 index 0000000..a2ae030 --- /dev/null +++ b/spacy/lang/tt/lex_attrs.py @@ -0,0 +1,58 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "нуль", + "ноль", + "бер", + "ике", + "өч", + "дүрт", + "биш", + "алты", + "җиде", + "сигез", + "тугыз", + "ун", + "унбер", + "унике", + "унөч", + "ундүрт", + "унбиш", + "уналты", + "унҗиде", + "унсигез", + "унтугыз", + "егерме", + "утыз", + "кырык", + "илле", + "алтмыш", + "җитмеш", + "сиксән", + "туксан", + "йөз", + "мең", + "төмән", + "миллион", + "миллиард", + "триллион", + "триллиард", +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/tt/punctuation.py b/spacy/lang/tt/punctuation.py new file mode 100644 index 0000000..5c233df --- /dev/null +++ b/spacy/lang/tt/punctuation.py @@ -0,0 +1,27 @@ +from ..char_classes import ( + ALPHA, + ALPHA_LOWER, + ALPHA_UPPER, + CONCAT_QUOTES, + HYPHENS, + LIST_ELLIPSES, + LIST_ICONS, +) + +_hyphens_no_dash = HYPHENS.replace("-", "").strip("|").replace("||", "") +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[{al}])\.(?=[{au}])".format(al=ALPHA_LOWER, au=ALPHA_UPPER), + r"(?<=[{a}])[,!?/()]+(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}{q}])[:<>=](?=[{a}])".format(a=ALPHA, q=CONCAT_QUOTES), + r"(?<=[{a}])--(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}])([{q}\)\]\(\[])(?=[\-{a}])".format(a=ALPHA, q=CONCAT_QUOTES), + r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=_hyphens_no_dash), + r"(?<=[0-9])-(?=[0-9])", + ] +) + +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/tt/stop_words.py b/spacy/lang/tt/stop_words.py new file mode 100644 index 0000000..44169b7 --- /dev/null +++ b/spacy/lang/tt/stop_words.py @@ -0,0 +1,173 @@ +# Tatar stopwords are from https://github.com/aliiae/stopwords-tt + +STOP_WORDS = set( + """алай алайса алар аларга аларда алардан аларны аларның аларча +алары аларын аларынга аларында аларыннан аларының алтмыш алтмышынчы алтмышынчыга +алтмышынчыда алтмышынчыдан алтмышынчылар алтмышынчыларга алтмышынчыларда +алтмышынчылардан алтмышынчыларны алтмышынчыларның алтмышынчыны алтмышынчының +алты алтылап алтынчы алтынчыга алтынчыда алтынчыдан алтынчылар алтынчыларга +алтынчыларда алтынчылардан алтынчыларны алтынчыларның алтынчыны алтынчының +алтышар анда андагы андай андый андыйга андыйда андыйдан андыйны андыйның аннан +ансы анча аны аныкы аныкын аныкынга аныкында аныкыннан аныкының анысы анысын +анысынга анысында анысыннан анысының аның аныңча аркылы ары аша аңа аңар аңарга +аңарда аңардагы аңардан + +бар бара барлык барча барчасы барчасын барчасына барчасында барчасыннан +барчасының бары башка башкача бе­лән без безгә бездә бездән безне безнең безнеңчә +белдерүенчә белән бер бергә беренче беренчегә беренчедә беренчедән беренчеләр +беренчеләргә беренчеләрдә беренчеләрдән беренчеләрне беренчеләрнең беренчене +беренченең беркайда беркайсы беркая беркаян беркем беркемгә беркемдә беркемне +беркемнең беркемнән берлән берни бернигә бернидә бернидән бернинди бернине +бернинең берничек берничә бернәрсә бернәрсәгә бернәрсәдә бернәрсәдән бернәрсәне +бернәрсәнең беррәттән берсе берсен берсенгә берсендә берсенең берсеннән берәр +берәрсе берәрсен берәрсендә берәрсенең берәрсеннән берәрсенә берәү бигрәк бик +бирле бит биш бишенче бишенчегә бишенчедә бишенчедән бишенчеләр бишенчеләргә +бишенчеләрдә бишенчеләрдән бишенчеләрне бишенчеләрнең бишенчене бишенченең +бишләп болай болар боларга боларда болардан боларны боларның болары боларын +боларынга боларында боларыннан боларының бу буе буена буенда буенча буйлап +буларак булачак булды булмый булса булып булыр булырга бусы бүтән бәлки бән +бәрабәренә бөтен бөтенесе бөтенесен бөтенесендә бөтенесенең бөтенесеннән +бөтенесенә + +вә + +гел генә гына гүя гүяки гәрчә + +да ди дигән диде дип дистәләгән дистәләрчә дүрт дүртенче дүртенчегә дүртенчедә +дүртенчедән дүртенчеләр дүртенчеләргә дүртенчеләрдә дүртенчеләрдән дүртенчеләрне +дүртенчеләрнең дүртенчене дүртенченең дүртләп дә + +егерме егерменче егерменчегә егерменчедә егерменчедән егерменчеләр +егерменчеләргә егерменчеләрдә егерменчеләрдән егерменчеләрне егерменчеләрнең +егерменчене егерменченең ел елда + +иде идек идем ике икенче икенчегә икенчедә икенчедән икенчеләр икенчеләргә +икенчеләрдә икенчеләрдән икенчеләрне икенчеләрнең икенчене икенченең икешәр икән +илле илленче илленчегә илленчедә илленчедән илленчеләр илленчеләргә +илленчеләрдә илленчеләрдән илленчеләрне илленчеләрнең илленчене илленченең илә +илән инде исә итеп иткән итте итү итә итәргә иң + +йөз йөзенче йөзенчегә йөзенчедә йөзенчедән йөзенчеләр йөзенчеләргә йөзенчеләрдә +йөзенчеләрдән йөзенчеләрне йөзенчеләрнең йөзенчене йөзенченең йөзләгән йөзләрчә +йөзәрләгән + +кадәр кай кайбер кайберләре кайберсе кайберәү кайберәүгә кайберәүдә кайберәүдән +кайберәүне кайберәүнең кайдагы кайсы кайсыбер кайсын кайсына кайсында кайсыннан +кайсының кайчангы кайчандагы кайчаннан караганда карамастан карамый карата каршы +каршына каршында каршындагы кебек кем кемгә кемдә кемне кемнең кемнән кенә ки +килеп килә кирәк кына кырыгынчы кырыгынчыга кырыгынчыда кырыгынчыдан +кырыгынчылар кырыгынчыларга кырыгынчыларда кырыгынчылардан кырыгынчыларны +кырыгынчыларның кырыгынчыны кырыгынчының кырык күк күпләгән күпме күпмеләп +күпмешәр күпмешәрләп күптән күрә + +ләкин + +максатында менә мең меңенче меңенчегә меңенчедә меңенчедән меңенчеләр +меңенчеләргә меңенчеләрдә меңенчеләрдән меңенчеләрне меңенчеләрнең меңенчене +меңенченең меңләгән меңләп меңнәрчә меңәрләгән меңәрләп миллиард миллиардлаган +миллиардларча миллион миллионлаган миллионнарча миллионынчы миллионынчыга +миллионынчыда миллионынчыдан миллионынчылар миллионынчыларга миллионынчыларда +миллионынчылардан миллионынчыларны миллионынчыларның миллионынчыны +миллионынчының мин миндә мине минем минемчә миннән миңа монда мондагы мондые +мондыен мондыенгә мондыендә мондыеннән мондыеның мондый мондыйга мондыйда +мондыйдан мондыйлар мондыйларга мондыйларда мондыйлардан мондыйларны +мондыйларның мондыйлары мондыйларын мондыйларынга мондыйларында мондыйларыннан +мондыйларының мондыйны мондыйның моннан монсыз монча моны моныкы моныкын +моныкынга моныкында моныкыннан моныкының монысы монысын монысынга монысында +монысыннан монысының моның моңа моңар моңарга мәгълүматынча мәгәр мән мөмкин + +ни нибарысы никадәре нинди ниндие ниндиен ниндиенгә ниндиендә ниндиенең +ниндиеннән ниндиләр ниндиләргә ниндиләрдә ниндиләрдән ниндиләрен ниндиләренн +ниндиләреннгә ниндиләренндә ниндиләреннең ниндиләренннән ниндиләрне ниндиләрнең +ниндирәк нихәтле ничаклы ничек ничәшәр ничәшәрләп нуль нче нчы нәрсә нәрсәгә +нәрсәдә нәрсәдән нәрсәне нәрсәнең + +саен сез сезгә сездә сездән сезне сезнең сезнеңчә сигез сигезенче сигезенчегә +сигезенчедә сигезенчедән сигезенчеләр сигезенчеләргә сигезенчеләрдә +сигезенчеләрдән сигезенчеләрне сигезенчеләрнең сигезенчене сигезенченең +сиксән син синдә сине синең синеңчә синнән сиңа соң сыман сүзенчә сүзләренчә + +та таба теге тегеләй тегеләр тегеләргә тегеләрдә тегеләрдән тегеләре тегеләрен +тегеләренгә тегеләрендә тегеләренең тегеләреннән тегеләрне тегеләрнең тегенди +тегендигә тегендидә тегендидән тегендине тегендинең тегендә тегендәге тегене +тегенеке тегенекен тегенекенгә тегенекендә тегенекенең тегенекеннән тегенең +тегеннән тегесе тегесен тегесенгә тегесендә тегесенең тегесеннән тегеңә тиеш тик +тикле тора триллиард триллион тугыз тугызлап тугызлашып тугызынчы тугызынчыга +тугызынчыда тугызынчыдан тугызынчылар тугызынчыларга тугызынчыларда +тугызынчылардан тугызынчыларны тугызынчыларның тугызынчыны тугызынчының туксан +туксанынчы туксанынчыга туксанынчыда туксанынчыдан туксанынчылар туксанынчыларга +туксанынчыларда туксанынчылардан туксанынчыларны туксанынчыларның туксанынчыны +туксанынчының турында тыш түгел тә тәгаенләнгән төмән + +уенча уйлавынча ук ул ун уналты уналтынчы уналтынчыга уналтынчыда уналтынчыдан +уналтынчылар уналтынчыларга уналтынчыларда уналтынчылардан уналтынчыларны +уналтынчыларның уналтынчыны уналтынчының унарлаган унарлап унаула унаулап унбер +унберенче унберенчегә унберенчедә унберенчедән унберенчеләр унберенчеләргә +унберенчеләрдә унберенчеләрдән унберенчеләрне унберенчеләрнең унберенчене +унберенченең унбиш унбишенче унбишенчегә унбишенчедә унбишенчедән унбишенчеләр +унбишенчеләргә унбишенчеләрдә унбишенчеләрдән унбишенчеләрне унбишенчеләрнең +унбишенчене унбишенченең ундүрт ундүртенче ундүртенчегә ундүртенчедә +ундүртенчедән ундүртенчеләр ундүртенчеләргә ундүртенчеләрдә ундүртенчеләрдән +ундүртенчеләрне ундүртенчеләрнең ундүртенчене ундүртенченең унике уникенче +уникенчегә уникенчедә уникенчедән уникенчеләр уникенчеләргә уникенчеләрдә +уникенчеләрдән уникенчеләрне уникенчеләрнең уникенчене уникенченең унлаган +унлап уннарча унсигез унсигезенче унсигезенчегә унсигезенчедә унсигезенчедән +унсигезенчеләр унсигезенчеләргә унсигезенчеләрдә унсигезенчеләрдән +унсигезенчеләрне унсигезенчеләрнең унсигезенчене унсигезенченең унтугыз +унтугызынчы унтугызынчыга унтугызынчыда унтугызынчыдан унтугызынчылар +унтугызынчыларга унтугызынчыларда унтугызынчылардан унтугызынчыларны +унтугызынчыларның унтугызынчыны унтугызынчының унынчы унынчыга унынчыда +унынчыдан унынчылар унынчыларга унынчыларда унынчылардан унынчыларны +унынчыларның унынчыны унынчының унҗиде унҗиденче унҗиденчегә унҗиденчедә +унҗиденчедән унҗиденчеләр унҗиденчеләргә унҗиденчеләрдә унҗиденчеләрдән +унҗиденчеләрне унҗиденчеләрнең унҗиденчене унҗиденченең унөч унөченче унөченчегә +унөченчедә унөченчедән унөченчеләр унөченчеләргә унөченчеләрдә унөченчеләрдән +унөченчеләрне унөченчеләрнең унөченчене унөченченең утыз утызынчы утызынчыга +утызынчыда утызынчыдан утызынчылар утызынчыларга утызынчыларда утызынчылардан +утызынчыларны утызынчыларның утызынчыны утызынчының + +фикеренчә фәкать + +хакында хәбәр хәлбуки хәтле хәтта + +чаклы чакта чөнки + +шикелле шул шулай шулар шуларга шуларда шулардан шуларны шуларның шулары шуларын +шуларынга шуларында шуларыннан шуларының шулкадәр шултикле шултиклем шулхәтле +шулчаклы шунда шундагы шундый шундыйга шундыйда шундыйдан шундыйны шундыйның +шунлыктан шуннан шунсы шунча шуны шуныкы шуныкын шуныкынга шуныкында шуныкыннан +шуныкының шунысы шунысын шунысынга шунысында шунысыннан шунысының шуның шушы +шушында шушыннан шушыны шушының шушыңа шуңа шуңар шуңарга + +элек + +югыйсә юк юкса + +я ягъни язуынча яисә яки яктан якын ярашлы яхут яшь яшьлек + +җиде җиделәп җиденче җиденчегә җиденчедә җиденчедән җиденчеләр җиденчеләргә +җиденчеләрдә җиденчеләрдән җиденчеләрне җиденчеләрнең җиденчене җиденченең +җидешәр җитмеш җитмешенче җитмешенчегә җитмешенчедә җитмешенчедән җитмешенчеләр +җитмешенчеләргә җитмешенчеләрдә җитмешенчеләрдән җитмешенчеләрне +җитмешенчеләрнең җитмешенчене җитмешенченең җыенысы + +үз үзе үзем үземдә үземне үземнең үземнән үземә үзен үзендә үзенең үзеннән үзенә +үк + +һичбер һичбере һичберен һичберендә һичберенең һичбереннән һичберенә һичберсе +һичберсен һичберсендә һичберсенең һичберсеннән һичберсенә һичберәү һичберәүгә +һичберәүдә һичберәүдән һичберәүне һичберәүнең һичкайсы һичкайсыга һичкайсыда +һичкайсыдан һичкайсыны һичкайсының һичкем һичкемгә һичкемдә һичкемне һичкемнең +һичкемнән һични һичнигә һичнидә һичнидән һичнинди һичнине һичнинең һичнәрсә +һичнәрсәгә һичнәрсәдә һичнәрсәдән һичнәрсәне һичнәрсәнең һәм һәммә һәммәсе +һәммәсен һәммәсендә һәммәсенең һәммәсеннән һәммәсенә һәр һәрбер һәрбере һәрберсе +һәркайсы һәркайсыга һәркайсыда һәркайсыдан һәркайсыны һәркайсының һәркем +һәркемгә һәркемдә һәркемне һәркемнең һәркемнән һәрни һәрнәрсә һәрнәрсәгә +һәрнәрсәдә һәрнәрсәдән һәрнәрсәне һәрнәрсәнең һәртөрле + +ә әгәр әйтүенчә әйтүләренчә әлбәттә әле әлеге әллә әмма әнә + +өстәп өч өчен өченче өченчегә өченчедә өченчедән өченчеләр өченчеләргә +өченчеләрдә өченчеләрдән өченчеләрне өченчеләрнең өченчене өченченең өчләп +өчәрләп""".split() +) diff --git a/spacy/lang/tt/tokenizer_exceptions.py b/spacy/lang/tt/tokenizer_exceptions.py new file mode 100644 index 0000000..280b9f8 --- /dev/null +++ b/spacy/lang/tt/tokenizer_exceptions.py @@ -0,0 +1,47 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +_abbrev_exc = [ + # Weekdays abbreviations + {ORTH: "дш", NORM: "дүшәмбе"}, + {ORTH: "сш", NORM: "сишәмбе"}, + {ORTH: "чш", NORM: "чәршәмбе"}, + {ORTH: "пш", NORM: "пәнҗешәмбе"}, + {ORTH: "җм", NORM: "җомга"}, + {ORTH: "шб", NORM: "шимбә"}, + {ORTH: "яш", NORM: "якшәмбе"}, + # Months abbreviations + {ORTH: "гый", NORM: "гыйнвар"}, + {ORTH: "фев", NORM: "февраль"}, + {ORTH: "мар", NORM: "март"}, + {ORTH: "мар", NORM: "март"}, + {ORTH: "апр", NORM: "апрель"}, + {ORTH: "июн", NORM: "июнь"}, + {ORTH: "июл", NORM: "июль"}, + {ORTH: "авг", NORM: "август"}, + {ORTH: "сен", NORM: "сентябрь"}, + {ORTH: "окт", NORM: "октябрь"}, + {ORTH: "ноя", NORM: "ноябрь"}, + {ORTH: "дек", NORM: "декабрь"}, + # Number abbreviations + {ORTH: "млрд", NORM: "миллиард"}, + {ORTH: "млн", NORM: "миллион"}, +] + +for abbr in _abbrev_exc: + for orth in (abbr[ORTH], abbr[ORTH].capitalize(), abbr[ORTH].upper()): + _exc[orth] = [{ORTH: orth, NORM: abbr[NORM]}] + _exc[orth + "."] = [{ORTH: orth + ".", NORM: abbr[NORM]}] + +for exc_data in [ # "etc." abbreviations + {ORTH: "һ.б.ш.", NORM: "һәм башка шундыйлар"}, + {ORTH: "һ.б.", NORM: "һәм башка"}, + {ORTH: "б.э.к.", NORM: "безнең эрага кадәр"}, + {ORTH: "б.э.", NORM: "безнең эра"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py new file mode 100644 index 0000000..5dd75a2 --- /dev/null +++ b/spacy/lang/uk/__init__.py @@ -0,0 +1,53 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...language import BaseDefaults, Language +from ..punctuation import ( + COMBINING_DIACRITICS_TOKENIZER_INFIXES, + COMBINING_DIACRITICS_TOKENIZER_SUFFIXES, +) +from .lemmatizer import UkrainianLemmatizer +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS + + +class UkrainianDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + suffixes = COMBINING_DIACRITICS_TOKENIZER_SUFFIXES + infixes = COMBINING_DIACRITICS_TOKENIZER_INFIXES + + +class Ukrainian(Language): + lang = "uk" + Defaults = UkrainianDefaults + + +@Ukrainian.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "pymorphy3", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, +) +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return UkrainianLemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +__all__ = ["Ukrainian"] diff --git a/spacy/lang/uk/examples.py b/spacy/lang/uk/examples.py new file mode 100644 index 0000000..3335c82 --- /dev/null +++ b/spacy/lang/uk/examples.py @@ -0,0 +1,18 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.uk.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Ніч на середу буде морозною.", + "Чим кращі книги ти читав, тим гірше спиш.", # Serhiy Zhadan + "Найстаріші ґудзики, відомі людству, археологи знайшли в долині ріки Інд.", + "Слов'янське слово «Україна» вперше згадується у Київському літописному зводі за Іпатіївським списком під 1187 роком.", # wikipedia + "Де у Києві найсмачніша кава?", + "Від Нижнього озера довгими дерев’яними сходами, над якими синьо й біло горіли маленькі коробочки-ліхтарики, підіймалися до нього двоє стовусів: найкращий друг Вертутій і його дванадцятилітній онук Чублик.", # blyznets_viktor_semenovych/zemlia_svitliachkiv + "Китайський космічний зонд \"Чан'е-4\" вперше в історії здійснив м'яку посадку на зворотному боці Місяця.", + "Коли до губ твоїх лишається півподиху, коли до губ твоїх лишається півкроку – зіниці твої виткані із подиву, в очах у тебе синьо і широко.", # Hryhorij Czubaj + "Дорогу сестру збираю у дорогу, а брати вирішили не брати машину.", # homographs +] diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py new file mode 100644 index 0000000..9ec582b --- /dev/null +++ b/spacy/lang/uk/lemmatizer.py @@ -0,0 +1,45 @@ +from typing import Callable, Optional + +from thinc.api import Model + +from ...pipeline.lemmatizer import lemmatizer_score +from ...vocab import Vocab +from ..ru.lemmatizer import RussianLemmatizer + + +class UkrainianLemmatizer(RussianLemmatizer): + def __init__( + self, + vocab: Vocab, + model: Optional[Model], + name: str = "lemmatizer", + *, + mode: str = "pymorphy3", + overwrite: bool = False, + scorer: Optional[Callable] = lemmatizer_score, + ) -> None: + if mode in {"pymorphy2", "pymorphy2_lookup"}: + try: + from pymorphy2 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Ukrainian lemmatizer mode 'pymorphy2' requires the " + "pymorphy2 library and dictionaries. Install them with: " + "pip install pymorphy2 pymorphy2-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="uk") + elif mode in {"pymorphy3", "pymorphy3_lookup"}: + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Ukrainian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3 pymorphy3-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="uk") + super().__init__( + vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) diff --git a/spacy/lang/uk/lex_attrs.py b/spacy/lang/uk/lex_attrs.py new file mode 100644 index 0000000..510e5b8 --- /dev/null +++ b/spacy/lang/uk/lex_attrs.py @@ -0,0 +1,70 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "більйон", + "вісім", + "вісімдесят", + "вісімнадцять", + "вісімсот", + "восьмий", + "два", + "двадцять", + "дванадцять", + "двісті", + "дев'яносто", + "дев'ятнадцять", + "дев'ятсот", + "дев'ять", + "десять", + "децильйон", + "квадрильйон", + "квінтильйон", + "мільйон", + "мільярд", + "нонильйон", + "один", + "одинадцять", + "октильйон", + "п'ятий", + "п'ятисотий", + "п'ятнадцять", + "п'ятсот", + "п'ять", + "секстильйон", + "септильйон", + "сім", + "сімдесят", + "сімнадцять", + "сімсот", + "сорок", + "сто", + "тисяча", + "три", + "тридцять", + "трильйон", + "тринадцять", + "триста", + "чотири", + "чотириста", + "чотирнадцять", + "шістдесят", + "шістнадцять", + "шістсот", + "шість", +] + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/uk/stop_words.py b/spacy/lang/uk/stop_words.py new file mode 100644 index 0000000..b11d7a0 --- /dev/null +++ b/spacy/lang/uk/stop_words.py @@ -0,0 +1,469 @@ +STOP_WORDS = set( + """а +або +адже +аж +але +алло +б +багато +без +безперервно +би +більш +більше +біля +близько +бо +був +буває +буде +будемо +будете +будеш +буду +будуть +будь +була +були +було +бути +в +вам +вами +вас +ваш +ваша +ваше +вашим +вашими +ваших +ваші +вашій +вашого +вашої +вашому +вашою +вашу +вгорі +вгору +вдалині +весь +вже +ви +від +відсотків +він +вісім +вісімнадцятий +вісімнадцять +вниз +внизу +вона +вони +воно +восьмий +все +всею +всі +всім +всіх +всього +всьому +всю +вся +втім +г +геть +говорив +говорить +давно +далеко +далі +дарма +два +двадцятий +двадцять +дванадцятий +дванадцять +дві +двох +де +дев'ятий +дев'ятнадцятий +дев'ятнадцять +дев'ять +декілька +день +десятий +десять +дійсно +для +дня +до +добре +довго +доки +досить +другий +дуже +дякую +е +є +ж +же +з +за +завжди +зазвичай +занадто +зараз +зате +звичайно +звідси +звідусіль +здається +зі +значить +знову +зовсім +і +із +її +їй +їм +іноді +інша +інше +інший +інших +інші +їх +й +його +йому +каже +ким +кілька +кого +кожен +кожна +кожне +кожні +коли +кому +краще +крім +куди +ласка +ледве +лише +м +має +майже +мало +мати +мене +мені +менш +менше +ми +мимо +міг +між +мій +мільйонів +мною +мого +могти +моє +моєї +моєму +моєю +може +можна +можно +можуть +мої +моїй +моїм +моїми +моїх +мою +моя +на +навіть +навіщо +навколо +навкруги +нагорі +над +назад +найбільш +нам +нами +нарешті +нас +наш +наша +наше +нашим +нашими +наших +наші +нашій +нашого +нашої +нашому +нашою +нашу +не +небагато +небудь +недалеко +неї +немає +нерідко +нещодавно +нею +нибудь +нижче +низько +ним +ними +них +ні +ніби +ніж +ній +ніколи +нікуди +нім +нічого +ну +нього +ньому +о +обидва +обоє +один +одинадцятий +одинадцять +однак +однієї +одній +одного +означає +окрім +он +особливо +ось +п'ятий +п'ятнадцятий +п'ятнадцять +п'ять +перед +перший +під +пізніше +пір +після +по +повинно +подів +поки +пора +поруч +посеред +потім +потрібно +почала +початку +при +про +просто +проте +проти +раз +разу +раніше +рано +раптом +рік +роки +років +року +році +сам +сама +саме +самим +самими +самих +самі +самій +само +самого +самому +саму +свого +своє +своєї +свої +своїй +своїх +свою +себе +сих +сім +сімнадцятий +сімнадцять +сказав +сказала +сказати +скільки +скрізь +собі +собою +спасибі +спочатку +справ +став +суть +сьогодні +сьомий +т +та +так +така +таке +такий +такі +також +там +твій +твого +твоє +твоєї +твоєму +твоєю +твої +твоїй +твоїм +твоїми +твоїх +твою +твоя +те +тебе +теж +тепер +ти +тим +тими +тисяч +тих +ті +тієї +тією +тій +тільки +тім +то +тобі +тобою +того +тоді +той +тому +тою +треба +третій +три +тринадцятий +тринадцять +трохи +ту +туди +тут +у +увесь +уміти +усе +усі +усім +усіма +усіх +усього +усьому +усю +усюди +уся +хіба +хотіти +хоч +хоча +хочеш +хто +це +цей +цим +цими +цих +ці +цієї +цій +цього +цьому +цю +ця +час +частіше +часто +часу +через +четвертий +чи +чиє +чиєї +чиєму +чиї +чиїй +чиїм +чиїми +чиїх +чий +чийого +чийому +чим +численна +численне +численний +численні +чию +чия +чого +чому +чотири +чотирнадцятий +чотирнадцять +шістнадцятий +шістнадцять +шість +шостий +ще +що +щоб +щодо +щось +я +як +яка +який +яких +які +якій +якого +якої +якщо""".split() +) diff --git a/spacy/lang/uk/tokenizer_exceptions.py b/spacy/lang/uk/tokenizer_exceptions.py new file mode 100644 index 0000000..07dd941 --- /dev/null +++ b/spacy/lang/uk/tokenizer_exceptions.py @@ -0,0 +1,36 @@ +from ...symbols import NORM, ORTH +from ...util import update_exc +from ..tokenizer_exceptions import BASE_EXCEPTIONS + +_exc = {} + +for exc_data in [ + {ORTH: "обл.", NORM: "область"}, + {ORTH: "р-н.", NORM: "район"}, + {ORTH: "р-н", NORM: "район"}, + {ORTH: "м.", NORM: "місто"}, + {ORTH: "вул.", NORM: "вулиця"}, + {ORTH: "просп.", NORM: "проспект"}, + {ORTH: "пр-кт", NORM: "проспект"}, + {ORTH: "бул.", NORM: "бульвар"}, + {ORTH: "пров.", NORM: "провулок"}, + {ORTH: "пл.", NORM: "площа"}, + {ORTH: "майд.", NORM: "майдан"}, + {ORTH: "мкр.", NORM: "мікрорайон"}, + {ORTH: "ст.", NORM: "станція"}, + {ORTH: "ж/м", NORM: "житловий масив"}, + {ORTH: "наб.", NORM: "набережна"}, + {ORTH: "в/ч", NORM: "військова частина"}, + {ORTH: "в/м", NORM: "військове містечко"}, + {ORTH: "оз.", NORM: "озеро"}, + {ORTH: "ім.", NORM: "імені"}, + {ORTH: "г.", NORM: "гора"}, + {ORTH: "п.", NORM: "пан"}, + {ORTH: "проф.", NORM: "професор"}, + {ORTH: "акад.", NORM: "академік"}, + {ORTH: "доц.", NORM: "доцент"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/lang/ur/__init__.py b/spacy/lang/ur/__init__.py new file mode 100644 index 0000000..4f20ac9 --- /dev/null +++ b/spacy/lang/ur/__init__.py @@ -0,0 +1,19 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_SUFFIXES +from .stop_words import STOP_WORDS + + +class UrduDefaults(BaseDefaults): + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + writing_system = {"direction": "rtl", "has_case": False, "has_letters": True} + + +class Urdu(Language): + lang = "ur" + Defaults = UrduDefaults + + +__all__ = ["Urdu"] diff --git a/spacy/lang/ur/examples.py b/spacy/lang/ur/examples.py new file mode 100644 index 0000000..f612c6b --- /dev/null +++ b/spacy/lang/ur/examples.py @@ -0,0 +1,11 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.da.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "اردو ہے جس کا نام ہم جانتے ہیں داغ", + "سارے جہاں میں دھوم ہماری زباں کی ہے", +] diff --git a/spacy/lang/ur/lex_attrs.py b/spacy/lang/ur/lex_attrs.py new file mode 100644 index 0000000..e590ed3 --- /dev/null +++ b/spacy/lang/ur/lex_attrs.py @@ -0,0 +1,45 @@ +from ...attrs import LIKE_NUM + +# Source https://quizlet.com/4271889/1-100-urdu-number-wordsurdu-numerals-flash-cards/ +# http://www.urduword.com/lessons.php?lesson=numbers +# https://en.wikibooks.org/wiki/Urdu/Vocabulary/Numbers +# https://www.urdu-english.com/lessons/beginner/numbers + +_num_words = """ایک دو تین چار پانچ چھ سات آٹھ نو دس گیارہ بارہ تیرہ چودہ پندرہ سولہ سترہ + اٹهارا انیس بیس اکیس بائیس تئیس چوبیس پچیس چھببیس +ستایس اٹھائس انتيس تیس اکتیس بتیس تینتیس چونتیس پینتیس + چھتیس سینتیس ارتیس انتالیس چالیس اکتالیس بیالیس تیتالیس +چوالیس پیتالیس چھیالیس سینتالیس اڑتالیس انچالیس پچاس اکاون باون + تریپن چون پچپن چھپن ستاون اٹھاون انسٹھ ساثھ +اکسٹھ باسٹھ تریسٹھ چوسٹھ پیسٹھ چھیاسٹھ سڑسٹھ اڑسٹھ +انھتر ستر اکھتر بھتتر تیھتر چوھتر تچھتر چھیتر ستتر +اٹھتر انیاسی اسی اکیاسی بیاسی تیراسی چوراسی پچیاسی چھیاسی + سٹیاسی اٹھیاسی نواسی نوے اکانوے بانوے ترانوے +چورانوے پچانوے چھیانوے ستانوے اٹھانوے ننانوے سو +""".split() + +# source https://www.google.com/intl/ur/inputtools/try/ + +_ordinal_words = """پہلا دوسرا تیسرا چوتھا پانچواں چھٹا ساتواں آٹھواں نواں دسواں گیارہواں بارہواں تیرھواں چودھواں + پندرھواں سولہواں سترھواں اٹھارواں انیسواں بسیواں +""".split() + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _num_words: + return True + if text in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/ur/punctuation.py b/spacy/lang/ur/punctuation.py new file mode 100644 index 0000000..382bfc7 --- /dev/null +++ b/spacy/lang/ur/punctuation.py @@ -0,0 +1,3 @@ +from ..punctuation import TOKENIZER_SUFFIXES + +_suffixes = TOKENIZER_SUFFIXES diff --git a/spacy/lang/ur/stop_words.py b/spacy/lang/ur/stop_words.py new file mode 100644 index 0000000..abfa364 --- /dev/null +++ b/spacy/lang/ur/stop_words.py @@ -0,0 +1,513 @@ +# Source: collected from different resource on internet +STOP_WORDS = set( + """ +ثھی +خو +گی +اپٌے +گئے +ثہت +طرف +ہوبری +پبئے +اپٌب +دوضری +گیب +کت +گب +ثھی +ضے +ہر +پر +اش +دی +گے +لگیں +ہے +ثعذ +ضکتے +تھی +اى +دیب +لئے +والے +یہ +ثدبئے +ضکتی +تھب +اًذر +رریعے +لگی +ہوبرا +ہوًے +ثبہر +ضکتب +ًہیں +تو +اور +رہب +لگے +ہوضکتب +ہوں +کب +ہوبرے +توبم +کیب +ایطے +رہی +هگر +ہوضکتی +ہیں +کریں +ہو +تک +کی +ایک +رہے +هیں +ہوضکتے +کیطے +ہوًب +تت +کہ +ہوا +آئے +ضبت +تھے +کیوں +ہو +تب +کے +پھر +ثغیر +خبر +ہے +رکھ +کی +طب +کوئی + رریعے +ثبرے +خب +اضطرذ +ثلکہ +خجکہ +رکھ +تب +کی +طرف +ثراں +خبر +رریعہ +اضکب +ثٌذ +خص +کی +لئے +توہیں +دوضرے +کررہی +اضکی +ثیچ +خوکہ +رکھتی +کیوًکہ +دوًوں +کر +رہے +خبر +ہی +ثرآں +اضکے +پچھلا +خیطب +رکھتے +کے +ثعذ +تو +ہی + دورى +کر +یہبں +آش +تھوڑا +چکے +زکویہ +دوضروں +ضکب +اوًچب +ثٌب +پل +تھوڑی +چلا +خبهوظ +دیتب +ضکٌب +اخبزت +اوًچبئی +ثٌبرہب +پوچھب +تھوڑے +چلو +ختن +دیتی +ضکی +اچھب +اوًچی +ثٌبرہی +پوچھتب +تیي +چلیں +در +دیتے +ضکے +اچھی +اوًچے +ثٌبرہے +پوچھتی +خبًب +چلے +درخبت +دیر +ضلطلہ +اچھے +اٹھبًب +ثٌبًب +پوچھتے +خبًتب +چھوٹب +درخہ +دیکھٌب +ضوچ +اختتبم +اہن +ثٌذ +پوچھٌب +خبًتی +چھوٹوں +درخے +دیکھو +ضوچب +ادھر +آئی +ثٌذکرًب +پوچھو +خبًتے +چھوٹی +درزقیقت +دیکھی +ضوچتب +ارد +آئے +ثٌذکرو +پوچھوں +خبًٌب +چھوٹے +درضت +دیکھیں +ضوچتی +اردگرد +آج +ثٌذی +پوچھیں +خططرذ +چھہ +دش +دیٌب +ضوچتے +ارکبى +آخر +ثڑا +پورا +خگہ +چیسیں +دفعہ +دے +ضوچٌب +اضتعوبل +آخر +پہلا +خگہوں +زبصل +دکھبئیں +راضتوں +ضوچو +اضتعوبلات +آدهی +ثڑی +پہلی +خگہیں +زبضر +دکھبتب +راضتہ +ضوچی +اغیب +آًب +ثڑے +پہلےضی +خلذی +زبل +دکھبتی +راضتے +ضوچیں +اطراف +آٹھ +ثھر +خٌبة +زبل +دکھبتے +رکي +ضیذھب +افراد +آیب +ثھرا +پہلے +خواى +زبلات +دکھبًب +رکھب +ضیذھی +اکثر +ثب +ہوا +پیع +خوًہی +زبلیہ +دکھبو +رکھی +ضیذھے +اکٹھب +ثھرپور +تبزٍ +خیطبکہ +زصوں +رکھے +ضیکٌڈ +اکٹھی +ثبری +ثہتر +تر +چبر +زصہ +دلچطپ +زیبدٍ +غبیذ +اکٹھے +ثبلا +ثہتری +ترتیت +چبہب +زصے +دلچطپی +ضبت +غخص +اکیلا +ثبلترتیت +ثہتریي +تریي +چبہٌب +زقبئق +دلچطپیبں +ضبدٍ +غذ +اکیلی +ثرش +پبش +تعذاد +چبہے +زقیتیں +هٌبضت +ضبرا +غروع +اکیلے +ثغیر +پبًب +چکب +زقیقت +دو +ضبرے +غروعبت +اگرچہ +ثلٌذ +پبًچ +تن +چکی +زکن +دور +ضبل +غے +الگ +پراًب +تٌہب +چکیں +دوضرا +ضبلوں +صبف +صسیر +قجیلہ +کوًطے +لازهی +هطئلے +ًیب +طریق +کرتی +کہتے +صفر +قطن +کھولا +لگتب +هطبئل +وار +طریقوں +کرتے +کہٌب +صورت +کئی +کھولٌب +لگتی +هطتعول +وار +طریقہ +کرتے +ہو +کہٌب +صورتسبل +کئے +کھولو +لگتے +هػتول +ٹھیک +طریقے +کرًب +کہو +صورتوں +کبفی +هطلق +ڈھوًڈا +طور +کرو +کہوں +صورتیں +کبم +کھولیں +لگی +هعلوم +ڈھوًڈلیب +طورپر +کریں +کہی +ضرور +کجھی +کھولے +لگے +هکول +ڈھوًڈًب +ظبہر +کرے +کہیں +ضرورت +کرا +کہب +لوجب +هلا +ڈھوًڈو +عذد +کل +کہیں +کرتب +کہتب +لوجی +هوکي +ڈھوًڈی +عظین +کن +کہے +ضروری +کرتبہوں +کہتی +لوجے +هوکٌبت +ڈھوًڈیں +علاقوں +کوتر +کیے +لوسبت +هوکٌہ +ہن +لے +ًبپطٌذ +ہورہے +علاقہ +کورا +کے +رریعے +لوسہ +هڑا +ہوئی +هتعلق +ًبگسیر +ہوگئی +علاقے +کوروں +گئی +لو +هڑًب +ہوئے +هسترم +ًطجت +ہو +گئے +علاوٍ +کورٍ +گرد +لوگ +هڑے +ہوتی +هسترهہ +ًقطہ +ہوگیب +کورے +گروپ +لوگوں +هہرثبى +ہوتے +هسطوش +ًکبلٌب +ہوًی +عووهی +کوطي +گروٍ +لڑکپي +هیرا +ہوچکب +هختلف +ًکتہ +ہی +فرد +کوى +گروہوں +لی +هیری +ہوچکی +هسیذ +فی +کوًطب +گٌتی +لیب +هیرے +ہوچکے +هطئلہ +ًوخواى +یقیٌی +قجل +کوًطی +لیٌب +ًئی +ہورہب +لیں +ًئے +ہورہی +ثبعث +ضت +""".split() +) diff --git a/spacy/lang/vi/__init__.py b/spacy/lang/vi/__init__.py new file mode 100644 index 0000000..ae1fa46 --- /dev/null +++ b/spacy/lang/vi/__init__.py @@ -0,0 +1,167 @@ +import re +import string +from pathlib import Path +from typing import Any, Dict, Union + +import srsly + +from ... import util +from ...language import BaseDefaults, Language +from ...tokens import Doc +from ...util import DummyTokenizer, load_config_from_str, registry +from ...vocab import Vocab +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + +DEFAULT_CONFIG = """ +[nlp] + +[nlp.tokenizer] +@tokenizers = "spacy.vi.VietnameseTokenizer" +use_pyvi = true +""" + + +def create_vietnamese_tokenizer(use_pyvi: bool = True): + def vietnamese_tokenizer_factory(nlp): + return VietnameseTokenizer(nlp.vocab, use_pyvi=use_pyvi) + + return vietnamese_tokenizer_factory + + +class VietnameseTokenizer(DummyTokenizer): + def __init__(self, vocab: Vocab, use_pyvi: bool = False): + self.vocab = vocab + self.use_pyvi = use_pyvi + if self.use_pyvi: + try: + from pyvi import ViTokenizer + + self.ViTokenizer = ViTokenizer + except ImportError: + msg = ( + "Pyvi not installed. Either set use_pyvi = False, " + "or install it https://pypi.python.org/pypi/pyvi" + ) + raise ImportError(msg) from None + + def __reduce__(self): + return VietnameseTokenizer, (self.vocab, self.use_pyvi) + + def __call__(self, text: str) -> Doc: + if self.use_pyvi: + words = self.pyvi_tokenize(text) + words, spaces = util.get_words_and_spaces(words, text) + return Doc(self.vocab, words=words, spaces=spaces) + else: + words, spaces = util.get_words_and_spaces(text.split(), text) + return Doc(self.vocab, words=words, spaces=spaces) + + # The methods pyvi_sylabelize_with_ws and pyvi_tokenize are adapted from + # pyvi v0.1, MIT License, Copyright (c) 2016 Viet-Trung Tran. + # See licenses/3rd_party_licenses.txt + def pyvi_sylabelize_with_ws(self, text): + """Modified from pyvi to preserve whitespace and skip unicode + normalization.""" + specials = [r"==>", r"->", r"\.\.\.", r">>"] + digit = r"\d+([\.,_]\d+)+" + email = r"([a-zA-Z0-9_.+-]+@([a-zA-Z0-9-]+\.)+[a-zA-Z0-9-]+)" + web = r"\w+://[^\s]+" + word = r"\w+" + non_word = r"[^\w\s]" + abbreviations = [ + r"[A-ZĐ]+\.", + r"Tp\.", + r"Mr\.", + r"Mrs\.", + r"Ms\.", + r"Dr\.", + r"ThS\.", + ] + + patterns = [] + patterns.extend(abbreviations) + patterns.extend(specials) + patterns.extend([web, email]) + patterns.extend([digit, non_word, word]) + + patterns = r"(\s+|" + "|".join(patterns) + ")" + tokens = re.findall(patterns, text, re.UNICODE) + + return [token[0] for token in tokens] + + def pyvi_tokenize(self, text): + """Modified from pyvi to preserve text and whitespace.""" + if len(text) == 0: + return [] + elif text.isspace(): + return [text] + segs = self.pyvi_sylabelize_with_ws(text) + words = [] + preceding_ws = [] + for i, token in enumerate(segs): + if not token.isspace(): + words.append(token) + preceding_ws.append( + "" if (i == 0 or not segs[i - 1].isspace()) else segs[i - 1] + ) + labels = self.ViTokenizer.ViTokenizer.model.predict( + [self.ViTokenizer.ViTokenizer.sent2features(words, False)] + ) + token = words[0] + tokens = [] + for i in range(1, len(labels[0])): + if ( + labels[0][i] == "I_W" + and words[i] not in string.punctuation + and words[i - 1] not in string.punctuation + and not words[i][0].isdigit() + and not words[i - 1][0].isdigit() + and not (words[i][0].istitle() and not words[i - 1][0].istitle()) + ): + token = token + preceding_ws[i] + words[i] + else: + tokens.append(token) + token = words[i] + tokens.append(token) + return tokens + + def _get_config(self) -> Dict[str, Any]: + return {"use_pyvi": self.use_pyvi} + + def _set_config(self, config: Dict[str, Any] = {}) -> None: + self.use_pyvi = config.get("use_pyvi", False) + + def to_bytes(self, **kwargs) -> bytes: + serializers = {"cfg": lambda: srsly.json_dumps(self._get_config())} + return util.to_bytes(serializers, []) + + def from_bytes(self, data: bytes, **kwargs) -> "VietnameseTokenizer": + deserializers = {"cfg": lambda b: self._set_config(srsly.json_loads(b))} + util.from_bytes(data, deserializers, []) + return self + + def to_disk(self, path: Union[str, Path], **kwargs) -> None: + path = util.ensure_path(path) + serializers = {"cfg": lambda p: srsly.write_json(p, self._get_config())} + util.to_disk(path, serializers, []) + + def from_disk(self, path: Union[str, Path], **kwargs) -> "VietnameseTokenizer": + path = util.ensure_path(path) + serializers = {"cfg": lambda p: self._set_config(srsly.read_json(p))} + util.from_disk(path, serializers, []) + return self + + +class VietnameseDefaults(BaseDefaults): + config = load_config_from_str(DEFAULT_CONFIG) + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Vietnamese(Language): + lang = "vi" + Defaults = VietnameseDefaults + + +__all__ = ["Vietnamese"] diff --git a/spacy/lang/vi/examples.py b/spacy/lang/vi/examples.py new file mode 100644 index 0000000..5f2a9b2 --- /dev/null +++ b/spacy/lang/vi/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. +>>> from spacy.lang.vi.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +sentences = [ + "Đây là đâu, tôi là ai?", + "Căn phòng có nhiều cửa sổ nên nó khá sáng", + "Đại dịch COVID vừa qua đã gây ảnh hưởng rất lớn tới nhiều doanh nghiệp lớn nhỏ.", + "Thành phố Hồ Chí Minh đã bị ảnh hưởng nặng nề trong thời gian vừa qua.", + "Ông bạn đang ở đâu thế?", + "Ai là người giải phóng đất nước Việt Nam khỏi ách đô hộ?", + "Vị tướng nào là người đã làm nên chiến thắng lịch sử Điện Biên Phủ?", + "Làm việc nhiều chán quá, đi chơi đâu đi?", +] diff --git a/spacy/lang/vi/lex_attrs.py b/spacy/lang/vi/lex_attrs.py new file mode 100644 index 0000000..82997a1 --- /dev/null +++ b/spacy/lang/vi/lex_attrs.py @@ -0,0 +1,45 @@ +from ...attrs import LIKE_NUM + +_num_words = [ + "không", # Zero + "một", # One + "mốt", # Also one, irreplacable in niché cases for unit digit such as "51"="năm mươi mốt" + "hai", # Two + "ba", # Three + "bốn", # Four + "tư", # Also four, used in certain cases for unit digit such as "54"="năm mươi tư" + "năm", # Five + "lăm", # Also five, irreplacable in niché cases for unit digit such as "55"="năm mươi lăm" + "sáu", # Six + "bảy", # Seven + "bẩy", # Also seven, old fashioned + "tám", # Eight + "chín", # Nine + "mười", # Ten + "chục", # Also ten, used for counting in tens such as "20 eggs"="hai chục trứng" + "trăm", # Hundred + "nghìn", # Thousand + "ngàn", # Also thousand, used in the south + "vạn", # Ten thousand + "triệu", # Million + "tỷ", # Billion + "tỉ", # Also billion, used in combinatorics such as "tỉ_phú"="billionaire" +] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text.lower() in _num_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/vi/stop_words.py b/spacy/lang/vi/stop_words.py new file mode 100644 index 0000000..3481701 --- /dev/null +++ b/spacy/lang/vi/stop_words.py @@ -0,0 +1,1947 @@ +# Source: https://github.com/stopwords/vietnamese-stopwords +STOP_WORDS = set( + """ +a_lô +a_ha +ai +ai_ai +ai_nấy +ai_đó +alô +amen +anh +anh_ấy +ba +ba_bau +ba_bản +ba_cùng +ba_họ +ba_ngày +ba_ngôi +ba_tăng +bao_giờ +bao_lâu +bao_nhiêu +bao_nả +bay_biến +biết +biết_bao +biết_bao_nhiêu +biết_chắc +biết_chừng_nào +biết_mình +biết_mấy +biết_thế +biết_trước +biết_việc +biết_đâu +biết_đâu_chừng +biết_đâu_đấy +biết_được +buổi +buổi_làm +buổi_mới +buổi_ngày +buổi_sớm +bà +bà_ấy +bài +bài_bác +bài_bỏ +bài_cái +bác +bán +bán_cấp +bán_dạ +bán_thế +bây_bẩy +bây_chừ +bây_giờ +bây_nhiêu +bèn +béng +bên +bên_bị +bên_có +bên_cạnh +bông +bước +bước_khỏi +bước_tới +bước_đi +bạn +bản +bản_bộ +bản_riêng +bản_thân +bản_ý +bất_chợt +bất_cứ +bất_giác +bất_kì +bất_kể +bất_kỳ +bất_luận +bất_ngờ +bất_nhược +bất_quá +bất_quá_chỉ +bất_thình_lình +bất_tử +bất_đồ +bấy +bấy_chầy +bấy_chừ +bấy_giờ +bấy_lâu +bấy_lâu_nay +bấy_nay +bấy_nhiêu +bập_bà_bập_bõm +bập_bõm +bắt_đầu +bắt_đầu_từ +bằng +bằng_cứ +bằng_không +bằng_người +bằng_nhau +bằng_như +bằng_nào +bằng_nấy +bằng_vào +bằng_được +bằng_ấy +bển +bệt +bị +bị_chú +bị_vì +bỏ +bỏ_bà +bỏ_cha +bỏ_cuộc +bỏ_không +bỏ_lại +bỏ_mình +bỏ_mất +bỏ_mẹ +bỏ_nhỏ +bỏ_quá +bỏ_ra +bỏ_riêng +bỏ_việc +bỏ_xa +bỗng +bỗng_chốc +bỗng_dưng +bỗng_không +bỗng_nhiên +bỗng_nhưng +bỗng_thấy +bỗng_đâu +bộ +bộ_thuộc +bộ_điều +bội_phần +bớ +bởi +bởi_ai +bởi_chưng +bởi_nhưng +bởi_sao +bởi_thế +bởi_thế_cho_nên +bởi_tại +bởi_vì +bởi_vậy +bởi_đâu +bức +cao +cao_lâu +cao_ráo +cao_răng +cao_sang +cao_số +cao_thấp +cao_thế +cao_xa +cha +cha_chả +chao_ôi +chia_sẻ +chiếc +cho +cho_biết +cho_chắc +cho_hay +cho_nhau +cho_nên +cho_rằng +cho_rồi +cho_thấy +cho_tin +cho_tới +cho_tới_khi +cho_về +cho_ăn +cho_đang +cho_được +cho_đến +cho_đến_khi +cho_đến_nỗi +choa +chu_cha +chui_cha +chung +chung_cho +chung_chung +chung_cuộc +chung_cục +chung_nhau +chung_qui +chung_quy +chung_quy_lại +chung_ái +chuyển +chuyển_tự +chuyển_đạt +chuyện +chuẩn_bị +chành_chạnh +chí_chết +chính +chính_bản +chính_giữa +chính_là +chính_thị +chính_điểm +chùn_chùn +chùn_chũn +chú +chú_dẫn +chú_khách +chú_mày +chú_mình +chúng +chúng_mình +chúng_ta +chúng_tôi +chúng_ông +chăn_chắn +chăng +chăng_chắc +chăng_nữa +chơi +chơi_họ +chưa +chưa_bao_giờ +chưa_chắc +chưa_có +chưa_cần +chưa_dùng +chưa_dễ +chưa_kể +chưa_tính +chưa_từng +chầm_chập +chậc +chắc +chắc_chắn +chắc_dạ +chắc_hẳn +chắc_lòng +chắc_người +chắc_vào +chắc_ăn +chẳng_lẽ +chẳng_những +chẳng_nữa +chẳng_phải +chết_nỗi +chết_thật +chết_tiệt +chỉ +chỉ_chính +chỉ_có +chỉ_là +chỉ_tên +chỉn +chị +chị_bộ +chị_ấy +chịu +chịu_chưa +chịu_lời +chịu_tốt +chịu_ăn +chọn +chọn_bên +chọn_ra +chốc_chốc +chớ +chớ_chi +chớ_gì +chớ_không +chớ_kể +chớ_như +chợt +chợt_nghe +chợt_nhìn +chủn +chứ +chứ_ai +chứ_còn +chứ_gì +chứ_không +chứ_không_phải +chứ_lại +chứ_lị +chứ_như +chứ_sao +coi_bộ +coi_mòi +con +con_con +con_dạ +con_nhà +con_tính +cu_cậu +cuối +cuối_cùng +cuối_điểm +cuốn +cuộc +càng +càng_càng +càng_hay +cá_nhân +các +các_cậu +cách +cách_bức +cách_không +cách_nhau +cách_đều +cái +cái_gì +cái_họ +cái_đã +cái_đó +cái_ấy +câu_hỏi +cây +cây_nước +còn +còn_như +còn_nữa +còn_thời_gian +còn_về +có +có_ai +có_chuyện +có_chăng +có_chăng_là +có_chứ +có_cơ +có_dễ +có_họ +có_khi +có_ngày +có_người +có_nhiều +có_nhà +có_phải +có_số +có_tháng +có_thế +có_thể +có_vẻ +có_ý +có_ăn +có_điều +có_điều_kiện +có_đáng +có_đâu +có_được +cóc_khô +cô +cô_mình +cô_quả +cô_tăng +cô_ấy +công_nhiên +cùng +cùng_chung +cùng_cực +cùng_nhau +cùng_tuổi +cùng_tột +cùng_với +cùng_ăn +căn +căn_cái +căn_cắt +căn_tính +cũng +cũng_như +cũng_nên +cũng_thế +cũng_vậy +cũng_vậy_thôi +cũng_được +cơ +cơ_chỉ +cơ_chừng +cơ_cùng +cơ_dẫn +cơ_hồ +cơ_hội +cơ_mà +cơn +cả +cả_nghe +cả_nghĩ +cả_ngày +cả_người +cả_nhà +cả_năm +cả_thảy +cả_thể +cả_tin +cả_ăn +cả_đến +cảm_thấy +cảm_ơn +cấp +cấp_số +cấp_trực_tiếp +cần +cần_cấp +cần_gì +cần_số +cật_lực +cật_sức +cậu +cổ_lai +cụ_thể +cụ_thể_là +cụ_thể_như +của +của_ngọt +của_tin +cứ +cứ_như +cứ_việc +cứ_điểm +cực_lực +do +do_vì +do_vậy +do_đó +duy +duy_chỉ +duy_có +dài +dài_lời +dài_ra +dành +dành_dành +dào +dì +dù +dù_cho +dù_dì +dù_gì +dù_rằng +dù_sao +dùng +dùng_cho +dùng_hết +dùng_làm +dùng_đến +dưới +dưới_nước +dạ +dạ_bán +dạ_con +dạ_dài +dạ_dạ +dạ_khách +dần_dà +dần_dần +dầu_sao +dẫn +dẫu +dẫu_mà +dẫu_rằng +dẫu_sao +dễ +dễ_dùng +dễ_gì +dễ_khiến +dễ_nghe +dễ_ngươi +dễ_như_chơi +dễ_sợ +dễ_sử_dụng +dễ_thường +dễ_thấy +dễ_ăn +dễ_đâu +dở_chừng +dữ +dữ_cách +em +em_em +giá_trị +giá_trị_thực_tế +giảm +giảm_chính +giảm_thấp +giảm_thế +giống +giống_người +giống_nhau +giống_như +giờ +giờ_lâu +giờ_này +giờ_đi +giờ_đây +giờ_đến +giữ +giữ_lấy +giữ_ý +giữa +giữa_lúc +gây +gây_cho +gây_giống +gây_ra +gây_thêm +gì +gì_gì +gì_đó +gần +gần_bên +gần_hết +gần_ngày +gần_như +gần_xa +gần_đây +gần_đến +gặp +gặp_khó_khăn +gặp_phải +gồm +hay +hay_biết +hay_hay +hay_không +hay_là +hay_làm +hay_nhỉ +hay_nói +hay_sao +hay_tin +hay_đâu +hiểu +hiện_nay +hiện_tại +hoàn_toàn +hoặc +hoặc_là +hãy +hãy_còn +hơn +hơn_cả +hơn_hết +hơn_là +hơn_nữa +hơn_trước +hầu_hết +hết +hết_chuyện +hết_cả +hết_của +hết_nói +hết_ráo +hết_rồi +hết_ý +họ +họ_gần +họ_xa +hỏi +hỏi_lại +hỏi_xem +hỏi_xin +hỗ_trợ +khi +khi_khác +khi_không +khi_nào +khi_nên +khi_trước +khiến +khoảng +khoảng_cách +khoảng_không +khá +khá_tốt +khác +khác_gì +khác_khác +khác_nhau +khác_nào +khác_thường +khác_xa +khách +khó +khó_biết +khó_chơi +khó_khăn +khó_làm +khó_mở +khó_nghe +khó_nghĩ +khó_nói +khó_thấy +khó_tránh +không +không_ai +không_bao_giờ +không_bao_lâu +không_biết +không_bán +không_chỉ +không_còn +không_có +không_có_gì +không_cùng +không_cần +không_cứ +không_dùng +không_gì +không_hay +không_khỏi +không_kể +không_ngoài +không_nhận +không_những +không_phải +không_phải_không +không_thể +không_tính +không_điều_kiện +không_được +không_đầy +không_để +khẳng_định +khỏi +khỏi_nói +kể +kể_cả +kể_như +kể_tới +kể_từ +liên_quan +loại +loại_từ +luôn +luôn_cả +luôn_luôn +luôn_tay +là +là_cùng +là_là +là_nhiều +là_phải +là_thế_nào +là_vì +là_ít +làm +làm_bằng +làm_cho +làm_dần_dần +làm_gì +làm_lòng +làm_lại +làm_lấy +làm_mất +làm_ngay +làm_như +làm_nên +làm_ra +làm_riêng +làm_sao +làm_theo +làm_thế_nào +làm_tin +làm_tôi +làm_tăng +làm_tại +làm_tắp_lự +làm_vì +làm_đúng +làm_được +lâu +lâu_các +lâu_lâu +lâu_nay +lâu_ngày +lên +lên_cao +lên_cơn +lên_mạnh +lên_ngôi +lên_nước +lên_số +lên_xuống +lên_đến +lòng +lòng_không +lúc +lúc_khác +lúc_lâu +lúc_nào +lúc_này +lúc_sáng +lúc_trước +lúc_đi +lúc_đó +lúc_đến +lúc_ấy +lý_do +lượng +lượng_cả +lượng_số +lượng_từ +lại +lại_bộ +lại_cái +lại_còn +lại_giống +lại_làm +lại_người +lại_nói +lại_nữa +lại_quả +lại_thôi +lại_ăn +lại_đây +lấy +lấy_có +lấy_cả +lấy_giống +lấy_làm +lấy_lý_do +lấy_lại +lấy_ra +lấy_ráo +lấy_sau +lấy_số +lấy_thêm +lấy_thế +lấy_vào +lấy_xuống +lấy_được +lấy_để +lần +lần_khác +lần_lần +lần_nào +lần_này +lần_sang +lần_sau +lần_theo +lần_trước +lần_tìm +lớn +lớn_lên +lớn_nhỏ +lời +lời_chú +lời_nói +mang +mang_lại +mang_mang +mang_nặng +mang_về +muốn +mà +mà_cả +mà_không +mà_lại +mà_thôi +mà_vẫn +mình +mạnh +mất +mất_còn +mọi +mọi_giờ +mọi_khi +mọi_lúc +mọi_người +mọi_nơi +mọi_sự +mọi_thứ +mọi_việc +mối +mỗi +mỗi_lúc +mỗi_lần +mỗi_một +mỗi_ngày +mỗi_người +một +một_cách +một_cơn +một_khi +một_lúc +một_số +một_vài +một_ít +mới +mới_hay +mới_rồi +mới_đây +mở +mở_mang +mở_nước +mở_ra +mợ +mức +nay +ngay +ngay_bây_giờ +ngay_cả +ngay_khi +ngay_khi_đến +ngay_lúc +ngay_lúc_này +ngay_lập_tức +ngay_thật +ngay_tức_khắc +ngay_tức_thì +ngay_từ +nghe +nghe_chừng +nghe_hiểu +nghe_không +nghe_lại +nghe_nhìn +nghe_như +nghe_nói +nghe_ra +nghe_rõ +nghe_thấy +nghe_tin +nghe_trực_tiếp +nghe_đâu +nghe_đâu_như +nghe_được +nghen +nghiễm_nhiên +nghĩ +nghĩ_lại +nghĩ_ra +nghĩ_tới +nghĩ_xa +nghĩ_đến +nghỉm +ngoài +ngoài_này +ngoài_ra +ngoài_xa +ngoải +nguồn +ngày +ngày_càng +ngày_cấp +ngày_giờ +ngày_ngày +ngày_nào +ngày_này +ngày_nọ +ngày_qua +ngày_rày +ngày_tháng +ngày_xưa +ngày_xửa +ngày_đến +ngày_ấy +ngôi +ngôi_nhà +ngôi_thứ +ngõ_hầu +ngăn_ngắt +ngươi +người +người_hỏi +người_khác +người_khách +người_mình +người_nghe +người_người +người_nhận +ngọn +ngọn_nguồn +ngọt +ngồi +ngồi_bệt +ngồi_không +ngồi_sau +ngồi_trệt +ngộ_nhỡ +nhanh +nhanh_lên +nhanh_tay +nhau +nhiên_hậu +nhiều +nhiều_ít +nhiệt_liệt +nhung_nhăng +nhà +nhà_chung +nhà_khó +nhà_làm +nhà_ngoài +nhà_ngươi +nhà_tôi +nhà_việc +nhân_dịp +nhân_tiện +nhé +nhìn +nhìn_chung +nhìn_lại +nhìn_nhận +nhìn_theo +nhìn_thấy +nhìn_xuống +nhóm +nhón_nhén +như +như_ai +như_chơi +như_không +như_là +như_nhau +như_quả +như_sau +như_thường +như_thế +như_thế_nào +như_thể +như_trên +như_trước +như_tuồng +như_vậy +như_ý +nhưng +nhưng_mà +nhược_bằng +nhất +nhất_loạt +nhất_luật +nhất_là +nhất_mực +nhất_nhất +nhất_quyết +nhất_sinh +nhất_thiết +nhất_thì +nhất_tâm +nhất_tề +nhất_đán +nhất_định +nhận +nhận_biết +nhận_họ +nhận_làm +nhận_nhau +nhận_ra +nhận_thấy +nhận_việc +nhận_được +nhằm +nhằm_khi +nhằm_lúc +nhằm_vào +nhằm_để +nhỉ +nhỏ +nhỏ_người +nhớ +nhớ_bập_bõm +nhớ_lại +nhớ_lấy +nhớ_ra +nhờ +nhờ_chuyển +nhờ_có +nhờ_nhờ +nhờ_đó +nhỡ_ra +những +những_ai +những_khi +những_là +những_lúc +những_muốn +những_như +nào +nào_cũng +nào_hay +nào_là +nào_phải +nào_đâu +nào_đó +này +này_nọ +nên +nên_chi +nên_chăng +nên_làm +nên_người +nên_tránh +nó +nóc +nói +nói_bông +nói_chung +nói_khó +nói_là +nói_lên +nói_lại +nói_nhỏ +nói_phải +nói_qua +nói_ra +nói_riêng +nói_rõ +nói_thêm +nói_thật +nói_toẹt +nói_trước +nói_tốt +nói_với +nói_xa +nói_ý +nói_đến +nói_đủ +năm +năm_tháng +nơi +nơi_nơi +nước +nước_bài +nước_cùng +nước_lên +nước_nặng +nước_quả +nước_xuống +nước_ăn +nước_đến +nấy +nặng +nặng_căn +nặng_mình +nặng_về +nếu +nếu_có +nếu_cần +nếu_không +nếu_mà +nếu_như +nếu_thế +nếu_vậy +nếu_được +nền +nọ +nớ +nức_nở +nữa +nữa_khi +nữa_là +nữa_rồi +oai_oái +oái +pho +phè +phè_phè +phía +phía_bên +phía_bạn +phía_dưới +phía_sau +phía_trong +phía_trên +phía_trước +phóc +phót +phù_hợp +phăn_phắt +phương_chi +phải +phải_biết +phải_chi +phải_chăng +phải_cách +phải_cái +phải_giờ +phải_khi +phải_không +phải_lại +phải_lời +phải_người +phải_như +phải_rồi +phải_tay +phần +phần_lớn +phần_nhiều +phần_nào +phần_sau +phần_việc +phắt +phỉ_phui +phỏng +phỏng_như +phỏng_nước +phỏng_theo +phỏng_tính +phốc +phụt +phứt +qua +qua_chuyện +qua_khỏi +qua_lại +qua_lần +qua_ngày +qua_tay +qua_thì +qua_đi +quan_trọng +quan_trọng_vấn_đề +quan_tâm +quay +quay_bước +quay_lại +quay_số +quay_đi +quá +quá_bán +quá_bộ +quá_giờ +quá_lời +quá_mức +quá_nhiều +quá_tay +quá_thì +quá_tin +quá_trình +quá_tuổi +quá_đáng +quá_ư +quả +quả_là +quả_thật +quả_thế +quả_vậy +quận +ra +ra_bài +ra_bộ +ra_chơi +ra_gì +ra_lại +ra_lời +ra_ngôi +ra_người +ra_sao +ra_tay +ra_vào +ra_ý +ra_điều +ra_đây +ren_rén +riu_ríu +riêng +riêng_từng +riệt +rày +ráo +ráo_cả +ráo_nước +ráo_trọi +rén +rén_bước +rích +rón_rén +rõ +rõ_là +rõ_thật +rút_cục +răng +răng_răng +rất +rất_lâu +rằng +rằng_là +rốt_cuộc +rốt_cục +rồi +rồi_nữa +rồi_ra +rồi_sao +rồi_sau +rồi_tay +rồi_thì +rồi_xem +rồi_đây +rứa +sa_sả +sang +sang_năm +sang_sáng +sang_tay +sao +sao_bản +sao_bằng +sao_cho +sao_vậy +sao_đang +sau +sau_chót +sau_cuối +sau_cùng +sau_hết +sau_này +sau_nữa +sau_sau +sau_đây +sau_đó +so +so_với +song_le +suýt +suýt_nữa +sáng +sáng_ngày +sáng_rõ +sáng_thế +sáng_ý +sì +sì_sì +sất +sắp +sắp_đặt +sẽ +sẽ_biết +sẽ_hay +số +số_cho_biết +số_cụ_thể +số_loại +số_là +số_người +số_phần +số_thiếu +sốt_sột +sớm +sớm_ngày +sở_dĩ +sử_dụng +sự +sự_thế +sự_việc +tanh +tanh_tanh +tay +tay_quay +tha_hồ +tha_hồ_chơi +tha_hồ_ăn +than_ôi +thanh +thanh_ba +thanh_chuyển +thanh_không +thanh_thanh +thanh_tính +thanh_điều_kiện +thanh_điểm +thay_đổi +thay_đổi_tình_trạng +theo +theo_bước +theo_như +theo_tin +thi_thoảng +thiếu +thiếu_gì +thiếu_điểm +thoạt +thoạt_nghe +thoạt_nhiên +thoắt +thuần +thuần_ái +thuộc +thuộc_bài +thuộc_cách +thuộc_lại +thuộc_từ +thà +thà_là +thà_rằng +thành_ra +thành_thử +thái_quá +tháng +tháng_ngày +tháng_năm +tháng_tháng +thêm +thêm_chuyện +thêm_giờ +thêm_vào +thì +thì_giờ +thì_là +thì_phải +thì_ra +thì_thôi +thình_lình +thích +thích_cứ +thích_thuộc +thích_tự +thích_ý +thím +thôi +thôi_việc +thúng_thắng +thương_ôi +thường +thường_bị +thường_hay +thường_khi +thường_số +thường_sự +thường_thôi +thường_thường +thường_tính +thường_tại +thường_xuất_hiện +thường_đến +thảo_hèn +thảo_nào +thấp +thấp_cơ +thấp_thỏm +thấp_xuống +thấy +thấy_tháng +thẩy +thậm +thậm_chí +thậm_cấp +thậm_từ +thật +thật_chắc +thật_là +thật_lực +thật_quả +thật_ra +thật_sự +thật_thà +thật_tốt +thật_vậy +thế +thế_chuẩn_bị +thế_là +thế_lại +thế_mà +thế_nào +thế_nên +thế_ra +thế_sự +thế_thì +thế_thôi +thế_thường +thế_thế +thế_à +thế_đó +thếch +thỉnh_thoảng +thỏm +thốc +thốc_tháo +thốt +thốt_nhiên +thốt_nói +thốt_thôi +thộc +thời_gian +thời_gian_sử_dụng +thời_gian_tính +thời_điểm +thục_mạng +thứ +thứ_bản +thứ_đến +thửa +thực_hiện +thực_hiện_đúng +thực_ra +thực_sự +thực_tế +thực_vậy +tin +tin_thêm +tin_vào +tiếp_theo +tiếp_tục +tiếp_đó +tiện_thể +toà +toé_khói +toẹt +trong +trong_khi +trong_lúc +trong_mình +trong_ngoài +trong_này +trong_số +trong_vùng +trong_đó +trong_ấy +tránh +tránh_khỏi +tránh_ra +tránh_tình_trạng +tránh_xa +trên +trên_bộ +trên_dưới +trước +trước_hết +trước_khi +trước_kia +trước_nay +trước_ngày +trước_nhất +trước_sau +trước_tiên +trước_tuổi +trước_đây +trước_đó +trả +trả_của +trả_lại +trả_ngay +trả_trước +trếu_tráo +trển +trệt +trệu_trạo +trỏng +trời_đất_ơi +trở_thành +trừ_phi +trực_tiếp +trực_tiếp_làm +tuy +tuy_có +tuy_là +tuy_nhiên +tuy_rằng +tuy_thế +tuy_vậy +tuy_đã +tuyệt_nhiên +tuần_tự +tuốt_luốt +tuốt_tuồn_tuột +tuốt_tuột +tuổi +tuổi_cả +tuổi_tôi +tà_tà +tên +tên_chính +tên_cái +tên_họ +tên_tự +tênh +tênh_tênh +tìm +tìm_bạn +tìm_cách +tìm_hiểu +tìm_ra +tìm_việc +tình_trạng +tính +tính_cách +tính_căn +tính_người +tính_phỏng +tính_từ +tít_mù +tò_te +tôi +tôi_con +tông_tốc +tù_tì +tăm_tắp +tăng +tăng_chúng +tăng_cấp +tăng_giảm +tăng_thêm +tăng_thế +tại +tại_lòng +tại_nơi +tại_sao +tại_tôi +tại_vì +tại_đâu +tại_đây +tại_đó +tạo +tạo_cơ_hội +tạo_nên +tạo_ra +tạo_ý +tạo_điều_kiện +tấm +tấm_bản +tấm_các +tấn +tấn_tới +tất_cả +tất_cả_bao_nhiêu +tất_thảy +tất_tần_tật +tất_tật +tập_trung +tắp +tắp_lự +tắp_tắp +tọt +tỏ_ra +tỏ_vẻ +tốc_tả +tối_ư +tốt +tốt_bạn +tốt_bộ +tốt_hơn +tốt_mối +tốt_ngày +tột +tột_cùng +tớ +tới +tới_gần +tới_mức +tới_nơi +tới_thì +tức_thì +tức_tốc +từ +từ_căn +từ_giờ +từ_khi +từ_loại +từ_nay +từ_thế +từ_tính +từ_tại +từ_từ +từ_ái +từ_điều +từ_đó +từ_ấy +từng +từng_cái +từng_giờ +từng_nhà +từng_phần +từng_thời_gian +từng_đơn_vị +từng_ấy +tự +tự_cao +tự_khi +tự_lượng +tự_tính +tự_tạo +tự_vì +tự_ý +tự_ăn +tựu_trung +veo +veo_veo +việc +việc_gì +vung_thiên_địa +vung_tàn_tán +vung_tán_tàn +và +vài +vài_ba +vài_người +vài_nhà +vài_nơi +vài_tên +vài_điều +vào +vào_gặp +vào_khoảng +vào_lúc +vào_vùng +vào_đến +vâng +vâng_chịu +vâng_dạ +vâng_vâng +vâng_ý +vèo +vèo_vèo +vì +vì_chưng +vì_rằng +vì_sao +vì_thế +vì_vậy +ví_bằng +ví_dù +ví_phỏng +ví_thử +vô_hình_trung +vô_kể +vô_luận +vô_vàn +vùng +vùng_lên +vùng_nước +văng_tê +vượt +vượt_khỏi +vượt_quá +vạn_nhất +vả_chăng +vả_lại +vấn_đề +vấn_đề_quan_trọng +vẫn +vẫn_thế +vậy +vậy_là +vậy_mà +vậy_nên +vậy_ra +vậy_thì +vậy_ư +về +về_không +về_nước +về_phần +về_sau +về_tay +vị_trí +vị_tất +vốn_dĩ +với +với_lại +với_nhau +vở +vụt +vừa +vừa_khi +vừa_lúc +vừa_mới +vừa_qua +vừa_rồi +vừa_vừa +xa +xa_cách +xa_gần +xa_nhà +xa_tanh +xa_tắp +xa_xa +xa_xả +xem +xem_lại +xem_ra +xem_số +xin +xin_gặp +xin_vâng +xiết_bao +xon_xón +xoành_xoạch +xoét +xoẳn +xoẹt +xuất_hiện +xuất_kì_bất_ý +xuất_kỳ_bất_ý +xuể +xuống +xăm_xúi +xăm_xăm +xăm_xắm +xảy_ra +xềnh_xệch +xệp +xử_lý +yêu_cầu +à +à_này +à_ơi +ào +ào_vào +ào_ào +á +á_à +ái +ái_chà +ái_dà +áng +áng_như +âu_là +ít +ít_biết +ít_có +ít_hơn +ít_khi +ít_lâu +ít_nhiều +ít_nhất +ít_nữa +ít_quá +ít_ra +ít_thôi +ít_thấy +ô_hay +ô_hô +ô_kê +ô_kìa +ôi_chao +ôi_thôi +ông +ông_nhỏ +ông_tạo +ông_từ +ông_ấy +ông_ổng +úi +úi_chà +úi_dào +ý +ý_chừng +ý_da +ý_hoặc +ăn +ăn_chung +ăn_chắc +ăn_chịu +ăn_cuộc +ăn_hết +ăn_hỏi +ăn_làm +ăn_người +ăn_ngồi +ăn_quá +ăn_riêng +ăn_sáng +ăn_tay +ăn_trên +ăn_về +đang +đang_tay +đang_thì +điều +điều_gì +điều_kiện +điểm +điểm_chính +điểm_gặp +điểm_đầu_tiên +đành_đạch +đáng +đáng_kể +đáng_lí +đáng_lý +đáng_lẽ +đáng_số +đánh_giá +đánh_đùng +đáo_để +đâu +đâu_có +đâu_cũng +đâu_như +đâu_nào +đâu_phải +đâu_đâu +đâu_đây +đâu_đó +đây +đây_này +đây_rồi +đây_đó +đã +đã_hay +đã_không +đã_là +đã_lâu +đã_thế +đã_vậy +đã_đủ +đó +đó_đây +đúng +đúng_ngày +đúng_ra +đúng_tuổi +đúng_với +đơn_vị +đưa +đưa_cho +đưa_chuyện +đưa_em +đưa_ra +đưa_tay +đưa_tin +đưa_tới +đưa_vào +đưa_về +đưa_xuống +đưa_đến +được +được_cái +được_lời +được_nước +được_tin +đại_loại +đại_nhân +đại_phàm +đại_để +đạt +đảm_bảo +đầu_tiên +đầy +đầy_năm +đầy_phè +đầy_tuổi +đặc_biệt +đặt +đặt_làm +đặt_mình +đặt_mức +đặt_ra +đặt_trước +đặt_để +đến +đến_bao_giờ +đến_cùng +đến_cùng_cực +đến_cả +đến_giờ +đến_gần +đến_hay +đến_khi +đến_lúc +đến_lời +đến_nay +đến_ngày +đến_nơi +đến_nỗi +đến_thì +đến_thế +đến_tuổi +đến_xem +đến_điều +đến_đâu +đều +đều_bước +đều_nhau +đều_đều +để +để_cho +để_giống +để_không +để_lòng +để_lại +để_mà +để_phần +để_được +để_đến_nỗi +đối_với +đồng_thời +đủ +đủ_dùng +đủ_nơi +đủ_số +đủ_điều +đủ_điểm +ơ +ơ_hay +ơ_kìa +ơi +ơi_là +ư +ạ +ạ_ơi +ấy +ấy_là +ầu_ơ +ắt +ắt_hẳn +ắt_là +ắt_phải +ắt_thật +ối_dào +ối_giời +ối_giời_ơi +ồ +ồ_ồ +ổng +ớ +ớ_này +ờ +ờ_ờ +ở +ở_lại +ở_như +ở_nhờ +ở_năm +ở_trên +ở_vào +ở_đây +ở_đó +ở_được +ủa +ứ_hự +ứ_ừ +ừ +ừ_nhé +ừ_thì +ừ_ào +ừ_ừ +ử +""".split("\n") +) diff --git a/spacy/lang/xx/__init__.py b/spacy/lang/xx/__init__.py new file mode 100644 index 0000000..aff8403 --- /dev/null +++ b/spacy/lang/xx/__init__.py @@ -0,0 +1,12 @@ +from ...language import Language + + +class MultiLanguage(Language): + """Language class to be used for models that support multiple languages. + This module allows models to specify their language ID as 'xx'. + """ + + lang = "xx" + + +__all__ = ["MultiLanguage"] diff --git a/spacy/lang/xx/examples.py b/spacy/lang/xx/examples.py new file mode 100644 index 0000000..34570d7 --- /dev/null +++ b/spacy/lang/xx/examples.py @@ -0,0 +1,95 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.de.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +# combined examples from de/en/es/fr/it/nl/pl/pt/ru + +sentences = [ + "Die ganze Stadt ist ein Startup: Shenzhen ist das Silicon Valley für Hardware-Firmen", + "Wie deutsche Startups die Technologie vorantreiben wollen: Künstliche Intelligenz", + "Trend zum Urlaub in Deutschland beschert Gastwirten mehr Umsatz", + "Bundesanwaltschaft erhebt Anklage gegen mutmaßlichen Schweizer Spion", + "San Francisco erwägt Verbot von Lieferrobotern", + "Autonome Fahrzeuge verlagern Haftpflicht auf Hersteller", + "Wo bist du?", + "Was ist die Hauptstadt von Deutschland?", + "Apple is looking at buying U.K. startup for $1 billion", + "Autonomous cars shift insurance liability toward manufacturers", + "San Francisco considers banning sidewalk delivery robots", + "London is a big city in the United Kingdom.", + "Where are you?", + "Who is the president of France?", + "What is the capital of the United States?", + "When was Barack Obama born?", + "Apple está buscando comprar una startup del Reino Unido por mil millones de dólares.", + "Los coches autónomos delegan la responsabilidad del seguro en sus fabricantes.", + "San Francisco analiza prohibir los robots delivery.", + "Londres es una gran ciudad del Reino Unido.", + "El gato come pescado.", + "Veo al hombre con el telescopio.", + "La araña come moscas.", + "El pingüino incuba en su nido.", + "Apple cherche à acheter une start-up anglaise pour 1 milliard de dollars", + "Les voitures autonomes déplacent la responsabilité de l'assurance vers les constructeurs", + "San Francisco envisage d'interdire les robots coursiers sur les trottoirs", + "Londres est une grande ville du Royaume-Uni", + "L’Italie choisit ArcelorMittal pour reprendre la plus grande aciérie d’Europe", + "Apple lance HomePod parce qu'il se sent menacé par l'Echo d'Amazon", + "La France ne devrait pas manquer d'électricité cet été, même en cas de canicule", + "Nouvelles attaques de Trump contre le maire de Londres", + "Où es-tu ?", + "Qui est le président de la France ?", + "Où est la capitale des États-Unis ?", + "Quand est né Barack Obama ?", + "Apple vuole comprare una startup del Regno Unito per un miliardo di dollari", + "Le automobili a guida autonoma spostano la responsabilità assicurativa verso i produttori", + "San Francisco prevede di bandire i robot di consegna porta a porta", + "Londra è una grande città del Regno Unito.", + "Apple overweegt om voor 1 miljard een U.K. startup te kopen", + "Autonome auto's verschuiven de verzekeringverantwoordelijkheid naar producenten", + "San Francisco overweegt robots op voetpaden te verbieden", + "Londen is een grote stad in het Verenigd Koninkrijk", + "Poczuł przyjemną woń mocnej kawy.", + "Istnieje wiele dróg oddziaływania substancji psychoaktywnej na układ nerwowy.", + "Powitał mnie biało-czarny kot, płosząc siedzące na płocie trzy dorodne dudki.", + "Nowy abonament pod lupą Komisji Europejskiej", + "Czy w ciągu ostatnich 48 godzin spożyłeś leki zawierające paracetamol?", + "Kto ma ochotę zapoznać się z innymi niż w książkach przygodami Muminków i ich przyjaciół, temu polecam komiks Tove Jansson „Muminki i morze”.", + "Apple está querendo comprar uma startup do Reino Unido por 100 milhões de dólares.", + "Carros autônomos empurram a responsabilidade do seguro para os fabricantes..", + "São Francisco considera banir os robôs de entrega que andam pelas calçadas.", + "Londres é a maior cidade do Reino Unido.", + # Translations from English: + "Apple рассматривает возможность покупки стартапа из Соединённого Королевства за $1 млрд", + "Беспилотные автомобили перекладывают страховую ответственность на производителя", + "В Сан-Франциско рассматривается возможность запрета роботов-курьеров, которые перемещаются по тротуару", + "Лондон — это большой город в Соединённом Королевстве", + # Native Russian sentences: + # Colloquial: + "Да, нет, наверное!", # Typical polite refusal + "Обратите внимание на необыкновенную красоту этого города-героя Москвы, столицы нашей Родины!", # From a tour guide speech + # Examples of Bookish Russian: + # Quote from "The Golden Calf" + "Рио-де-Жанейро — это моя мечта, и не смейте касаться её своими грязными лапами!", + # Quotes from "Ivan Vasilievich changes his occupation" + "Ты пошто боярыню обидел, смерд?!!", + "Оставь меня, старушка, я в печали!", + # Quotes from Dostoevsky: + "Уж коли я, такой же, как и ты, человек грешный, над тобой умилился и пожалел тебя, кольми паче бог", + "В мечтах я нередко, говорит, доходил до страстных помыслов о служении человечеству и может быть действительно пошел бы на крест за людей, если б это вдруг как-нибудь потребовалось, а между тем я двух дней не в состоянии прожить ни с кем в одной комнате, о чем знаю из опыта", + "Зато всегда так происходило, что чем более я ненавидел людей в частности, тем пламеннее становилась любовь моя к человечеству вообще", + # Quotes from Chekhov: + "Ненужные дела и разговоры всё об одном отхватывают на свою долю лучшую часть времени, лучшие силы, и в конце концов остается какая-то куцая, бескрылая жизнь, какая-то чепуха, и уйти и бежать нельзя, точно сидишь в сумасшедшем доме или в арестантских ротах!", + # Quotes from Turgenev: + "Нравится тебе женщина, старайся добиться толку; а нельзя — ну, не надо, отвернись — земля не клином сошлась", + "Узенькое местечко, которое я занимаю, до того крохотно в сравнении с остальным пространством, где меня нет и где дела до меня нет; и часть времени, которую мне удастся прожить, так ничтожна перед вечностью, где меня не было и не будет...", + # Quotes from newspapers: + # Komsomolskaya Pravda: + "На заседании президиума правительства Москвы принято решение присвоить статус инвестиционного приоритетного проекта города Москвы киностудии Союзмультфильм", + "Глава Минобороны Сергей Шойгу заявил, что обстановка на этом стратегическом направлении требует непрерывного совершенствования боевого состава войск", + # Argumenty i Facty: + "На реплику лже-Говина — дескать, он (Волков) будет лучшим революционером — Стамп с энтузиазмом ответил: Непременно!", +] diff --git a/spacy/lang/yo/__init__.py b/spacy/lang/yo/__init__.py new file mode 100644 index 0000000..93c4ca4 --- /dev/null +++ b/spacy/lang/yo/__init__.py @@ -0,0 +1,16 @@ +from ...language import BaseDefaults, Language +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + + +class YorubaDefaults(BaseDefaults): + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + + +class Yoruba(Language): + lang = "yo" + Defaults = YorubaDefaults + + +__all__ = ["Yoruba"] diff --git a/spacy/lang/yo/examples.py b/spacy/lang/yo/examples.py new file mode 100644 index 0000000..0a610f1 --- /dev/null +++ b/spacy/lang/yo/examples.py @@ -0,0 +1,22 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.yo.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +# 1. https://yo.wikipedia.org/wiki/Wikipedia:%C3%80y%E1%BB%8Dk%C3%A0_p%C3%A0t%C3%A0k%C3%AC +# 2.https://yo.wikipedia.org/wiki/Oj%C3%BAew%C3%A9_%C3%80k%E1%BB%8D%CC%81k%E1%BB%8D%CC%81 +# 3. https://www.bbc.com/yoruba + +sentences = [ + "Ìjọba Tanzania fi Ajìjàgbara Ọmọ Orílẹ̀-èdèe Uganda sí àtìmọ́lé", + "Olúṣẹ́gun Ọbásanjọ́, tí ó jẹ́ Ààrẹ ìjọba ológun àná (láti ọdún 1976 sí 1979), tí ó sì tún ṣe Ààrẹ ìjọba alágbádá tí ìbò gbé wọlé (ní ọdún 1999 sí 2007), kúndùn láti máa bu ẹnu àtẹ́ lu àwọn " + "ètò ìjọba Ààrẹ orílẹ̀-èdè Nàìjíríà tí ó jẹ tẹ̀lé e.", + "Akin Alabi rán ẹnu mọ́ agbárá Adárí Òsìsẹ̀, àwọn ọmọ Nàìjíríà dẹnu bò ó", + "Ta ló leè dúró s'ẹ́gbẹ̀ẹ́ Okunnu láì rẹ́rìín?", + "Dídarapọ̀ mọ́n ìpolongo", + "Bi a se n so, omobinrin ni oruko ni ojo kejo bee naa ni omokunrin ni oruko ni ojo kesan.", + "Oríṣìíríṣìí nǹkan ló le yọrí sí orúkọ tí a sọ ọmọ", + "Gbogbo won ni won ni oriki ti won", +] diff --git a/spacy/lang/yo/lex_attrs.py b/spacy/lang/yo/lex_attrs.py new file mode 100644 index 0000000..5f33e06 --- /dev/null +++ b/spacy/lang/yo/lex_attrs.py @@ -0,0 +1,111 @@ +import unicodedata + +from ...attrs import LIKE_NUM + +_num_words = [ + "ení", + "oókàn", + "ọ̀kanlá", + "ẹ́ẹdọ́gbọ̀n", + "àádọ́fà", + "ẹ̀walélúɡba", + "egbèje", + "ẹgbàárin", + "èjì", + "eéjì", + "èjìlá", + "ọgbọ̀n,", + "ọgọ́fà", + "ọ̀ọ́dúrún", + "ẹgbẹ̀jọ", + "ẹ̀ẹ́dẹ́ɡbàárùn", + "ẹ̀ta", + "ẹẹ́ta", + "ẹ̀talá", + "aárùndílogójì", + "àádóje", + "irinwó", + "ẹgbẹ̀sàn", + "ẹgbàárùn", + "ẹ̀rin", + "ẹẹ́rin", + "ẹ̀rinlá", + "ogójì", + "ogóje", + "ẹ̀ẹ́dẹ́gbẹ̀ta", + "ẹgbàá", + "ẹgbàájọ", + "àrún", + "aárùn", + "ẹ́ẹdógún", + "àádọ́ta", + "àádọ́jọ", + "ẹgbẹ̀ta", + "ẹgboókànlá", + "ẹgbàawǎ", + "ẹ̀fà", + "ẹẹ́fà", + "ẹẹ́rìndílógún", + "ọgọ́ta", + "ọgọ́jọ", + "ọ̀ọ́dẹ́gbẹ̀rin", + "ẹgbẹ́ẹdógún", + "ọkẹ́marun", + "èje", + "etàdílógún", + "àádọ́rin", + "àádọ́sán", + "ẹgbẹ̀rin", + "ẹgbàajì", + "ẹgbẹ̀ẹgbẹ̀rún", + "ẹ̀jọ", + "ẹẹ́jọ", + "eéjìdílógún", + "ọgọ́rin", + "ọgọsàn", + "ẹ̀ẹ́dẹ́gbẹ̀rún", + "ẹgbẹ́ẹdọ́gbọ̀n", + "ọgọ́rùn ọkẹ́", + "ẹ̀sán", + "ẹẹ́sàn", + "oókàndílógún", + "àádọ́rùn", + "ẹ̀wadilúɡba", + "ẹgbẹ̀rún", + "ẹgbàáta", + "ẹ̀wá", + "ẹẹ́wàá", + "ogún", + "ọgọ́rùn", + "igba", + "ẹgbẹ̀fà", + "ẹ̀ẹ́dẹ́ɡbarin", +] + + +def strip_accents_text(text): + """ + Converts the string to NFD, separates & returns only the base characters + :param text: + :return: input string without diacritic adornments on base characters + """ + return "".join( + c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn" + ) + + +def like_num(text): + text = text.replace(",", "").replace(".", "") + num_markers = ["dí", "dọ", "lé", "dín", "di", "din", "le", "do"] + if any(mark in text for mark in num_markers): + return True + text = strip_accents_text(text) + _num_words_stripped = [strip_accents_text(num) for num in _num_words] + if text.isdigit(): + return True + if text in _num_words_stripped or text.lower() in _num_words_stripped: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/yo/stop_words.py b/spacy/lang/yo/stop_words.py new file mode 100644 index 0000000..5c7a7fc --- /dev/null +++ b/spacy/lang/yo/stop_words.py @@ -0,0 +1,9 @@ +# stop words as whitespace-separated list. +# Source: https://raw.githubusercontent.com/dohliam/more-stoplists/master/yo/yo.txt + +STOP_WORDS = set( + "a an b bá bí bẹ̀rẹ̀ d e f fún fẹ́ g gbogbo i inú j jù jẹ jẹ́ k kan kì kí kò " + "l láti lè lọ m mi mo máa mọ̀ n ni náà ní nígbà nítorí nǹkan o p padà pé " + "púpọ̀ pẹ̀lú r rẹ̀ s sì sí sínú t ti tí u w wà wá wọn wọ́n y yìí à àti àwọn á " + "è é ì í ò òun ó ù ú ń ńlá ǹ ̀ ́ ̣ ṣ ṣe ṣé ṣùgbọ́n ẹ ẹmọ́ ọ ọjọ́ ọ̀pọ̀lọpọ̀".split() +) diff --git a/spacy/lang/zh/__init__.py b/spacy/lang/zh/__init__.py new file mode 100644 index 0000000..748e9ac --- /dev/null +++ b/spacy/lang/zh/__init__.py @@ -0,0 +1,336 @@ +import tempfile +import warnings +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional + +import srsly + +from ... import util +from ...errors import Errors, Warnings +from ...language import BaseDefaults, Language +from ...scorer import Scorer +from ...tokens import Doc +from ...training import Example, validate_examples +from ...util import DummyTokenizer, load_config_from_str, registry +from ...vocab import Vocab +from .lex_attrs import LEX_ATTRS +from .stop_words import STOP_WORDS + +# fmt: off +_PKUSEG_INSTALL_MSG = "install spacy-pkuseg with `pip install \"spacy-pkuseg>=0.0.27,<0.1.0\"` or `conda install -c conda-forge \"spacy-pkuseg>=0.0.27,<0.1.0\"`" +# fmt: on + +DEFAULT_CONFIG = """ +[nlp] + +[nlp.tokenizer] +@tokenizers = "spacy.zh.ChineseTokenizer" +segmenter = "char" + +[initialize] + +[initialize.tokenizer] +pkuseg_model = null +pkuseg_user_dict = "default" +""" + + +class Segmenter(str, Enum): + char = "char" + jieba = "jieba" + pkuseg = "pkuseg" + + @classmethod + def values(cls): + return list(cls.__members__.keys()) + + +def create_chinese_tokenizer(segmenter: Segmenter = Segmenter.char): + def chinese_tokenizer_factory(nlp): + return ChineseTokenizer(nlp.vocab, segmenter=segmenter) + + return chinese_tokenizer_factory + + +class ChineseTokenizer(DummyTokenizer): + def __init__(self, vocab: Vocab, segmenter: Segmenter = Segmenter.char): + self.vocab = vocab + self.segmenter = ( + segmenter.value if isinstance(segmenter, Segmenter) else segmenter + ) + self.pkuseg_seg = None + self.jieba_seg = None + if self.segmenter not in Segmenter.values(): + warn_msg = Warnings.W103.format( + lang="Chinese", + segmenter=self.segmenter, + supported=", ".join(Segmenter.values()), + default="'char' (character segmentation)", + ) + warnings.warn(warn_msg) + self.segmenter = Segmenter.char + if self.segmenter == Segmenter.jieba: + self.jieba_seg = try_jieba_import() + + def initialize( + self, + get_examples: Optional[Callable[[], Iterable[Example]]] = None, + *, + nlp: Optional[Language] = None, + pkuseg_model: Optional[str] = None, + pkuseg_user_dict: Optional[str] = "default", + ): + if self.segmenter == Segmenter.pkuseg: + if pkuseg_user_dict is None: + pkuseg_user_dict = pkuseg_model + self.pkuseg_seg = try_pkuseg_import( + pkuseg_model=pkuseg_model, pkuseg_user_dict=pkuseg_user_dict + ) + + def __call__(self, text: str) -> Doc: + if self.segmenter == Segmenter.jieba: + words = list([x for x in self.jieba_seg.cut(text, cut_all=False) if x]) # type: ignore[union-attr] + words, spaces = util.get_words_and_spaces(words, text) + return Doc(self.vocab, words=words, spaces=spaces) + elif self.segmenter == Segmenter.pkuseg: + if self.pkuseg_seg is None: + raise ValueError(Errors.E1000) + words = self.pkuseg_seg.cut(text) + words, spaces = util.get_words_and_spaces(words, text) + return Doc(self.vocab, words=words, spaces=spaces) + + # warn if segmenter setting is not the only remaining option "char" + if self.segmenter != Segmenter.char: + warn_msg = Warnings.W103.format( + lang="Chinese", + segmenter=self.segmenter, + supported=", ".join(Segmenter.values()), + default="'char' (character segmentation)", + ) + warnings.warn(warn_msg) + + # split into individual characters + words = list(text) + words, spaces = util.get_words_and_spaces(words, text) + return Doc(self.vocab, words=words, spaces=spaces) + + def pkuseg_update_user_dict(self, words: List[str], reset: bool = False): + if self.segmenter == Segmenter.pkuseg: + if reset: + try: + import spacy_pkuseg + + self.pkuseg_seg.preprocesser = spacy_pkuseg.Preprocesser(None) # type: ignore[attr-defined] + except ImportError: + msg = ( + "spacy_pkuseg not installed: unable to reset pkuseg " + "user dict. Please " + _PKUSEG_INSTALL_MSG + ) + raise ImportError(msg) from None + for word in words: + self.pkuseg_seg.preprocesser.insert(word.strip(), "") # type: ignore[attr-defined] + else: + warn_msg = Warnings.W104.format(target="pkuseg", current=self.segmenter) + warnings.warn(warn_msg) + + def score(self, examples): + validate_examples(examples, "ChineseTokenizer.score") + return Scorer.score_tokenization(examples) + + def _get_config(self) -> Dict[str, Any]: + return { + "segmenter": self.segmenter, + } + + def _set_config(self, config: Dict[str, Any] = {}) -> None: + self.segmenter = config.get("segmenter", Segmenter.char) + + def to_bytes(self, **kwargs): + pkuseg_features_b = b"" + pkuseg_weights_b = b"" + pkuseg_processors_data = None + if self.pkuseg_seg: + with tempfile.TemporaryDirectory() as tempdir: + self.pkuseg_seg.feature_extractor.save(tempdir) + self.pkuseg_seg.model.save(tempdir) + tempdir = Path(tempdir) + with open(tempdir / "features.msgpack", "rb") as fileh: + pkuseg_features_b = fileh.read() + with open(tempdir / "weights.npz", "rb") as fileh: + pkuseg_weights_b = fileh.read() + pkuseg_processors_data = ( + _get_pkuseg_trie_data(self.pkuseg_seg.preprocesser.trie), + self.pkuseg_seg.postprocesser.do_process, + sorted(list(self.pkuseg_seg.postprocesser.common_words)), + sorted(list(self.pkuseg_seg.postprocesser.other_words)), + ) + serializers = { + "cfg": lambda: srsly.json_dumps(self._get_config()), + "pkuseg_features": lambda: pkuseg_features_b, + "pkuseg_weights": lambda: pkuseg_weights_b, + "pkuseg_processors": lambda: srsly.msgpack_dumps(pkuseg_processors_data), + } + return util.to_bytes(serializers, []) + + def from_bytes(self, data, **kwargs): + pkuseg_data = {"features_b": b"", "weights_b": b"", "processors_data": None} + + def deserialize_pkuseg_features(b): + pkuseg_data["features_b"] = b + + def deserialize_pkuseg_weights(b): + pkuseg_data["weights_b"] = b + + def deserialize_pkuseg_processors(b): + pkuseg_data["processors_data"] = srsly.msgpack_loads(b) + + deserializers = { + "cfg": lambda b: self._set_config(srsly.json_loads(b)), + "pkuseg_features": deserialize_pkuseg_features, + "pkuseg_weights": deserialize_pkuseg_weights, + "pkuseg_processors": deserialize_pkuseg_processors, + } + util.from_bytes(data, deserializers, []) + + if pkuseg_data["features_b"] and pkuseg_data["weights_b"]: + with tempfile.TemporaryDirectory() as tempdir: + tempdir = Path(tempdir) + with open(tempdir / "features.msgpack", "wb") as fileh: + fileh.write(pkuseg_data["features_b"]) + with open(tempdir / "weights.npz", "wb") as fileh: + fileh.write(pkuseg_data["weights_b"]) + try: + import spacy_pkuseg + except ImportError: + raise ImportError( + "spacy-pkuseg not installed. To use this model, " + + _PKUSEG_INSTALL_MSG + ) from None + self.pkuseg_seg = spacy_pkuseg.pkuseg(str(tempdir)) + if pkuseg_data["processors_data"]: + processors_data = pkuseg_data["processors_data"] + user_dict, do_process, common_words, other_words = processors_data + self.pkuseg_seg.preprocesser = spacy_pkuseg.Preprocesser(user_dict) + self.pkuseg_seg.postprocesser.do_process = do_process + self.pkuseg_seg.postprocesser.common_words = set(common_words) + self.pkuseg_seg.postprocesser.other_words = set(other_words) + + return self + + def to_disk(self, path, **kwargs): + path = util.ensure_path(path) + + def save_pkuseg_model(path): + if self.pkuseg_seg: + if not path.exists(): + path.mkdir(parents=True) + self.pkuseg_seg.model.save(path) + self.pkuseg_seg.feature_extractor.save(path) + + def save_pkuseg_processors(path): + if self.pkuseg_seg: + data = ( + _get_pkuseg_trie_data(self.pkuseg_seg.preprocesser.trie), + self.pkuseg_seg.postprocesser.do_process, + sorted(list(self.pkuseg_seg.postprocesser.common_words)), + sorted(list(self.pkuseg_seg.postprocesser.other_words)), + ) + srsly.write_msgpack(path, data) + + serializers = { + "cfg": lambda p: srsly.write_json(p, self._get_config()), + "pkuseg_model": lambda p: save_pkuseg_model(p), + "pkuseg_processors": lambda p: save_pkuseg_processors(p), + } + return util.to_disk(path, serializers, []) + + def from_disk(self, path, **kwargs): + path = util.ensure_path(path) + + def load_pkuseg_model(path): + try: + import spacy_pkuseg + except ImportError: + if self.segmenter == Segmenter.pkuseg: + raise ImportError( + "spacy-pkuseg not installed. To use this model, " + + _PKUSEG_INSTALL_MSG + ) from None + if path.exists(): + self.pkuseg_seg = spacy_pkuseg.pkuseg(path) + + def load_pkuseg_processors(path): + try: + import spacy_pkuseg + except ImportError: + if self.segmenter == Segmenter.pkuseg: + raise ImportError(self._pkuseg_install_msg) from None + if self.segmenter == Segmenter.pkuseg: + data = srsly.read_msgpack(path) + user_dict, do_process, common_words, other_words = data + self.pkuseg_seg.preprocesser = spacy_pkuseg.Preprocesser(user_dict) + self.pkuseg_seg.postprocesser.do_process = do_process + self.pkuseg_seg.postprocesser.common_words = set(common_words) + self.pkuseg_seg.postprocesser.other_words = set(other_words) + + serializers = { + "cfg": lambda p: self._set_config(srsly.read_json(p)), + "pkuseg_model": lambda p: load_pkuseg_model(p), + "pkuseg_processors": lambda p: load_pkuseg_processors(p), + } + util.from_disk(path, serializers, []) + + +class ChineseDefaults(BaseDefaults): + config = load_config_from_str(DEFAULT_CONFIG) + lex_attr_getters = LEX_ATTRS + stop_words = STOP_WORDS + writing_system = {"direction": "ltr", "has_case": False, "has_letters": False} + + +class Chinese(Language): + lang = "zh" + Defaults = ChineseDefaults + + +def try_jieba_import(): + try: + import jieba + + # segment a short text to have jieba initialize its cache in advance + list(jieba.cut("作为", cut_all=False)) + + return jieba + except ImportError: + msg = ( + "Jieba not installed. To use jieba, install it with `pip " + " install jieba` or from https://github.com/fxsjy/jieba" + ) + raise ImportError(msg) from None + + +def try_pkuseg_import(pkuseg_model: Optional[str], pkuseg_user_dict: Optional[str]): + try: + import spacy_pkuseg + + except ImportError: + msg = "spacy-pkuseg not installed. To use pkuseg, " + _PKUSEG_INSTALL_MSG + raise ImportError(msg) from None + try: + return spacy_pkuseg.pkuseg(pkuseg_model, user_dict=pkuseg_user_dict) + except FileNotFoundError: + msg = "Unable to load pkuseg model from: " + str(pkuseg_model or "") + raise FileNotFoundError(msg) from None + + +def _get_pkuseg_trie_data(node, path=""): + data = [] + for c, child_node in sorted(node.children.items()): + data.extend(_get_pkuseg_trie_data(child_node, path + c)) + if node.isword: + data.append((path, node.usertag)) + return data + + +__all__ = ["Chinese"] diff --git a/spacy/lang/zh/examples.py b/spacy/lang/zh/examples.py new file mode 100644 index 0000000..8be1336 --- /dev/null +++ b/spacy/lang/zh/examples.py @@ -0,0 +1,16 @@ +""" +Example sentences to test spaCy and its language models. + +>>> from spacy.lang.zh.examples import sentences +>>> docs = nlp.pipe(sentences) +""" + +# from https://zh.wikipedia.org/wiki/汉语 +sentences = [ + "作为语言而言，为世界使用人数最多的语言，目前世界有五分之一人口做为母语。", + "汉语有多种分支，当中官话最为流行，为中华人民共和国的国家通用语言（又称为普通话）、以及中华民国的国语。", + "此外，中文还是联合国正式语文，并被上海合作组织等国际组织采用为官方语言。", + "在中国大陆，汉语通称为“汉语”。", + "在联合国、台湾、香港及澳门，通称为“中文”。", + "在新加坡及马来西亚，通称为“华语”。", +] diff --git a/spacy/lang/zh/lex_attrs.py b/spacy/lang/zh/lex_attrs.py new file mode 100644 index 0000000..36fa731 --- /dev/null +++ b/spacy/lang/zh/lex_attrs.py @@ -0,0 +1,97 @@ +import re + +from ...attrs import LIKE_NUM + +_single_num_words = [ + "〇", + "一", + "二", + "三", + "四", + "五", + "六", + "七", + "八", + "九", + "十", + "十一", + "十二", + "十三", + "十四", + "十五", + "十六", + "十七", + "十八", + "十九", + "廿", + "卅", + "卌", + "皕", + "零", + "壹", + "贰", + "叁", + "肆", + "伍", + "陆", + "柒", + "捌", + "玖", + "拾", + "拾壹", + "拾贰", + "拾叁", + "拾肆", + "拾伍", + "拾陆", + "拾柒", + "拾捌", + "拾玖", +] + +_count_num_words = [ + "一", + "二", + "三", + "四", + "五", + "六", + "七", + "八", + "九", + "壹", + "贰", + "叁", + "肆", + "伍", + "陆", + "柒", + "捌", + "玖", +] + +_base_num_words = ["十", "百", "千", "万", "亿", "兆", "拾", "佰", "仟"] + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "").replace("，", "").replace("。", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + if text in _single_num_words: + return True + # fmt: off + if re.match('^((' + '|'.join(_count_num_words) + '){1}' + + '(' + '|'.join(_base_num_words) + '){1})+' + + '(' + '|'.join(_count_num_words) + ')?$', text): + return True + # fmt: on + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/zh/stop_words.py b/spacy/lang/zh/stop_words.py new file mode 100644 index 0000000..42ae4a1 --- /dev/null +++ b/spacy/lang/zh/stop_words.py @@ -0,0 +1,1899 @@ +# stop words as whitespace-separated list +# Chinese stop words,maybe not enough +STOP_WORDS = set( + """ +! +" +# +$ +% +& +' +( +) +* ++ +, +- +-- +. +.. +... +...... +................... +./ +.一 +.数 +.日 +/ +// +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +:// +:: +; +< += +> +>> +? +@ +A +Lex +[ +\ +] +^ +_ +` +exp +sub +sup +| +} +~ +~~~~ +· +× +××× +Δ +Ψ +γ +μ +φ +φ． +В +— +—— +——— +‘ +’ +’‘ +“ +” +”， +… +…… +…………………………………………………③ +′∈ +′｜ +℃ +Ⅲ +↑ +→ +∈［ +∪φ∈ +≈ +① +② +②ｃ +③ +③］ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +── +■ +▲ +　 +、 +。 +〈 +〉 +《 +》 +》）， +」 +『 +』 +【 +】 +〔 +〕 +〕〔 +㈧ +一 +一. +一一 +一下 +一个 +一些 +一何 +一切 +一则 +一则通过 +一天 +一定 +一方面 +一旦 +一时 +一来 +一样 +一次 +一片 +一番 +一直 +一致 +一般 +一起 +一转眼 +一边 +一面 +七 +万一 +三 +三天两头 +三番两次 +三番五次 +上 +上下 +上升 +上去 +上来 +上述 +上面 +下 +下列 +下去 +下来 +下面 +不 +不一 +不下 +不久 +不了 +不亦乐乎 +不仅 +不仅...而且 +不仅仅 +不仅仅是 +不会 +不但 +不但...而且 +不光 +不免 +不再 +不力 +不单 +不变 +不只 +不可 +不可开交 +不可抗拒 +不同 +不外 +不外乎 +不够 +不大 +不如 +不妨 +不定 +不对 +不少 +不尽 +不尽然 +不巧 +不已 +不常 +不得 +不得不 +不得了 +不得已 +不必 +不怎么 +不怕 +不惟 +不成 +不拘 +不择手段 +不敢 +不料 +不断 +不日 +不时 +不是 +不曾 +不止 +不止一次 +不比 +不消 +不满 +不然 +不然的话 +不特 +不独 +不由得 +不知不觉 +不管 +不管怎样 +不经意 +不胜 +不能 +不能不 +不至于 +不若 +不要 +不论 +不起 +不足 +不过 +不迭 +不问 +不限 +与 +与其 +与其说 +与否 +与此同时 +专门 +且 +且不说 +且说 +两者 +严格 +严重 +个 +个人 +个别 +中小 +中间 +丰富 +串行 +临 +临到 +为 +为主 +为了 +为什么 +为什麽 +为何 +为止 +为此 +为着 +主张 +主要 +举凡 +举行 +乃 +乃至 +乃至于 +么 +之 +之一 +之前 +之后 +之後 +之所以 +之类 +乌乎 +乎 +乒 +乘 +乘势 +乘机 +乘胜 +乘虚 +乘隙 +九 +也 +也好 +也就是说 +也是 +也罢 +了 +了解 +争取 +二 +二来 +二话不说 +二话没说 +于 +于是 +于是乎 +云云 +云尔 +互 +互相 +五 +些 +交口 +亦 +产生 +亲口 +亲手 +亲眼 +亲自 +亲身 +人 +人人 +人们 +人家 +人民 +什么 +什么样 +什麽 +仅 +仅仅 +今 +今后 +今天 +今年 +今後 +介于 +仍 +仍旧 +仍然 +从 +从不 +从严 +从中 +从事 +从今以后 +从优 +从古到今 +从古至今 +从头 +从宽 +从小 +从新 +从无到有 +从早到晚 +从未 +从来 +从此 +从此以后 +从而 +从轻 +从速 +从重 +他 +他人 +他们 +他是 +他的 +代替 +以 +以上 +以下 +以为 +以便 +以免 +以前 +以及 +以后 +以外 +以後 +以故 +以期 +以来 +以至 +以至于 +以致 +们 +任 +任何 +任凭 +任务 +企图 +伙同 +会 +伟大 +传 +传说 +传闻 +似乎 +似的 +但 +但凡 +但愿 +但是 +何 +何乐而不为 +何以 +何况 +何处 +何妨 +何尝 +何必 +何时 +何止 +何苦 +何须 +余外 +作为 +你 +你们 +你是 +你的 +使 +使得 +使用 +例如 +依 +依据 +依照 +依靠 +便 +便于 +促进 +保持 +保管 +保险 +俺 +俺们 +倍加 +倍感 +倒不如 +倒不如说 +倒是 +倘 +倘使 +倘或 +倘然 +倘若 +借 +借以 +借此 +假使 +假如 +假若 +偏偏 +做到 +偶尔 +偶而 +傥然 +像 +儿 +允许 +元／吨 +充其极 +充其量 +充分 +先不先 +先后 +先後 +先生 +光 +光是 +全体 +全力 +全年 +全然 +全身心 +全部 +全都 +全面 +八 +八成 +公然 +六 +兮 +共 +共同 +共总 +关于 +其 +其一 +其中 +其二 +其他 +其余 +其后 +其它 +其实 +其次 +具体 +具体地说 +具体来说 +具体说来 +具有 +兼之 +内 +再 +再其次 +再则 +再有 +再次 +再者 +再者说 +再说 +冒 +冲 +决不 +决定 +决非 +况且 +准备 +凑巧 +凝神 +几 +几乎 +几度 +几时 +几番 +几经 +凡 +凡是 +凭 +凭借 +出 +出于 +出去 +出来 +出现 +分别 +分头 +分期 +分期分批 +切 +切不可 +切切 +切勿 +切莫 +则 +则甚 +刚 +刚好 +刚巧 +刚才 +初 +别 +别人 +别处 +别是 +别的 +别管 +别说 +到 +到了儿 +到处 +到头 +到头来 +到底 +到目前为止 +前后 +前此 +前者 +前进 +前面 +加上 +加之 +加以 +加入 +加强 +动不动 +动辄 +勃然 +匆匆 +十分 +千 +千万 +千万千万 +半 +单 +单单 +单纯 +即 +即令 +即使 +即便 +即刻 +即如 +即将 +即或 +即是说 +即若 +却 +却不 +历 +原来 +去 +又 +又及 +及 +及其 +及时 +及至 +双方 +反之 +反之亦然 +反之则 +反倒 +反倒是 +反应 +反手 +反映 +反而 +反过来 +反过来说 +取得 +取道 +受到 +变成 +古来 +另 +另一个 +另一方面 +另外 +另悉 +另方面 +另行 +只 +只当 +只怕 +只是 +只有 +只消 +只要 +只限 +叫 +叫做 +召开 +叮咚 +叮当 +可 +可以 +可好 +可是 +可能 +可见 +各 +各个 +各人 +各位 +各地 +各式 +各种 +各级 +各自 +合理 +同 +同一 +同时 +同样 +后 +后来 +后者 +后面 +向 +向使 +向着 +吓 +吗 +否则 +吧 +吧哒 +吱 +呀 +呃 +呆呆地 +呐 +呕 +呗 +呜 +呜呼 +呢 +周围 +呵 +呵呵 +呸 +呼哧 +呼啦 +咋 +和 +咚 +咦 +咧 +咱 +咱们 +咳 +哇 +哈 +哈哈 +哉 +哎 +哎呀 +哎哟 +哗 +哗啦 +哟 +哦 +哩 +哪 +哪个 +哪些 +哪儿 +哪天 +哪年 +哪怕 +哪样 +哪边 +哪里 +哼 +哼唷 +唉 +唯有 +啊 +啊呀 +啊哈 +啊哟 +啐 +啥 +啦 +啪达 +啷当 +喀 +喂 +喏 +喔唷 +喽 +嗡 +嗡嗡 +嗬 +嗯 +嗳 +嘎 +嘎嘎 +嘎登 +嘘 +嘛 +嘻 +嘿 +嘿嘿 +四 +因 +因为 +因了 +因此 +因着 +因而 +固 +固然 +在 +在下 +在于 +地 +均 +坚决 +坚持 +基于 +基本 +基本上 +处在 +处处 +处理 +复杂 +多 +多么 +多亏 +多多 +多多少少 +多多益善 +多少 +多年前 +多年来 +多数 +多次 +够瞧的 +大 +大不了 +大举 +大事 +大体 +大体上 +大凡 +大力 +大多 +大多数 +大大 +大家 +大张旗鼓 +大批 +大抵 +大概 +大略 +大约 +大致 +大都 +大量 +大面儿上 +失去 +奇 +奈 +奋勇 +她 +她们 +她是 +她的 +好 +好在 +好的 +好象 +如 +如上 +如上所述 +如下 +如今 +如何 +如其 +如前所述 +如同 +如常 +如是 +如期 +如果 +如次 +如此 +如此等等 +如若 +始而 +姑且 +存在 +存心 +孰料 +孰知 +宁 +宁可 +宁愿 +宁肯 +它 +它们 +它们的 +它是 +它的 +安全 +完全 +完成 +定 +实现 +实际 +宣布 +容易 +密切 +对 +对于 +对应 +对待 +对方 +对比 +将 +将才 +将要 +将近 +小 +少数 +尔 +尔后 +尔尔 +尔等 +尚且 +尤其 +就 +就地 +就是 +就是了 +就是说 +就此 +就算 +就要 +尽 +尽可能 +尽如人意 +尽心尽力 +尽心竭力 +尽快 +尽早 +尽然 +尽管 +尽管如此 +尽量 +局外 +居然 +届时 +属于 +屡 +屡屡 +屡次 +屡次三番 +岂 +岂但 +岂止 +岂非 +川流不息 +左右 +巨大 +巩固 +差一点 +差不多 +己 +已 +已矣 +已经 +巴 +巴巴 +带 +帮助 +常 +常常 +常言说 +常言说得好 +常言道 +平素 +年复一年 +并 +并不 +并不是 +并且 +并排 +并无 +并没 +并没有 +并肩 +并非 +广大 +广泛 +应当 +应用 +应该 +庶乎 +庶几 +开外 +开始 +开展 +引起 +弗 +弹指之间 +强烈 +强调 +归 +归根到底 +归根结底 +归齐 +当 +当下 +当中 +当儿 +当前 +当即 +当口儿 +当地 +当场 +当头 +当庭 +当时 +当然 +当真 +当着 +形成 +彻夜 +彻底 +彼 +彼时 +彼此 +往 +往往 +待 +待到 +很 +很多 +很少 +後来 +後面 +得 +得了 +得出 +得到 +得天独厚 +得起 +心里 +必 +必定 +必将 +必然 +必要 +必须 +快 +快要 +忽地 +忽然 +怎 +怎么 +怎么办 +怎么样 +怎奈 +怎样 +怎麽 +怕 +急匆匆 +怪 +怪不得 +总之 +总是 +总的来看 +总的来说 +总的说来 +总结 +总而言之 +恍然 +恐怕 +恰似 +恰好 +恰如 +恰巧 +恰恰 +恰恰相反 +恰逢 +您 +您们 +您是 +惟其 +惯常 +意思 +愤然 +愿意 +慢说 +成为 +成年 +成年累月 +成心 +我 +我们 +我是 +我的 +或 +或则 +或多或少 +或是 +或曰 +或者 +或许 +战斗 +截然 +截至 +所 +所以 +所在 +所幸 +所有 +所谓 +才 +才能 +扑通 +打 +打从 +打开天窗说亮话 +扩大 +把 +抑或 +抽冷子 +拦腰 +拿 +按 +按时 +按期 +按照 +按理 +按说 +挨个 +挨家挨户 +挨次 +挨着 +挨门挨户 +挨门逐户 +换句话说 +换言之 +据 +据实 +据悉 +据我所知 +据此 +据称 +据说 +掌握 +接下来 +接着 +接著 +接连不断 +放量 +故 +故意 +故此 +故而 +敞开儿 +敢 +敢于 +敢情 +数/ +整个 +断然 +方 +方便 +方才 +方能 +方面 +旁人 +无 +无宁 +无法 +无论 +既 +既...又 +既往 +既是 +既然 +日复一日 +日渐 +日益 +日臻 +日见 +时候 +昂然 +明显 +明确 +是 +是不是 +是以 +是否 +是的 +显然 +显著 +普通 +普遍 +暗中 +暗地里 +暗自 +更 +更为 +更加 +更进一步 +曾 +曾经 +替 +替代 +最 +最后 +最大 +最好 +最後 +最近 +最高 +有 +有些 +有关 +有利 +有力 +有及 +有所 +有效 +有时 +有点 +有的 +有的是 +有着 +有著 +望 +朝 +朝着 +末##末 +本 +本人 +本地 +本着 +本身 +权时 +来 +来不及 +来得及 +来看 +来着 +来自 +来讲 +来说 +极 +极为 +极了 +极其 +极力 +极大 +极度 +极端 +构成 +果然 +果真 +某 +某个 +某些 +某某 +根据 +根本 +格外 +梆 +概 +次第 +欢迎 +欤 +正值 +正在 +正如 +正巧 +正常 +正是 +此 +此中 +此后 +此地 +此处 +此外 +此时 +此次 +此间 +殆 +毋宁 +每 +每个 +每天 +每年 +每当 +每时每刻 +每每 +每逢 +比 +比及 +比如 +比如说 +比方 +比照 +比起 +比较 +毕竟 +毫不 +毫无 +毫无例外 +毫无保留地 +汝 +沙沙 +没 +没奈何 +没有 +沿 +沿着 +注意 +活 +深入 +清楚 +满 +满足 +漫说 +焉 +然 +然则 +然后 +然後 +然而 +照 +照着 +牢牢 +特别是 +特殊 +特点 +犹且 +犹自 +独 +独自 +猛然 +猛然间 +率尔 +率然 +现代 +现在 +理应 +理当 +理该 +瑟瑟 +甚且 +甚么 +甚或 +甚而 +甚至 +甚至于 +用 +用来 +甫 +甭 +由 +由于 +由是 +由此 +由此可见 +略 +略为 +略加 +略微 +白 +白白 +的 +的确 +的话 +皆可 +目前 +直到 +直接 +相似 +相信 +相反 +相同 +相对 +相对而言 +相应 +相当 +相等 +省得 +看 +看上去 +看出 +看到 +看来 +看样子 +看看 +看见 +看起来 +真是 +真正 +眨眼 +着 +着呢 +矣 +矣乎 +矣哉 +知道 +砰 +确定 +碰巧 +社会主义 +离 +种 +积极 +移动 +究竟 +穷年累月 +突出 +突然 +窃 +立 +立刻 +立即 +立地 +立时 +立马 +竟 +竟然 +竟而 +第 +第二 +等 +等到 +等等 +策略地 +简直 +简而言之 +简言之 +管 +类如 +粗 +精光 +紧接着 +累年 +累次 +纯 +纯粹 +纵 +纵令 +纵使 +纵然 +练习 +组成 +经 +经常 +经过 +结合 +结果 +给 +绝 +绝不 +绝对 +绝非 +绝顶 +继之 +继后 +继续 +继而 +维持 +综上所述 +缕缕 +罢了 +老 +老大 +老是 +老老实实 +考虑 +者 +而 +而且 +而况 +而又 +而后 +而外 +而已 +而是 +而言 +而论 +联系 +联袂 +背地里 +背靠背 +能 +能否 +能够 +腾 +自 +自个儿 +自从 +自各儿 +自后 +自家 +自己 +自打 +自身 +臭 +至 +至于 +至今 +至若 +致 +般的 +良好 +若 +若夫 +若是 +若果 +若非 +范围 +莫 +莫不 +莫不然 +莫如 +莫若 +莫非 +获得 +藉以 +虽 +虽则 +虽然 +虽说 +蛮 +行为 +行动 +表明 +表示 +被 +要 +要不 +要不是 +要不然 +要么 +要是 +要求 +见 +规定 +觉得 +譬喻 +譬如 +认为 +认真 +认识 +让 +许多 +论 +论说 +设使 +设或 +设若 +诚如 +诚然 +话说 +该 +该当 +说明 +说来 +说说 +请勿 +诸 +诸位 +诸如 +谁 +谁人 +谁料 +谁知 +谨 +豁然 +贼死 +赖以 +赶 +赶快 +赶早不赶晚 +起 +起先 +起初 +起头 +起来 +起见 +起首 +趁 +趁便 +趁势 +趁早 +趁机 +趁热 +趁着 +越是 +距 +跟 +路经 +转动 +转变 +转贴 +轰然 +较 +较为 +较之 +较比 +边 +达到 +达旦 +迄 +迅速 +过 +过于 +过去 +过来 +运用 +近 +近几年来 +近年来 +近来 +还 +还是 +还有 +还要 +这 +这一来 +这个 +这么 +这么些 +这么样 +这么点儿 +这些 +这会儿 +这儿 +这就是说 +这时 +这样 +这次 +这点 +这种 +这般 +这边 +这里 +这麽 +进入 +进去 +进来 +进步 +进而 +进行 +连 +连同 +连声 +连日 +连日来 +连袂 +连连 +迟早 +迫于 +适应 +适当 +适用 +逐步 +逐渐 +通常 +通过 +造成 +逢 +遇到 +遭到 +遵循 +遵照 +避免 +那 +那个 +那么 +那么些 +那么样 +那些 +那会儿 +那儿 +那时 +那末 +那样 +那般 +那边 +那里 +那麽 +部分 +都 +鄙人 +采取 +里面 +重大 +重新 +重要 +鉴于 +针对 +长期以来 +长此下去 +长线 +长话短说 +问题 +间或 +防止 +阿 +附近 +陈年 +限制 +陡然 +除 +除了 +除却 +除去 +除外 +除开 +除此 +除此之外 +除此以外 +除此而外 +除非 +随 +随后 +随时 +随着 +随著 +隔夜 +隔日 +难得 +难怪 +难说 +难道 +难道说 +集中 +零 +需要 +非但 +非常 +非徒 +非得 +非特 +非独 +靠 +顶多 +顷 +顷刻 +顷刻之间 +顷刻间 +顺 +顺着 +顿时 +颇 +风雨无阻 +饱 +首先 +马上 +高低 +高兴 +默然 +默默地 +齐 +︿ +！ +＃ +＄ +％ +＆ +＇ +（ +） +）÷（１－ +）、 +＊ +＋ +＋ξ +＋＋ +， +，也 +－ +－β +－－ +－［＊］－ +． +／ +０ +０：２ +１ +１． +１２％ +２ +２．３％ +３ +４ +５ +５：０ +６ +７ +８ +９ +： +； +＜ +＜± +＜Δ +＜λ +＜φ +＜＜ +＝ +＝″ +＝☆ +＝（ +＝－ +＝［ +＝｛ +＞ +＞λ +？ +＠ +Ａ +ＬＩ +Ｒ．Ｌ． +ＺＸＦＩＴＬ +［ +［①①］ +［①②］ +［①③］ +［①④］ +［①⑤］ +［①⑥］ +［①⑦］ +［①⑧］ +［①⑨］ +［①Ａ］ +［①Ｂ］ +［①Ｃ］ +［①Ｄ］ +［①Ｅ］ +［①］ +［①ａ］ +［①ｃ］ +［①ｄ］ +［①ｅ］ +［①ｆ］ +［①ｇ］ +［①ｈ］ +［①ｉ］ +［①ｏ］ +［② +［②①］ +［②②］ +［②③］ +［②④ +［②⑤］ +［②⑥］ +［②⑦］ +［②⑧］ +［②⑩］ +［②Ｂ］ +［②Ｇ］ +［②］ +［②ａ］ +［②ｂ］ +［②ｃ］ +［②ｄ］ +［②ｅ］ +［②ｆ］ +［②ｇ］ +［②ｈ］ +［②ｉ］ +［②ｊ］ +［③①］ +［③⑩］ +［③Ｆ］ +［③］ +［③ａ］ +［③ｂ］ +［③ｃ］ +［③ｄ］ +［③ｅ］ +［③ｇ］ +［③ｈ］ +［④］ +［④ａ］ +［④ｂ］ +［④ｃ］ +［④ｄ］ +［④ｅ］ +［⑤］ +［⑤］］ +［⑤ａ］ +［⑤ｂ］ +［⑤ｄ］ +［⑤ｅ］ +［⑤ｆ］ +［⑥］ +［⑦］ +［⑧］ +［⑨］ +［⑩］ +［＊］ +［－ +［］ +］ +］∧′＝［ +］［ +＿ +ａ］ +ｂ］ +ｃ］ +ｅ］ +ｆ］ +ｎｇ昉 +｛ +｛－ +｜ +｝ +｝＞ +～ +～± +～＋ +￥ +""".split() +) diff --git a/spacy/language.py b/spacy/language.py new file mode 100644 index 0000000..8e91018 --- /dev/null +++ b/spacy/language.py @@ -0,0 +1,2466 @@ +import functools +import inspect +import itertools +import multiprocessing as mp +import random +import traceback +import warnings +from contextlib import ExitStack, contextmanager +from copy import deepcopy +from dataclasses import dataclass +from itertools import chain, cycle +from pathlib import Path +from timeit import default_timer as timer +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + NoReturn, + Optional, + Pattern, + Sequence, + Set, + Tuple, + TypeVar, + Union, + cast, + overload, +) + +import numpy +import srsly +from cymem.cymem import Pool +from thinc.api import Config, CupyOps, Optimizer, get_current_ops +from thinc.util import convert_recursive + +from . import about, ty, util +from .compat import Literal +from .errors import Errors, Warnings +from .git_info import GIT_VERSION +from .lang.punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES +from .lang.tokenizer_exceptions import BASE_EXCEPTIONS, URL_MATCH +from .lookups import load_lookups +from .pipe_analysis import analyze_pipes, print_pipe_analysis, validate_attrs +from .schemas import ( + ConfigSchema, + ConfigSchemaInit, + ConfigSchemaNlp, + ConfigSchemaPretrain, + validate_init_settings, +) +from .scorer import Scorer +from .tokenizer import Tokenizer +from .tokens import Doc +from .tokens.underscore import Underscore +from .training import Example, validate_examples +from .training.initialize import init_tok2vec, init_vocab +from .util import ( + _DEFAULT_EMPTY_PIPES, + CONFIG_SECTION_ORDER, + SimpleFrozenDict, + SimpleFrozenList, + _pipe, + combine_score_weights, + raise_error, + registry, + warn_if_jupyter_cupy, +) +from .vectors import BaseVectors +from .vocab import Vocab, create_vocab + +PipeCallable = Callable[[Doc], Doc] + + +# This is the base config will all settings (training etc.) +DEFAULT_CONFIG_PATH = Path(__file__).parent / "default_config.cfg" +DEFAULT_CONFIG = util.load_config(DEFAULT_CONFIG_PATH) +# This is the base config for the [pretraining] block and currently not included +# in the main config and only added via the 'init fill-config' command +DEFAULT_CONFIG_PRETRAIN_PATH = Path(__file__).parent / "default_config_pretraining.cfg" + +# Type variable for contexts piped with documents +_AnyContext = TypeVar("_AnyContext") + + +class BaseDefaults: + """Language data defaults, available via Language.Defaults. Can be + overwritten by language subclasses by defining their own subclasses of + Language.Defaults. + """ + + config: Config = Config(section_order=CONFIG_SECTION_ORDER) + tokenizer_exceptions: Dict[str, List[dict]] = BASE_EXCEPTIONS + prefixes: Optional[Sequence[Union[str, Pattern]]] = TOKENIZER_PREFIXES + suffixes: Optional[Sequence[Union[str, Pattern]]] = TOKENIZER_SUFFIXES + infixes: Optional[Sequence[Union[str, Pattern]]] = TOKENIZER_INFIXES + token_match: Optional[Callable] = None + url_match: Optional[Callable] = URL_MATCH + syntax_iterators: Dict[str, Callable] = {} + lex_attr_getters: Dict[int, Callable[[str], Any]] = {} + stop_words: Set[str] = set() + writing_system = {"direction": "ltr", "has_case": True, "has_letters": True} + + +def create_tokenizer() -> Callable[["Language"], Tokenizer]: + """Registered function to create a tokenizer. Returns a factory that takes + the nlp object and returns a Tokenizer instance using the language detaults. + """ + + def tokenizer_factory(nlp: "Language") -> Tokenizer: + prefixes = nlp.Defaults.prefixes + suffixes = nlp.Defaults.suffixes + infixes = nlp.Defaults.infixes + prefix_search = util.compile_prefix_regex(prefixes).search if prefixes else None + suffix_search = util.compile_suffix_regex(suffixes).search if suffixes else None + infix_finditer = util.compile_infix_regex(infixes).finditer if infixes else None + return Tokenizer( + nlp.vocab, + rules=nlp.Defaults.tokenizer_exceptions, + prefix_search=prefix_search, + suffix_search=suffix_search, + infix_finditer=infix_finditer, + token_match=nlp.Defaults.token_match, + url_match=nlp.Defaults.url_match, + ) + + return tokenizer_factory + + +def load_lookups_data(lang, tables): + util.logger.debug("Loading lookups from spacy-lookups-data: %s", tables) + lookups = load_lookups(lang=lang, tables=tables) + return lookups + + +class Language: + """A text-processing pipeline. Usually you'll load this once per process, + and pass the instance around your application. + + Defaults (class): Settings, data and factory methods for creating the `nlp` + object and processing pipeline. + lang (str): Two-letter ISO 639-1 or three-letter ISO 639-3 language codes, such as 'en' and 'eng'. + + DOCS: https://spacy.io/api/language + """ + + Defaults = BaseDefaults + lang: Optional[str] = None + default_config = DEFAULT_CONFIG + + factories = SimpleFrozenDict(error=Errors.E957) + _factory_meta: Dict[str, "FactoryMeta"] = {} # meta by factory + + def __init__( + self, + vocab: Union[Vocab, bool] = True, + *, + max_length: int = 10**6, + meta: Dict[str, Any] = {}, + create_tokenizer: Optional[Callable[["Language"], Callable[[str], Doc]]] = None, + create_vectors: Optional[Callable[["Vocab"], BaseVectors]] = None, + batch_size: int = 1000, + **kwargs, + ) -> None: + """Initialise a Language object. + + vocab (Vocab): A `Vocab` object. If `True`, a vocab is created. + meta (dict): Custom meta data for the Language class. Is written to by + models to add model meta data. + max_length (int): Maximum number of characters in a single text. The + current models may run out memory on extremely long texts, due to + large internal allocations. You should segment these texts into + meaningful units, e.g. paragraphs, subsections etc, before passing + them to spaCy. Default maximum length is 1,000,000 charas (1mb). As + a rule of thumb, if all pipeline components are enabled, spaCy's + default models currently requires roughly 1GB of temporary memory per + 100,000 characters in one text. + create_tokenizer (Callable): Function that takes the nlp object and + returns a tokenizer. + batch_size (int): Default batch size for pipe and evaluate. + + DOCS: https://spacy.io/api/language#init + """ + from .pipeline.factories import register_factories + + register_factories() + # We're only calling this to import all factories provided via entry + # points. The factory decorator applied to these functions takes care + # of the rest. + util.registry._entry_point_factories.get_all() + + self._config = DEFAULT_CONFIG.merge(self.default_config) + self._meta = dict(meta) + self._path = None + self._optimizer: Optional[Optimizer] = None + # Component meta and configs are only needed on the instance + self._pipe_meta: Dict[str, "FactoryMeta"] = {} # meta by component + self._pipe_configs: Dict[str, Config] = {} # config by component + + if not isinstance(vocab, Vocab) and vocab is not True: + raise ValueError(Errors.E918.format(vocab=vocab, vocab_type=type(Vocab))) + if vocab is True: + vectors_name = meta.get("vectors", {}).get("name") + vocab = create_vocab(self.lang, self.Defaults, vectors_name=vectors_name) + if not create_vectors: + vectors_cfg = {"vectors": self._config["nlp"]["vectors"]} + create_vectors = registry.resolve(vectors_cfg)["vectors"] + vocab.vectors = create_vectors(vocab) + else: + if (self.lang and vocab.lang) and (self.lang != vocab.lang): + raise ValueError(Errors.E150.format(nlp=self.lang, vocab=vocab.lang)) + self.vocab: Vocab = vocab + if self.lang is None: + self.lang = self.vocab.lang + self._components: List[Tuple[str, PipeCallable]] = [] + self._disabled: Set[str] = set() + self.max_length = max_length + # Create the default tokenizer from the default config + if not create_tokenizer: + tokenizer_cfg = {"tokenizer": self._config["nlp"]["tokenizer"]} + create_tokenizer = registry.resolve(tokenizer_cfg)["tokenizer"] + self.tokenizer = create_tokenizer(self) + self.batch_size = batch_size + self.default_error_handler = raise_error + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.default_config = DEFAULT_CONFIG.merge(cls.Defaults.config) + cls.default_config["nlp"]["lang"] = cls.lang + + @property + def path(self): + return self._path + + @property + def meta(self) -> Dict[str, Any]: + """Custom meta data of the language class. If a model is loaded, this + includes details from the model's meta.json. + + RETURNS (Dict[str, Any]): The meta. + + DOCS: https://spacy.io/api/language#meta + """ + spacy_version = util.get_minor_version_range(about.__version__) + if self.vocab.lang: + self._meta.setdefault("lang", self.vocab.lang) + else: + self._meta.setdefault("lang", self.lang) + self._meta.setdefault("name", "pipeline") + self._meta.setdefault("version", "0.0.0") + self._meta.setdefault("spacy_version", spacy_version) + self._meta.setdefault("description", "") + self._meta.setdefault("author", "") + self._meta.setdefault("email", "") + self._meta.setdefault("url", "") + self._meta.setdefault("license", "") + self._meta.setdefault("spacy_git_version", GIT_VERSION) + self._meta["vectors"] = { + "width": self.vocab.vectors_length, + "vectors": len(self.vocab.vectors), + "keys": self.vocab.vectors.n_keys, + "name": self.vocab.vectors.name, + "mode": self.vocab.vectors.mode, + } + self._meta["labels"] = dict(self.pipe_labels) + # TODO: Adding this back to prevent breaking people's code etc., but + # we should consider removing it + self._meta["pipeline"] = list(self.pipe_names) + self._meta["components"] = list(self.component_names) + self._meta["disabled"] = list(self.disabled) + return self._meta + + @meta.setter + def meta(self, value: Dict[str, Any]) -> None: + self._meta = value + + @property + def config(self) -> Config: + """Trainable config for the current language instance. Includes the + current pipeline components, as well as default training config. + + RETURNS (thinc.api.Config): The config. + + DOCS: https://spacy.io/api/language#config + """ + self._config.setdefault("nlp", {}) + self._config.setdefault("training", {}) + self._config["nlp"]["lang"] = self.lang + # We're storing the filled config for each pipeline component and so + # we can populate the config again later + pipeline = {} + score_weights = [] + for pipe_name in self.component_names: + pipe_meta = self.get_pipe_meta(pipe_name) + pipe_config = self.get_pipe_config(pipe_name) + pipeline[pipe_name] = {"factory": pipe_meta.factory, **pipe_config} + if pipe_meta.default_score_weights: + score_weights.append(pipe_meta.default_score_weights) + self._config["nlp"]["pipeline"] = list(self.component_names) + self._config["nlp"]["disabled"] = list(self.disabled) + self._config["components"] = pipeline + # We're merging the existing score weights back into the combined + # weights to make sure we're preserving custom settings in the config + # but also reflect updates (e.g. new components added) + prev_weights = self._config["training"].get("score_weights", {}) + combined_score_weights = combine_score_weights(score_weights, prev_weights) + self._config["training"]["score_weights"] = combined_score_weights + if not srsly.is_json_serializable(self._config): + raise ValueError(Errors.E961.format(config=self._config)) + return self._config + + @config.setter + def config(self, value: Config) -> None: + self._config = value + + @property + def disabled(self) -> List[str]: + """Get the names of all disabled components. + + RETURNS (List[str]): The disabled components. + """ + # Make sure the disabled components are returned in the order they + # appear in the pipeline (which isn't guaranteed by the set) + names = [name for name, _ in self._components if name in self._disabled] + return SimpleFrozenList(names, error=Errors.E926.format(attr="disabled")) + + @property + def factory_names(self) -> List[str]: + """Get names of all available factories. + + RETURNS (List[str]): The factory names. + """ + names = list(self.factories.keys()) + return SimpleFrozenList(names) + + @property + def components(self) -> List[Tuple[str, PipeCallable]]: + """Get all (name, component) tuples in the pipeline, including the + currently disabled components. + """ + return SimpleFrozenList( + self._components, error=Errors.E926.format(attr="components") + ) + + @property + def component_names(self) -> List[str]: + """Get the names of the available pipeline components. Includes all + active and inactive pipeline components. + + RETURNS (List[str]): List of component name strings, in order. + """ + names = [pipe_name for pipe_name, _ in self._components] + return SimpleFrozenList(names, error=Errors.E926.format(attr="component_names")) + + @property + def pipeline(self) -> List[Tuple[str, PipeCallable]]: + """The processing pipeline consisting of (name, component) tuples. The + components are called on the Doc in order as it passes through the + pipeline. + + RETURNS (List[Tuple[str, Callable[[Doc], Doc]]]): The pipeline. + """ + pipes = [(n, p) for n, p in self._components if n not in self._disabled] + return SimpleFrozenList(pipes, error=Errors.E926.format(attr="pipeline")) + + @property + def pipe_names(self) -> List[str]: + """Get names of available active pipeline components. + + RETURNS (List[str]): List of component name strings, in order. + """ + names = [pipe_name for pipe_name, _ in self.pipeline] + return SimpleFrozenList(names, error=Errors.E926.format(attr="pipe_names")) + + @property + def pipe_factories(self) -> Dict[str, str]: + """Get the component factories for the available pipeline components. + + RETURNS (Dict[str, str]): Factory names, keyed by component names. + """ + factories = {} + for pipe_name, pipe in self._components: + factories[pipe_name] = self.get_pipe_meta(pipe_name).factory + return SimpleFrozenDict(factories) + + @property + def pipe_labels(self) -> Dict[str, List[str]]: + """Get the labels set by the pipeline components, if available (if + the component exposes a labels property and the labels are not + hidden). + + RETURNS (Dict[str, List[str]]): Labels keyed by component name. + """ + labels = {} + for name, pipe in self._components: + if hasattr(pipe, "hide_labels") and pipe.hide_labels is True: + continue + if hasattr(pipe, "labels"): + labels[name] = list(pipe.labels) + return SimpleFrozenDict(labels) + + @classmethod + def has_factory(cls, name: str) -> bool: + """RETURNS (bool): Whether a factory of that name is registered.""" + internal_name = cls.get_factory_name(name) + return name in registry.factories or internal_name in registry.factories + + @classmethod + def get_factory_name(cls, name: str) -> str: + """Get the internal factory name based on the language subclass. + + name (str): The factory name. + RETURNS (str): The internal factory name. + """ + if cls.lang is None: + return name + return f"{cls.lang}.{name}" + + @classmethod + def get_factory_meta(cls, name: str) -> "FactoryMeta": + """Get the meta information for a given factory name. + + name (str): The component factory name. + RETURNS (FactoryMeta): The meta for the given factory name. + """ + internal_name = cls.get_factory_name(name) + if internal_name in cls._factory_meta: + return cls._factory_meta[internal_name] + if name in cls._factory_meta: + return cls._factory_meta[name] + raise ValueError(Errors.E967.format(meta="factory", name=name)) + + @classmethod + def set_factory_meta(cls, name: str, value: "FactoryMeta") -> None: + """Set the meta information for a given factory name. + + name (str): The component factory name. + value (FactoryMeta): The meta to set. + """ + cls._factory_meta[cls.get_factory_name(name)] = value + + def get_pipe_meta(self, name: str) -> "FactoryMeta": + """Get the meta information for a given component name. + + name (str): The component name. + RETURNS (FactoryMeta): The meta for the given component name. + """ + if name not in self._pipe_meta: + raise ValueError(Errors.E967.format(meta="component", name=name)) + return self._pipe_meta[name] + + def get_pipe_config(self, name: str) -> Config: + """Get the config used to create a pipeline component. + + name (str): The component name. + RETURNS (Config): The config used to create the pipeline component. + """ + if name not in self._pipe_configs: + raise ValueError(Errors.E960.format(name=name)) + pipe_config = self._pipe_configs[name] + return pipe_config + + @classmethod + def factory( + cls, + name: str, + *, + default_config: Dict[str, Any] = SimpleFrozenDict(), + assigns: Iterable[str] = SimpleFrozenList(), + requires: Iterable[str] = SimpleFrozenList(), + retokenizes: bool = False, + default_score_weights: Dict[str, Optional[float]] = SimpleFrozenDict(), + func: Optional[Callable] = None, + ) -> Callable: + """Register a new pipeline component factory. Can be used as a decorator + on a function or classmethod, or called as a function with the factory + provided as the func keyword argument. To create a component and add + it to the pipeline, you can use nlp.add_pipe(name). + + name (str): The name of the component factory. + default_config (Dict[str, Any]): Default configuration, describing the + default values of the factory arguments. + assigns (Iterable[str]): Doc/Token attributes assigned by this component, + e.g. "token.ent_id". Used for pipeline analysis. + requires (Iterable[str]): Doc/Token attributes required by this component, + e.g. "token.ent_id". Used for pipeline analysis. + retokenizes (bool): Whether the component changes the tokenization. + Used for pipeline analysis. + default_score_weights (Dict[str, Optional[float]]): The scores to report during + training, and their default weight towards the final score used to + select the best model. Weights should sum to 1.0 per component and + will be combined and normalized for the whole pipeline. If None, + the score won't be shown in the logs or be weighted. + func (Optional[Callable]): Factory function if not used as a decorator. + + DOCS: https://spacy.io/api/language#factory + """ + if not isinstance(name, str): + raise ValueError(Errors.E963.format(decorator="factory")) + if "." in name: + raise ValueError(Errors.E853.format(name=name)) + if not isinstance(default_config, dict): + err = Errors.E962.format( + style="default config", name=name, cfg_type=type(default_config) + ) + raise ValueError(err) + + def add_factory(factory_func: Callable) -> Callable: + internal_name = cls.get_factory_name(name) + if internal_name in registry.factories: + # We only check for the internal name here – it's okay if it's a + # subclass and the base class has a factory of the same name. We + # also only raise if the function is different to prevent raising + # if module is reloaded. + existing_func = registry.factories.get(internal_name) + if not util.is_same_func(factory_func, existing_func): + err = Errors.E004.format( + name=name, func=existing_func, new_func=factory_func + ) + raise ValueError(err) + + arg_names = util.get_arg_names(factory_func) + if "nlp" not in arg_names or "name" not in arg_names: + raise ValueError(Errors.E964.format(name=name)) + # Officially register the factory so we can later call + # registry.resolve and refer to it in the config as + # @factories = "spacy.Language.xyz". We use the class name here so + # different classes can have different factories. + registry.factories.register(internal_name, func=factory_func) + factory_meta = FactoryMeta( + factory=name, + default_config=default_config, + assigns=validate_attrs(assigns), + requires=validate_attrs(requires), + scores=list(default_score_weights.keys()), + default_score_weights=default_score_weights, + retokenizes=retokenizes, + ) + cls.set_factory_meta(name, factory_meta) + # We're overwriting the class attr with a frozen dict to handle + # backwards-compat (writing to Language.factories directly). This + # wouldn't work with an instance property and just produce a + # confusing error – here we can show a custom error + cls.factories = SimpleFrozenDict( + registry.factories.get_all(), error=Errors.E957 + ) + return factory_func + + if func is not None: # Support non-decorator use cases + return add_factory(func) + return add_factory + + @classmethod + def component( + cls, + name: str, + *, + assigns: Iterable[str] = SimpleFrozenList(), + requires: Iterable[str] = SimpleFrozenList(), + retokenizes: bool = False, + func: Optional[PipeCallable] = None, + ) -> Callable[..., Any]: + """Register a new pipeline component. Can be used for stateless function + components that don't require a separate factory. Can be used as a + decorator on a function or classmethod, or called as a function with the + factory provided as the func keyword argument. To create a component and + add it to the pipeline, you can use nlp.add_pipe(name). + + name (str): The name of the component factory. + assigns (Iterable[str]): Doc/Token attributes assigned by this component, + e.g. "token.ent_id". Used for pipeline analysis. + requires (Iterable[str]): Doc/Token attributes required by this component, + e.g. "token.ent_id". Used for pipeline analysis. + retokenizes (bool): Whether the component changes the tokenization. + Used for pipeline analysis. + func (Optional[Callable[[Doc], Doc]): Factory function if not used as a decorator. + + DOCS: https://spacy.io/api/language#component + """ + if name is not None: + if not isinstance(name, str): + raise ValueError(Errors.E963.format(decorator="component")) + if "." in name: + raise ValueError(Errors.E853.format(name=name)) + component_name = name if name is not None else util.get_object_name(func) + + def add_component(component_func: PipeCallable) -> Callable: + if isinstance(func, type): # function is a class + raise ValueError(Errors.E965.format(name=component_name)) + + def factory_func(nlp, name: str) -> PipeCallable: + return component_func + + internal_name = cls.get_factory_name(name) + if internal_name in registry.factories: + # We only check for the internal name here – it's okay if it's a + # subclass and the base class has a factory of the same name. We + # also only raise if the function is different to prevent raising + # if module is reloaded. It's hacky, but we need to check the + # existing functure for a closure and whether that's identical + # to the component function (because factory_func created above + # will always be different, even for the same function) + existing_func = registry.factories.get(internal_name) + closure = existing_func.__closure__ + wrapped = [c.cell_contents for c in closure][0] if closure else None + if util.is_same_func(wrapped, component_func): + factory_func = existing_func # noqa: F811 + + cls.factory( + component_name, + assigns=assigns, + requires=requires, + retokenizes=retokenizes, + func=factory_func, + ) + return component_func + + if func is not None: # Support non-decorator use cases + return add_component(func) + return add_component + + def analyze_pipes( + self, + *, + keys: List[str] = ["assigns", "requires", "scores", "retokenizes"], + pretty: bool = False, + ) -> Optional[Dict[str, Any]]: + """Analyze the current pipeline components, print a summary of what + they assign or require and check that all requirements are met. + + keys (List[str]): The meta values to display in the table. Corresponds + to values in FactoryMeta, defined by @Language.factory decorator. + pretty (bool): Pretty-print the results. + RETURNS (dict): The data. + """ + analysis = analyze_pipes(self, keys=keys) + if pretty: + print_pipe_analysis(analysis, keys=keys) + return analysis + + def get_pipe(self, name: str) -> PipeCallable: + """Get a pipeline component for a given component name. + + name (str): Name of pipeline component to get. + RETURNS (callable): The pipeline component. + + DOCS: https://spacy.io/api/language#get_pipe + """ + for pipe_name, component in self._components: + if pipe_name == name: + return component + raise KeyError(Errors.E001.format(name=name, opts=self.component_names)) + + def create_pipe( + self, + factory_name: str, + name: Optional[str] = None, + *, + config: Dict[str, Any] = SimpleFrozenDict(), + raw_config: Optional[Config] = None, + validate: bool = True, + ) -> PipeCallable: + """Create a pipeline component. Mostly used internally. To create and + add a component to the pipeline, you can use nlp.add_pipe. + + factory_name (str): Name of component factory. + name (Optional[str]): Optional name to assign to component instance. + Defaults to factory name if not set. + config (Dict[str, Any]): Config parameters to use for this component. + Will be merged with default config, if available. + raw_config (Optional[Config]): Internals: the non-interpolated config. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. + RETURNS (Callable[[Doc], Doc]): The pipeline component. + + DOCS: https://spacy.io/api/language#create_pipe + """ + name = name if name is not None else factory_name + if not isinstance(config, dict): + err = Errors.E962.format(style="config", name=name, cfg_type=type(config)) + raise ValueError(err) + if not srsly.is_json_serializable(config): + raise ValueError(Errors.E961.format(config=config)) + if not self.has_factory(factory_name): + err = Errors.E002.format( + name=factory_name, + opts=", ".join(self.factory_names), + method="create_pipe", + lang=util.get_object_name(self), + lang_code=self.lang, + ) + raise ValueError(err) + pipe_meta = self.get_factory_meta(factory_name) + # This is unideal, but the alternative would mean you always need to + # specify the full config settings, which is not really viable. + if pipe_meta.default_config: + config = Config(pipe_meta.default_config).merge(config) + internal_name = self.get_factory_name(factory_name) + # If the language-specific factory doesn't exist, try again with the + # not-specific name + if internal_name not in registry.factories: + internal_name = factory_name + # The name allows components to know their pipe name and use it in the + # losses etc. (even if multiple instances of the same factory are used) + config = {"nlp": self, "name": name, **config, "@factories": internal_name} + # We need to create a top-level key because Thinc doesn't allow resolving + # top-level references to registered functions. Also gives nicer errors. + cfg = {factory_name: config} + # We're calling the internal _fill here to avoid constructing the + # registered functions twice + resolved = registry.resolve(cfg, validate=validate) + filled = registry.fill({"cfg": cfg[factory_name]}, validate=validate)["cfg"] + filled = Config(filled) + filled["factory"] = factory_name + filled.pop("@factories", None) + # Remove the extra values we added because we don't want to keep passing + # them around, copying them etc. + filled.pop("nlp", None) + filled.pop("name", None) + # Merge the final filled config with the raw config (including non- + # interpolated variables) + if raw_config: + filled = filled.merge(raw_config) + self._pipe_configs[name] = filled + return resolved[factory_name] + + def create_pipe_from_source( + self, source_name: str, source: "Language", *, name: str + ) -> Tuple[PipeCallable, str]: + """Create a pipeline component by copying it from an existing model. + + source_name (str): Name of the component in the source pipeline. + source (Language): The source nlp object to copy from. + name (str): Optional alternative name to use in current pipeline. + RETURNS (Tuple[Callable[[Doc], Doc], str]): The component and its factory name. + """ + # Check source type + if not isinstance(source, Language): + raise ValueError(Errors.E945.format(name=source_name, source=type(source))) + if self.vocab.vectors != source.vocab.vectors: + warnings.warn(Warnings.W113.format(name=source_name)) + if source_name not in source.component_names: + raise KeyError( + Errors.E944.format( + name=source_name, + model=f"{source.meta['lang']}_{source.meta['name']}", + opts=", ".join(source.component_names), + ) + ) + pipe = source.get_pipe(source_name) + # There is no actual solution here. Either the component has the right + # name for the source pipeline or the component has the right name for + # the current pipeline. This prioritizes the current pipeline. + if hasattr(pipe, "name"): + pipe.name = name + # Make sure the source config is interpolated so we don't end up with + # orphaned variables in our final config + source_config = source.config.interpolate() + pipe_config = util.copy_config(source_config["components"][source_name]) + self._pipe_configs[name] = pipe_config + if self.vocab.strings != source.vocab.strings: + for s in source.vocab.strings: + self.vocab.strings.add(s) + return pipe, pipe_config["factory"] + + def add_pipe( + self, + factory_name: str, + name: Optional[str] = None, + *, + before: Optional[Union[str, int]] = None, + after: Optional[Union[str, int]] = None, + first: Optional[bool] = None, + last: Optional[bool] = None, + source: Optional["Language"] = None, + config: Dict[str, Any] = SimpleFrozenDict(), + raw_config: Optional[Config] = None, + validate: bool = True, + ) -> PipeCallable: + """Add a component to the processing pipeline. Valid components are + callables that take a `Doc` object, modify it and return it. Only one + of before/after/first/last can be set. Default behaviour is "last". + + factory_name (str): Name of the component factory. + name (str): Name of pipeline component. Overwrites existing + component.name attribute if available. If no name is set and + the component exposes no name attribute, component.__name__ is + used. An error is raised if a name already exists in the pipeline. + before (Union[str, int]): Name or index of the component to insert new + component directly before. + after (Union[str, int]): Name or index of the component to insert new + component directly after. + first (bool): If True, insert component first in the pipeline. + last (bool): If True, insert component last in the pipeline. + source (Language): Optional loaded nlp object to copy the pipeline + component from. + config (Dict[str, Any]): Config parameters to use for this component. + Will be merged with default config, if available. + raw_config (Optional[Config]): Internals: the non-interpolated config. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. + RETURNS (Callable[[Doc], Doc]): The pipeline component. + + DOCS: https://spacy.io/api/language#add_pipe + """ + if not isinstance(factory_name, str): + bad_val = repr(factory_name) + err = Errors.E966.format(component=bad_val, name=name) + raise ValueError(err) + name = name if name is not None else factory_name + if name in self.component_names: + raise ValueError(Errors.E007.format(name=name, opts=self.component_names)) + # Overriding pipe name in the config is not supported and will be ignored. + if "name" in config: + warnings.warn(Warnings.W119.format(name_in_config=config.pop("name"))) + if source is not None: + # We're loading the component from a model. After loading the + # component, we know its real factory name + pipe_component, factory_name = self.create_pipe_from_source( + factory_name, source, name=name + ) + else: + pipe_component = self.create_pipe( + factory_name, + name=name, + config=config, + raw_config=raw_config, + validate=validate, + ) + pipe_index = self._get_pipe_index(before, after, first, last) + self._pipe_meta[name] = self.get_factory_meta(factory_name) + self._components.insert(pipe_index, (name, pipe_component)) + self._link_components() + return pipe_component + + def _get_pipe_index( + self, + before: Optional[Union[str, int]] = None, + after: Optional[Union[str, int]] = None, + first: Optional[bool] = None, + last: Optional[bool] = None, + ) -> int: + """Determine where to insert a pipeline component based on the before/ + after/first/last values. + + before (str): Name or index of the component to insert directly before. + after (str): Name or index of component to insert directly after. + first (bool): If True, insert component first in the pipeline. + last (bool): If True, insert component last in the pipeline. + RETURNS (int): The index of the new pipeline component. + """ + all_args = {"before": before, "after": after, "first": first, "last": last} + if sum(arg is not None for arg in [before, after, first, last]) >= 2: + raise ValueError( + Errors.E006.format(args=all_args, opts=self.component_names) + ) + if last or not any(value is not None for value in [first, before, after]): + return len(self._components) + elif first: + return 0 + elif isinstance(before, str): + if before not in self.component_names: + raise ValueError( + Errors.E001.format(name=before, opts=self.component_names) + ) + return self.component_names.index(before) + elif isinstance(after, str): + if after not in self.component_names: + raise ValueError( + Errors.E001.format(name=after, opts=self.component_names) + ) + return self.component_names.index(after) + 1 + # We're only accepting indices referring to components that exist + # (can't just do isinstance here because bools are instance of int, too) + elif type(before) == int: + if before >= len(self._components) or before < 0: + err = Errors.E959.format( + dir="before", idx=before, opts=self.component_names + ) + raise ValueError(err) + return before + elif type(after) == int: + if after >= len(self._components) or after < 0: + err = Errors.E959.format( + dir="after", idx=after, opts=self.component_names + ) + raise ValueError(err) + return after + 1 + raise ValueError(Errors.E006.format(args=all_args, opts=self.component_names)) + + def has_pipe(self, name: str) -> bool: + """Check if a component name is present in the pipeline. Equivalent to + `name in nlp.pipe_names`. + + name (str): Name of the component. + RETURNS (bool): Whether a component of the name exists in the pipeline. + + DOCS: https://spacy.io/api/language#has_pipe + """ + return name in self.pipe_names + + def replace_pipe( + self, + name: str, + factory_name: str, + *, + config: Dict[str, Any] = SimpleFrozenDict(), + validate: bool = True, + ) -> PipeCallable: + """Replace a component in the pipeline. + + name (str): Name of the component to replace. + factory_name (str): Factory name of replacement component. + config (Optional[Dict[str, Any]]): Config parameters to use for this + component. Will be merged with default config, if available. + validate (bool): Whether to validate the component config against the + arguments and types expected by the factory. + RETURNS (Callable[[Doc], Doc]): The new pipeline component. + + DOCS: https://spacy.io/api/language#replace_pipe + """ + if name not in self.component_names: + raise ValueError(Errors.E001.format(name=name, opts=self.pipe_names)) + if hasattr(factory_name, "__call__"): + err = Errors.E968.format(component=repr(factory_name), name=name) + raise ValueError(err) + # We need to delegate to Language.add_pipe here instead of just writing + # to Language.pipeline to make sure the configs are handled correctly + pipe_index = self.component_names.index(name) + self.remove_pipe(name) + if not len(self._components) or pipe_index == len(self._components): + # we have no components to insert before/after, or we're replacing the last component + return self.add_pipe( + factory_name, name=name, config=config, validate=validate + ) + else: + return self.add_pipe( + factory_name, + name=name, + before=pipe_index, + config=config, + validate=validate, + ) + + def rename_pipe(self, old_name: str, new_name: str) -> None: + """Rename a pipeline component. + + old_name (str): Name of the component to rename. + new_name (str): New name of the component. + + DOCS: https://spacy.io/api/language#rename_pipe + """ + if old_name not in self.component_names: + raise ValueError( + Errors.E001.format(name=old_name, opts=self.component_names) + ) + if new_name in self.component_names: + raise ValueError( + Errors.E007.format(name=new_name, opts=self.component_names) + ) + i = self.component_names.index(old_name) + self._components[i] = (new_name, self._components[i][1]) + self._pipe_meta[new_name] = self._pipe_meta.pop(old_name) + self._pipe_configs[new_name] = self._pipe_configs.pop(old_name) + # Make sure [initialize] config is adjusted + if old_name in self._config["initialize"]["components"]: + init_cfg = self._config["initialize"]["components"].pop(old_name) + self._config["initialize"]["components"][new_name] = init_cfg + self._link_components() + + def remove_pipe(self, name: str) -> Tuple[str, PipeCallable]: + """Remove a component from the pipeline. + + name (str): Name of the component to remove. + RETURNS (Tuple[str, Callable[[Doc], Doc]]): A `(name, component)` tuple of the removed component. + + DOCS: https://spacy.io/api/language#remove_pipe + """ + if name not in self.component_names: + raise ValueError(Errors.E001.format(name=name, opts=self.component_names)) + removed = self._components.pop(self.component_names.index(name)) + # We're only removing the component itself from the metas/configs here + # because factory may be used for something else + self._pipe_meta.pop(name) + self._pipe_configs.pop(name) + self.meta.get("_sourced_vectors_hashes", {}).pop(name, None) + # Make sure name is removed from the [initialize] config + if name in self._config["initialize"]["components"]: + self._config["initialize"]["components"].pop(name) + # Make sure the name is also removed from the set of disabled components + if name in self.disabled: + self._disabled.remove(name) + self._link_components() + return removed + + def disable_pipe(self, name: str) -> None: + """Disable a pipeline component. The component will still exist on + the nlp object, but it won't be run as part of the pipeline. Does + nothing if the component is already disabled. + + name (str): The name of the component to disable. + """ + if name not in self.component_names: + raise ValueError(Errors.E001.format(name=name, opts=self.component_names)) + self._disabled.add(name) + + def enable_pipe(self, name: str) -> None: + """Enable a previously disabled pipeline component so it's run as part + of the pipeline. Does nothing if the component is already enabled. + + name (str): The name of the component to enable. + """ + if name not in self.component_names: + raise ValueError(Errors.E001.format(name=name, opts=self.component_names)) + if name in self.disabled: + self._disabled.remove(name) + + def __call__( + self, + text: Union[str, Doc], + *, + disable: Iterable[str] = SimpleFrozenList(), + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Doc: + """Apply the pipeline to some text. The text can span multiple sentences, + and can contain arbitrary whitespace. Alignment into the original string + is preserved. + + text (Union[str, Doc]): If `str`, the text to be processed. If `Doc`, + the doc will be passed directly to the pipeline, skipping + `Language.make_doc`. + disable (List[str]): Names of the pipeline components to disable. + component_cfg (Dict[str, dict]): An optional dictionary with extra + keyword arguments for specific components. + RETURNS (Doc): A container for accessing the annotations. + + DOCS: https://spacy.io/api/language#call + """ + doc = self._ensure_doc(text) + if component_cfg is None: + component_cfg = {} + for name, proc in self.pipeline: + if name in disable: + continue + if not hasattr(proc, "__call__"): + raise ValueError(Errors.E003.format(component=type(proc), name=name)) + error_handler = self.default_error_handler + if hasattr(proc, "get_error_handler"): + error_handler = proc.get_error_handler() + try: + doc = proc(doc, **component_cfg.get(name, {})) # type: ignore[call-arg] + except KeyError as e: + # This typically happens if a component is not initialized + raise ValueError(Errors.E109.format(name=name)) from e + except Exception as e: + error_handler(name, proc, [doc], e) + if not isinstance(doc, Doc): + raise ValueError(Errors.E005.format(name=name, returned_type=type(doc))) + return doc + + def disable_pipes(self, *names) -> "DisabledPipes": + """Disable one or more pipeline components. If used as a context + manager, the pipeline will be restored to the initial state at the end + of the block. Otherwise, a DisabledPipes object is returned, that has + a `.restore()` method you can use to undo your changes. + + This method has been deprecated since 3.0 + """ + warnings.warn(Warnings.W096, DeprecationWarning) + if len(names) == 1 and isinstance(names[0], (list, tuple)): + names = names[0] # type: ignore[assignment] # support list of names instead of spread + return self.select_pipes(disable=names) + + def select_pipes( + self, + *, + disable: Optional[Union[str, Iterable[str]]] = None, + enable: Optional[Union[str, Iterable[str]]] = None, + ) -> "DisabledPipes": + """Disable one or more pipeline components. If used as a context + manager, the pipeline will be restored to the initial state at the end + of the block. Otherwise, a DisabledPipes object is returned, that has + a `.restore()` method you can use to undo your changes. + + disable (str or iterable): The name(s) of the pipes to disable + enable (str or iterable): The name(s) of the pipes to enable - all others will be disabled + + DOCS: https://spacy.io/api/language#select_pipes + """ + if enable is None and disable is None: + raise ValueError(Errors.E991) + if isinstance(disable, str): + disable = [disable] + if enable is not None: + if isinstance(enable, str): + enable = [enable] + to_disable = [pipe for pipe in self.pipe_names if pipe not in enable] + # raise an error if the enable and disable keywords are not consistent + if disable is not None and disable != to_disable: + raise ValueError( + Errors.E992.format( + enable=enable, disable=disable, names=self.pipe_names + ) + ) + disable = to_disable + assert disable is not None + # DisabledPipes will restore the pipes in 'disable' when it's done, so we need to exclude + # those pipes that were already disabled. + disable = [d for d in disable if d not in self._disabled] + return DisabledPipes(self, disable) + + def make_doc(self, text: str) -> Doc: + """Turn a text into a Doc object. + + text (str): The text to process. + RETURNS (Doc): The processed doc. + """ + if len(text) > self.max_length: + raise ValueError( + Errors.E088.format(length=len(text), max_length=self.max_length) + ) + return self.tokenizer(text) + + def _ensure_doc(self, doc_like: Union[str, Doc, bytes]) -> Doc: + """Create a Doc if need be, or raise an error if the input is not + a Doc, string, or a byte array (generated by Doc.to_bytes()).""" + if isinstance(doc_like, Doc): + return doc_like + if isinstance(doc_like, str): + return self.make_doc(doc_like) + if isinstance(doc_like, bytes): + return Doc(self.vocab).from_bytes(doc_like) + raise ValueError(Errors.E1041.format(type=type(doc_like))) + + def _ensure_doc_with_context( + self, doc_like: Union[str, Doc, bytes], context: _AnyContext + ) -> Doc: + """Call _ensure_doc to generate a Doc and set its context object.""" + doc = self._ensure_doc(doc_like) + doc._context = context + return doc + + def update( + self, + examples: Iterable[Example], + _: Optional[Any] = None, + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + exclude: Iterable[str] = SimpleFrozenList(), + annotates: Iterable[str] = SimpleFrozenList(), + ): + """Update the models in the pipeline. + + examples (Iterable[Example]): A batch of examples + _: Should not be set - serves to catch backwards-incompatible scripts. + drop (float): The dropout rate. + sgd (Optimizer): An optimizer. + losses (Dict[str, float]): Dictionary to update with the loss, keyed by + component. + component_cfg (Dict[str, Dict]): Config parameters for specific pipeline + components, keyed by component name. + exclude (Iterable[str]): Names of components that shouldn't be updated. + annotates (Iterable[str]): Names of components that should set + annotations on the predicted examples after updating. + RETURNS (Dict[str, float]): The updated losses dictionary + + DOCS: https://spacy.io/api/language#update + """ + if _ is not None: + raise ValueError(Errors.E989) + if losses is None: + losses = {} + if isinstance(examples, list) and len(examples) == 0: + return losses + validate_examples(examples, "Language.update") + examples = _copy_examples(examples) + if sgd is None: + if self._optimizer is None: + self._optimizer = self.create_optimizer() + sgd = self._optimizer + if component_cfg is None: + component_cfg = {} + pipe_kwargs = {} + for i, (name, proc) in enumerate(self.pipeline): + component_cfg.setdefault(name, {}) + pipe_kwargs[name] = deepcopy(component_cfg[name]) + component_cfg[name].setdefault("drop", drop) + pipe_kwargs[name].setdefault("batch_size", self.batch_size) + for name, proc in self.pipeline: + # ignore statements are used here because mypy ignores hasattr + if name not in exclude and hasattr(proc, "update"): + proc.update(examples, sgd=None, losses=losses, **component_cfg[name]) # type: ignore + if sgd not in (None, False): + if ( + name not in exclude + and isinstance(proc, ty.TrainableComponent) + and proc.is_trainable + and proc.model not in (True, False, None) + ): + proc.finish_update(sgd) + if name in annotates: + for doc, eg in zip( + _pipe( + (eg.predicted for eg in examples), + proc=proc, + name=name, + default_error_handler=self.default_error_handler, + kwargs=pipe_kwargs[name], + ), + examples, + ): + eg.predicted = doc + return _replace_numpy_floats(losses) + + def rehearse( + self, + examples: Iterable[Example], + *, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + exclude: Iterable[str] = SimpleFrozenList(), + ) -> Dict[str, float]: + """Make a "rehearsal" update to the models in the pipeline, to prevent + forgetting. Rehearsal updates run an initial copy of the model over some + data, and update the model so its current predictions are more like the + initial ones. This is useful for keeping a pretrained model on-track, + even if you're updating it with a smaller set of examples. + + examples (Iterable[Example]): A batch of `Example` objects. + sgd (Optional[Optimizer]): An optimizer. + component_cfg (Dict[str, Dict]): Config parameters for specific pipeline + components, keyed by component name. + exclude (Iterable[str]): Names of components that shouldn't be updated. + RETURNS (dict): Results from the update. + + EXAMPLE: + >>> raw_text_batches = minibatch(raw_texts) + >>> for labelled_batch in minibatch(examples): + >>> nlp.update(labelled_batch) + >>> raw_batch = [Example.from_dict(nlp.make_doc(text), {}) for text in next(raw_text_batches)] + >>> nlp.rehearse(raw_batch) + + DOCS: https://spacy.io/api/language#rehearse + """ + if losses is None: + losses = {} + if isinstance(examples, list) and len(examples) == 0: + return losses + validate_examples(examples, "Language.rehearse") + if sgd is None: + if self._optimizer is None: + self._optimizer = self.create_optimizer() + sgd = self._optimizer + pipes = list(self.pipeline) + random.shuffle(pipes) + if component_cfg is None: + component_cfg = {} + grads = {} + + def get_grads(key, W, dW): + grads[key] = (W, dW) + return W, dW + + get_grads.learn_rate = sgd.learn_rate # type: ignore[attr-defined, union-attr] + get_grads.b1 = sgd.b1 # type: ignore[attr-defined, union-attr] + get_grads.b2 = sgd.b2 # type: ignore[attr-defined, union-attr] + for name, proc in pipes: + if name in exclude or not hasattr(proc, "rehearse"): + continue + grads = {} + proc.rehearse( # type: ignore[attr-defined] + examples, sgd=get_grads, losses=losses, **component_cfg.get(name, {}) + ) + for key, (W, dW) in grads.items(): + sgd(key, W, dW) # type: ignore[call-arg, misc] + return losses + + def begin_training( + self, + get_examples: Optional[Callable[[], Iterable[Example]]] = None, + *, + sgd: Optional[Optimizer] = None, + ) -> Optimizer: + warnings.warn(Warnings.W089, DeprecationWarning) + return self.initialize(get_examples, sgd=sgd) + + def initialize( + self, + get_examples: Optional[Callable[[], Iterable[Example]]] = None, + *, + sgd: Optional[Optimizer] = None, + ) -> Optimizer: + """Initialize the pipe for training, using data examples if available. + + get_examples (Callable[[], Iterable[Example]]): Optional function that + returns gold-standard Example objects. + sgd (Optional[Optimizer]): An optimizer to use for updates. If not + provided, will be created using the .create_optimizer() method. + RETURNS (thinc.api.Optimizer): The optimizer. + + DOCS: https://spacy.io/api/language#initialize + """ + if get_examples is None: + util.logger.debug( + "No 'get_examples' callback provided to 'Language.initialize', creating dummy examples" + ) + doc = Doc(self.vocab, words=["x", "y", "z"]) + + def get_examples(): + return [Example.from_dict(doc, {})] + + if not hasattr(get_examples, "__call__"): + err = Errors.E930.format( + method="Language.initialize", obj=type(get_examples) + ) + raise TypeError(err) + # Make sure the config is interpolated so we can resolve subsections + config = self.config.interpolate() + # These are the settings provided in the [initialize] block in the config + I = registry.resolve(config["initialize"], schema=ConfigSchemaInit) # type: ignore[arg-type] + before_init = I["before_init"] + if before_init is not None: + before_init(self) + try: + init_vocab( + self, data=I["vocab_data"], lookups=I["lookups"], vectors=I["vectors"] + ) + except IOError: + raise IOError(Errors.E884.format(vectors=I["vectors"])) + if self.vocab.vectors.shape[1] >= 1: + ops = get_current_ops() + self.vocab.vectors.to_ops(ops) + if hasattr(self.tokenizer, "initialize"): + tok_settings = validate_init_settings( + self.tokenizer.initialize, # type: ignore[union-attr] + I["tokenizer"], + section="tokenizer", + name="tokenizer", + ) + self.tokenizer.initialize(get_examples, nlp=self, **tok_settings) # type: ignore[union-attr] + for name, proc in self.pipeline: + if isinstance(proc, ty.InitializableComponent): + p_settings = I["components"].get(name, {}) + p_settings = validate_init_settings( + proc.initialize, p_settings, section="components", name=name + ) + proc.initialize(get_examples, nlp=self, **p_settings) + pretrain_cfg = config.get("pretraining") + if pretrain_cfg: + P = registry.resolve(pretrain_cfg, schema=ConfigSchemaPretrain) # type: ignore[arg-type] + init_tok2vec(self, P, I) + self._link_components() + self._optimizer = sgd + if sgd is not None: + self._optimizer = sgd + elif self._optimizer is None: + self._optimizer = self.create_optimizer() + after_init = I["after_init"] + if after_init is not None: + after_init(self) + return self._optimizer + + def resume_training(self, *, sgd: Optional[Optimizer] = None) -> Optimizer: + """Continue training a pretrained model. + + Create and return an optimizer, and initialize "rehearsal" for any pipeline + component that has a .rehearse() method. Rehearsal is used to prevent + models from "forgetting" their initialized "knowledge". To perform + rehearsal, collect samples of text you want the models to retain performance + on, and call nlp.rehearse() with a batch of Example objects. + + RETURNS (Optimizer): The optimizer. + + DOCS: https://spacy.io/api/language#resume_training + """ + ops = get_current_ops() + if self.vocab.vectors.shape[1] >= 1: + self.vocab.vectors.to_ops(ops) + for name, proc in self.pipeline: + if hasattr(proc, "_rehearsal_model"): + proc._rehearsal_model = deepcopy(proc.model) # type: ignore[attr-defined] + if sgd is not None: + self._optimizer = sgd + elif self._optimizer is None: + self._optimizer = self.create_optimizer() + return self._optimizer + + def set_error_handler( + self, + error_handler: Callable[[str, PipeCallable, List[Doc], Exception], NoReturn], + ): + """Set an error handler object for all the components in the pipeline + that implement a set_error_handler function. + + error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], NoReturn]): + Function that deals with a failing batch of documents. This callable + function should take in the component's name, the component itself, + the offending batch of documents, and the exception that was thrown. + DOCS: https://spacy.io/api/language#set_error_handler + """ + self.default_error_handler = error_handler + for name, pipe in self.pipeline: + if hasattr(pipe, "set_error_handler"): + pipe.set_error_handler(error_handler) + + def evaluate( + self, + examples: Iterable[Example], + *, + batch_size: Optional[int] = None, + scorer: Optional[Scorer] = None, + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + scorer_cfg: Optional[Dict[str, Any]] = None, + per_component: bool = False, + ) -> Dict[str, Any]: + """Evaluate a model's pipeline components. + + examples (Iterable[Example]): `Example` objects. + batch_size (Optional[int]): Batch size to use. + scorer (Optional[Scorer]): Scorer to use. If not passed in, a new one + will be created. + component_cfg (dict): An optional dictionary with extra keyword + arguments for specific components. + scorer_cfg (dict): An optional dictionary with extra keyword arguments + for the scorer. + per_component (bool): Whether to return the scores keyed by component + name. Defaults to False. + + RETURNS (Scorer): The scorer containing the evaluation results. + + DOCS: https://spacy.io/api/language#evaluate + """ + examples = list(examples) + validate_examples(examples, "Language.evaluate") + examples = _copy_examples(examples) + if batch_size is None: + batch_size = self.batch_size + if component_cfg is None: + component_cfg = {} + if scorer_cfg is None: + scorer_cfg = {} + if scorer is None: + kwargs = dict(scorer_cfg) + kwargs.setdefault("nlp", self) + scorer = Scorer(**kwargs) + # reset annotation in predicted docs and time tokenization + start_time = timer() + # this is purely for timing + for eg in examples: + self.make_doc(eg.reference.text) + # apply all pipeline components + docs = self.pipe( + (eg.predicted for eg in examples), + batch_size=batch_size, + component_cfg=component_cfg, + ) + for eg, doc in zip(examples, docs): + eg.predicted = doc + end_time = timer() + results = scorer.score(examples, per_component=per_component) + n_words = sum(len(eg.predicted) for eg in examples) + results["speed"] = n_words / (end_time - start_time) + return _replace_numpy_floats(results) + + def create_optimizer(self): + """Create an optimizer, usually using the [training.optimizer] config.""" + subconfig = {"optimizer": self.config["training"]["optimizer"]} + return registry.resolve(subconfig)["optimizer"] + + @contextmanager + def use_params(self, params: Optional[dict]): + """Replace weights of models in the pipeline with those provided in the + params dictionary. Can be used as a contextmanager, in which case, + models go back to their original weights after the block. + + params (dict): A dictionary of parameters keyed by model ID. + + EXAMPLE: + >>> with nlp.use_params(optimizer.averages): + >>> nlp.to_disk("/tmp/checkpoint") + + DOCS: https://spacy.io/api/language#use_params + """ + if not params: + yield + else: + contexts = [ + pipe.use_params(params) # type: ignore[attr-defined] + for name, pipe in self.pipeline + if hasattr(pipe, "use_params") and hasattr(pipe, "model") + ] + # TODO: Having trouble with contextlib + # Workaround: these aren't actually context managers atm. + for context in contexts: + try: + next(context) + except StopIteration: + pass + yield + for context in contexts: + try: + next(context) + except StopIteration: + pass + + @overload + def pipe( + self, + texts: Iterable[Union[str, Doc]], + *, + as_tuples: Literal[False] = ..., + batch_size: Optional[int] = ..., + disable: Iterable[str] = ..., + component_cfg: Optional[Dict[str, Dict[str, Any]]] = ..., + n_process: int = ..., + ) -> Iterator[Doc]: ... + + @overload + def pipe( # noqa: F811 + self, + texts: Iterable[Tuple[Union[str, Doc], _AnyContext]], + *, + as_tuples: Literal[True] = ..., + batch_size: Optional[int] = ..., + disable: Iterable[str] = ..., + component_cfg: Optional[Dict[str, Dict[str, Any]]] = ..., + n_process: int = ..., + ) -> Iterator[Tuple[Doc, _AnyContext]]: ... + + def pipe( # noqa: F811 + self, + texts: Union[ + Iterable[Union[str, Doc]], Iterable[Tuple[Union[str, Doc], _AnyContext]] + ], + *, + as_tuples: bool = False, + batch_size: Optional[int] = None, + disable: Iterable[str] = SimpleFrozenList(), + component_cfg: Optional[Dict[str, Dict[str, Any]]] = None, + n_process: int = 1, + ) -> Union[Iterator[Doc], Iterator[Tuple[Doc, _AnyContext]]]: + """Process texts as a stream, and yield `Doc` objects in order. + + texts (Iterable[Union[str, Doc]]): A sequence of texts or docs to + process. + as_tuples (bool): If set to True, inputs should be a sequence of + (text, context) tuples. Output will then be a sequence of + (doc, context) tuples. Defaults to False. + batch_size (Optional[int]): The number of texts to buffer. + disable (List[str]): Names of the pipeline components to disable. + component_cfg (Dict[str, Dict]): An optional dictionary with extra keyword + arguments for specific components. + n_process (int): Number of processors to process texts. If -1, set `multiprocessing.cpu_count()`. + YIELDS (Doc): Documents in the order of the original text. + + DOCS: https://spacy.io/api/language#pipe + """ + if as_tuples: + texts = cast(Iterable[Tuple[Union[str, Doc], _AnyContext]], texts) + docs_with_contexts = ( + self._ensure_doc_with_context(text, context) for text, context in texts + ) + docs = self.pipe( + docs_with_contexts, + batch_size=batch_size, + disable=disable, + n_process=n_process, + component_cfg=component_cfg, + ) + for doc in docs: + context = doc._context + doc._context = None + yield (doc, context) + return + + texts = cast(Iterable[Union[str, Doc]], texts) + + # Set argument defaults + if n_process == -1: + n_process = mp.cpu_count() + if component_cfg is None: + component_cfg = {} + if batch_size is None: + batch_size = self.batch_size + + pipes = [] # contains functools.partial objects to easily create multiprocess worker. + for name, proc in self.pipeline: + if name in disable: + continue + kwargs = component_cfg.get(name, {}) + # Allow component_cfg to overwrite the top-level kwargs. + kwargs.setdefault("batch_size", batch_size) + f = functools.partial( + _pipe, + proc=proc, + name=name, + kwargs=kwargs, + default_error_handler=self.default_error_handler, + ) + pipes.append(f) + + if n_process != 1: + if self._has_gpu_model(disable): + warnings.warn(Warnings.W114) + + docs = self._multiprocessing_pipe(texts, pipes, n_process, batch_size) + else: + # if n_process == 1, no processes are forked. + docs = (self._ensure_doc(text) for text in texts) + for pipe in pipes: + docs = pipe(docs) + for doc in docs: + yield doc + + def _has_gpu_model(self, disable: Iterable[str]): + for name, proc in self.pipeline: + is_trainable = hasattr(proc, "is_trainable") and proc.is_trainable # type: ignore + if name in disable or not is_trainable: + continue + + if ( + hasattr(proc, "model") + and hasattr(proc.model, "ops") + and isinstance(proc.model.ops, CupyOps) + ): # type: ignore + return True + + return False + + def _multiprocessing_pipe( + self, + texts: Iterable[Union[str, Doc]], + pipes: Iterable[Callable[..., Iterator[Doc]]], + n_process: int, + batch_size: int, + ) -> Iterator[Doc]: + def prepare_input( + texts: Iterable[Union[str, Doc]], + ) -> Iterable[Tuple[Union[str, bytes], _AnyContext]]: + # Serialize Doc inputs to bytes to avoid incurring pickling + # overhead when they are passed to child processes. Also yield + # any context objects they might have separately (as they are not serialized). + for doc_like in texts: + if isinstance(doc_like, Doc): + yield (doc_like.to_bytes(), cast(_AnyContext, doc_like._context)) + else: + yield (doc_like, cast(_AnyContext, None)) + + serialized_texts_with_ctx = prepare_input(texts) # type: ignore + # raw_texts is used later to stop iteration. + texts, raw_texts = itertools.tee(serialized_texts_with_ctx) # type: ignore + # for sending texts to worker + texts_q: List[mp.Queue] = [mp.Queue() for _ in range(n_process)] + # for receiving byte-encoded docs from worker + bytedocs_recv_ch, bytedocs_send_ch = zip( + *[mp.Pipe(False) for _ in range(n_process)] + ) + + batch_texts = util.minibatch(texts, batch_size) + # Sender sends texts to the workers. + # This is necessary to properly handle infinite length of texts. + # (In this case, all data cannot be sent to the workers at once) + sender = _Sender(batch_texts, texts_q, chunk_size=n_process) + # send twice to make process busy + sender.send() + sender.send() + + procs = [ + mp.Process( + target=_apply_pipes, + args=( + self._ensure_doc_with_context, + pipes, + rch, + sch, + Underscore.get_state(), + ), + ) + for rch, sch in zip(texts_q, bytedocs_send_ch) + ] + for proc in procs: + proc.start() + + # Close writing-end of channels. This is needed to avoid that reading + # from the channel blocks indefinitely when the worker closes the + # channel. + for tx in bytedocs_send_ch: + tx.close() + + # Cycle channels not to break the order of docs. + # The received object is a batch of byte-encoded docs, so flatten them with chain.from_iterable. + byte_tuples = chain.from_iterable( + recv.recv() for recv in cycle(bytedocs_recv_ch) + ) + try: + for i, (_, (byte_doc, context, byte_error)) in enumerate( + zip(raw_texts, byte_tuples), 1 + ): + if byte_doc is not None: + doc = Doc(self.vocab).from_bytes(byte_doc) + doc._context = context + yield doc + elif byte_error is not None: + error = srsly.msgpack_loads(byte_error) + self.default_error_handler( + None, None, None, ValueError(Errors.E871.format(error=error)) + ) + if i % batch_size == 0: + # tell `sender` that one batch was consumed. + sender.step() + finally: + # If we are stopping in an orderly fashion, the workers' queues + # are empty. Put the sentinel in their queues to signal that work + # is done, so that they can exit gracefully. + for q in texts_q: + q.put(_WORK_DONE_SENTINEL) + q.close() + + # Otherwise, we are stopping because the error handler raised an + # exception. The sentinel will be last to go out of the queue. + # To avoid doing unnecessary work or hanging on platforms that + # block on sending (Windows), we'll close our end of the channel. + # This signals to the worker that it can exit the next time it + # attempts to send data down the channel. + for r in bytedocs_recv_ch: + r.close() + + for proc in procs: + proc.join() + + if not all(proc.exitcode == 0 for proc in procs): + warnings.warn(Warnings.W127) + + def _link_components(self) -> None: + """Register 'listeners' within pipeline components, to allow them to + effectively share weights. + """ + # I had thought, "Why do we do this inside the Language object? Shouldn't + # it be the tok2vec/transformer/etc's job? + # The problem is we need to do it during deserialization...And the + # components don't receive the pipeline then. So this does have to be + # here :( + # First, fix up all the internal component names in case they have + # gotten out of sync due to sourcing components from different + # pipelines, since find_listeners uses proc2.name for the listener + # map. + for name, proc in self.pipeline: + if hasattr(proc, "name"): + proc.name = name + for i, (name1, proc1) in enumerate(self.pipeline): + if isinstance(proc1, ty.ListenedToComponent): + proc1.listener_map = {} + for name2, proc2 in self.pipeline[i + 1 :]: + proc1.find_listeners(proc2) + + @classmethod + def from_config( + cls, + config: Union[Dict[str, Any], Config] = {}, + *, + vocab: Union[Vocab, bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + meta: Dict[str, Any] = SimpleFrozenDict(), + auto_fill: bool = True, + validate: bool = True, + ) -> "Language": + """Create the nlp object from a loaded config. Will set up the tokenizer + and language data, add pipeline components etc. If no config is provided, + the default config of the given language is used. + + config (Dict[str, Any] / Config): The loaded config. + vocab (Vocab): A Vocab object. If True, a vocab is created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. + Disabled pipes will be loaded but they won't be run unless you + explicitly enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. + Excluded components won't be loaded. + meta (Dict[str, Any]): Meta overrides for nlp.meta. + auto_fill (bool): Automatically fill in missing values in config based + on defaults and function argument annotations. + validate (bool): Validate the component config and arguments against + the types expected by the factory. + RETURNS (Language): The initialized Language class. + + DOCS: https://spacy.io/api/language#from_config + """ + if auto_fill: + config = Config( + cls.default_config, section_order=CONFIG_SECTION_ORDER + ).merge(config) + if "nlp" not in config: + raise ValueError(Errors.E985.format(config=config)) + # fill in [nlp.vectors] if not present (as a narrower alternative to + # auto-filling [nlp] from the default config) + if "vectors" not in config["nlp"]: + config["nlp"]["vectors"] = {"@vectors": "spacy.Vectors.v1"} + config_lang = config["nlp"].get("lang") + if config_lang is not None and config_lang != cls.lang: + raise ValueError( + Errors.E958.format( + bad_lang_code=config["nlp"]["lang"], + lang_code=cls.lang, + lang=util.get_object_name(cls), + ) + ) + config["nlp"]["lang"] = cls.lang + # This isn't very elegant, but we remove the [components] block here to prevent + # it from getting resolved (causes problems because we expect to pass in + # the nlp and name args for each component). If we're auto-filling, we're + # using the nlp.config with all defaults. + config = util.copy_config(config) + orig_pipeline = config.pop("components", {}) + orig_pretraining = config.pop("pretraining", None) + config["components"] = {} + if auto_fill: + filled = registry.fill(config, validate=validate, schema=ConfigSchema) # type: ignore[arg-type] + else: + filled = config + filled["components"] = orig_pipeline + config["components"] = orig_pipeline + if orig_pretraining is not None: + filled["pretraining"] = orig_pretraining + config["pretraining"] = orig_pretraining + resolved_nlp = registry.resolve( + filled["nlp"], + validate=validate, + schema=ConfigSchemaNlp, # type: ignore[arg-type] + ) + create_tokenizer = resolved_nlp["tokenizer"] + create_vectors = resolved_nlp["vectors"] + before_creation = resolved_nlp["before_creation"] + after_creation = resolved_nlp["after_creation"] + after_pipeline_creation = resolved_nlp["after_pipeline_creation"] + lang_cls = cls + if before_creation is not None: + lang_cls = before_creation(cls) + if ( + not isinstance(lang_cls, type) + or not issubclass(lang_cls, cls) + or lang_cls is not cls + ): + raise ValueError(Errors.E943.format(value=type(lang_cls))) + + # Warn about require_gpu usage in jupyter notebook + warn_if_jupyter_cupy() + + # Note that we don't load vectors here, instead they get loaded explicitly + # inside stuff like the spacy train function. If we loaded them here, + # then we would load them twice at runtime: once when we make from config, + # and then again when we load from disk. + nlp = lang_cls( + vocab=vocab, + create_tokenizer=create_tokenizer, + create_vectors=create_vectors, + meta=meta, + ) + if after_creation is not None: + nlp = after_creation(nlp) + if not isinstance(nlp, cls): + raise ValueError(Errors.E942.format(name="creation", value=type(nlp))) + # To create the components we need to use the final interpolated config + # so all values are available (if component configs use variables). + # Later we replace the component config with the raw config again. + interpolated = filled.interpolate() if not filled.is_interpolated else filled + pipeline = interpolated.get("components", {}) + # If components are loaded from a source (existing models), we cache + # them here so they're only loaded once + source_nlps = {} + source_nlp_vectors_hashes = {} + vocab_b = None + for pipe_name in config["nlp"]["pipeline"]: + if pipe_name not in pipeline: + opts = ", ".join(pipeline.keys()) + raise ValueError(Errors.E956.format(name=pipe_name, opts=opts)) + pipe_cfg = util.copy_config(pipeline[pipe_name]) + raw_config = Config(filled["components"][pipe_name]) + if pipe_name not in exclude: + if "factory" not in pipe_cfg and "source" not in pipe_cfg: + err = Errors.E984.format(name=pipe_name, config=pipe_cfg) + raise ValueError(err) + if "factory" in pipe_cfg: + factory = pipe_cfg.pop("factory") + # The pipe name (key in the config) here is the unique name + # of the component, not necessarily the factory + nlp.add_pipe( + factory, + name=pipe_name, + config=pipe_cfg, + validate=validate, + raw_config=raw_config, + ) + else: + assert "source" in pipe_cfg + # We need the sourced components to reference the same + # vocab without modifying the current vocab state **AND** + # we still want to load the source model vectors to perform + # the vectors check. Since the source vectors clobber the + # current ones, we save the original vocab state and + # restore after this loop. Existing strings are preserved + # during deserialization, so they do not need any + # additional handling. + if vocab_b is None: + vocab_b = nlp.vocab.to_bytes(exclude=["lookups", "strings"]) + model = pipe_cfg["source"] + if model not in source_nlps: + # Load with the same vocab, adding any strings + source_nlps[model] = util.load_model( + model, vocab=nlp.vocab, exclude=["lookups"] + ) + source_name = pipe_cfg.get("component", pipe_name) + listeners_replaced = False + if "replace_listeners" in pipe_cfg: + # Make sure that the listened-to component has the + # state of the source pipeline listener map so that the + # replace_listeners method below works as intended. + source_nlps[model]._link_components() + for name, proc in source_nlps[model].pipeline: + if source_name in getattr(proc, "listening_components", []): + source_nlps[model].replace_listeners( + name, source_name, pipe_cfg["replace_listeners"] + ) + listeners_replaced = True + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W113\\]") + nlp.add_pipe( + source_name, source=source_nlps[model], name=pipe_name + ) + # At this point after nlp.add_pipe, the listener map + # corresponds to the new pipeline. + if model not in source_nlp_vectors_hashes: + source_nlp_vectors_hashes[model] = hash( + source_nlps[model].vocab.vectors.to_bytes( + exclude=["strings"] + ) + ) + if "_sourced_vectors_hashes" not in nlp.meta: + nlp.meta["_sourced_vectors_hashes"] = {} + nlp.meta["_sourced_vectors_hashes"][pipe_name] = ( + source_nlp_vectors_hashes[model] + ) + # Delete from cache if listeners were replaced + if listeners_replaced: + del source_nlps[model] + # Restore the original vocab after sourcing if necessary + if vocab_b is not None: + nlp.vocab.from_bytes(vocab_b) + + # Resolve disabled/enabled settings. + if isinstance(disable, str): + disable = [disable] + if isinstance(enable, str): + enable = [enable] + if isinstance(exclude, str): + exclude = [exclude] + + # `enable` should not be merged with `enabled` (the opposite is true for `disable`/`disabled`). If the config + # specifies values for `enabled` not included in `enable`, emit warning. + if id(enable) != id(_DEFAULT_EMPTY_PIPES): + enabled = config["nlp"].get("enabled", []) + if len(enabled) and not set(enabled).issubset(enable): + warnings.warn( + Warnings.W123.format( + enable=enable, + enabled=enabled, + ) + ) + + # Ensure sets of disabled/enabled pipe names are not contradictory. + disabled_pipes = cls._resolve_component_status( + list({*disable, *config["nlp"].get("disabled", [])}), + enable, + config["nlp"]["pipeline"], + ) + nlp._disabled = set(p for p in disabled_pipes if p not in exclude) + + nlp.batch_size = config["nlp"]["batch_size"] + nlp.config = filled if auto_fill else config + if after_pipeline_creation is not None: + nlp = after_pipeline_creation(nlp) + if not isinstance(nlp, cls): + raise ValueError( + Errors.E942.format(name="pipeline_creation", value=type(nlp)) + ) + return nlp + + def replace_listeners( + self, + tok2vec_name: str, + pipe_name: str, + listeners: Iterable[str], + ) -> None: + """Find listener layers (connecting to a token-to-vector embedding + component) of a given pipeline component model and replace + them with a standalone copy of the token-to-vector layer. This can be + useful when training a pipeline with components sourced from an existing + pipeline: if multiple components (e.g. tagger, parser, NER) listen to + the same tok2vec component, but some of them are frozen and not updated, + their performance may degrade significantly as the tok2vec component is + updated with new data. To prevent this, listeners can be replaced with + a standalone tok2vec layer that is owned by the component and doesn't + change if the component isn't updated. + + tok2vec_name (str): Name of the token-to-vector component, typically + "tok2vec" or "transformer". + pipe_name (str): Name of pipeline component to replace listeners for. + listeners (Iterable[str]): The paths to the listeners, relative to the + component config, e.g. ["model.tok2vec"]. Typically, implementations + will only connect to one tok2vec component, [model.tok2vec], but in + theory, custom models can use multiple listeners. The value here can + either be an empty list to not replace any listeners, or a complete + (!) list of the paths to all listener layers used by the model. + + DOCS: https://spacy.io/api/language#replace_listeners + """ + if tok2vec_name not in self.pipe_names: + err = Errors.E889.format( + tok2vec=tok2vec_name, + name=pipe_name, + unknown=tok2vec_name, + opts=", ".join(self.pipe_names), + ) + raise ValueError(err) + if pipe_name not in self.pipe_names: + err = Errors.E889.format( + tok2vec=tok2vec_name, + name=pipe_name, + unknown=pipe_name, + opts=", ".join(self.pipe_names), + ) + raise ValueError(err) + tok2vec = self.get_pipe(tok2vec_name) + tok2vec_cfg = self.get_pipe_config(tok2vec_name) + if not isinstance(tok2vec, ty.ListenedToComponent): + raise ValueError(Errors.E888.format(name=tok2vec_name, pipe=type(tok2vec))) + tok2vec_model = tok2vec.model + pipe_listeners = tok2vec.listener_map.get(pipe_name, []) + pipe = self.get_pipe(pipe_name) + pipe_cfg = self._pipe_configs[pipe_name] + if listeners: + util.logger.debug("Replacing listeners of component '%s'", pipe_name) + if len(list(listeners)) != len(pipe_listeners): + # The number of listeners defined in the component model doesn't + # match the listeners to replace, so we won't be able to update + # the nodes and generate a matching config + err = Errors.E887.format( + name=pipe_name, + tok2vec=tok2vec_name, + paths=listeners, + n_listeners=len(pipe_listeners), + ) + raise ValueError(err) + # Update the config accordingly by copying the tok2vec model to all + # sections defined in the listener paths + for listener_path in listeners: + # Check if the path actually exists in the config + try: + util.dot_to_object(pipe_cfg, listener_path) + except KeyError: + err = Errors.E886.format( + name=pipe_name, tok2vec=tok2vec_name, path=listener_path + ) + raise ValueError(err) + new_config = tok2vec_cfg["model"] + if "replace_listener_cfg" in tok2vec_model.attrs: + replace_func = tok2vec_model.attrs["replace_listener_cfg"] + new_config = replace_func( + tok2vec_cfg["model"], pipe_cfg["model"]["tok2vec"] + ) + util.set_dot_to_object(pipe_cfg, listener_path, new_config) + # Go over the listener layers and replace them + for listener in pipe_listeners: + new_model = tok2vec_model.copy() + replace_listener_func = tok2vec_model.attrs.get("replace_listener") + if replace_listener_func is not None: + # Pass the extra args to the callback without breaking compatibility with + # old library versions that only expect a single parameter. + num_params = len( + inspect.signature(replace_listener_func).parameters + ) + if num_params == 1: + new_model = replace_listener_func(new_model) + elif num_params == 3: + new_model = replace_listener_func(new_model, listener, tok2vec) + else: + raise ValueError(Errors.E1055.format(num_params=num_params)) + + util.replace_model_node(pipe.model, listener, new_model) # type: ignore[attr-defined] + tok2vec.remove_listener(listener, pipe_name) + + @contextmanager + def memory_zone(self, mem: Optional[Pool] = None) -> Iterator[Pool]: + """Begin a block where all resources allocated during the block will + be freed at the end of it. If a resources was created within the + memory zone block, accessing it outside the block is invalid. + Behaviour of this invalid access is undefined. Memory zones should + not be nested. + + The memory zone is helpful for services that need to process large + volumes of text with a defined memory budget. + + Example + ------- + >>> with nlp.memory_zone(): + ... for doc in nlp.pipe(texts): + ... process_my_doc(doc) + >>> # use_doc(doc) <-- Invalid: doc was allocated in the memory zone + """ + if mem is None: + mem = Pool() + # The ExitStack allows programmatic nested context managers. + # We don't know how many we need, so it would be awkward to have + # them as nested blocks. + with ExitStack() as stack: + contexts = [stack.enter_context(self.vocab.memory_zone(mem))] + if hasattr(self.tokenizer, "memory_zone"): + contexts.append(stack.enter_context(self.tokenizer.memory_zone(mem))) + for _, pipe in self.pipeline: + if hasattr(pipe, "memory_zone"): + contexts.append(stack.enter_context(pipe.memory_zone(mem))) + yield mem + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Save the current state to a directory. If a model is loaded, this + will include the model. + + path (str / Path): Path to a directory, which will be created if + it doesn't exist. + exclude (Iterable[str]): Names of components or serialization fields to exclude. + + DOCS: https://spacy.io/api/language#to_disk + """ + path = util.ensure_path(path) + serializers = {} + serializers["tokenizer"] = lambda p: self.tokenizer.to_disk( # type: ignore[union-attr] + p, exclude=["vocab"] + ) + serializers["meta.json"] = lambda p: srsly.write_json( + p, _replace_numpy_floats(self.meta) + ) + serializers["config.cfg"] = lambda p: self.config.to_disk(p) + for name, proc in self._components: + if name in exclude: + continue + if not hasattr(proc, "to_disk"): + continue + serializers[name] = lambda p, proc=proc: proc.to_disk(p, exclude=["vocab"]) # type: ignore[misc] + serializers["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) + util.to_disk(path, serializers, exclude) + + @staticmethod + def _resolve_component_status( + disable: Union[str, Iterable[str]], + enable: Union[str, Iterable[str]], + pipe_names: Iterable[str], + ) -> Tuple[str, ...]: + """Derives whether (1) `disable` and `enable` values are consistent and (2) + resolves those to a single set of disabled components. Raises an error in + case of inconsistency. + + disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. + pipe_names (Iterable[str]): Names of all pipeline components. + + RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t. + specified includes and excludes. + """ + + if isinstance(disable, str): + disable = [disable] + to_disable = disable + + if enable: + if isinstance(enable, str): + enable = [enable] + to_disable = { + *[pipe_name for pipe_name in pipe_names if pipe_name not in enable], + *disable, + } + # If any pipe to be enabled is in to_disable, the specification is inconsistent. + if len(set(enable) & to_disable): + raise ValueError(Errors.E1042.format(enable=enable, disable=disable)) + + return tuple(to_disable) + + def from_disk( + self, + path: Union[str, Path], + *, + exclude: Iterable[str] = SimpleFrozenList(), + overrides: Dict[str, Any] = SimpleFrozenDict(), + ) -> "Language": + """Loads state from a directory. Modifies the object in place and + returns it. If the saved `Language` object contains a model, the + model will be loaded. + + path (str / Path): A path to a directory. + exclude (Iterable[str]): Names of components or serialization fields to exclude. + RETURNS (Language): The modified `Language` object. + + DOCS: https://spacy.io/api/language#from_disk + """ + + def deserialize_meta(path: Path) -> None: + if path.exists(): + data = srsly.read_json(path) + self.meta.update(data) + # self.meta always overrides meta["vectors"] with the metadata + # from self.vocab.vectors, so set the name directly + self.vocab.vectors.name = data.get("vectors", {}).get("name") + + def deserialize_vocab(path: Path) -> None: + if path.exists(): + self.vocab.from_disk(path, exclude=exclude) + + path = util.ensure_path(path) + deserializers = {} + if Path(path / "config.cfg").exists(): # type: ignore[operator] + deserializers["config.cfg"] = lambda p: self.config.from_disk( + p, interpolate=False, overrides=overrides + ) + deserializers["meta.json"] = deserialize_meta # type: ignore[assignment] + deserializers["vocab"] = deserialize_vocab # type: ignore[assignment] + deserializers["tokenizer"] = lambda p: self.tokenizer.from_disk( # type: ignore[union-attr] + p, exclude=["vocab"] + ) + for name, proc in self._components: + if name in exclude: + continue + if not hasattr(proc, "from_disk"): + continue + deserializers[name] = lambda p, proc=proc: proc.from_disk( # type: ignore[misc] + p, exclude=["vocab"] + ) + if not (path / "vocab").exists() and "vocab" not in exclude: # type: ignore[operator] + # Convert to list here in case exclude is (default) tuple + exclude = list(exclude) + ["vocab"] + util.from_disk(path, deserializers, exclude) # type: ignore[arg-type] + self._path = path # type: ignore[assignment] + self._link_components() + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the current state to a binary string. + + exclude (Iterable[str]): Names of components or serialization fields to exclude. + RETURNS (bytes): The serialized form of the `Language` object. + + DOCS: https://spacy.io/api/language#to_bytes + """ + serializers: Dict[str, Callable[[], bytes]] = {} + serializers["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serializers["tokenizer"] = lambda: self.tokenizer.to_bytes(exclude=["vocab"]) # type: ignore[union-attr] + serializers["meta.json"] = lambda: srsly.json_dumps( + _replace_numpy_floats(self.meta) + ) + serializers["config.cfg"] = lambda: self.config.to_bytes() + for name, proc in self._components: + if name in exclude: + continue + if not hasattr(proc, "to_bytes"): + continue + serializers[name] = lambda proc=proc: proc.to_bytes(exclude=["vocab"]) # type: ignore[misc] + return util.to_bytes(serializers, exclude) + + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "Language": + """Load state from a binary string. + + bytes_data (bytes): The data to load from. + exclude (Iterable[str]): Names of components or serialization fields to exclude. + RETURNS (Language): The `Language` object. + + DOCS: https://spacy.io/api/language#from_bytes + """ + + def deserialize_meta(b): + data = srsly.json_loads(b) + self.meta.update(data) + # self.meta always overrides meta["vectors"] with the metadata + # from self.vocab.vectors, so set the name directly + self.vocab.vectors.name = data.get("vectors", {}).get("name") + + deserializers: Dict[str, Callable[[bytes], Any]] = {} + deserializers["config.cfg"] = lambda b: self.config.from_bytes( + b, interpolate=False + ) + deserializers["meta.json"] = deserialize_meta + deserializers["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) + deserializers["tokenizer"] = lambda b: self.tokenizer.from_bytes( # type: ignore[union-attr] + b, exclude=["vocab"] + ) + for name, proc in self._components: + if name in exclude: + continue + if not hasattr(proc, "from_bytes"): + continue + deserializers[name] = lambda b, proc=proc: proc.from_bytes( # type: ignore[misc] + b, exclude=["vocab"] + ) + util.from_bytes(bytes_data, deserializers, exclude) + self._link_components() + return self + + +def _replace_numpy_floats(meta_dict: dict) -> dict: + return convert_recursive( + lambda v: isinstance(v, numpy.floating), lambda v: float(v), dict(meta_dict) + ) + + +@dataclass +class FactoryMeta: + """Dataclass containing information about a component and its defaults + provided by the @Language.component or @Language.factory decorator. It's + created whenever a component is defined and stored on the Language class for + each component instance and factory instance. + """ + + factory: str + default_config: Optional[Dict[str, Any]] = None # noqa: E704 + assigns: Iterable[str] = tuple() + requires: Iterable[str] = tuple() + retokenizes: bool = False + scores: Iterable[str] = tuple() + default_score_weights: Optional[Dict[str, Optional[float]]] = None # noqa: E704 + + +class DisabledPipes(list): + """Manager for temporary pipeline disabling.""" + + def __init__(self, nlp: Language, names: List[str]) -> None: + self.nlp = nlp + self.names = names + for name in self.names: + self.nlp.disable_pipe(name) + list.__init__(self) + self.extend(self.names) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.restore() + + def restore(self) -> None: + """Restore the pipeline to its state when DisabledPipes was created.""" + for name in self.names: + if name not in self.nlp.component_names: + raise ValueError(Errors.E008.format(name=name)) + self.nlp.enable_pipe(name) + self[:] = [] + + +def _copy_examples(examples: Iterable[Example]) -> List[Example]: + """Make a copy of a batch of examples, copying the predicted Doc as well. + This is used in contexts where we need to take ownership of the examples + so that they can be mutated, for instance during Language.evaluate and + Language.update. + """ + return [Example(eg.x.copy(), eg.y) for eg in examples] + + +def _apply_pipes( + ensure_doc: Callable[[Union[str, Doc, bytes], _AnyContext], Doc], + pipes: Iterable[Callable[..., Iterator[Doc]]], + receiver, + sender, + underscore_state: Tuple[dict, dict, dict], +) -> None: + """Worker for Language.pipe + + ensure_doc (Callable[[Union[str, Doc]], Doc]): Function to create Doc from text + or raise an error if the input is neither a Doc nor a string. + pipes (Iterable[Pipe]): The components to apply. + receiver (multiprocessing.Connection): Pipe to receive text. Usually + created by `multiprocessing.Pipe()` + sender (multiprocessing.Connection): Pipe to send doc. Usually created by + `multiprocessing.Pipe()` + underscore_state (Tuple[dict, dict, dict]): The data in the Underscore class + of the parent. + """ + Underscore.load_state(underscore_state) + while True: + try: + texts_with_ctx = receiver.get() + + # Stop working if we encounter the end-of-work sentinel. + if isinstance(texts_with_ctx, _WorkDoneSentinel): + sender.close() + receiver.close() + return + + docs = ( + ensure_doc(doc_like, context) for doc_like, context in texts_with_ctx + ) + for pipe in pipes: + docs = pipe(docs) # type: ignore[arg-type, assignment] + # Connection does not accept unpickable objects, so send list. + byte_docs = [(doc.to_bytes(), doc._context, None) for doc in docs] + padding = [(None, None, None)] * (len(texts_with_ctx) - len(byte_docs)) + data: Sequence[Tuple[Optional[bytes], Optional[Any], Optional[bytes]]] = ( + byte_docs + padding # type: ignore[operator] + ) + except Exception: + error_msg = [(None, None, srsly.msgpack_dumps(traceback.format_exc()))] + padding = [(None, None, None)] * (len(texts_with_ctx) - 1) + data = error_msg + padding + + try: + sender.send(data) + except BrokenPipeError: + # Parent has closed the pipe prematurely. This happens when a + # worker encounters an error and the error handler is set to + # stop processing. + sender.close() + receiver.close() + return + + +class _Sender: + """Util for sending data to multiprocessing workers in Language.pipe""" + + def __init__( + self, data: Iterable[Any], queues: List[mp.Queue], chunk_size: int + ) -> None: + self.data = iter(data) + self.queues = iter(cycle(queues)) + self.chunk_size = chunk_size + self.count = 0 + + def send(self) -> None: + """Send chunk_size items from self.data to channels.""" + for item, q in itertools.islice( + zip(self.data, cycle(self.queues)), self.chunk_size + ): + # cycle channels so that distribute the texts evenly + q.put(item) + + def step(self) -> None: + """Tell sender that comsumed one item. Data is sent to the workers after + every chunk_size calls. + """ + self.count += 1 + if self.count >= self.chunk_size: + self.count = 0 + self.send() + + +class _WorkDoneSentinel: + pass + + +_WORK_DONE_SENTINEL = _WorkDoneSentinel() diff --git a/spacy/lexeme.pxd b/spacy/lexeme.pxd new file mode 100644 index 0000000..a16a14f --- /dev/null +++ b/spacy/lexeme.pxd @@ -0,0 +1,98 @@ +from numpy cimport ndarray + +from .attrs cimport ( + ID, + LANG, + LENGTH, + LOWER, + NORM, + ORTH, + PREFIX, + SHAPE, + SUFFIX, + attr_id_t, +) +from .strings cimport StringStore +from .structs cimport LexemeC +from .typedefs cimport attr_t, flags_t, hash_t, len_t, tag_t +from .vocab cimport Vocab + + +cdef LexemeC EMPTY_LEXEME +cdef attr_t OOV_RANK + +cdef class Lexeme: + cdef LexemeC* c + cdef readonly Vocab vocab + cdef readonly attr_t orth + + @staticmethod + cdef inline Lexeme from_ptr(LexemeC* lex, Vocab vocab): + cdef Lexeme self = Lexeme.__new__(Lexeme, vocab, lex.orth) + self.c = lex + self.vocab = vocab + self.orth = lex.orth + return self + + @staticmethod + cdef inline void set_struct_attr(LexemeC* lex, attr_id_t name, attr_t value) noexcept nogil: + if name < (sizeof(flags_t) * 8): + Lexeme.c_set_flag(lex, name, value) + elif name == ID: + lex.id = value + elif name == LOWER: + lex.lower = value + elif name == NORM: + lex.norm = value + elif name == SHAPE: + lex.shape = value + elif name == PREFIX: + lex.prefix = value + elif name == SUFFIX: + lex.suffix = value + elif name == LANG: + lex.lang = value + + @staticmethod + cdef inline attr_t get_struct_attr(const LexemeC* lex, attr_id_t feat_name) noexcept nogil: + if feat_name < (sizeof(flags_t) * 8): + if Lexeme.c_check_flag(lex, feat_name): + return 1 + else: + return 0 + elif feat_name == ID: + return lex.id + elif feat_name == ORTH: + return lex.orth + elif feat_name == LOWER: + return lex.lower + elif feat_name == NORM: + return lex.norm + elif feat_name == SHAPE: + return lex.shape + elif feat_name == PREFIX: + return lex.prefix + elif feat_name == SUFFIX: + return lex.suffix + elif feat_name == LENGTH: + return lex.length + elif feat_name == LANG: + return lex.lang + else: + return 0 + + @staticmethod + cdef inline bint c_check_flag(const LexemeC* lexeme, attr_id_t flag_id) noexcept nogil: + cdef flags_t one = 1 + if lexeme.flags & (one << flag_id): + return True + else: + return False + + @staticmethod + cdef inline bint c_set_flag(LexemeC* lex, attr_id_t flag_id, bint value) noexcept nogil: + cdef flags_t one = 1 + if value: + lex.flags |= one << flag_id + else: + lex.flags &= ~(one << flag_id) diff --git a/spacy/lexeme.pyi b/spacy/lexeme.pyi new file mode 100644 index 0000000..9980b9f --- /dev/null +++ b/spacy/lexeme.pyi @@ -0,0 +1,61 @@ +from typing import Any, Union + +from thinc.types import Floats1d + +from .tokens import Doc, Span, Token +from .vocab import Vocab + +class Lexeme: + def __init__(self, vocab: Vocab, orth: int) -> None: ... + def __richcmp__(self, other: Lexeme, op: int) -> bool: ... + def __hash__(self) -> int: ... + def set_attrs(self, **attrs: Any) -> None: ... + def set_flag(self, flag_id: int, value: bool) -> None: ... + def check_flag(self, flag_id: int) -> bool: ... + def similarity(self, other: Union[Doc, Span, Token, Lexeme]) -> float: ... + @property + def has_vector(self) -> bool: ... + @property + def vector_norm(self) -> float: ... + vector: Floats1d + rank: int + sentiment: float + @property + def orth_(self) -> str: ... + @property + def text(self) -> str: ... + orth: int + lower: int + norm: int + shape: int + prefix: int + suffix: int + cluster: int + lang: int + prob: float + lower_: str + norm_: str + shape_: str + prefix_: str + suffix_: str + lang_: str + flags: int + @property + def is_oov(self) -> bool: ... + is_stop: bool + is_alpha: bool + is_ascii: bool + is_digit: bool + is_lower: bool + is_upper: bool + is_title: bool + is_punct: bool + is_space: bool + is_bracket: bool + is_quote: bool + is_left_punct: bool + is_right_punct: bool + is_currency: bool + like_url: bool + like_num: bool + like_email: bool diff --git a/spacy/lexeme.pyx b/spacy/lexeme.pyx new file mode 100644 index 0000000..8886dde --- /dev/null +++ b/spacy/lexeme.pyx @@ -0,0 +1,556 @@ +# cython: embedsignature=True +# cython: profile=False +# Compiler crashes on memory view coercion without this. Should report bug. +cimport numpy as np +from libc.string cimport memset + +np.import_array() + +import warnings + +import numpy +from thinc.api import get_array_module + +from .attrs cimport ( + IS_ALPHA, + IS_ASCII, + IS_BRACKET, + IS_CURRENCY, + IS_DIGIT, + IS_LEFT_PUNCT, + IS_LOWER, + IS_PUNCT, + IS_QUOTE, + IS_RIGHT_PUNCT, + IS_SPACE, + IS_STOP, + IS_TITLE, + IS_UPPER, + LIKE_EMAIL, + LIKE_NUM, + LIKE_URL, +) +from .typedefs cimport attr_t, flags_t + +from .attrs import intify_attrs +from .errors import Errors, Warnings + +OOV_RANK = 0xffffffffffffffff # UINT64_MAX +memset(&EMPTY_LEXEME, 0, sizeof(LexemeC)) +EMPTY_LEXEME.id = OOV_RANK + + +cdef class Lexeme: + """An entry in the vocabulary. A `Lexeme` has no string context – it's a + word-type, as opposed to a word token. It therefore has no part-of-speech + tag, dependency parse, or lemma (lemmatization depends on the + part-of-speech tag). + + DOCS: https://spacy.io/api/lexeme + """ + def __init__(self, Vocab vocab, attr_t orth): + """Create a Lexeme object. + + vocab (Vocab): The parent vocabulary + orth (uint64): The orth id of the lexeme. + Returns (Lexeme): The newly constructd object. + """ + self.vocab = vocab + self.orth = orth + self.c = vocab.get_by_orth(vocab.mem, orth) + if self.c.orth != orth: + raise ValueError(Errors.E071.format(orth=orth, vocab_orth=self.c.orth)) + + def __richcmp__(self, other, int op): + if other is None: + if op == 0 or op == 1 or op == 2: + return False + else: + return True + if isinstance(other, Lexeme): + a = self.orth + b = other.orth + elif isinstance(other, int): + a = self.orth + b = other + elif isinstance(other, str): + a = self.orth_ + b = other + else: + a = 0 + b = 1 + if op == 2: # == + return a == b + elif op == 3: # != + return a != b + elif op == 0: # < + return a < b + elif op == 1: # <= + return a <= b + elif op == 4: # > + return a > b + elif op == 5: # >= + return a >= b + else: + raise NotImplementedError(op) + + def __hash__(self): + return self.c.orth + + def set_attrs(self, **attrs): + cdef attr_id_t attr + attrs = intify_attrs(attrs) + for attr, value in attrs.items(): + # skip PROB, e.g. from lexemes.jsonl + if isinstance(value, float): + continue + elif isinstance(value, int): + Lexeme.set_struct_attr(self.c, attr, value) + else: + Lexeme.set_struct_attr(self.c, attr, self.vocab.strings.add(value)) + + def set_flag(self, attr_id_t flag_id, bint value): + """Change the value of a boolean flag. + + flag_id (int): The attribute ID of the flag to set. + value (bool): The new value of the flag. + """ + Lexeme.c_set_flag(self.c, flag_id, value) + + def check_flag(self, attr_id_t flag_id): + """Check the value of a boolean flag. + + flag_id (int): The attribute ID of the flag to query. + RETURNS (bool): The value of the flag. + """ + return True if Lexeme.c_check_flag(self.c, flag_id) else False + + def similarity(self, other): + """Compute a semantic similarity estimate. Defaults to cosine over + vectors. + + other (object): The object to compare with. By default, accepts `Doc`, + `Span`, `Token` and `Lexeme` objects. + RETURNS (float): A scalar similarity score. Higher is more similar. + """ + # Return 1.0 similarity for matches + if hasattr(other, "orth"): + if self.c.orth == other.orth: + return 1.0 + elif ( + hasattr(other, "__len__") and len(other) == 1 + and hasattr(other[0], "orth") + and self.c.orth == other[0].orth + ): + return 1.0 + if self.vector_norm == 0 or other.vector_norm == 0: + warnings.warn(Warnings.W008.format(obj="Lexeme")) + return 0.0 + vector = self.vector + xp = get_array_module(vector) + result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm) + # ensure we get a scalar back (numpy does this automatically but cupy doesn't) + return result.item() + + @property + def has_vector(self): + """RETURNS (bool): Whether a word vector is associated with the object. + """ + return self.vocab.has_vector(self.c.orth) + + @property + def vector_norm(self): + """RETURNS (float): The L2 norm of the vector representation.""" + vector = self.vector + return numpy.sqrt((vector**2).sum()) + + @property + def vector(self): + """A real-valued meaning representation. + + RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array + representing the lexeme's semantics. + """ + cdef int length = self.vocab.vectors_length + if length == 0: + raise ValueError(Errors.E010) + return self.vocab.get_vector(self.c.orth) + + @vector.setter + def vector(self, vector): + if len(vector) != self.vocab.vectors_length: + raise ValueError(Errors.E073.format(new_length=len(vector), + length=self.vocab.vectors_length)) + self.vocab.set_vector(self.c.orth, vector) + + @property + def rank(self): + """RETURNS (str): Sequential ID of the lexeme's lexical type, used + to index into tables, e.g. for word vectors.""" + return self.c.id + + @rank.setter + def rank(self, value): + self.c.id = value + + @property + def sentiment(self): + """RETURNS (float): A scalar value indicating the positivity or + negativity of the lexeme.""" + sentiment_table = self.vocab.lookups.get_table("lexeme_sentiment", {}) + return sentiment_table.get(self.c.orth, 0.0) + + @sentiment.setter + def sentiment(self, float x): + if "lexeme_sentiment" not in self.vocab.lookups: + self.vocab.lookups.add_table("lexeme_sentiment") + sentiment_table = self.vocab.lookups.get_table("lexeme_sentiment") + sentiment_table[self.c.orth] = x + + @property + def orth_(self): + """RETURNS (str): The original verbatim text of the lexeme + (identical to `Lexeme.text`). Exists mostly for consistency with + the other attributes.""" + return self.vocab.strings[self.c.orth] + + @property + def text(self): + """RETURNS (str): The original verbatim text of the lexeme.""" + return self.orth_ + + @property + def lower(self): + """RETURNS (uint64): Lowercase form of the lexeme.""" + return self.c.lower + + @lower.setter + def lower(self, attr_t x): + self.c.lower = x + + @property + def norm(self): + """RETURNS (uint64): The lexeme's norm, i.e. a normalised form of the + lexeme text. + """ + return self.c.norm + + @norm.setter + def norm(self, attr_t x): + if "lexeme_norm" not in self.vocab.lookups: + self.vocab.lookups.add_table("lexeme_norm") + norm_table = self.vocab.lookups.get_table("lexeme_norm") + norm_table[self.c.orth] = self.vocab.strings[x] + self.c.norm = x + + @property + def shape(self): + """RETURNS (uint64): Transform of the word's string, to show + orthographic features. + """ + return self.c.shape + + @shape.setter + def shape(self, attr_t x): + self.c.shape = x + + @property + def prefix(self): + """RETURNS (uint64): Length-N substring from the start of the word. + Defaults to `N=1`. + """ + return self.c.prefix + + @prefix.setter + def prefix(self, attr_t x): + self.c.prefix = x + + @property + def suffix(self): + """RETURNS (uint64): Length-N substring from the end of the word. + Defaults to `N=3`. + """ + return self.c.suffix + + @suffix.setter + def suffix(self, attr_t x): + self.c.suffix = x + + @property + def cluster(self): + """RETURNS (int): Brown cluster ID.""" + cluster_table = self.vocab.lookups.get_table("lexeme_cluster", {}) + return cluster_table.get(self.c.orth, 0) + + @cluster.setter + def cluster(self, int x): + cluster_table = self.vocab.lookups.get_table("lexeme_cluster", {}) + cluster_table[self.c.orth] = x + + @property + def lang(self): + """RETURNS (uint64): Language of the parent vocabulary.""" + return self.c.lang + + @lang.setter + def lang(self, attr_t x): + self.c.lang = x + + @property + def prob(self): + """RETURNS (float): Smoothed log probability estimate of the lexeme's + type.""" + prob_table = self.vocab.lookups.get_table("lexeme_prob", {}) + settings_table = self.vocab.lookups.get_table("lexeme_settings", {}) + default_oov_prob = settings_table.get("oov_prob", -20.0) + return prob_table.get(self.c.orth, default_oov_prob) + + @prob.setter + def prob(self, float x): + prob_table = self.vocab.lookups.get_table("lexeme_prob", {}) + prob_table[self.c.orth] = x + + @property + def lower_(self): + """RETURNS (str): Lowercase form of the word.""" + return self.vocab.strings[self.c.lower] + + @lower_.setter + def lower_(self, str x): + self.c.lower = self.vocab.strings.add(x) + + @property + def norm_(self): + """RETURNS (str): The lexeme's norm, i.e. a normalised form of the + lexeme text. + """ + return self.vocab.strings[self.c.norm] + + @norm_.setter + def norm_(self, str x): + self.norm = self.vocab.strings.add(x) + + @property + def shape_(self): + """RETURNS (str): Transform of the word's string, to show + orthographic features. + """ + return self.vocab.strings[self.c.shape] + + @shape_.setter + def shape_(self, str x): + self.c.shape = self.vocab.strings.add(x) + + @property + def prefix_(self): + """RETURNS (str): Length-N substring from the start of the word. + Defaults to `N=1`. + """ + return self.vocab.strings[self.c.prefix] + + @prefix_.setter + def prefix_(self, str x): + self.c.prefix = self.vocab.strings.add(x) + + @property + def suffix_(self): + """RETURNS (str): Length-N substring from the end of the word. + Defaults to `N=3`. + """ + return self.vocab.strings[self.c.suffix] + + @suffix_.setter + def suffix_(self, str x): + self.c.suffix = self.vocab.strings.add(x) + + @property + def lang_(self): + """RETURNS (str): Language of the parent vocabulary.""" + return self.vocab.strings[self.c.lang] + + @lang_.setter + def lang_(self, str x): + self.c.lang = self.vocab.strings.add(x) + + @property + def flags(self): + """RETURNS (uint64): Container of the lexeme's binary flags.""" + return self.c.flags + + @flags.setter + def flags(self, flags_t x): + self.c.flags = x + + @property + def is_oov(self): + """RETURNS (bool): Whether the lexeme is out-of-vocabulary.""" + return self.orth not in self.vocab.vectors + + @property + def is_stop(self): + """RETURNS (bool): Whether the lexeme is a stop word.""" + return Lexeme.c_check_flag(self.c, IS_STOP) + + @is_stop.setter + def is_stop(self, bint x): + Lexeme.c_set_flag(self.c, IS_STOP, x) + + @property + def is_alpha(self): + """RETURNS (bool): Whether the lexeme consists of alphabetic + characters. Equivalent to `lexeme.text.isalpha()`. + """ + return Lexeme.c_check_flag(self.c, IS_ALPHA) + + @is_alpha.setter + def is_alpha(self, bint x): + Lexeme.c_set_flag(self.c, IS_ALPHA, x) + + @property + def is_ascii(self): + """RETURNS (bool): Whether the lexeme consists of ASCII characters. + Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. + """ + return Lexeme.c_check_flag(self.c, IS_ASCII) + + @is_ascii.setter + def is_ascii(self, bint x): + Lexeme.c_set_flag(self.c, IS_ASCII, x) + + @property + def is_digit(self): + """RETURNS (bool): Whether the lexeme consists of digits. Equivalent + to `lexeme.text.isdigit()`. + """ + return Lexeme.c_check_flag(self.c, IS_DIGIT) + + @is_digit.setter + def is_digit(self, bint x): + Lexeme.c_set_flag(self.c, IS_DIGIT, x) + + @property + def is_lower(self): + """RETURNS (bool): Whether the lexeme is in lowercase. Equivalent to + `lexeme.text.islower()`. + """ + return Lexeme.c_check_flag(self.c, IS_LOWER) + + @is_lower.setter + def is_lower(self, bint x): + Lexeme.c_set_flag(self.c, IS_LOWER, x) + + @property + def is_upper(self): + """RETURNS (bool): Whether the lexeme is in uppercase. Equivalent to + `lexeme.text.isupper()`. + """ + return Lexeme.c_check_flag(self.c, IS_UPPER) + + @is_upper.setter + def is_upper(self, bint x): + Lexeme.c_set_flag(self.c, IS_UPPER, x) + + @property + def is_title(self): + """RETURNS (bool): Whether the lexeme is in titlecase. Equivalent to + `lexeme.text.istitle()`. + """ + return Lexeme.c_check_flag(self.c, IS_TITLE) + + @is_title.setter + def is_title(self, bint x): + Lexeme.c_set_flag(self.c, IS_TITLE, x) + + @property + def is_punct(self): + """RETURNS (bool): Whether the lexeme is punctuation.""" + return Lexeme.c_check_flag(self.c, IS_PUNCT) + + @is_punct.setter + def is_punct(self, bint x): + Lexeme.c_set_flag(self.c, IS_PUNCT, x) + + @property + def is_space(self): + """RETURNS (bool): Whether the lexeme consist of whitespace characters. + Equivalent to `lexeme.text.isspace()`. + """ + return Lexeme.c_check_flag(self.c, IS_SPACE) + + @is_space.setter + def is_space(self, bint x): + Lexeme.c_set_flag(self.c, IS_SPACE, x) + + @property + def is_bracket(self): + """RETURNS (bool): Whether the lexeme is a bracket.""" + return Lexeme.c_check_flag(self.c, IS_BRACKET) + + @is_bracket.setter + def is_bracket(self, bint x): + Lexeme.c_set_flag(self.c, IS_BRACKET, x) + + @property + def is_quote(self): + """RETURNS (bool): Whether the lexeme is a quotation mark.""" + return Lexeme.c_check_flag(self.c, IS_QUOTE) + + @is_quote.setter + def is_quote(self, bint x): + Lexeme.c_set_flag(self.c, IS_QUOTE, x) + + @property + def is_left_punct(self): + """RETURNS (bool): Whether the lexeme is left punctuation, e.g. (.""" + return Lexeme.c_check_flag(self.c, IS_LEFT_PUNCT) + + @is_left_punct.setter + def is_left_punct(self, bint x): + Lexeme.c_set_flag(self.c, IS_LEFT_PUNCT, x) + + @property + def is_right_punct(self): + """RETURNS (bool): Whether the lexeme is right punctuation, e.g. ).""" + return Lexeme.c_check_flag(self.c, IS_RIGHT_PUNCT) + + @is_right_punct.setter + def is_right_punct(self, bint x): + Lexeme.c_set_flag(self.c, IS_RIGHT_PUNCT, x) + + @property + def is_currency(self): + """RETURNS (bool): Whether the lexeme is a currency symbol, e.g. $, €.""" + return Lexeme.c_check_flag(self.c, IS_CURRENCY) + + @is_currency.setter + def is_currency(self, bint x): + Lexeme.c_set_flag(self.c, IS_CURRENCY, x) + + @property + def like_url(self): + """RETURNS (bool): Whether the lexeme resembles a URL.""" + return Lexeme.c_check_flag(self.c, LIKE_URL) + + @like_url.setter + def like_url(self, bint x): + Lexeme.c_set_flag(self.c, LIKE_URL, x) + + @property + def like_num(self): + """RETURNS (bool): Whether the lexeme represents a number, e.g. "10.9", + "10", "ten", etc. + """ + return Lexeme.c_check_flag(self.c, LIKE_NUM) + + @like_num.setter + def like_num(self, bint x): + Lexeme.c_set_flag(self.c, LIKE_NUM, x) + + @property + def like_email(self): + """RETURNS (bool): Whether the lexeme resembles an email address.""" + return Lexeme.c_check_flag(self.c, LIKE_EMAIL) + + @like_email.setter + def like_email(self, bint x): + Lexeme.c_set_flag(self.c, LIKE_EMAIL, x) diff --git a/spacy/lookups.py b/spacy/lookups.py new file mode 100644 index 0000000..fd404ed --- /dev/null +++ b/spacy/lookups.py @@ -0,0 +1,314 @@ +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import srsly +from preshed.bloom import BloomFilter + +from .errors import Errors +from .strings import get_string_id +from .util import SimpleFrozenDict, ensure_path, load_language_data, registry + +UNSET = object() + + +def load_lookups(lang: str, tables: List[str], strict: bool = True) -> "Lookups": + """Load the data from the spacy-lookups-data package for a given language, + if available. Returns an empty `Lookups` container if there's no data or if the package + is not installed. + + lang (str): The language code (corresponds to entry point exposed by + the spacy-lookups-data package). + tables (List[str]): Name of tables to load, e.g. ["lemma_lookup", "lemma_exc"] + strict (bool): Whether to raise an error if a table doesn't exist. + RETURNS (Lookups): The lookups container containing the loaded tables. + """ + # TODO: import spacy_lookups_data instead of going via entry points here? + lookups = Lookups() + if lang not in registry.lookups: + if strict and len(tables) > 0: + raise ValueError(Errors.E955.format(table=", ".join(tables), lang=lang)) + return lookups + data = registry.lookups.get(lang) + for table in tables: + if table not in data: + if strict: + raise ValueError(Errors.E955.format(table=table, lang=lang)) + language_data = {} # type: ignore[var-annotated] + else: + language_data = load_language_data(data[table]) # type: ignore[assignment] + lookups.add_table(table, language_data) + return lookups + + +class Table(OrderedDict): + """A table in the lookups. Subclass of builtin dict that implements a + slightly more consistent and unified API. + + Includes a Bloom filter to speed up missed lookups. + """ + + @classmethod + def from_dict(cls, data: dict, name: Optional[str] = None) -> "Table": + """Initialize a new table from a dict. + + data (dict): The dictionary. + name (str): Optional table name for reference. + + DOCS: https://spacy.io/api/lookups#table.from_dict + """ + self = cls(name=name) + self.update(data) + return self + + def __init__(self, name: Optional[str] = None, data: Optional[dict] = None) -> None: + """Initialize a new table. + + name (str): Optional table name for reference. + data (dict): Initial data, used to hint Bloom Filter. + + DOCS: https://spacy.io/api/lookups#table.init + """ + OrderedDict.__init__(self) + self.name = name + # Assume a default size of 1M items + self.default_size = 1e6 + size = max(len(data), 1) if data is not None else self.default_size + self.bloom = BloomFilter.from_error_rate(size) + if data: + self.update(data) + + def __setitem__(self, key: Union[str, int], value: Any) -> None: + """Set new key/value pair. String keys will be hashed. + + key (str / int): The key to set. + value: The value to set. + """ + key = get_string_id(key) + OrderedDict.__setitem__(self, key, value) # type: ignore[assignment] + self.bloom.add(key) + + def set(self, key: Union[str, int], value: Any) -> None: + """Set new key/value pair. String keys will be hashed. + Same as table[key] = value. + + key (str / int): The key to set. + value: The value to set. + """ + self[key] = value + + def __getitem__(self, key: Union[str, int]) -> Any: + """Get the value for a given key. String keys will be hashed. + + key (str / int): The key to get. + RETURNS: The value. + """ + key = get_string_id(key) + return OrderedDict.__getitem__(self, key) # type: ignore[index] + + def get(self, key: Union[str, int], default: Optional[Any] = None) -> Any: + """Get the value for a given key. String keys will be hashed. + + key (str / int): The key to get. + default: The default value to return. + RETURNS: The value. + """ + key = get_string_id(key) + return OrderedDict.get(self, key, default) # type: ignore[arg-type] + + def __contains__(self, key: Union[str, int]) -> bool: # type: ignore[override] + """Check whether a key is in the table. String keys will be hashed. + + key (str / int): The key to check. + RETURNS (bool): Whether the key is in the table. + """ + key = get_string_id(key) + # This can give a false positive, so we need to check it after + if key not in self.bloom: + return False + return OrderedDict.__contains__(self, key) + + def to_bytes(self) -> bytes: + """Serialize table to a bytestring. + + RETURNS (bytes): The serialized table. + + DOCS: https://spacy.io/api/lookups#table.to_bytes + """ + data = { + "name": self.name, + "dict": dict(self.items()), + "bloom": self.bloom.to_bytes(), + } + return srsly.msgpack_dumps(data) + + def from_bytes(self, bytes_data: bytes) -> "Table": + """Load a table from a bytestring. + + bytes_data (bytes): The data to load. + RETURNS (Table): The loaded table. + + DOCS: https://spacy.io/api/lookups#table.from_bytes + """ + loaded = srsly.msgpack_loads(bytes_data) + data = loaded.get("dict", {}) + self.name = loaded["name"] + self.bloom = BloomFilter().from_bytes(loaded["bloom"]) + self.clear() + self.update(data) + return self + + +class Lookups: + """Container for large lookup tables and dictionaries, e.g. lemmatization + data or tokenizer exception lists. Lookups are available via vocab.lookups, + so they can be accessed before the pipeline components are applied (e.g. + in the tokenizer and lemmatizer), as well as within the pipeline components + via doc.vocab.lookups. + """ + + def __init__(self) -> None: + """Initialize the Lookups object. + + DOCS: https://spacy.io/api/lookups#init + """ + self._tables: Dict[str, Table] = {} + + def __contains__(self, name: str) -> bool: + """Check if the lookups contain a table of a given name. Delegates to + Lookups.has_table. + + name (str): Name of the table. + RETURNS (bool): Whether a table of that name is in the lookups. + """ + return self.has_table(name) + + def __len__(self) -> int: + """RETURNS (int): The number of tables in the lookups.""" + return len(self._tables) + + @property + def tables(self) -> List[str]: + """RETURNS (List[str]): Names of all tables in the lookups.""" + return list(self._tables.keys()) + + def add_table(self, name: str, data: dict = SimpleFrozenDict()) -> Table: + """Add a new table to the lookups. Raises an error if the table exists. + + name (str): Unique name of table. + data (dict): Optional data to add to the table. + RETURNS (Table): The newly added table. + + DOCS: https://spacy.io/api/lookups#add_table + """ + if name in self.tables: + raise ValueError(Errors.E158.format(name=name)) + table = Table(name=name, data=data) + self._tables[name] = table + return table + + def set_table(self, name: str, table: Table) -> None: + """Set a table. + + name (str): Name of the table to set. + table (Table): The Table to set. + + DOCS: https://spacy.io/api/lookups#set_table + """ + self._tables[name] = table + + def get_table(self, name: str, default: Any = UNSET) -> Table: + """Get a table. Raises an error if the table doesn't exist and no + default value is provided. + + name (str): Name of the table. + default (Any): Optional default value to return if table doesn't exist. + RETURNS (Table): The table. + + DOCS: https://spacy.io/api/lookups#get_table + """ + if name not in self._tables: + if default == UNSET: + raise KeyError(Errors.E159.format(name=name, tables=self.tables)) + return default + return self._tables[name] + + def remove_table(self, name: str) -> Table: + """Remove a table. Raises an error if the table doesn't exist. + + name (str): Name of the table to remove. + RETURNS (Table): The removed table. + + DOCS: https://spacy.io/api/lookups#remove_table + """ + if name not in self._tables: + raise KeyError(Errors.E159.format(name=name, tables=self.tables)) + return self._tables.pop(name) + + def has_table(self, name: str) -> bool: + """Check if the lookups contain a table of a given name. + + name (str): Name of the table. + RETURNS (bool): Whether a table of that name exists. + + DOCS: https://spacy.io/api/lookups#has_table + """ + return name in self._tables + + def to_bytes(self, **kwargs) -> bytes: + """Serialize the lookups to a bytestring. + + RETURNS (bytes): The serialized Lookups. + + DOCS: https://spacy.io/api/lookups#to_bytes + """ + return srsly.msgpack_dumps(self._tables) + + def from_bytes(self, bytes_data: bytes, **kwargs) -> "Lookups": + """Load the lookups from a bytestring. + + bytes_data (bytes): The data to load. + RETURNS (Lookups): The loaded Lookups. + + DOCS: https://spacy.io/api/lookups#from_bytes + """ + self._tables = {} + for key, value in srsly.msgpack_loads(bytes_data).items(): + self._tables[key] = Table(key, value) + return self + + def to_disk( + self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs + ) -> None: + """Save the lookups to a directory as lookups.bin. Expects a path to a + directory, which will be created if it doesn't exist. + + path (str / Path): The file path. + + DOCS: https://spacy.io/api/lookups#to_disk + """ + path = ensure_path(path) + if not path.exists(): + path.mkdir() + filepath = path / filename + with filepath.open("wb") as file_: + file_.write(self.to_bytes()) + + def from_disk( + self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs + ) -> "Lookups": + """Load lookups from a directory containing a lookups.bin. Will skip + loading if the file doesn't exist. + + path (str / Path): The directory path. + RETURNS (Lookups): The loaded lookups. + + DOCS: https://spacy.io/api/lookups#from_disk + """ + path = ensure_path(path) + filepath = path / filename + if filepath.exists(): + with filepath.open("rb") as file_: + data = file_.read() + return self.from_bytes(data) + return self diff --git a/spacy/matcher/__init__.py b/spacy/matcher/__init__.py new file mode 100644 index 0000000..b6d6d70 --- /dev/null +++ b/spacy/matcher/__init__.py @@ -0,0 +1,6 @@ +from .dependencymatcher import DependencyMatcher +from .levenshtein import levenshtein +from .matcher import Matcher +from .phrasematcher import PhraseMatcher + +__all__ = ["DependencyMatcher", "Matcher", "PhraseMatcher", "levenshtein"] diff --git a/spacy/matcher/dependencymatcher.pyi b/spacy/matcher/dependencymatcher.pyi new file mode 100644 index 0000000..3d744df --- /dev/null +++ b/spacy/matcher/dependencymatcher.pyi @@ -0,0 +1,67 @@ +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from ..tokens.doc import Doc +from ..tokens.span import Span +from ..vocab import Vocab +from .matcher import Matcher + +class DependencyMatcher: + """Match dependency parse tree based on pattern rules.""" + + _patterns: Dict[str, List[Any]] + _raw_patterns: Dict[str, List[Any]] + _tokens_to_key: Dict[str, List[Any]] + _root: Dict[str, List[Any]] + _tree: Dict[str, List[Any]] + _callbacks: Dict[ + Any, Callable[[DependencyMatcher, Doc, int, List[Tuple[int, List[int]]]], Any] + ] + _ops: Dict[str, Any] + vocab: Vocab + _matcher: Matcher + def __init__(self, vocab: Vocab, *, validate: bool = ...) -> None: ... + def __reduce__( + self, + ) -> Tuple[ + Callable[ + [Vocab, Dict[str, Any], Dict[str, Callable[..., Any]]], DependencyMatcher + ], + Tuple[ + Vocab, + Dict[str, List[Any]], + Dict[ + str, + Callable[ + [DependencyMatcher, Doc, int, List[Tuple[int, List[int]]]], Any + ], + ], + ], + None, + None, + ]: ... + def __len__(self) -> int: ... + def __contains__(self, key: Union[str, int]) -> bool: ... + def add( + self, + key: Union[str, int], + patterns: List[List[Dict[str, Any]]], + *, + on_match: Optional[ + Callable[[DependencyMatcher, Doc, int, List[Tuple[int, List[int]]]], Any] + ] = ..., + ) -> None: ... + def has_key(self, key: Union[str, int]) -> bool: ... + def get( + self, key: Union[str, int], default: Optional[Any] = ... + ) -> Tuple[ + Optional[ + Callable[[DependencyMatcher, Doc, int, List[Tuple[int, List[int]]]], Any] + ], + List[List[Dict[str, Any]]], + ]: ... + def remove(self, key: Union[str, int]) -> None: ... + def __call__(self, doclike: Union[Doc, Span]) -> List[Tuple[int, List[int]]]: ... + +def unpickle_matcher( + vocab: Vocab, patterns: Dict[str, Any], callbacks: Dict[str, Callable[..., Any]] +) -> DependencyMatcher: ... diff --git a/spacy/matcher/dependencymatcher.pyx b/spacy/matcher/dependencymatcher.pyx new file mode 100644 index 0000000..ab5f5d5 --- /dev/null +++ b/spacy/matcher/dependencymatcher.pyx @@ -0,0 +1,494 @@ +# cython: infer_types=True +import warnings +from collections import defaultdict +from itertools import product +from typing import List + +from ..tokens.doc cimport Doc +from ..vocab cimport Vocab +from .matcher cimport Matcher + +from ..errors import Errors, Warnings +from ..tokens import Span + +DELIMITER = "||" +INDEX_HEAD = 1 +INDEX_RELOP = 0 + + +cdef class DependencyMatcher: + """Match dependency parse tree based on pattern rules.""" + cdef readonly Vocab vocab + cdef readonly Matcher _matcher + cdef public object _patterns + cdef public object _raw_patterns + cdef public object _tokens_to_key + cdef public object _root + cdef public object _callbacks + cdef public object _tree + cdef public object _ops + + def __init__(self, vocab, *, validate=False): + """Create the DependencyMatcher. + + vocab (Vocab): The vocabulary object, which must be shared with the + documents the matcher will operate on. + validate (bool): Whether patterns should be validated, passed to + Matcher as `validate` + """ + self._matcher = Matcher(vocab, validate=validate) + + # Associates each key to the raw patterns list added by the user + # e.g. {'key': [[{'RIGHT_ID': ..., 'RIGHT_ATTRS': ... }, ... ], ... ], ... } + self._raw_patterns = defaultdict(list) + + # Associates each key to a list of lists of 'RIGHT_ATTRS' + # e.g. {'key': [[{'POS': ... }, ... ], ... ], ... } + self._patterns = defaultdict(list) + + # Associates each key to a list of lists where each list associates + # a token position within the pattern to the key used by the internal matcher) + # e.g. {'key': [ ['token_key', ... ], ... ], ... } + self._tokens_to_key = defaultdict(list) + + # Associates each key to a list of ints where each int corresponds + # to the position of the root token within the pattern + # e.g. {'key': [3, 1, 4, ... ], ... } + self._root = defaultdict(list) + + # Associates each key to a list of dicts where each dict describes all + # branches from a token (identifed by its position) to other tokens as + # a list of tuples: ('REL_OP', 'LEFT_ID') + # e.g. {'key': [{2: [('<', 'left_id'), ...] ...}, ... ], ...} + self._tree = defaultdict(list) + + # Associates each key to its on_match callback + # e.g. {'key': on_match_callback, ...} + self._callbacks = {} + + self.vocab = vocab + self._ops = { + "<": self._dep, + ">": self._gov, + "<<": self._dep_chain, + ">>": self._gov_chain, + ".": self._imm_precede, + ".*": self._precede, + ";": self._imm_follow, + ";*": self._follow, + "$+": self._imm_right_sib, + "$-": self._imm_left_sib, + "$++": self._right_sib, + "$--": self._left_sib, + ">+": self._imm_right_child, + ">-": self._imm_left_child, + ">++": self._right_child, + ">--": self._left_child, + "<+": self._imm_right_parent, + "<-": self._imm_left_parent, + "<++": self._right_parent, + "<--": self._left_parent, + } + + def __reduce__(self): + data = (self.vocab, self._raw_patterns, self._callbacks) + return (unpickle_matcher, data, None, None) + + def __len__(self): + """Get the number of rules, which are edges, added to the dependency + tree matcher. + + RETURNS (int): The number of rules. + """ + return len(self._patterns) + + def __contains__(self, key): + """Check whether the matcher contains rules for a match ID. + + key (str): The match ID. + RETURNS (bool): Whether the matcher contains rules for this match ID. + """ + return self.has_key(key) # no-cython-lint: W601 + + def _validate_input(self, pattern, key): + idx = 0 + visited_nodes = {} + for relation in pattern: + if not isinstance(relation, dict): + raise ValueError(Errors.E1008) + if "RIGHT_ATTRS" not in relation and "RIGHT_ID" not in relation: + raise ValueError(Errors.E098.format(key=key)) + if idx == 0: + if not( + "RIGHT_ID" in relation + and "REL_OP" not in relation + and "LEFT_ID" not in relation + ): + raise ValueError(Errors.E099.format(key=key)) + visited_nodes[relation["RIGHT_ID"]] = True + else: + required_keys = {"RIGHT_ID", "RIGHT_ATTRS", "REL_OP", "LEFT_ID"} + relation_keys = set(relation.keys()) + # Identify required keys that have not been specified + missing = required_keys - relation_keys + if missing: + missing_txt = ", ".join(list(missing)) + raise ValueError(Errors.E100.format( + required=required_keys, + missing=missing_txt + )) + # Identify additional, unsupported keys + unsupported = relation_keys - required_keys + if unsupported: + unsupported_txt = ", ".join(list(unsupported)) + warnings.warn(Warnings.W126.format( + unsupported=unsupported_txt + )) + if ( + relation["RIGHT_ID"] in visited_nodes + or relation["LEFT_ID"] not in visited_nodes + ): + raise ValueError(Errors.E101.format(key=key)) + if relation["REL_OP"] not in self._ops: + raise ValueError(Errors.E1007.format(op=relation["REL_OP"])) + visited_nodes[relation["RIGHT_ID"]] = True + visited_nodes[relation["LEFT_ID"]] = True + if relation["RIGHT_ATTRS"].get("OP", "") in ("?", "*", "+"): + raise ValueError(Errors.E1016.format(node=relation)) + idx = idx + 1 + + def _get_matcher_key(self, key, pattern_idx, token_idx): + """ + Creates a token key to be used by the matcher + """ + return self._normalize_key( + str(key) + DELIMITER + + str(pattern_idx) + DELIMITER + + str(token_idx) + ) + + def add(self, key, patterns, *, on_match=None): + """Add a new matcher rule to the matcher. + + key (str): The match ID. + patterns (list): The patterns to add for the given key. + on_match (callable): Optional callback executed on match. + """ + if on_match is not None and not hasattr(on_match, "__call__"): + raise ValueError(Errors.E171.format(arg_type=type(on_match))) + if patterns is None or not isinstance(patterns, List): # old API + raise ValueError(Errors.E948.format(arg_type=type(patterns))) + for pattern in patterns: + if len(pattern) == 0: + raise ValueError(Errors.E012.format(key=key)) + self._validate_input(pattern, key) + + key = self._normalize_key(key) + + # Save raw patterns and on_match callback + self._raw_patterns[key].extend(patterns) + self._callbacks[key] = on_match + + # Add 'RIGHT_ATTRS' to self._patterns[key] + _patterns = [[[pat["RIGHT_ATTRS"]] for pat in pattern] for pattern in patterns] + pattern_offset = len(self._patterns[key]) + self._patterns[key].extend(_patterns) + + # Add each node pattern of all the input patterns individually to the + # matcher. This enables only a single instance of Matcher to be used. + # Multiple adds are required to track each node pattern. + tokens_to_key_list = [] + for i, current_patterns in enumerate(_patterns, start=pattern_offset): + + # Preallocate list space + tokens_to_key = [None] * len(current_patterns) + + # TODO: Better ways to hash edges in pattern? + for j, _pattern in enumerate(current_patterns): + k = self._get_matcher_key(key, i, j) + self._matcher.add(k, [_pattern]) + tokens_to_key[j] = k + + tokens_to_key_list.append(tokens_to_key) + + self._tokens_to_key[key].extend(tokens_to_key_list) + + # Create an object tree to traverse later on. This data structure + # enables easy tree pattern match. Doc-Token based tree cannot be + # reused since it is memory-heavy and tightly coupled with the Doc. + self._retrieve_tree(patterns, key) + + def _retrieve_tree(self, patterns, key): + + # New trees belonging to this key + tree_list = [] + + # List of indices to the root nodes + root_list = [] + + # Group token id to create trees + for i, pattern in enumerate(patterns): + tree = defaultdict(list) + root = -1 + + right_id_to_token = {} + for j, token in enumerate(pattern): + right_id_to_token[token["RIGHT_ID"]] = j + + for j, token in enumerate(pattern): + if "REL_OP" in token: + # Add tree branch + tree[right_id_to_token[token["LEFT_ID"]]].append( + (token["REL_OP"], j), + ) + else: + # No 'REL_OP', this is the root + root = j + + tree_list.append(tree) + root_list.append(root) + + self._tree[key].extend(tree_list) + self._root[key].extend(root_list) + + def has_key(self, key): + """Check whether the matcher has a rule with a given key. + + key (string or int): The key to check. + RETURNS (bool): Whether the matcher has the rule. + """ + return self._normalize_key(key) in self._patterns + + def get(self, key, default=None): + """Retrieve the pattern stored for a key. + + key (str / int): The key to retrieve. + RETURNS (tuple): The rule, as an (on_match, patterns) tuple. + """ + key = self._normalize_key(key) + if key not in self._raw_patterns: + return default + return (self._callbacks[key], self._raw_patterns[key]) + + def remove(self, key): + key = self._normalize_key(key) + if key not in self._patterns: + raise ValueError(Errors.E175.format(key=key)) + self._patterns.pop(key) + self._raw_patterns.pop(key) + self._tree.pop(key) + self._root.pop(key) + for mklist in self._tokens_to_key.pop(key): + for mkey in mklist: + self._matcher.remove(mkey) + + def _get_keys_to_position_maps(self, doc): + """ + Processes the doc and groups all matches by their root and match id. + Returns a dict mapping each (root, match id) pair to the list of + tokens indices which are descendants of root and match the token + pattern identified by the given match id. + + e.g. keys_to_position_maps[root_index][match_id] = [...] + """ + keys_to_position_maps = defaultdict(lambda: defaultdict(list)) + for match_id, start, end in self._matcher(doc): + if start + 1 != end: + warnings.warn(Warnings.W110.format(tokens=[t.text for t in doc[start:end]], pattern=self._matcher.get(match_id)[1][0][0])) + token = doc[start] + root = ([token] + list(token.ancestors))[-1] + keys_to_position_maps[root.i][match_id].append(start) + return keys_to_position_maps + + def __call__(self, object doclike): + """Find all token sequences matching the supplied pattern. + + doclike (Doc or Span): The document to match over. + RETURNS (list): A list of `(key, start, end)` tuples, + describing the matches. A match tuple describes a span + `doc[start:end]`. The `label_id` and `key` are both integers. + """ + if isinstance(doclike, Doc): + doc = doclike + elif isinstance(doclike, Span): + doc = doclike.as_doc(copy_user_data=True) + else: + raise ValueError(Errors.E195.format(good="Doc or Span", got=type(doclike).__name__)) + + matched_key_trees = [] + keys_to_position_maps = self._get_keys_to_position_maps(doc) + cache = {} + + for key in self._patterns.keys(): + for root, tree, tokens_to_key in zip( + self._root[key], + self._tree[key], + self._tokens_to_key[key], + ): + for keys_to_position in keys_to_position_maps.values(): + for matched_tree in self._get_matches(cache, doc, tree, tokens_to_key, keys_to_position): + matched_key_trees.append((key, matched_tree)) + + for i, (match_id, nodes) in enumerate(matched_key_trees): + on_match = self._callbacks.get(match_id) + if on_match is not None: + on_match(self, doc, i, matched_key_trees) + + return matched_key_trees + + def _get_matches(self, cache, doc, tree, tokens_to_key, keys_to_position): + cdef bint is_valid + + all_positions = [keys_to_position[key] for key in tokens_to_key] + + # Generate all potential matches by computing the cartesian product of all + # position of the matched tokens + for candidate_match in product(*all_positions): + + # A potential match is a valid match if all relationships between the + # matched tokens are satisfied. + is_valid = True + for left_idx in range(len(candidate_match)): + is_valid = self._check_relationships(cache, doc, candidate_match, left_idx, tree) + + if not is_valid: + break + + if is_valid: + yield list(candidate_match) + + def _check_relationships(self, cache, doc, candidate_match, left_idx, tree): + # Position of the left operand within the document + left_pos = candidate_match[left_idx] + + for relop, right_idx in tree[left_idx]: + + # Position of the left operand within the document + right_pos = candidate_match[right_idx] + + # List of valid right matches + valid_right_matches = self._resolve_node_operator(cache, doc, left_pos, relop) + + # If the proposed right token is not within the valid ones, fail + if right_pos not in valid_right_matches: + return False + + return True + + def _resolve_node_operator(self, cache, doc, node, operator): + """ + Given a doc, a node (as a index in the doc) and a REL_OP operator + returns the list of nodes from the doc that belong to node+operator. + """ + key = (node, operator) + if key not in cache: + cache[key] = [token.i for token in self._ops[operator](doc, node)] + return cache[key] + + def _dep(self, doc, node): + if doc[node].head == doc[node]: + return [] + return [doc[node].head] + + def _gov(self, doc, node): + return list(doc[node].children) + + def _dep_chain(self, doc, node): + return list(doc[node].ancestors) + + def _gov_chain(self, doc, node): + return [t for t in doc[node].subtree if t != doc[node]] + + def _imm_precede(self, doc, node): + sent = self._get_sent(doc[node]) + if node < len(doc) - 1 and doc[node + 1] in sent: + return [doc[node + 1]] + return [] + + def _precede(self, doc, node): + sent = self._get_sent(doc[node]) + return [doc[i] for i in range(node + 1, sent.end)] + + def _imm_follow(self, doc, node): + sent = self._get_sent(doc[node]) + if node > 0 and doc[node - 1] in sent: + return [doc[node - 1]] + return [] + + def _follow(self, doc, node): + sent = self._get_sent(doc[node]) + return [doc[i] for i in range(sent.start, node)] + + def _imm_right_sib(self, doc, node): + for child in list(doc[node].head.children): + if child.i == node + 1: + return [doc[child.i]] + return [] + + def _imm_left_sib(self, doc, node): + for child in list(doc[node].head.children): + if child.i == node - 1: + return [doc[child.i]] + return [] + + def _right_sib(self, doc, node): + return [doc[child.i] for child in doc[node].head.children if child.i > node] + + def _left_sib(self, doc, node): + return [doc[child.i] for child in doc[node].head.children if child.i < node] + + def _imm_right_child(self, doc, node): + for child in doc[node].rights: + if child.i == node + 1: + return [doc[child.i]] + return [] + + def _imm_left_child(self, doc, node): + for child in doc[node].lefts: + if child.i == node - 1: + return [doc[child.i]] + return [] + + def _right_child(self, doc, node): + return [child for child in doc[node].rights] + + def _left_child(self, doc, node): + return [child for child in doc[node].lefts] + + def _imm_right_parent(self, doc, node): + if doc[node].head.i == node + 1: + return [doc[node].head] + return [] + + def _imm_left_parent(self, doc, node): + if doc[node].head.i == node - 1: + return [doc[node].head] + return [] + + def _right_parent(self, doc, node): + if doc[node].head.i > node: + return [doc[node].head] + return [] + + def _left_parent(self, doc, node): + if doc[node].head.i < node: + return [doc[node].head] + return [] + + def _normalize_key(self, key): + if isinstance(key, str): + return self.vocab.strings.add(key) + else: + return key + + def _get_sent(self, token): + root = (list(token.ancestors) or [token])[-1] + return token.doc[root.left_edge.i:root.right_edge.i + 1] + + +def unpickle_matcher(vocab, patterns, callbacks): + matcher = DependencyMatcher(vocab) + for key, pattern in patterns.items(): + callback = callbacks.get(key, None) + matcher.add(key, pattern, on_match=callback) + return matcher diff --git a/spacy/matcher/levenshtein.pyx b/spacy/matcher/levenshtein.pyx new file mode 100644 index 0000000..1bafdbb --- /dev/null +++ b/spacy/matcher/levenshtein.pyx @@ -0,0 +1,31 @@ +# cython: binding=True, infer_types=True, language_level=3 +from cpython.object cimport PyObject +from libc.stdint cimport int64_t + +from typing import Optional + +from ..util import registry + + +cdef extern from "polyleven.c": + int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k) + + +cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None): + if k is None: + k = -1 + return polyleven(a, b, k) + + +cpdef bint levenshtein_compare(input_text: str, pattern_text: str, fuzzy: int = -1): + if fuzzy >= 0: + max_edits = fuzzy + else: + # allow at least two edits (to allow at least one transposition) and up + # to 30% of the pattern string length + max_edits = max(2, round(0.3 * len(pattern_text))) + return levenshtein(input_text, pattern_text, max_edits) <= max_edits + + +def make_levenshtein_compare(): + return levenshtein_compare diff --git a/spacy/matcher/matcher.pxd b/spacy/matcher/matcher.pxd new file mode 100644 index 0000000..2c82cea --- /dev/null +++ b/spacy/matcher/matcher.pxd @@ -0,0 +1,80 @@ +from cymem.cymem cimport Pool +from libc.stdint cimport int32_t +from libcpp.vector cimport vector + +from ..lexeme cimport attr_id_t +from ..structs cimport TokenC +from ..typedefs cimport attr_t, hash_t +from ..vocab cimport Vocab + + +cdef enum action_t: + REJECT = 0000 + MATCH = 1000 + ADVANCE = 0100 + RETRY = 0010 + RETRY_EXTEND = 0011 + RETRY_ADVANCE = 0110 + MATCH_EXTEND = 1001 + MATCH_REJECT = 2000 + MATCH_DOUBLE = 3000 + + +cdef enum quantifier_t: + ZERO + ZERO_ONE + ZERO_PLUS + ONE + ONE_PLUS + FINAL_ID + + +cdef struct AttrValueC: + attr_id_t attr + attr_t value + +cdef struct IndexValueC: + int32_t index + attr_t value + +cdef struct TokenPatternC: + AttrValueC* attrs + int32_t* py_predicates + IndexValueC* extra_attrs + int32_t nr_attr + int32_t nr_extra_attr + int32_t nr_py + quantifier_t quantifier + hash_t key + int32_t token_idx + + +cdef struct MatchAlignmentC: + int32_t token_idx + int32_t length + + +cdef struct PatternStateC: + TokenPatternC* pattern + int32_t start + int32_t length + + +cdef struct MatchC: + attr_t pattern_id + int32_t start + int32_t length + + +cdef class Matcher: + cdef Pool mem + cdef vector[TokenPatternC*] patterns + cdef readonly Vocab vocab + cdef public object validate + cdef public object _patterns + cdef public object _callbacks + cdef public object _filter + cdef public object _extensions + cdef public object _extra_predicates + cdef public object _seen_attrs + cdef public object _fuzzy_compare diff --git a/spacy/matcher/matcher.pyi b/spacy/matcher/matcher.pyi new file mode 100644 index 0000000..e474d25 --- /dev/null +++ b/spacy/matcher/matcher.pyi @@ -0,0 +1,70 @@ +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + Union, + overload, +) + +from ..compat import Literal +from ..tokens import Doc, Span +from ..vocab import Vocab + +class Matcher: + def __init__( + self, + vocab: Vocab, + validate: bool = ..., + fuzzy_compare: Callable[[str, str, int], bool] = ..., + ) -> None: ... + def __reduce__(self) -> Any: ... + def __len__(self) -> int: ... + def __contains__(self, key: str) -> bool: ... + def add( + self, + key: Union[str, int], + patterns: List[List[Dict[str, Any]]], + *, + on_match: Optional[ + Callable[[Matcher, Doc, int, List[Tuple[Any, ...]]], Any] + ] = ..., + greedy: Optional[str] = ..., + ) -> None: ... + def remove(self, key: str) -> None: ... + def has_key(self, key: Union[str, int]) -> bool: ... + def get( + self, key: Union[str, int], default: Optional[Any] = ... + ) -> Tuple[Optional[Callable[[Any], Any]], List[List[Dict[Any, Any]]]]: ... + def pipe( + self, + docs: Iterable[Tuple[Doc, Any]], + batch_size: int = ..., + return_matches: bool = ..., + as_tuples: bool = ..., + ) -> Union[ + Iterator[Tuple[Tuple[Doc, Any], Any]], Iterator[Tuple[Doc, Any]], Iterator[Doc] + ]: ... + @overload + def __call__( + self, + doclike: Union[Doc, Span], + *, + as_spans: Literal[False] = ..., + allow_missing: bool = ..., + with_alignments: bool = ..., + ) -> List[Tuple[int, int, int]]: ... + @overload + def __call__( + self, + doclike: Union[Doc, Span], + *, + as_spans: Literal[True], + allow_missing: bool = ..., + with_alignments: bool = ..., + ) -> List[Span]: ... + def _normalize_key(self, key: Any) -> Any: ... diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx new file mode 100644 index 0000000..64c26c8 --- /dev/null +++ b/spacy/matcher/matcher.pyx @@ -0,0 +1,1219 @@ +# cython: binding=True, infer_types=True +from typing import Iterable, List + +from cymem.cymem cimport Pool +from libc.stdint cimport int8_t, int32_t +from libc.string cimport memcmp, memset +from libcpp.vector cimport vector +from murmurhash.mrmr cimport hash64 + +import re +import warnings + +import srsly + +from ..attrs cimport DEP, ENT_IOB, ID, LEMMA, MORPH, NULL_ATTR, POS, TAG +from ..structs cimport TokenC +from ..tokens.doc cimport Doc, get_token_attr_for_matcher +from ..tokens.morphanalysis cimport MorphAnalysis +from ..tokens.span cimport Span +from ..tokens.token cimport Token +from ..typedefs cimport attr_t + +from ..attrs import IDS +from ..errors import Errors, MatchPatternError, Warnings +from ..schemas import validate_token_pattern +from ..strings import get_string_id +from .levenshtein import levenshtein_compare + +DEF PADDING = 5 + + +cdef class Matcher: + """Match sequences of tokens, based on pattern rules. + + DOCS: https://spacy.io/api/matcher + USAGE: https://spacy.io/usage/rule-based-matching + """ + + def __init__(self, vocab, validate=True, *, fuzzy_compare=levenshtein_compare): + """Create the Matcher. + + vocab (Vocab): The vocabulary object, which must be shared with the + validate (bool): Validate all patterns added to this matcher. + fuzzy_compare (Callable[[str, str, int], bool]): The comparison method + for the FUZZY operators. + """ + self._extra_predicates = [] + self._patterns = {} + self._callbacks = {} + self._filter = {} + self._extensions = {} + self._seen_attrs = set() + self.vocab = vocab + self.mem = Pool() + self.validate = validate + self._fuzzy_compare = fuzzy_compare + + def __reduce__(self): + data = (self.vocab, self._patterns, self._callbacks, self.validate, self._fuzzy_compare) + return (unpickle_matcher, data, None, None) + + def __len__(self): + """Get the number of rules added to the matcher. Note that this only + returns the number of rules (identical with the number of IDs), not the + number of individual patterns. + + RETURNS (int): The number of rules. + """ + return len(self._patterns) + + def __contains__(self, key): + """Check whether the matcher contains rules for a match ID. + + key (str): The match ID. + RETURNS (bool): Whether the matcher contains rules for this match ID. + """ + return self.has_key(key) # no-cython-lint: W601 + + def add(self, key, patterns, *, on_match=None, greedy: str = None): + """Add a match-rule to the matcher. A match-rule consists of: an ID + key, an on_match callback, and one or more patterns. + + If the key exists, the patterns are appended to the previous ones, and + the previous on_match callback is replaced. The `on_match` callback + will receive the arguments `(matcher, doc, i, matches)`. You can also + set `on_match` to `None` to not perform any actions. + + A pattern consists of one or more `token_specs`, where a `token_spec` + is a dictionary mapping attribute IDs to values, and optionally a + quantifier operator under the key "op". The available quantifiers are: + + '!': Negate the pattern, by requiring it to match exactly 0 times. + '?': Make the pattern optional, by allowing it to match 0 or 1 times. + '+': Require the pattern to match 1 or more times. + '*': Allow the pattern to zero or more times. + '{n}': Require the pattern to match exactly _n_ times. + '{n,m}': Require the pattern to match at least _n_ but not more than _m_ times. + '{n,}': Require the pattern to match at least _n_ times. + '{,m}': Require the pattern to match at most _m_ times. + + The + and * operators return all possible matches (not just the greedy + ones). However, the "greedy" argument can filter the final matches + by returning a non-overlapping set per key, either taking preference to + the first greedy match ("FIRST"), or the longest ("LONGEST"). + + Since spaCy v2.2.2, Matcher.add takes a list of patterns as the second + argument, and the on_match callback is an optional keyword argument. + + key (Union[str, int]): The match ID. + patterns (list): The patterns to add for the given key. + on_match (callable): Optional callback executed on match. + greedy (str): Optional filter: "FIRST" or "LONGEST". + """ + errors = {} + if on_match is not None and not hasattr(on_match, "__call__"): + raise ValueError(Errors.E171.format(arg_type=type(on_match))) + if patterns is None or not isinstance(patterns, List): # old API + raise ValueError(Errors.E948.format(arg_type=type(patterns))) + if greedy is not None and greedy not in ["FIRST", "LONGEST"]: + raise ValueError(Errors.E947.format(expected=["FIRST", "LONGEST"], arg=greedy)) + for i, pattern in enumerate(patterns): + if len(pattern) == 0: + raise ValueError(Errors.E012.format(key=key)) + if not isinstance(pattern, list): + raise ValueError(Errors.E178.format(pat=pattern, key=key)) + if self.validate: + errors[i] = validate_token_pattern(pattern) + if any(err for err in errors.values()): + raise MatchPatternError(key, errors) + key = self._normalize_key(key) + for pattern in patterns: + try: + specs = _preprocess_pattern( + pattern, + self.vocab, + self._extensions, + self._extra_predicates, + self._fuzzy_compare + ) + self.patterns.push_back(init_pattern(self.mem, key, specs)) + for spec in specs: + for attr, _ in spec[1]: + self._seen_attrs.add(attr) + except OverflowError, AttributeError: + raise ValueError(Errors.E154.format()) from None + self._patterns.setdefault(key, []) + self._callbacks[key] = on_match + self._filter[key] = greedy + self._patterns[key].extend(patterns) + + def _require_patterns(self) -> None: + """Raise a warning if this component has no patterns defined.""" + if len(self) == 0: + warnings.warn(Warnings.W036.format(name="matcher")) + + def remove(self, key): + """Remove a rule from the matcher. A KeyError is raised if the key does + not exist. + + key (str): The ID of the match rule. + """ + norm_key = self._normalize_key(key) + if norm_key not in self._patterns: + raise ValueError(Errors.E175.format(key=key)) + self._patterns.pop(norm_key) + self._callbacks.pop(norm_key) + cdef int i = 0 + while i < self.patterns.size(): + pattern_key = get_ent_id(self.patterns.at(i)) + if pattern_key == norm_key: + self.patterns.erase(self.patterns.begin()+i) + else: + i += 1 + + def has_key(self, key): + """Check whether the matcher has a rule with a given key. + + key (string or int): The key to check. + RETURNS (bool): Whether the matcher has the rule. + """ + return self._normalize_key(key) in self._patterns + + def get(self, key, default=None): + """Retrieve the pattern stored for a key. + + key (str / int): The key to retrieve. + RETURNS (tuple): The rule, as an (on_match, patterns) tuple. + """ + key = self._normalize_key(key) + if key not in self._patterns: + return default + return (self._callbacks[key], self._patterns[key]) + + def pipe(self, docs, batch_size=1000, return_matches=False, as_tuples=False): + """Match a stream of documents, yielding them in turn. Deprecated as of + spaCy v3.0. + """ + warnings.warn(Warnings.W105.format(matcher="Matcher"), DeprecationWarning) + if as_tuples: + for doc, context in docs: + matches = self(doc) + if return_matches: + yield ((doc, matches), context) + else: + yield (doc, context) + else: + for doc in docs: + matches = self(doc) + if return_matches: + yield (doc, matches) + else: + yield doc + + def __call__(self, object doclike, *, as_spans=False, allow_missing=False, with_alignments=False): + """Find all token sequences matching the supplied pattern. + + doclike (Doc or Span): The document to match over. + as_spans (bool): Return Span objects with labels instead of (match_id, + start, end) tuples. + allow_missing (bool): Whether to skip checks for missing annotation for + attributes included in patterns. Defaults to False. + with_alignments (bool): Return match alignment information, which is + `List[int]` with length of matched span. Each entry denotes the + corresponding index of token pattern. If as_spans is set to True, + this setting is ignored. + RETURNS (list): A list of `(match_id, start, end)` tuples, + describing the matches. A match tuple describes a span + `doc[start:end]`. The `match_id` is an integer. If as_spans is set + to True, a list of Span objects is returned. + If with_alignments is set to True and as_spans is set to False, + A list of `(match_id, start, end, alignments)` tuples is returned. + """ + self._require_patterns() + if isinstance(doclike, Doc): + doc = doclike + length = len(doc) + elif isinstance(doclike, Span): + doc = doclike.doc + length = doclike.end - doclike.start + else: + raise ValueError(Errors.E195.format(good="Doc or Span", got=type(doclike).__name__)) + # Skip alignments calculations if as_spans is set + if as_spans: + with_alignments = False + cdef Pool tmp_pool = Pool() + if not allow_missing: + for attr in (TAG, POS, MORPH, LEMMA, DEP): + if attr in self._seen_attrs and not doc.has_annotation(attr): + if attr == TAG: + pipe = "tagger" + elif attr in (POS, MORPH): + pipe = "morphologizer or tagger+attribute_ruler" + elif attr == LEMMA: + pipe = "lemmatizer" + elif attr == DEP: + pipe = "parser" + error_msg = Errors.E155.format(pipe=pipe, attr=self.vocab.strings.as_string(attr)) + raise ValueError(error_msg) + + if self.patterns.empty(): + matches = [] + else: + matches = find_matches( + &self.patterns[0], + self.patterns.size(), + doclike, + length, + extensions=self._extensions, + predicates=self._extra_predicates, + with_alignments=with_alignments + ) + final_matches = [] + pairs_by_id = {} + # For each key, either add all matches, or only the filtered, + # non-overlapping ones this `match` can be either (start, end) or + # (start, end, alignments) depending on `with_alignments=` option. + for key, *match in matches: + span_filter = self._filter.get(key) + if span_filter is not None: + pairs = pairs_by_id.get(key, []) + pairs.append(match) + pairs_by_id[key] = pairs + else: + final_matches.append((key, *match)) + matched = tmp_pool.alloc(length, sizeof(char)) + empty = tmp_pool.alloc(length, sizeof(char)) + for key, pairs in pairs_by_id.items(): + memset(matched, 0, length * sizeof(matched[0])) + span_filter = self._filter.get(key) + if span_filter == "FIRST": + sorted_pairs = sorted(pairs, key=lambda x: (x[0], -x[1]), reverse=False) # sort by start + elif span_filter == "LONGEST": + sorted_pairs = sorted(pairs, key=lambda x: (x[1]-x[0], -x[0]), reverse=True) # reverse sort by length + else: + raise ValueError(Errors.E947.format(expected=["FIRST", "LONGEST"], arg=span_filter)) + for match in sorted_pairs: + start, end = match[:2] + assert 0 <= start < end # Defend against segfaults + span_len = end-start + # If no tokens in the span have matched + if memcmp(&matched[start], &empty[start], span_len * sizeof(matched[0])) == 0: + final_matches.append((key, *match)) + # Mark tokens that have matched + memset(&matched[start], 1, span_len * sizeof(matched[0])) + if as_spans: + final_results = [] + for key, start, end, *_ in final_matches: + if isinstance(doclike, Span): + start += doclike.start + end += doclike.start + final_results.append(Span(doc, start, end, label=key)) + elif with_alignments: + # convert alignments List[Dict[str, int]] --> List[int] + # when multiple alignment (belongs to the same length) is found, + # keeps the alignment that has largest token_idx + final_results = [] + for key, start, end, alignments in final_matches: + sorted_alignments = sorted(alignments, key=lambda x: (x['length'], x['token_idx']), reverse=False) + alignments = [0] * (end-start) + for align in sorted_alignments: + if align['length'] >= end-start: + continue + # Since alignments are sorted in order of (length, token_idx) + # this overwrites smaller token_idx when they have same length. + alignments[align['length']] = align['token_idx'] + final_results.append((key, start, end, alignments)) + final_matches = final_results # for callbacks + else: + final_results = final_matches + # perform the callbacks on the filtered set of results + for i, (key, *_) in enumerate(final_matches): + on_match = self._callbacks.get(key, None) + if on_match is not None: + on_match(self, doc, i, final_matches) + return final_results + + def _normalize_key(self, key): + if isinstance(key, str): + return self.vocab.strings.add(key) + else: + return key + + +def unpickle_matcher(vocab, patterns, callbacks, validate, fuzzy_compare): + matcher = Matcher(vocab, validate=validate, fuzzy_compare=fuzzy_compare) + for key, pattern in patterns.items(): + callback = callbacks.get(key, None) + matcher.add(key, pattern, on_match=callback) + return matcher + + +cdef find_matches(TokenPatternC** patterns, int n, object doclike, int length, extensions=None, predicates=tuple(), bint with_alignments=0): + """Find matches in a doc, with a compiled array of patterns. Matches are + returned as a list of (id, start, end) tuples or (id, start, end, alignments) tuples (if with_alignments != 0) + + To augment the compiled patterns, we optionally also take two Python lists. + + The "predicates" list contains functions that take a Python list and return a + boolean value. It's mostly used for regular expressions. + + The "extensions" list contains functions that take a Python list and return + an attr ID. It's mostly used for extension attributes. + """ + cdef vector[PatternStateC] states + cdef vector[MatchC] matches + cdef vector[vector[MatchAlignmentC]] align_states + cdef vector[vector[MatchAlignmentC]] align_matches + cdef int i, j, nr_extra_attr + cdef Pool mem = Pool() + output = [] + if length == 0: + # avoid any processing or mem alloc if the document is empty + return output + if len(predicates) > 0: + predicate_cache = mem.alloc(length * len(predicates), sizeof(int8_t)) + if extensions is not None and len(extensions) >= 1: + nr_extra_attr = max(extensions.values()) + 1 + extra_attr_values = mem.alloc(length * nr_extra_attr, sizeof(attr_t)) + else: + nr_extra_attr = 0 + extra_attr_values = mem.alloc(length, sizeof(attr_t)) + for i, token in enumerate(doclike): + for name, index in extensions.items(): + value = token._.get(name) + if isinstance(value, str): + value = token.vocab.strings[value] + extra_attr_values[i * nr_extra_attr + index] = value + # Main loop + for i in range(length): + for j in range(n): + states.push_back(PatternStateC(patterns[j], i, 0)) + if with_alignments != 0: + align_states.resize(states.size()) + transition_states( + states, + matches, + align_states, + align_matches, + predicate_cache, + doclike[i], + extra_attr_values, + predicates, + with_alignments + ) + extra_attr_values += nr_extra_attr + predicate_cache += len(predicates) + # Handle matches that end in 0-width patterns + finish_states(matches, states, align_matches, align_states, with_alignments) + seen = set() + for i in range(matches.size()): + match = ( + matches[i].pattern_id, + matches[i].start, + matches[i].start+matches[i].length + ) + # We need to deduplicate, because we could otherwise arrive at the same + # match through two paths, e.g. .?.? matching 'a'. Are we matching the + # first .?, or the second .? -- it doesn't matter, it's just one match. + # Skip 0-length matches. (TODO: fix algorithm) + if match not in seen and matches[i].length > 0: + if with_alignments != 0: + # since the length of align_matches equals to that of match, we can share same 'i' + output.append(match + (align_matches[i],)) + else: + output.append(match) + seen.add(match) + return output + + +cdef void transition_states( + vector[PatternStateC]& states, + vector[MatchC]& matches, + vector[vector[MatchAlignmentC]]& align_states, + vector[vector[MatchAlignmentC]]& align_matches, + int8_t* cached_py_predicates, + Token token, + const attr_t* extra_attrs, + py_predicates, + bint with_alignments +) except *: + cdef int q = 0 + cdef vector[PatternStateC] new_states + cdef vector[vector[MatchAlignmentC]] align_new_states + for i in range(states.size()): + if states[i].pattern.nr_py >= 1: + update_predicate_cache( + cached_py_predicates, + states[i].pattern, + token, + py_predicates + ) + action = get_action(states[i], token.c, extra_attrs, + cached_py_predicates) + if action == REJECT: + continue + # Keep only a subset of states (the active ones). Index q is the + # states which are still alive. If we reject a state, we overwrite + # it in the states list, because q doesn't advance. + state = states[i] + states[q] = state + # Separate from states, performance is guaranteed for users who only need basic options (without alignments). + # `align_states` always corresponds to `states` 1:1. + if with_alignments != 0: + align_state = align_states[i] + align_states[q] = align_state + while action in (RETRY, RETRY_ADVANCE, RETRY_EXTEND): + # Update alignment before the transition of current state + # 'MatchAlignmentC' maps 'original token index of current pattern' to 'current matching length' + if with_alignments != 0: + align_states[q].push_back(MatchAlignmentC(states[q].pattern.token_idx, states[q].length)) + if action == RETRY_EXTEND: + # This handles the 'extend' + new_states.push_back( + PatternStateC(pattern=states[q].pattern, start=state.start, + length=state.length+1)) + if with_alignments != 0: + align_new_states.push_back(align_states[q]) + if action == RETRY_ADVANCE: + # This handles the 'advance' + new_states.push_back( + PatternStateC(pattern=states[q].pattern+1, start=state.start, + length=state.length+1)) + if with_alignments != 0: + align_new_states.push_back(align_states[q]) + states[q].pattern += 1 + if states[q].pattern.nr_py != 0: + update_predicate_cache( + cached_py_predicates, + states[q].pattern, + token, + py_predicates + ) + action = get_action(states[q], token.c, extra_attrs, + cached_py_predicates) + # Update alignment before the transition of current state + if with_alignments != 0: + align_states[q].push_back(MatchAlignmentC(states[q].pattern.token_idx, states[q].length)) + if action == REJECT: + pass + elif action == ADVANCE: + states[q].pattern += 1 + states[q].length += 1 + q += 1 + else: + ent_id = get_ent_id(state.pattern) + if action == MATCH: + matches.push_back( + MatchC( + pattern_id=ent_id, + start=state.start, + length=state.length+1 + ) + ) + # `align_matches` always corresponds to `matches` 1:1 + if with_alignments != 0: + align_matches.push_back(align_states[q]) + elif action == MATCH_DOUBLE: + # push match without last token if length > 0 + if state.length > 0: + matches.push_back( + MatchC( + pattern_id=ent_id, + start=state.start, + length=state.length + ) + ) + # MATCH_DOUBLE emits matches twice, + # add one more to align_matches in order to keep 1:1 relationship + if with_alignments != 0: + align_matches.push_back(align_states[q]) + # push match with last token + matches.push_back( + MatchC( + pattern_id=ent_id, + start=state.start, + length=state.length + 1 + ) + ) + # `align_matches` always corresponds to `matches` 1:1 + if with_alignments != 0: + align_matches.push_back(align_states[q]) + elif action == MATCH_REJECT: + matches.push_back( + MatchC( + pattern_id=ent_id, + start=state.start, + length=state.length + ) + ) + # `align_matches` always corresponds to `matches` 1:1 + if with_alignments != 0: + align_matches.push_back(align_states[q]) + elif action == MATCH_EXTEND: + matches.push_back( + MatchC(pattern_id=ent_id, start=state.start, + length=state.length)) + # `align_matches` always corresponds to `matches` 1:1 + if with_alignments != 0: + align_matches.push_back(align_states[q]) + states[q].length += 1 + q += 1 + states.resize(q) + for i in range(new_states.size()): + states.push_back(new_states[i]) + # `align_states` always corresponds to `states` 1:1 + if with_alignments != 0: + align_states.resize(q) + for i in range(align_new_states.size()): + align_states.push_back(align_new_states[i]) + + +cdef int update_predicate_cache( + int8_t* cache, + const TokenPatternC* pattern, + Token token, + predicates +) except -1: + # If the state references any extra predicates, check whether they match. + # These are cached, so that we don't call these potentially expensive + # Python functions more than we need to. + for i in range(pattern.nr_py): + index = pattern.py_predicates[i] + if cache[index] == 0: + predicate = predicates[index] + result = predicate(token) + if result is True: + cache[index] = 1 + elif result is False: + cache[index] = -1 + elif result is None: + pass + else: + raise ValueError(Errors.E125.format(value=result)) + + +cdef void finish_states(vector[MatchC]& matches, vector[PatternStateC]& states, + vector[vector[MatchAlignmentC]]& align_matches, + vector[vector[MatchAlignmentC]]& align_states, + bint with_alignments) except *: + """Handle states that end in zero-width patterns.""" + cdef PatternStateC state + cdef vector[MatchAlignmentC] align_state + for i in range(states.size()): + state = states[i] + if with_alignments != 0: + align_state = align_states[i] + while get_quantifier(state) in (ZERO_PLUS, ZERO_ONE): + # Update alignment before the transition of current state + if with_alignments != 0: + align_state.push_back(MatchAlignmentC(state.pattern.token_idx, state.length)) + is_final = get_is_final(state) + if is_final: + ent_id = get_ent_id(state.pattern) + # `align_matches` always corresponds to `matches` 1:1 + if with_alignments != 0: + align_matches.push_back(align_state) + matches.push_back( + MatchC(pattern_id=ent_id, start=state.start, length=state.length)) + break + else: + state.pattern += 1 + +cdef action_t get_action( + PatternStateC state, + const TokenC * token, + const attr_t * extra_attrs, + const int8_t * predicate_matches +) noexcept nogil: + """We need to consider: + a) Does the token match the specification? [Yes, No] + b) What's the quantifier? [1, 0+, ?] + c) Is this the last specification? [final, non-final] + + We can transition in the following ways: + a) Do we emit a match? + b) Do we add a state with (next state, next token)? + c) Do we add a state with (next state, same token)? + d) Do we add a state with (same state, next token)? + + We'll code the actions as boolean strings, so 0000 means no to all 4, + 1000 means match but no states added, etc. + + 1: + Yes, final: + 1000 + Yes, non-final: + 0100 + No, final: + 0000 + No, non-final + 0000 + 0+: + Yes, final: + 1001 + Yes, non-final: + 0011 + No, final: + 1000 (note: Don't include last token!) + No, non-final: + 0010 + ?: + Yes, final: + 1000 + Yes, non-final: + 0100 + No, final: + 1000 (note: Don't include last token!) + No, non-final: + 0010 + + Possible combinations: 1000, 0100, 0000, 1001, 0110, 0011, 0010, + + We'll name the bits "match", "advance", "retry", "extend" + REJECT = 0000 + MATCH = 1000 + ADVANCE = 0100 + RETRY = 0010 + MATCH_EXTEND = 1001 + RETRY_ADVANCE = 0110 + RETRY_EXTEND = 0011 + MATCH_REJECT = 2000 # Match, but don't include last token + MATCH_DOUBLE = 3000 # Match both with and without last token + + Problem: If a quantifier is matching, we're adding a lot of open partials + """ + cdef int8_t is_match + is_match = get_is_match(state, token, extra_attrs, predicate_matches) + quantifier = get_quantifier(state) + is_final = get_is_final(state) + if quantifier == ZERO: + is_match = not is_match + quantifier = ONE + if quantifier == ONE: + if is_match and is_final: + # Yes, final: 1000 + return MATCH + elif is_match and not is_final: + # Yes, non-final: 0100 + return ADVANCE + elif not is_match and is_final: + # No, final: 0000 + return REJECT + else: + return REJECT + elif quantifier == ZERO_PLUS: + if is_match and is_final: + # Yes, final: 1001 + return MATCH_EXTEND + elif is_match and not is_final: + # Yes, non-final: 0011 + return RETRY_EXTEND + elif not is_match and is_final: + # No, final 2000 (note: Don't include last token!) + return MATCH_REJECT + else: + # No, non-final 0010 + return RETRY + elif quantifier == ZERO_ONE: + if is_match and is_final: + # Yes, final: 3000 + # To cater for a pattern ending in "?", we need to add + # a match both with and without the last token + return MATCH_DOUBLE + elif is_match and not is_final: + # Yes, non-final: 0110 + # We need both branches here, consider a pair like: + # pattern: .?b string: b + # If we 'ADVANCE' on the .?, we miss the match. + return RETRY_ADVANCE + elif not is_match and is_final: + # No, final 2000 (note: Don't include last token!) + return MATCH_REJECT + else: + # No, non-final 0010 + return RETRY + + +cdef int8_t get_is_match( + PatternStateC state, + const TokenC* token, + const attr_t* extra_attrs, + const int8_t* predicate_matches +) noexcept nogil: + for i in range(state.pattern.nr_py): + if predicate_matches[state.pattern.py_predicates[i]] == -1: + return 0 + spec = state.pattern + if spec.nr_attr > 0: + for attr in spec.attrs[:spec.nr_attr]: + if get_token_attr_for_matcher(token, attr.attr) != attr.value: + return 0 + for i in range(spec.nr_extra_attr): + if spec.extra_attrs[i].value != extra_attrs[spec.extra_attrs[i].index]: + return 0 + return True + + +cdef inline int8_t get_is_final(PatternStateC state) noexcept nogil: + if state.pattern[1].quantifier == FINAL_ID: + return 1 + else: + return 0 + + +cdef inline int8_t get_quantifier(PatternStateC state) noexcept nogil: + return state.pattern.quantifier + + +cdef TokenPatternC* init_pattern(Pool mem, attr_t entity_id, object token_specs) except NULL: + pattern = mem.alloc(len(token_specs) + 1, sizeof(TokenPatternC)) + cdef int i, index + for i, (quantifier, spec, extensions, predicates, token_idx) in enumerate(token_specs): + pattern[i].quantifier = quantifier + # Ensure attrs refers to a null pointer if nr_attr == 0 + if len(spec) > 0: + pattern[i].attrs = mem.alloc(len(spec), sizeof(AttrValueC)) + pattern[i].nr_attr = len(spec) + for j, (attr, value) in enumerate(spec): + pattern[i].attrs[j].attr = attr + pattern[i].attrs[j].value = value + if len(extensions) > 0: + pattern[i].extra_attrs = mem.alloc(len(extensions), sizeof(IndexValueC)) + for j, (index, value) in enumerate(extensions): + pattern[i].extra_attrs[j].index = index + pattern[i].extra_attrs[j].value = value + pattern[i].nr_extra_attr = len(extensions) + if len(predicates) > 0: + pattern[i].py_predicates = mem.alloc(len(predicates), sizeof(int32_t)) + for j, index in enumerate(predicates): + pattern[i].py_predicates[j] = index + pattern[i].nr_py = len(predicates) + pattern[i].key = hash64(pattern[i].attrs, pattern[i].nr_attr * sizeof(AttrValueC), 0) + pattern[i].token_idx = token_idx + i = len(token_specs) + # Use quantifier to identify final ID pattern node (rather than previous + # uninitialized quantifier == 0/ZERO + nr_attr == 0 + non-zero-length attrs) + pattern[i].quantifier = FINAL_ID + pattern[i].attrs = mem.alloc(1, sizeof(AttrValueC)) + pattern[i].attrs[0].attr = ID + pattern[i].attrs[0].value = entity_id + pattern[i].nr_attr = 1 + pattern[i].nr_extra_attr = 0 + pattern[i].nr_py = 0 + pattern[i].token_idx = -1 + return pattern + + +cdef attr_t get_ent_id(const TokenPatternC* pattern) noexcept nogil: + while pattern.quantifier != FINAL_ID: + pattern += 1 + id_attr = pattern[0].attrs[0] + if id_attr.attr != ID: + with gil: + raise ValueError(Errors.E074.format(attr=ID, bad_attr=id_attr.attr)) + return id_attr.value + + +def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates, fuzzy_compare): + """This function interprets the pattern, converting the various bits of + syntactic sugar before we compile it into a struct with init_pattern. + + We need to split the pattern up into four parts: + * Normal attribute/value pairs, which are stored on either the token or lexeme, + can be handled directly. + * Extension attributes are handled specially, as we need to prefetch the + values from Python for the doc before we begin matching. + * Extra predicates also call Python functions, so we have to create the + functions and store them. So we store these specially as well. + * Extension attributes that have extra predicates are stored within the + extra_predicates. + * Token index that this pattern belongs to. + """ + tokens = [] + string_store = vocab.strings + for token_idx, spec in enumerate(token_specs): + if not spec: + # Signifier for 'any token' + tokens.append((ONE, [(NULL_ATTR, 0)], [], [], token_idx)) + continue + if not isinstance(spec, dict): + raise ValueError(Errors.E154.format()) + ops = _get_operators(spec) + attr_values = _get_attr_values(spec, string_store) + extensions = _get_extensions(spec, string_store, extensions_table) + predicates = _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare) + for op in ops: + tokens.append((op, list(attr_values), list(extensions), list(predicates), token_idx)) + return tokens + + +def _get_attr_values(spec, string_store): + attr_values = [] + for attr, value in spec.items(): + input_attr = attr + if isinstance(attr, str): + attr = attr.upper() + if attr == '_': + continue + elif attr == "OP": + continue + if attr == "TEXT": + attr = "ORTH" + if attr == "IS_SENT_START": + attr = "SENT_START" + attr = IDS.get(attr) + if isinstance(value, str): + if attr == ENT_IOB and value in Token.iob_strings(): + value = Token.iob_strings().index(value) + else: + value = string_store.add(value) + elif isinstance(value, bool): + value = int(value) + elif isinstance(value, int): + pass + elif isinstance(value, dict): + continue + else: + raise ValueError(Errors.E153.format(vtype=type(value).__name__)) + if attr is not None: + attr_values.append((attr, value)) + else: + # should be caught in validation + raise ValueError(Errors.E152.format(attr=input_attr)) + return attr_values + + +def _predicate_cache_key(attr, predicate, value, *, regex=False, fuzzy=None): + # tuple order affects performance + return (attr, regex, fuzzy, predicate, srsly.json_dumps(value, sort_keys=True)) + + +# These predicate helper classes are used to match the REGEX, IN, >= etc +# extensions to the matcher introduced in #3173. + +class _FuzzyPredicate: + operators = ("FUZZY", "FUZZY1", "FUZZY2", "FUZZY3", "FUZZY4", "FUZZY5", + "FUZZY6", "FUZZY7", "FUZZY8", "FUZZY9") + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.value = value + self.predicate = predicate + self.is_extension = is_extension + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + fuzz = self.predicate[len("FUZZY"):] # number after prefix + self.fuzzy = int(fuzz) if fuzz else -1 + self.fuzzy_compare = fuzzy_compare + self.key = _predicate_cache_key(self.attr, self.predicate, value, fuzzy=self.fuzzy) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)] + if self.value == value: + return True + return self.fuzzy_compare(value, self.value, self.fuzzy) + + +class _RegexPredicate: + operators = ("REGEX",) + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.value = re.compile(value) + self.predicate = predicate + self.is_extension = is_extension + self.key = _predicate_cache_key(self.attr, self.predicate, value) + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)] + return bool(self.value.search(value)) + + +class _SetPredicate: + operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS") + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.vocab = vocab + self.regex = regex + self.fuzzy = fuzzy + self.fuzzy_compare = fuzzy_compare + if self.attr == MORPH: + # normalize morph strings + self.value = set(self.vocab.morphology.add(v) for v in value) + else: + if self.regex: + self.value = set(re.compile(v) for v in value) + elif self.fuzzy is not None: + # add to string store + self.value = set(self.vocab.strings.add(v) for v in value) + else: + self.value = set(get_string_id(v) for v in value) + self.predicate = predicate + self.is_extension = is_extension + self.key = _predicate_cache_key(self.attr, self.predicate, value, regex=self.regex, fuzzy=self.fuzzy) + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = get_token_attr_for_matcher(token.c, self.attr) + + if self.predicate in ("IN", "NOT_IN"): + if isinstance(value, (str, int)): + value = get_string_id(value) + else: + return False + elif self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"): + # ensure that all values are enclosed in a set + if self.attr == MORPH: + # break up MORPH into individual Feat=Val values + value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value)) + elif isinstance(value, (str, int)): + value = set((get_string_id(value),)) + elif isinstance(value, Iterable) and all(isinstance(v, (str, int)) for v in value): + value = set(get_string_id(v) for v in value) + else: + return False + + if self.predicate == "IN": + if self.regex: + value = self.vocab.strings[value] + return any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return True + else: + return False + elif self.predicate == "NOT_IN": + if self.regex: + value = self.vocab.strings[value] + return not any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return not any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return False + else: + return True + elif self.predicate == "IS_SUBSET": + return value <= self.value + elif self.predicate == "IS_SUPERSET": + return value >= self.value + elif self.predicate == "INTERSECTS": + return bool(value & self.value) + + def __repr__(self): + return repr(("SetPredicate", self.i, self.attr, self.value, self.predicate)) + + +class _ComparisonPredicate: + operators = ("==", "!=", ">=", "<=", ">", "<") + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.value = value + self.predicate = predicate + self.is_extension = is_extension + self.key = _predicate_cache_key(self.attr, self.predicate, value) + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = get_token_attr_for_matcher(token.c, self.attr) + if self.predicate == "==": + return value == self.value + if self.predicate == "!=": + return value != self.value + elif self.predicate == ">=": + return value >= self.value + elif self.predicate == "<=": + return value <= self.value + elif self.predicate == ">": + return value > self.value + elif self.predicate == "<": + return value < self.value + + +def _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare): + predicate_types = { + "REGEX": _RegexPredicate, + "IN": _SetPredicate, + "NOT_IN": _SetPredicate, + "IS_SUBSET": _SetPredicate, + "IS_SUPERSET": _SetPredicate, + "INTERSECTS": _SetPredicate, + "==": _ComparisonPredicate, + "!=": _ComparisonPredicate, + ">=": _ComparisonPredicate, + "<=": _ComparisonPredicate, + ">": _ComparisonPredicate, + "<": _ComparisonPredicate, + "FUZZY": _FuzzyPredicate, + "FUZZY1": _FuzzyPredicate, + "FUZZY2": _FuzzyPredicate, + "FUZZY3": _FuzzyPredicate, + "FUZZY4": _FuzzyPredicate, + "FUZZY5": _FuzzyPredicate, + "FUZZY6": _FuzzyPredicate, + "FUZZY7": _FuzzyPredicate, + "FUZZY8": _FuzzyPredicate, + "FUZZY9": _FuzzyPredicate, + } + seen_predicates = {pred.key: pred.i for pred in extra_predicates} + output = [] + for attr, value in spec.items(): + if isinstance(attr, str): + if attr == "_": + output.extend( + _get_extension_extra_predicates( + value, extra_predicates, predicate_types, + seen_predicates)) + continue + elif attr.upper() == "OP": + continue + if attr.upper() == "TEXT": + attr = "ORTH" + attr = IDS.get(attr.upper()) + if isinstance(value, dict): + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, fuzzy_compare=fuzzy_compare)) + return output + + +def _get_extra_predicates_dict(attr, value_dict, vocab, predicate_types, + extra_predicates, seen_predicates, regex=False, fuzzy=None, fuzzy_compare=None): + output = [] + for type_, value in value_dict.items(): + type_ = type_.upper() + cls = predicate_types.get(type_) + if cls is None: + warnings.warn(Warnings.W035.format(pattern=value_dict)) + # ignore unrecognized predicate type + continue + elif cls == _RegexPredicate: + if isinstance(value, dict): + # add predicates inside regex operator + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + regex=True)) + continue + elif cls == _FuzzyPredicate: + if isinstance(value, dict): + # add predicates inside fuzzy operator + fuzz = type_[len("FUZZY"):] # number after prefix + fuzzy_val = int(fuzz) if fuzz else -1 + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + fuzzy=fuzzy_val, fuzzy_compare=fuzzy_compare)) + continue + predicate = cls(len(extra_predicates), attr, value, type_, vocab=vocab, + regex=regex, fuzzy=fuzzy, fuzzy_compare=fuzzy_compare) + # Don't create redundant predicates. + # This helps with efficiency, as we're caching the results. + if predicate.key in seen_predicates: + output.append(seen_predicates[predicate.key]) + else: + extra_predicates.append(predicate) + output.append(predicate.i) + seen_predicates[predicate.key] = predicate.i + return output + + +def _get_extension_extra_predicates( + spec, extra_predicates, predicate_types, seen_predicates +): + output = [] + for attr, value in spec.items(): + if isinstance(value, dict): + for type_, cls in predicate_types.items(): + if type_ in value: + key = _predicate_cache_key(attr, type_, value[type_]) + if key in seen_predicates: + output.append(seen_predicates[key]) + else: + predicate = cls(len(extra_predicates), attr, value[type_], type_, + is_extension=True) + extra_predicates.append(predicate) + output.append(predicate.i) + seen_predicates[key] = predicate.i + return output + + +def _get_operators(spec): + # Support 'syntactic sugar' operator '+', as combination of ONE, ZERO_PLUS + lookup = {"*": (ZERO_PLUS,), "+": (ONE, ZERO_PLUS), + "?": (ZERO_ONE,), "1": (ONE,), "!": (ZERO,)} + # Fix casing + spec = {key.upper(): values for key, values in spec.items() + if isinstance(key, str)} + if "OP" not in spec: + return (ONE,) + elif spec["OP"] in lookup: + return lookup[spec["OP"]] + # Min_max {n,m} + elif spec["OP"].startswith("{") and spec["OP"].endswith("}"): + # {n} --> {n,n} exactly n ONE,(n) + # {n,m}--> {n,m} min of n, max of m ONE,(n),ZERO_ONE,(m) + # {,m} --> {0,m} min of zero, max of m ZERO_ONE,(m) + # {n,} --> {n,∞} min of n, max of inf ONE,(n),ZERO_PLUS + + min_max = spec["OP"][1:-1] + min_max = min_max if "," in min_max else f"{min_max},{min_max}" + n, m = min_max.split(",") + + # 1. Either n or m is a blank string and the other is numeric -->isdigit + # 2. Both are numeric and n <= m + if (not n.isdecimal() and not m.isdecimal()) or (n.isdecimal() and m.isdecimal() and int(n) > int(m)): + keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m " + raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys)) + + # if n is empty string, zero would be used + head = tuple(ONE for __ in range(int(n or 0))) + tail = tuple(ZERO_ONE for __ in range(int(m) - int(n or 0))) if m else (ZERO_PLUS,) + return head + tail + else: + keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m " + raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys)) + + +def _get_extensions(spec, string_store, name2index): + attr_values = [] + if not isinstance(spec.get("_", {}), dict): + raise ValueError(Errors.E154.format()) + for name, value in spec.get("_", {}).items(): + if isinstance(value, dict): + # Handle predicates (e.g. "IN", in the extra_predicates, not here. + continue + if isinstance(value, str): + value = string_store.add(value) + if name not in name2index: + name2index[name] = len(name2index) + attr_values.append((name2index[name], value)) + return attr_values diff --git a/spacy/matcher/phrasematcher.pxd b/spacy/matcher/phrasematcher.pxd new file mode 100644 index 0000000..bffc1ac --- /dev/null +++ b/spacy/matcher/phrasematcher.pxd @@ -0,0 +1,21 @@ +from cymem.cymem cimport Pool +from libcpp.vector cimport vector +from preshed.maps cimport MapStruct, key_t + +from ..attrs cimport attr_id_t +from ..structs cimport SpanC +from ..tokens.doc cimport Doc +from ..vocab cimport Vocab + + +cdef class PhraseMatcher: + cdef readonly Vocab vocab + cdef attr_id_t attr + cdef object _callbacks + cdef object _docs + cdef bint _validate + cdef MapStruct* c_map + cdef Pool mem + cdef key_t _terminal_hash + + cdef void find_matches(self, Doc doc, int start_idx, int end_idx, vector[SpanC] *matches) nogil diff --git a/spacy/matcher/phrasematcher.pyi b/spacy/matcher/phrasematcher.pyi new file mode 100644 index 0000000..0f56699 --- /dev/null +++ b/spacy/matcher/phrasematcher.pyi @@ -0,0 +1,38 @@ +from typing import Any, Callable, List, Optional, Tuple, Union, overload + +from ..compat import Literal +from ..tokens import Doc, Span +from ..vocab import Vocab +from .matcher import Matcher + +class PhraseMatcher: + def __init__( + self, vocab: Vocab, attr: Optional[Union[int, str]] = ..., validate: bool = ... + ) -> None: ... + def __reduce__(self) -> Any: ... + def __len__(self) -> int: ... + def __contains__(self, key: str) -> bool: ... + def add( + self, + key: str, + docs: List[Doc], + *, + on_match: Optional[ + Callable[[Matcher, Doc, int, List[Tuple[Any, ...]]], Any] + ] = ..., + ) -> None: ... + def remove(self, key: str) -> None: ... + @overload + def __call__( + self, + doclike: Union[Doc, Span], + *, + as_spans: Literal[False] = ..., + ) -> List[Tuple[int, int, int]]: ... + @overload + def __call__( + self, + doclike: Union[Doc, Span], + *, + as_spans: Literal[True], + ) -> List[Span]: ... diff --git a/spacy/matcher/phrasematcher.pyx b/spacy/matcher/phrasematcher.pyx new file mode 100644 index 0000000..a71f85f --- /dev/null +++ b/spacy/matcher/phrasematcher.pyx @@ -0,0 +1,357 @@ +# cython: infer_types=True +from preshed.maps cimport map_clear, map_get, map_init, map_iter, map_set + +import warnings + +from ..attrs cimport DEP, LEMMA, MORPH, POS, TAG + +from ..attrs import IDS + +from ..tokens.span cimport Span +from ..tokens.token cimport Token +from ..typedefs cimport attr_t + +from ..errors import Errors, Warnings +from ..schemas import TokenPattern + + +cdef class PhraseMatcher: + """Efficiently match large terminology lists. While the `Matcher` matches + sequences based on lists of token descriptions, the `PhraseMatcher` accepts + match patterns in the form of `Doc` objects. + + DOCS: https://spacy.io/api/phrasematcher + USAGE: https://spacy.io/usage/rule-based-matching#phrasematcher + + Adapted from FlashText: https://github.com/vi3k6i5/flashtext + MIT License (see `LICENSE`) + Copyright (c) 2017 Vikash Singh (vikash.duliajan@gmail.com) + """ + + def __init__(self, Vocab vocab, attr="ORTH", validate=False): + """Initialize the PhraseMatcher. + + vocab (Vocab): The shared vocabulary. + attr (int / str): Token attribute to match on. + validate (bool): Perform additional validation when patterns are added. + + DOCS: https://spacy.io/api/phrasematcher#init + """ + self.vocab = vocab + self._callbacks = {} + self._docs = {} + self._validate = validate + + self.mem = Pool() + self.c_map = self.mem.alloc(1, sizeof(MapStruct)) + self._terminal_hash = 826361138722620965 + map_init(self.mem, self.c_map, 8) + + if isinstance(attr, int): + self.attr = attr + else: + if attr is None: + attr = "ORTH" + attr = attr.upper() + if attr == "TEXT": + attr = "ORTH" + if attr == "IS_SENT_START": + attr = "SENT_START" + if attr.lower() not in TokenPattern().model_dump(): + raise ValueError(Errors.E152.format(attr=attr)) + self.attr = IDS.get(attr) + + def __len__(self): + """Get the number of match IDs added to the matcher. + + RETURNS (int): The number of rules. + + DOCS: https://spacy.io/api/phrasematcher#len + """ + return len(self._callbacks) + + def __contains__(self, key): + """Check whether the matcher contains rules for a match ID. + + key (str): The match ID. + RETURNS (bool): Whether the matcher contains rules for this match ID. + + DOCS: https://spacy.io/api/phrasematcher#contains + """ + return key in self._callbacks + + def __reduce__(self): + data = (self.vocab, self._docs, self._callbacks, self.attr) + return (unpickle_matcher, data, None, None) + + def remove(self, key): + """Remove a rule from the matcher by match ID. A KeyError is raised if + the key does not exist. + + key (str): The match ID. + + DOCS: https://spacy.io/api/phrasematcher#remove + """ + if key not in self._docs: + raise KeyError(key) + cdef MapStruct* current_node + cdef MapStruct* terminal_map + cdef MapStruct* node_pointer + cdef void* result + cdef key_t terminal_key + cdef void* value + cdef int c_i = 0 + cdef vector[MapStruct*] path_nodes + cdef vector[key_t] path_keys + cdef key_t key_to_remove + for keyword in sorted(self._docs[key], key=lambda x: len(x), reverse=True): + current_node = self.c_map + path_nodes.clear() + path_keys.clear() + for token in keyword: + result = map_get(current_node, token) + if result: + path_nodes.push_back(current_node) + path_keys.push_back(token) + current_node = result + else: + # if token is not found, break out of the loop + current_node = NULL + break + path_nodes.push_back(current_node) + path_keys.push_back(self._terminal_hash) + # remove the tokens from trie node if there are no other + # keywords with them + result = map_get(current_node, self._terminal_hash) + if current_node != NULL and result: + terminal_map = result + terminal_keys = [] + c_i = 0 + while map_iter(terminal_map, &c_i, &terminal_key, &value): + terminal_keys.append(self.vocab.strings[terminal_key]) + # if this is the only remaining key, remove unnecessary paths + if terminal_keys == [key]: + while not path_nodes.empty(): + node_pointer = path_nodes.back() + path_nodes.pop_back() + key_to_remove = path_keys.back() + path_keys.pop_back() + result = map_get(node_pointer, key_to_remove) + if node_pointer.filled == 1: + map_clear(node_pointer, key_to_remove) + self.mem.free(result) + else: + # more than one key means more than 1 path, + # delete not required path and keep the others + map_clear(node_pointer, key_to_remove) + self.mem.free(result) + break + # otherwise simply remove the key + else: + result = map_get(current_node, self._terminal_hash) + if result: + map_clear(result, self.vocab.strings[key]) + + del self._callbacks[key] + del self._docs[key] + + def add(self, key, docs, *_docs, on_match=None): + """Add a match-rule to the phrase-matcher. A match-rule consists of: an ID + key, an on_match callback, and one or more patterns. + + Since spaCy v2.2.2, PhraseMatcher.add takes a list of patterns as the + second argument, with the on_match callback as an optional keyword + argument. + + key (str): The match ID. + docs (list): List of `Doc` objects representing match patterns. + on_match (callable): Callback executed on match. + *_docs (Doc): For backwards compatibility: list of patterns to add + as variable arguments. Will be ignored if a list of patterns is + provided as the second argument. + + DOCS: https://spacy.io/api/phrasematcher#add + """ + if docs is None or hasattr(docs, "__call__"): # old API + on_match = docs + docs = _docs + + _ = self.vocab[key] + self._callbacks[key] = on_match + self._docs.setdefault(key, set()) + + cdef MapStruct* current_node + cdef MapStruct* internal_node + cdef void* result + + if isinstance(docs, Doc): + raise ValueError(Errors.E179.format(key=key)) + for doc in docs: + if len(doc) == 0: + continue + if isinstance(doc, Doc): + attrs = (TAG, POS, MORPH, LEMMA, DEP) + has_annotation = {attr: doc.has_annotation(attr) for attr in attrs} + for attr in attrs: + if self.attr == attr and not has_annotation[attr]: + if attr == TAG: + pipe = "tagger" + elif attr in (POS, MORPH): + pipe = "morphologizer or tagger+attribute_ruler" + elif attr == LEMMA: + pipe = "lemmatizer" + elif attr == DEP: + pipe = "parser" + error_msg = Errors.E155.format(pipe=pipe, attr=self.vocab.strings.as_string(attr)) + raise ValueError(error_msg) + if self._validate and any(has_annotation.values()) \ + and self.attr not in attrs: + string_attr = self.vocab.strings[self.attr] + warnings.warn(Warnings.W012.format(key=key, attr=string_attr)) + keyword = self._convert_to_array(doc) + else: + keyword = doc + self._docs[key].add(tuple(keyword)) + + current_node = self.c_map + for token in keyword: + if token == self._terminal_hash: + warnings.warn(Warnings.W021) + break + result = map_get(current_node, token) + if not result: + internal_node = self.mem.alloc(1, sizeof(MapStruct)) + map_init(self.mem, internal_node, 8) + map_set(self.mem, current_node, token, internal_node) + result = internal_node + current_node = result + result = map_get(current_node, self._terminal_hash) + if not result: + internal_node = self.mem.alloc(1, sizeof(MapStruct)) + map_init(self.mem, internal_node, 8) + map_set(self.mem, current_node, self._terminal_hash, internal_node) + result = internal_node + map_set(self.mem, result, self.vocab.strings[key], NULL) + + def __call__(self, object doclike, *, as_spans=False): + """Find all sequences matching the supplied patterns on the `Doc`. + + doclike (Doc or Span): The document to match over. + as_spans (bool): Return Span objects with labels instead of (match_id, + start, end) tuples. + RETURNS (list): A list of `(match_id, start, end)` tuples, + describing the matches. A match tuple describes a span + `doc[start:end]`. The `match_id` is an integer. If as_spans is set + to True, a list of Span objects is returned. + + DOCS: https://spacy.io/api/phrasematcher#call + """ + matches = [] + if doclike is None or len(doclike) == 0: + # if doc is empty or None just return empty list + return matches + if isinstance(doclike, Doc): + doc = doclike + start_idx = 0 + end_idx = len(doc) + elif isinstance(doclike, Span): + doc = doclike.doc + start_idx = doclike.start + end_idx = doclike.end + else: + raise ValueError(Errors.E195.format(good="Doc or Span", got=type(doclike).__name__)) + + cdef vector[SpanC] c_matches + self.find_matches(doc, start_idx, end_idx, &c_matches) + for i in range(c_matches.size()): + matches.append((c_matches[i].label, c_matches[i].start, c_matches[i].end)) + for i, (ent_id, start, end) in enumerate(matches): + on_match = self._callbacks.get(self.vocab.strings[ent_id]) + if on_match is not None: + on_match(self, doc, i, matches) + if as_spans: + return [Span(doc, start, end, label=key) for key, start, end in matches] + else: + return matches + + cdef void find_matches(self, Doc doc, int start_idx, int end_idx, vector[SpanC] *matches) nogil: + cdef MapStruct* current_node = self.c_map + cdef int start = 0 + cdef int idx = start_idx + cdef int idy = start_idx + cdef key_t key + cdef void* value + cdef int i = 0 + cdef SpanC ms + cdef void* result + while idx < end_idx: + start = idx + token = Token.get_struct_attr(&doc.c[idx], self.attr) + # look for sequences from this position + result = map_get(current_node, token) + if result: + current_node = result + idy = idx + 1 + while idy < end_idx: + result = map_get(current_node, self._terminal_hash) + if result: + i = 0 + while map_iter(result, &i, &key, &value): + ms = make_spanstruct(key, start, idy) + matches.push_back(ms) + inner_token = Token.get_struct_attr(&doc.c[idy], self.attr) + result = map_get(current_node, inner_token) + if result: + current_node = result + idy += 1 + else: + break + else: + # end of doc reached + result = map_get(current_node, self._terminal_hash) + if result: + i = 0 + while map_iter(result, &i, &key, &value): + ms = make_spanstruct(key, start, idy) + matches.push_back(ms) + current_node = self.c_map + idx += 1 + + def pipe(self, stream, batch_size=1000, return_matches=False, as_tuples=False): + """Match a stream of documents, yielding them in turn. Deprecated as of + spaCy v3.0. + """ + warnings.warn(Warnings.W105.format(matcher="PhraseMatcher"), DeprecationWarning) + if as_tuples: + for doc, context in stream: + matches = self(doc) + if return_matches: + yield ((doc, matches), context) + else: + yield (doc, context) + else: + for doc in stream: + matches = self(doc) + if return_matches: + yield (doc, matches) + else: + yield doc + + def _convert_to_array(self, Doc doc): + return [Token.get_struct_attr(&doc.c[i], self.attr) for i in range(len(doc))] + + +def unpickle_matcher(vocab, docs, callbacks, attr): + matcher = PhraseMatcher(vocab, attr=attr) + for key, specs in docs.items(): + callback = callbacks.get(key, None) + matcher.add(key, specs, on_match=callback) + return matcher + + +cdef SpanC make_spanstruct(attr_t label, int start, int end) nogil: + cdef SpanC spanc + spanc.label = label + spanc.start = start + spanc.end = end + return spanc diff --git a/spacy/ml/__init__.py b/spacy/ml/__init__.py new file mode 100644 index 0000000..fce8ae5 --- /dev/null +++ b/spacy/ml/__init__.py @@ -0,0 +1,2 @@ +from .callbacks import create_models_with_nvtx_range # noqa: F401 +from .models import * # noqa: F401, F403 diff --git a/spacy/ml/_character_embed.py b/spacy/ml/_character_embed.py new file mode 100644 index 0000000..8cc4d25 --- /dev/null +++ b/spacy/ml/_character_embed.py @@ -0,0 +1,58 @@ +from typing import List + +from thinc.api import Model +from thinc.types import Floats2d + +from ..tokens import Doc + + +def CharacterEmbed(nM: int, nC: int) -> Model[List[Doc], List[Floats2d]]: + # nM: Number of dimensions per character. nC: Number of characters. + return Model( + "charembed", + forward, + init=init, + dims={"nM": nM, "nC": nC, "nO": nM * nC, "nV": 256}, + params={"E": None}, + ) + + +def init(model: Model, X=None, Y=None): + vectors_table = model.ops.alloc3f( + model.get_dim("nC"), model.get_dim("nV"), model.get_dim("nM") + ) + model.set_param("E", vectors_table) + + +def forward(model: Model, docs: List[Doc], is_train: bool): + if docs is None: + return [] + ids = [] + output = [] + E = model.get_param("E") + nC = model.get_dim("nC") + nM = model.get_dim("nM") + nO = model.get_dim("nO") + # This assists in indexing; it's like looping over this dimension. + # Still consider this weird witch craft...But thanks to Mark Neumann + # for the tip. + nCv = model.ops.xp.arange(nC) + for doc in docs: + doc_ids = model.ops.asarray(doc.to_utf8_array(nr_char=nC)) + doc_vectors = model.ops.alloc3f(len(doc), nC, nM) + # Let's say I have a 2d array of indices, and a 3d table of data. What numpy + # incantation do I chant to get + # output[i, j, k] == data[j, ids[i, j], k]? + doc_vectors[:, nCv] = E[nCv, doc_ids[:, nCv]] # type: ignore[call-overload, index] + output.append(doc_vectors.reshape((len(doc), nO))) + ids.append(doc_ids) + + def backprop(d_output): + dE = model.ops.alloc(E.shape, dtype=E.dtype) + for doc_ids, d_doc_vectors in zip(ids, d_output): + d_doc_vectors = d_doc_vectors.reshape((len(doc_ids), nC, nM)) + dE[nCv, doc_ids[:, nCv]] += d_doc_vectors[:, nCv] + model.inc_grad("E", dE) + return [] + + return output, backprop diff --git a/spacy/ml/_precomputable_affine.py b/spacy/ml/_precomputable_affine.py new file mode 100644 index 0000000..464c325 --- /dev/null +++ b/spacy/ml/_precomputable_affine.py @@ -0,0 +1,161 @@ +from thinc.api import Model, normal_init + + +def PrecomputableAffine(nO, nI, nF, nP, dropout=0.1): + model = Model( + "precomputable_affine", + forward, + init=init, + dims={"nO": nO, "nI": nI, "nF": nF, "nP": nP}, + params={"W": None, "b": None, "pad": None}, + attrs={"dropout_rate": dropout}, + ) + return model + + +def forward(model, X, is_train): + nF = model.get_dim("nF") + nO = model.get_dim("nO") + nP = model.get_dim("nP") + nI = model.get_dim("nI") + W = model.get_param("W") + # Preallocate array for layer output, including padding. + Yf = model.ops.alloc2f(X.shape[0] + 1, nF * nO * nP, zeros=False) + model.ops.gemm(X, W.reshape((nF * nO * nP, nI)), trans2=True, out=Yf[1:]) + Yf = Yf.reshape((Yf.shape[0], nF, nO, nP)) + + # Set padding. Padding has shape (1, nF, nO, nP). Unfortunately, we cannot + # change its shape to (nF, nO, nP) without breaking existing models. So + # we'll squeeze the first dimension here. + Yf[0] = model.ops.xp.squeeze(model.get_param("pad"), 0) + + def backward(dY_ids): + # This backprop is particularly tricky, because we get back a different + # thing from what we put out. We put out an array of shape: + # (nB, nF, nO, nP), and get back: + # (nB, nO, nP) and ids (nB, nF) + # The ids tell us the values of nF, so we would have: + # + # dYf = zeros((nB, nF, nO, nP)) + # for b in range(nB): + # for f in range(nF): + # dYf[b, ids[b, f]] += dY[b] + # + # However, we avoid building that array for efficiency -- and just pass + # in the indices. + dY, ids = dY_ids + assert dY.ndim == 3 + assert dY.shape[1] == nO, dY.shape + assert dY.shape[2] == nP, dY.shape + # nB = dY.shape[0] + model.inc_grad("pad", _backprop_precomputable_affine_padding(model, dY, ids)) + Xf = X[ids] + Xf = Xf.reshape((Xf.shape[0], nF * nI)) + + model.inc_grad("b", dY.sum(axis=0)) + dY = dY.reshape((dY.shape[0], nO * nP)) + + Wopfi = W.transpose((1, 2, 0, 3)) + Wopfi = Wopfi.reshape((nO * nP, nF * nI)) + dXf = model.ops.gemm(dY.reshape((dY.shape[0], nO * nP)), Wopfi) + + dWopfi = model.ops.gemm(dY, Xf, trans1=True) + dWopfi = dWopfi.reshape((nO, nP, nF, nI)) + # (o, p, f, i) --> (f, o, p, i) + dWopfi = dWopfi.transpose((2, 0, 1, 3)) + model.inc_grad("W", dWopfi) + return dXf.reshape((dXf.shape[0], nF, nI)) + + return Yf, backward + + +def _backprop_precomputable_affine_padding(model, dY, ids): + nB = dY.shape[0] + nF = model.get_dim("nF") + nP = model.get_dim("nP") + nO = model.get_dim("nO") + # Backprop the "padding", used as a filler for missing values. + # Values that are missing are set to -1, and each state vector could + # have multiple missing values. The padding has different values for + # different missing features. The gradient of the padding vector is: + # + # for b in range(nB): + # for f in range(nF): + # if ids[b, f] < 0: + # d_pad[f] += dY[b] + # + # Which can be rewritten as: + # + # (ids < 0).T @ dY + mask = model.ops.asarray(ids < 0, dtype="f") + d_pad = model.ops.gemm(mask, dY.reshape(nB, nO * nP), trans1=True) + return d_pad.reshape((1, nF, nO, nP)) + + +def init(model, X=None, Y=None): + """This is like the 'layer sequential unit variance', but instead + of taking the actual inputs, we randomly generate whitened data. + + Why's this all so complicated? We have a huge number of inputs, + and the maxout unit makes guessing the dynamics tricky. Instead + we set the maxout weights to values that empirically result in + whitened outputs given whitened inputs. + """ + if model.has_param("W") and model.get_param("W").any(): + return + + nF = model.get_dim("nF") + nO = model.get_dim("nO") + nP = model.get_dim("nP") + nI = model.get_dim("nI") + W = model.ops.alloc4f(nF, nO, nP, nI) + b = model.ops.alloc2f(nO, nP) + pad = model.ops.alloc4f(1, nF, nO, nP) + + ops = model.ops + W = normal_init(ops, W.shape, mean=float(ops.xp.sqrt(1.0 / nF * nI))) + pad = normal_init(ops, pad.shape, mean=1.0) + model.set_param("W", W) + model.set_param("b", b) + model.set_param("pad", pad) + + ids = ops.alloc((5000, nF), dtype="f") + ids += ops.xp.random.uniform(0, 1000, ids.shape) + ids = ops.asarray(ids, dtype="i") + tokvecs = ops.alloc((5000, nI), dtype="f") + tokvecs += ops.xp.random.normal(loc=0.0, scale=1.0, size=tokvecs.size).reshape( + tokvecs.shape + ) + + def predict(ids, tokvecs): + # nS ids. nW tokvecs. Exclude the padding array. + hiddens = model.predict(tokvecs[:-1]) # (nW, f, o, p) + vectors = model.ops.alloc((ids.shape[0], nO * nP), dtype="f") + # need nS vectors + hiddens = hiddens.reshape((hiddens.shape[0] * nF, nO * nP)) + model.ops.scatter_add(vectors, ids.flatten(), hiddens) + vectors = vectors.reshape((vectors.shape[0], nO, nP)) + vectors += b + vectors = model.ops.asarray(vectors) + if nP >= 2: + return model.ops.maxout(vectors)[0] + else: + return vectors * (vectors >= 0) + + tol_var = 0.01 + tol_mean = 0.01 + t_max = 10 + W = model.get_param("W").copy() + b = model.get_param("b").copy() + for t_i in range(t_max): + acts1 = predict(ids, tokvecs) + var = model.ops.xp.var(acts1) + mean = model.ops.xp.mean(acts1) + if abs(var - 1.0) >= tol_var: + W /= model.ops.xp.sqrt(var) + model.set_param("W", W) + elif abs(mean) >= tol_mean: + b -= mean + model.set_param("b", b) + else: + break diff --git a/spacy/ml/callbacks.py b/spacy/ml/callbacks.py new file mode 100644 index 0000000..d9976ce --- /dev/null +++ b/spacy/ml/callbacks.py @@ -0,0 +1,120 @@ +import functools +import inspect +import types +import warnings +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Set + +from thinc.layers import with_nvtx_range +from thinc.util import use_nvtx_range + +from ..errors import Warnings + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from ..language import Language # noqa: F401 + + +DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS = [ + "pipe", + "predict", + "set_annotations", + "update", + "rehearse", + "get_loss", + "initialize", + "begin_update", + "finish_update", + "update", +] + + +def models_with_nvtx_range(nlp, forward_color: int, backprop_color: int): + pipes = [ + pipe + for _, pipe in nlp.components + if hasattr(pipe, "is_trainable") and pipe.is_trainable + ] + + seen_models: Set[int] = set() + for pipe in pipes: + for node in pipe.model.walk(): + if id(node) in seen_models: + continue + seen_models.add(id(node)) + with_nvtx_range( + node, forward_color=forward_color, backprop_color=backprop_color + ) + + return nlp + + +def create_models_with_nvtx_range( + forward_color: int = -1, backprop_color: int = -1 +) -> Callable[["Language"], "Language"]: + return functools.partial( + models_with_nvtx_range, + forward_color=forward_color, + backprop_color=backprop_color, + ) + + +def nvtx_range_wrapper_for_pipe_method(self, func, *args, **kwargs): + if isinstance(func, functools.partial): + return func(*args, **kwargs) + else: + with use_nvtx_range(f"{self.name} {func.__name__}"): + return func(*args, **kwargs) + + +def pipes_with_nvtx_range( + nlp, additional_pipe_functions: Optional[Dict[str, List[str]]] +): + for _, pipe in nlp.components: + if additional_pipe_functions: + extra_funcs = additional_pipe_functions.get(pipe.name, []) + else: + extra_funcs = [] + + for name in DEFAULT_NVTX_ANNOTATABLE_PIPE_METHODS + extra_funcs: + func = getattr(pipe, name, None) + if func is None: + if name in extra_funcs: + warnings.warn(Warnings.W121.format(method=name, pipe=pipe.name)) + continue + + wrapped_func = functools.partial( + types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func + ) + + # We need to preserve the original function signature so that + # the original parameters are passed to pydantic for validation downstream. + try: + wrapped_func.__signature__ = inspect.signature(func) # type: ignore + except: + # Can fail for Cython methods that do not have bindings. + warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name)) + continue + + try: + setattr( + pipe, + name, + wrapped_func, + ) + except AttributeError: + warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name)) + + return nlp + + +def create_models_and_pipes_with_nvtx_range( + forward_color: int = -1, + backprop_color: int = -1, + additional_pipe_functions: Optional[Dict[str, List[str]]] = None, +) -> Callable[["Language"], "Language"]: + def inner(nlp): + nlp = models_with_nvtx_range(nlp, forward_color, backprop_color) + nlp = pipes_with_nvtx_range(nlp, additional_pipe_functions) + return nlp + + return inner diff --git a/spacy/ml/extract_ngrams.py b/spacy/ml/extract_ngrams.py new file mode 100644 index 0000000..9f54b48 --- /dev/null +++ b/spacy/ml/extract_ngrams.py @@ -0,0 +1,32 @@ +from thinc.api import Model + +from ..attrs import LOWER + + +def extract_ngrams(ngram_size: int, attr: int = LOWER) -> Model: + model: Model = Model("extract_ngrams", forward) + model.attrs["ngram_size"] = ngram_size + model.attrs["attr"] = attr + return model + + +def forward(model: Model, docs, is_train: bool): + batch_keys = [] + batch_vals = [] + for doc in docs: + unigrams = model.ops.asarray(doc.to_array([model.attrs["attr"]])) + ngrams = [unigrams] + for n in range(2, model.attrs["ngram_size"] + 1): + ngrams.append(model.ops.ngrams(n, unigrams)) # type: ignore[arg-type] + keys = model.ops.xp.concatenate(ngrams) + keys, vals = model.ops.xp.unique(keys, return_counts=True) + batch_keys.append(keys) + batch_vals.append(vals) + lengths = model.ops.asarray([arr.shape[0] for arr in batch_keys], dtype="int32") + batch_keys = model.ops.xp.concatenate(batch_keys) + batch_vals = model.ops.asarray(model.ops.xp.concatenate(batch_vals), dtype="f") + + def backprop(dY): + return [] + + return (batch_keys, batch_vals, lengths), backprop diff --git a/spacy/ml/extract_spans.py b/spacy/ml/extract_spans.py new file mode 100644 index 0000000..925bfd4 --- /dev/null +++ b/spacy/ml/extract_spans.py @@ -0,0 +1,64 @@ +from typing import Callable, List, Tuple + +from thinc.api import Model, to_numpy +from thinc.types import Ints1d, Ragged + + +def extract_spans() -> Model[Tuple[Ragged, Ragged], Ragged]: + """Extract spans from a sequence of source arrays, as specified by an array + of (start, end) indices. The output is a ragged array of the + extracted spans. + """ + return Model( + "extract_spans", forward, layers=[], refs={}, attrs={}, dims={}, init=init + ) + + +def init(model, X=None, Y=None): + pass + + +def forward( + model: Model, source_spans: Tuple[Ragged, Ragged], is_train: bool +) -> Tuple[Ragged, Callable]: + """Get subsequences from source vectors.""" + ops = model.ops + X, spans = source_spans + assert spans.dataXd.ndim == 2 + indices = _get_span_indices(ops, spans, X.lengths) + if len(indices) > 0: + Y = Ragged(X.dataXd[indices], spans.dataXd[:, 1] - spans.dataXd[:, 0]) # type: ignore[arg-type, index] + else: + Y = Ragged( + ops.xp.zeros(X.dataXd.shape, dtype=X.dataXd.dtype), + ops.xp.zeros((len(X.lengths),), dtype="i"), + ) + x_shape = X.dataXd.shape + x_lengths = X.lengths + + def backprop_windows(dY: Ragged) -> Tuple[Ragged, Ragged]: + dX = Ragged(ops.alloc2f(*x_shape), x_lengths) + ops.scatter_add(dX.dataXd, indices, dY.dataXd) # type: ignore[arg-type] + return (dX, spans) + + return Y, backprop_windows + + +def _get_span_indices(ops, spans: Ragged, lengths: Ints1d) -> Ints1d: + """Construct a flat array that has the indices we want to extract from the + source data. For instance, if we want the spans (5, 9), (8, 10) the + indices will be [5, 6, 7, 8, 8, 9]. + """ + spans, lengths = _ensure_cpu(spans, lengths) + indices: List[int] = [] + offset = 0 + for i, length in enumerate(lengths): + spans_i = spans[i].dataXd + offset + for j in range(spans_i.shape[0]): + indices.extend(range(spans_i[j, 0], spans_i[j, 1])) # type: ignore[arg-type, call-overload] + offset += length + return ops.asarray1i(indices) + + +def _ensure_cpu(spans: Ragged, lengths: Ints1d) -> Tuple[Ragged, Ints1d]: + return Ragged(to_numpy(spans.dataXd), to_numpy(spans.lengths)), to_numpy(lengths) diff --git a/spacy/ml/featureextractor.py b/spacy/ml/featureextractor.py new file mode 100644 index 0000000..ad376e1 --- /dev/null +++ b/spacy/ml/featureextractor.py @@ -0,0 +1,30 @@ +from typing import Callable, List, Tuple, Union + +from thinc.api import Model +from thinc.types import Ints2d + +from ..tokens import Doc + + +def FeatureExtractor( + columns: Union[List[str], List[int], List[Union[int, str]]], +) -> Model[List[Doc], List[Ints2d]]: + return Model("extract_features", forward, attrs={"columns": columns}) + + +def forward( + model: Model[List[Doc], List[Ints2d]], docs, is_train: bool +) -> Tuple[List[Ints2d], Callable]: + columns = model.attrs["columns"] + features: List[Ints2d] = [] + for doc in docs: + if hasattr(doc, "to_array"): + attrs = doc.to_array(columns) + else: + attrs = doc.doc.to_array(columns)[doc.start : doc.end] + if attrs.ndim == 1: + attrs = attrs.reshape((attrs.shape[0], 1)) + features.append(model.ops.asarray2i(attrs, dtype="uint64")) + + backprop: Callable[[List[Ints2d]], List] = lambda d_features: [] + return features, backprop diff --git a/spacy/ml/models/__init__.py b/spacy/ml/models/__init__.py new file mode 100644 index 0000000..5125018 --- /dev/null +++ b/spacy/ml/models/__init__.py @@ -0,0 +1,8 @@ +from .entity_linker import * # noqa +from .multi_task import * # noqa +from .parser import * # noqa +from .span_finder import * # noqa +from .spancat import * # noqa +from .tagger import * # noqa +from .textcat import * # noqa +from .tok2vec import * # noqa diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py new file mode 100644 index 0000000..05ad9a2 --- /dev/null +++ b/spacy/ml/models/entity_linker.py @@ -0,0 +1,127 @@ +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Tuple + +from thinc.api import ( + Linear, + Maxout, + Model, + Ragged, + chain, + list2ragged, + reduce_mean, + residual, + tuplify, +) +from thinc.types import Floats2d + +from ...errors import Errors +from ...kb import ( + Candidate, + InMemoryLookupKB, + KnowledgeBase, + get_candidates, + get_candidates_batch, +) +from ...tokens import Doc, Span +from ...vocab import Vocab +from ..extract_spans import extract_spans + + +def build_nel_encoder( + tok2vec: Model, nO: Optional[int] = None +) -> Model[List[Doc], Floats2d]: + with Model.define_operators({">>": chain, "&": tuplify}): + token_width = tok2vec.maybe_get_dim("nO") + output_layer = Linear(nO=nO, nI=token_width) + model = ( + ((tok2vec >> list2ragged()) & build_span_maker()) + >> extract_spans() + >> reduce_mean() + >> residual(Maxout(nO=token_width, nI=token_width, nP=2, dropout=0.0)) # type: ignore + >> output_layer + ) + model.set_ref("output_layer", output_layer) + model.set_ref("tok2vec", tok2vec) + # flag to show this isn't legacy + model.attrs["include_span_maker"] = True + return model + + +def build_span_maker(n_sents: int = 0) -> Model: + model: Model = Model("span_maker", forward=span_maker_forward) + model.attrs["n_sents"] = n_sents + return model + + +def span_maker_forward(model, docs: List[Doc], is_train) -> Tuple[Ragged, Callable]: + ops = model.ops + n_sents = model.attrs["n_sents"] + candidates = [] + for doc in docs: + cands = [] + try: + sentences = [s for s in doc.sents] + except ValueError: + # no sentence info, normal in initialization + for tok in doc: + tok.is_sent_start = tok.i == 0 + sentences = [doc[:]] + for ent in doc.ents: + try: + # find the sentence in the list of sentences. + sent_index = sentences.index(ent.sent) + except AttributeError: + # Catch the exception when ent.sent is None and provide a user-friendly warning + raise RuntimeError(Errors.E030) from None + # get n previous sentences, if there are any + start_sentence = max(0, sent_index - n_sents) + # get n posterior sentences, or as many < n as there are + end_sentence = min(len(sentences) - 1, sent_index + n_sents) + # get token positions + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + # save positions for extraction + cands.append((start_token, end_token)) + + candidates.append(ops.asarray2i(cands)) + lengths = model.ops.asarray1i([len(cands) for cands in candidates]) + out = Ragged(model.ops.flatten(candidates), lengths) + # because this is just rearranging docs, the backprop does nothing + return out, lambda x: [] + + +def load_kb( + kb_path: Path, +) -> Callable[[Vocab], KnowledgeBase]: + def kb_from_file(vocab: Vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=1) + kb.from_disk(kb_path) + return kb + + return kb_from_file + + +def empty_kb_for_config() -> Callable[[Vocab, int], KnowledgeBase]: + def empty_kb_factory(vocab: Vocab, entity_vector_length: int): + return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length) + + return empty_kb_factory + + +def empty_kb( + entity_vector_length: int, +) -> Callable[[Vocab], KnowledgeBase]: + def empty_kb_factory(vocab: Vocab): + return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length) + + return empty_kb_factory + + +def create_candidates() -> Callable[[KnowledgeBase, Span], Iterable[Candidate]]: + return get_candidates + + +def create_candidates_batch() -> Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] +]: + return get_candidates_batch diff --git a/spacy/ml/models/multi_task.py b/spacy/ml/models/multi_task.py new file mode 100644 index 0000000..9beecf8 --- /dev/null +++ b/spacy/ml/models/multi_task.py @@ -0,0 +1,270 @@ +from functools import partial +from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, cast + +import numpy +from thinc.api import ( + CosineDistance, + L2Distance, + LayerNorm, + Linear, + Maxout, + Model, + MultiSoftmax, + Softmax, + chain, + list2array, + to_categorical, + zero_init, +) +from thinc.loss import Loss +from thinc.types import Floats2d, Ints1d + +from ...attrs import ID, ORTH +from ...errors import Errors +from ...util import OOV_RANK +from ...vectors import Mode as VectorsMode + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from ...tokens.doc import Doc # noqa: F401 + from ...vocab import Vocab # noqa: F401 + + +def create_pretrain_vectors( + maxout_pieces: int, hidden_size: int, loss: str +) -> Callable[["Vocab", Model], Model]: + def create_vectors_objective(vocab: "Vocab", tok2vec: Model) -> Model: + if vocab.vectors.shape[1] == 0: + raise ValueError(Errors.E875) + model = build_cloze_multi_task_model( + vocab, tok2vec, hidden_size=hidden_size, maxout_pieces=maxout_pieces + ) + model.attrs["loss"] = create_vectors_loss() + return model + + def create_vectors_loss() -> Callable: + distance: Loss + if loss == "cosine": + distance = CosineDistance(normalize=True, ignore_zeros=True) + return partial(get_vectors_loss, distance=distance) + elif loss == "L2": + distance = L2Distance(normalize=True) + return partial(get_vectors_loss, distance=distance) + else: + raise ValueError(Errors.E906.format(found=loss, supported="'cosine', 'L2'")) + + return create_vectors_objective + + +def create_pretrain_characters( + maxout_pieces: int, hidden_size: int, n_characters: int +) -> Callable[["Vocab", Model], Model]: + def create_characters_objective(vocab: "Vocab", tok2vec: Model) -> Model: + model = build_cloze_characters_multi_task_model( + vocab, + tok2vec, + hidden_size=hidden_size, + maxout_pieces=maxout_pieces, + nr_char=n_characters, + ) + model.attrs["loss"] = partial(get_characters_loss, nr_char=n_characters) + return model + + return create_characters_objective + + +def get_vectors_loss(ops, docs, prediction, distance): + """Compute a loss based on a distance between the documents' vectors and + the prediction. + """ + vocab = docs[0].vocab + if vocab.vectors.mode == VectorsMode.default: + # The simplest way to implement this would be to vstack the + # token.vector values, but that's a bit inefficient, especially on GPU. + # Instead we fetch the index into the vectors table for each of our + # tokens, and look them up all at once. This prevents data copying. + ids = ops.flatten([doc.to_array(ID).ravel() for doc in docs]) + target = docs[0].vocab.vectors.data[ids] + target[ids == OOV_RANK] = 0 + d_target, loss = distance(prediction, target) + elif vocab.vectors.mode == VectorsMode.floret: + keys = ops.flatten([cast(Ints1d, doc.to_array(ORTH)) for doc in docs]) + target = vocab.vectors.get_batch(keys) + target = ops.as_contig(target) + d_target, loss = distance(prediction, target) + else: + raise ValueError(Errors.E850.format(mode=vocab.vectors.mode)) + return loss, d_target + + +def get_characters_loss(ops, docs, prediction, nr_char): + """Compute a loss based on a number of characters predicted from the docs.""" + target_ids = numpy.vstack([doc.to_utf8_array(nr_char=nr_char) for doc in docs]) + target_ids = target_ids.reshape((-1,)) + target = ops.asarray(to_categorical(target_ids, n_classes=256), dtype="f") + target = target.reshape((-1, 256 * nr_char)) + diff = prediction - target + loss = (diff**2).sum() + d_target = diff / float(prediction.shape[0]) + return loss, d_target + + +def build_multi_task_model( + tok2vec: Model, + maxout_pieces: int, + token_vector_width: int, + nO: Optional[int] = None, +) -> Model: + softmax = Softmax(nO=nO, nI=token_vector_width * 2) + model = chain( + tok2vec, + Maxout( + nO=token_vector_width * 2, + nI=token_vector_width, + nP=maxout_pieces, + dropout=0.0, + ), + LayerNorm(token_vector_width * 2), + softmax, + ) + model.set_ref("tok2vec", tok2vec) + model.set_ref("output_layer", softmax) + return model + + +def build_cloze_multi_task_model( + vocab: "Vocab", tok2vec: Model, maxout_pieces: int, hidden_size: int +) -> Model: + nO = vocab.vectors.shape[1] + output_layer = chain( + cast(Model[List["Floats2d"], Floats2d], list2array()), + Maxout( + nO=hidden_size, + nI=tok2vec.get_dim("nO"), + nP=maxout_pieces, + normalize=True, + dropout=0.0, + ), + Linear(nO=nO, nI=hidden_size, init_W=zero_init), + ) + model = chain(tok2vec, output_layer) + model = build_masked_language_model(vocab, model) + model.set_ref("tok2vec", tok2vec) + model.set_ref("output_layer", output_layer) + return model + + +def build_cloze_characters_multi_task_model( + vocab: "Vocab", tok2vec: Model, maxout_pieces: int, hidden_size: int, nr_char: int +) -> Model: + output_layer = chain( + cast(Model[List["Floats2d"], Floats2d], list2array()), + Maxout(nO=hidden_size, nP=maxout_pieces), + LayerNorm(nI=hidden_size), + MultiSoftmax([256] * nr_char, nI=hidden_size), # type: ignore[arg-type] + ) + model = build_masked_language_model(vocab, chain(tok2vec, output_layer)) + model.set_ref("tok2vec", tok2vec) + model.set_ref("output_layer", output_layer) + return model + + +def build_masked_language_model( + vocab: "Vocab", wrapped_model: Model, mask_prob: float = 0.15 +) -> Model: + """Convert a model into a BERT-style masked language model""" + random_words = _RandomWords(vocab) + + def mlm_forward(model, docs, is_train): + mask, docs = _apply_mask(docs, random_words, mask_prob=mask_prob) + mask = model.ops.asarray(mask).reshape((mask.shape[0], 1)) + output, backprop = model.layers[0](docs, is_train) + + def mlm_backward(d_output): + d_output *= 1 - mask + return backprop(d_output) + + return output, mlm_backward + + def mlm_initialize(model: Model, X=None, Y=None): + wrapped = model.layers[0] + wrapped.initialize(X=X, Y=Y) + for dim in wrapped.dim_names: + if wrapped.has_dim(dim): + model.set_dim(dim, wrapped.get_dim(dim)) + + mlm_model: Model = Model( + "masked-language-model", + mlm_forward, + layers=[wrapped_model], + init=mlm_initialize, + refs={"wrapped": wrapped_model}, + dims=dict.fromkeys(wrapped_model.dim_names), + ) + mlm_model.set_ref("wrapped", wrapped_model) + return mlm_model + + +class _RandomWords: + def __init__(self, vocab: "Vocab") -> None: + # Extract lexeme representations + self.words = [lex.text for lex in vocab if lex.prob != 0.0] + self.words = self.words[:10000] + + # Compute normalized lexeme probabilities + probs = [lex.prob for lex in vocab if lex.prob != 0.0] + probs = probs[:10000] + probs: numpy.ndarray = numpy.exp(numpy.array(probs, dtype="f")) + probs /= probs.sum() + self.probs = probs + + # Initialize cache + self._cache: List[int] = [] + + def next(self) -> str: + if not self._cache: + self._cache.extend( + numpy.random.choice(len(self.words), 10000, p=self.probs) + ) + index = self._cache.pop() + return self.words[index] + + +def _apply_mask( + docs: Iterable["Doc"], random_words: _RandomWords, mask_prob: float = 0.15 +) -> Tuple[numpy.ndarray, List["Doc"]]: + # This needs to be here to avoid circular imports + from ...tokens.doc import Doc # noqa: F811 + + N = sum(len(doc) for doc in docs) + mask = numpy.random.uniform(0.0, 1.0, (N,)) + mask = mask >= mask_prob + i = 0 + masked_docs = [] + for doc in docs: + words = [] + for token in doc: + if not mask[i]: + word = _replace_word(token.text, random_words) + else: + word = token.text + words.append(word) + i += 1 + spaces = [bool(w.whitespace_) for w in doc] + # NB: If you change this implementation to instead modify + # the docs in place, take care that the IDs reflect the original + # words. Currently we use the original docs to make the vectors + # for the target, so we don't lose the original tokens. But if + # you modified the docs in place here, you would. + masked_docs.append(Doc(doc.vocab, words=words, spaces=spaces)) + return mask, masked_docs + + +def _replace_word(word: str, random_words: _RandomWords, mask: str = "[MASK]") -> str: + roll = numpy.random.random() + if roll < 0.8: + return mask + elif roll < 0.9: + return random_words.next() + else: + return word diff --git a/spacy/ml/models/parser.py b/spacy/ml/models/parser.py new file mode 100644 index 0000000..20b8f6d --- /dev/null +++ b/spacy/ml/models/parser.py @@ -0,0 +1,173 @@ +from typing import List, Optional + +from thinc.api import Linear, Model, chain, list2array, use_ops, zero_init +from thinc.types import Floats2d + +from ...compat import Literal +from ...errors import Errors +from ...tokens import Doc +from .._precomputable_affine import PrecomputableAffine +from ..tb_framework import TransitionModel + + +def build_tb_parser_model( + tok2vec: Model[List[Doc], List[Floats2d]], + state_type: Literal["parser", "ner"], + extra_state_tokens: bool, + hidden_width: int, + maxout_pieces: int, + use_upper: bool, + nO: Optional[int] = None, +) -> Model: + """ + Build a transition-based parser model. Can apply to NER or dependency-parsing. + + Transition-based parsing is an approach to structured prediction where the + task of predicting the structure is mapped to a series of state transitions. + You might find this tutorial helpful as background: + https://explosion.ai/blog/parsing-english-in-python + + The neural network state prediction model consists of either two or three + subnetworks: + + * tok2vec: Map each token into a vector representations. This subnetwork + is run once for each batch. + * lower: Construct a feature-specific vector for each (token, feature) pair. + This is also run once for each batch. Constructing the state + representation is then simply a matter of summing the component features + and applying the non-linearity. + * upper (optional): A feed-forward network that predicts scores from the + state representation. If not present, the output from the lower model is + used as action scores directly. + + tok2vec (Model[List[Doc], List[Floats2d]]): + Subnetwork to map tokens into vector representations. + state_type (str): + String value denoting the type of parser model: "parser" or "ner" + extra_state_tokens (bool): Whether or not to use additional tokens in the context + to construct the state vector. Defaults to `False`, which means 3 and 8 + for the NER and parser respectively. When set to `True`, this would become 6 + feature sets (for the NER) or 13 (for the parser). + hidden_width (int): The width of the hidden layer. + maxout_pieces (int): How many pieces to use in the state prediction layer. + Recommended values are 1, 2 or 3. If 1, the maxout non-linearity + is replaced with a ReLu non-linearity if use_upper=True, and no + non-linearity if use_upper=False. + use_upper (bool): Whether to use an additional hidden layer after the state + vector in order to predict the action scores. It is recommended to set + this to False for large pretrained models such as transformers, and True + for smaller networks. The upper layer is computed on CPU, which becomes + a bottleneck on larger GPU-based models, where it's also less necessary. + nO (int or None): The number of actions the model will predict between. + Usually inferred from data at the beginning of training, or loaded from + disk. + """ + if state_type == "parser": + nr_feature_tokens = 13 if extra_state_tokens else 8 + elif state_type == "ner": + nr_feature_tokens = 6 if extra_state_tokens else 3 + else: + raise ValueError(Errors.E917.format(value=state_type)) + t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None + tok2vec = chain( + tok2vec, + list2array(), + Linear(hidden_width, t2v_width), + ) + tok2vec.set_dim("nO", hidden_width) + lower = _define_lower( + nO=hidden_width if use_upper else nO, + nF=nr_feature_tokens, + nI=tok2vec.get_dim("nO"), + nP=maxout_pieces, + ) + upper = None + if use_upper: + with use_ops("cpu"): + # Initialize weights at zero, as it's a classification layer. + upper = _define_upper(nO=nO, nI=None) + return TransitionModel(tok2vec, lower, upper, resize_output) + + +def _define_upper(nO, nI): + return Linear(nO=nO, nI=nI, init_W=zero_init) + + +def _define_lower(nO, nF, nI, nP): + return PrecomputableAffine(nO=nO, nF=nF, nI=nI, nP=nP) + + +def resize_output(model, new_nO): + if model.attrs["has_upper"]: + return _resize_upper(model, new_nO) + return _resize_lower(model, new_nO) + + +def _resize_upper(model, new_nO): + upper = model.get_ref("upper") + if upper.has_dim("nO") is None: + upper.set_dim("nO", new_nO) + return model + elif new_nO == upper.get_dim("nO"): + return model + + smaller = upper + nI = smaller.maybe_get_dim("nI") + with use_ops("cpu"): + larger = _define_upper(nO=new_nO, nI=nI) + # it could be that the model is not initialized yet, then skip this bit + if smaller.has_param("W"): + larger_W = larger.ops.alloc2f(new_nO, nI) + larger_b = larger.ops.alloc1f(new_nO) + smaller_W = smaller.get_param("W") + smaller_b = smaller.get_param("b") + # Weights are stored in (nr_out, nr_in) format, so we're basically + # just adding rows here. + if smaller.has_dim("nO"): + old_nO = smaller.get_dim("nO") + larger_W[:old_nO] = smaller_W + larger_b[:old_nO] = smaller_b + for i in range(old_nO, new_nO): + model.attrs["unseen_classes"].add(i) + + larger.set_param("W", larger_W) + larger.set_param("b", larger_b) + model._layers[-1] = larger + model.set_ref("upper", larger) + return model + + +def _resize_lower(model, new_nO): + lower = model.get_ref("lower") + if lower.has_dim("nO") is None: + lower.set_dim("nO", new_nO) + return model + + smaller = lower + nI = smaller.maybe_get_dim("nI") + nF = smaller.maybe_get_dim("nF") + nP = smaller.maybe_get_dim("nP") + larger = _define_lower(nO=new_nO, nI=nI, nF=nF, nP=nP) + # it could be that the model is not initialized yet, then skip this bit + if smaller.has_param("W"): + larger_W = larger.ops.alloc4f(nF, new_nO, nP, nI) + larger_b = larger.ops.alloc2f(new_nO, nP) + larger_pad = larger.ops.alloc4f(1, nF, new_nO, nP) + smaller_W = smaller.get_param("W") + smaller_b = smaller.get_param("b") + smaller_pad = smaller.get_param("pad") + # Copy the old weights and padding into the new layer + if smaller.has_dim("nO"): + old_nO = smaller.get_dim("nO") + larger_W[:, 0:old_nO, :, :] = smaller_W + larger_pad[:, :, 0:old_nO, :] = smaller_pad + larger_b[0:old_nO, :] = smaller_b + for i in range(old_nO, new_nO): + model.attrs["unseen_classes"].add(i) + + larger.set_param("W", larger_W) + larger.set_param("b", larger_b) + larger.set_param("pad", larger_pad) + model._layers[1] = larger + model.set_ref("lower", larger) + return model diff --git a/spacy/ml/models/span_finder.py b/spacy/ml/models/span_finder.py new file mode 100644 index 0000000..226b736 --- /dev/null +++ b/spacy/ml/models/span_finder.py @@ -0,0 +1,39 @@ +from typing import Callable, List, Tuple + +from thinc.api import Model, chain, with_array +from thinc.types import Floats1d, Floats2d + +from ...tokens import Doc + +InT = List[Doc] +OutT = Floats2d + + +def build_finder_model( + tok2vec: Model[InT, List[Floats2d]], scorer: Model[OutT, OutT] +) -> Model[InT, OutT]: + + logistic_layer: Model[List[Floats2d], List[Floats2d]] = with_array(scorer) + model: Model[InT, OutT] = chain(tok2vec, logistic_layer, flattener()) + model.set_ref("tok2vec", tok2vec) + model.set_ref("scorer", scorer) + model.set_ref("logistic_layer", logistic_layer) + + return model + + +def flattener() -> Model[List[Floats2d], Floats2d]: + """Flattens the input to a 1-dimensional list of scores""" + + def forward( + model: Model[Floats1d, Floats1d], X: List[Floats2d], is_train: bool + ) -> Tuple[Floats2d, Callable[[Floats2d], List[Floats2d]]]: + lens = model.ops.asarray1i([len(doc) for doc in X]) + Y = model.ops.flatten(X) + + def backprop(dY: Floats2d) -> List[Floats2d]: + return model.ops.unflatten(dY, lens) + + return Y, backprop + + return Model("Flattener", forward=forward) diff --git a/spacy/ml/models/spancat.py b/spacy/ml/models/spancat.py new file mode 100644 index 0000000..697d1df --- /dev/null +++ b/spacy/ml/models/spancat.py @@ -0,0 +1,73 @@ +from typing import List, Tuple, cast + +from thinc.api import ( + Linear, + Logistic, + Maxout, + Model, + chain, + concatenate, + glorot_uniform_init, + list2ragged, + reduce_first, + reduce_last, + reduce_max, + reduce_mean, + with_getitem, +) +from thinc.types import Floats2d, Ragged + +from ...tokens import Doc +from ..extract_spans import extract_spans + + +def build_linear_logistic(nO=None, nI=None) -> Model[Floats2d, Floats2d]: + """An output layer for multi-label classification. It uses a linear layer + followed by a logistic activation. + """ + return chain(Linear(nO=nO, nI=nI, init_W=glorot_uniform_init), Logistic()) + + +def build_mean_max_reducer(hidden_size: int) -> Model[Ragged, Floats2d]: + """Reduce sequences by concatenating their mean and max pooled vectors, + and then combine the concatenated vectors with a hidden layer. + """ + return chain( + concatenate( + cast(Model[Ragged, Floats2d], reduce_last()), + cast(Model[Ragged, Floats2d], reduce_first()), + reduce_mean(), + reduce_max(), + ), + Maxout(nO=hidden_size, normalize=True, dropout=0.0), + ) + + +def build_spancat_model( + tok2vec: Model[List[Doc], List[Floats2d]], + reducer: Model[Ragged, Floats2d], + scorer: Model[Floats2d, Floats2d], +) -> Model[Tuple[List[Doc], Ragged], Floats2d]: + """Build a span categorizer model, given a token-to-vector model, a + reducer model to map the sequence of vectors for each span down to a single + vector, and a scorer model to map the vectors to probabilities. + + tok2vec (Model[List[Doc], List[Floats2d]]): The tok2vec model. + reducer (Model[Ragged, Floats2d]): The reducer model. + scorer (Model[Floats2d, Floats2d]): The scorer model. + """ + model = chain( + cast( + Model[Tuple[List[Doc], Ragged], Tuple[Ragged, Ragged]], + with_getitem( + 0, chain(tok2vec, cast(Model[List[Floats2d], Ragged], list2ragged())) + ), + ), + extract_spans(), + reducer, + scorer, + ) + model.set_ref("tok2vec", tok2vec) + model.set_ref("reducer", reducer) + model.set_ref("scorer", scorer) + return model diff --git a/spacy/ml/models/tagger.py b/spacy/ml/models/tagger.py new file mode 100644 index 0000000..d3b090d --- /dev/null +++ b/spacy/ml/models/tagger.py @@ -0,0 +1,29 @@ +from typing import List, Optional + +from thinc.api import Model, Softmax_v2, chain, with_array, zero_init +from thinc.types import Floats2d + +from ...tokens import Doc + + +def build_tagger_model( + tok2vec: Model[List[Doc], List[Floats2d]], nO: Optional[int] = None, normalize=False +) -> Model[List[Doc], List[Floats2d]]: + """Build a tagger model, using a provided token-to-vector component. The tagger + model simply adds a linear layer with softmax activation to predict scores + given the token vectors. + + tok2vec (Model[List[Doc], List[Floats2d]]): The token-to-vector subnetwork. + nO (int or None): The number of tags to output. Inferred from the data if None. + """ + # TODO: glorot_uniform_init seems to work a bit better than zero_init here?! + t2v_width = tok2vec.get_dim("nO") if tok2vec.has_dim("nO") else None + output_layer = Softmax_v2( + nO, t2v_width, init_W=zero_init, normalize_outputs=normalize + ) + softmax = with_array(output_layer) # type: ignore + model = chain(tok2vec, softmax) + model.set_ref("tok2vec", tok2vec) + model.set_ref("softmax", output_layer) + model.set_ref("output_layer", output_layer) + return model diff --git a/spacy/ml/models/textcat.py b/spacy/ml/models/textcat.py new file mode 100644 index 0000000..8194ab3 --- /dev/null +++ b/spacy/ml/models/textcat.py @@ -0,0 +1,363 @@ +from functools import partial +from typing import List, Optional, Tuple, cast + +from thinc.api import ( + Dropout, + Gelu, + LayerNorm, + Linear, + Logistic, + Maxout, + Model, + ParametricAttention, + ParametricAttention_v2, + Relu, + Softmax, + SparseLinear, + SparseLinear_v2, + chain, + clone, + concatenate, + list2ragged, + reduce_first, + reduce_last, + reduce_max, + reduce_mean, + reduce_sum, + residual, + resizable, + softmax_activation, + with_cpu, +) +from thinc.layers.chain import init as init_chain +from thinc.layers.resizable import resize_linear_weighted, resize_model +from thinc.types import ArrayXd, Floats2d + +from ...attrs import ORTH +from ...errors import Errors +from ...tokens import Doc +from ..extract_ngrams import extract_ngrams +from ..staticvectors import StaticVectors +from .tok2vec import get_tok2vec_width + +NEG_VALUE = -5000 + + +def build_simple_cnn_text_classifier( + tok2vec: Model, exclusive_classes: bool, nO: Optional[int] = None +) -> Model[List[Doc], Floats2d]: + """ + Build a simple CNN text classifier, given a token-to-vector model as inputs. + If exclusive_classes=True, a softmax non-linearity is applied, so that the + outputs sum to 1. If exclusive_classes=False, a logistic non-linearity + is applied instead, so that outputs are in the range [0, 1]. + """ + return build_reduce_text_classifier( + tok2vec=tok2vec, + exclusive_classes=exclusive_classes, + use_reduce_first=False, + use_reduce_last=False, + use_reduce_max=False, + use_reduce_mean=True, + nO=nO, + ) + + +def resize_and_set_ref(model, new_nO, resizable_layer): + resizable_layer = resize_model(resizable_layer, new_nO) + model.set_ref("output_layer", resizable_layer.layers[0]) + model.set_dim("nO", new_nO, force=True) + return model + + +def build_bow_text_classifier( + exclusive_classes: bool, + ngram_size: int, + no_output_layer: bool, + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + return _build_bow_text_classifier( + exclusive_classes=exclusive_classes, + ngram_size=ngram_size, + no_output_layer=no_output_layer, + nO=nO, + sparse_linear=SparseLinear(nO=nO), + ) + + +def build_bow_text_classifier_v3( + exclusive_classes: bool, + ngram_size: int, + no_output_layer: bool, + length: int = 262144, + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + if length < 1: + raise ValueError(Errors.E1056.format(length=length)) + + # Find k such that 2**(k-1) < length <= 2**k. + length = 2 ** (length - 1).bit_length() + + return _build_bow_text_classifier( + exclusive_classes=exclusive_classes, + ngram_size=ngram_size, + no_output_layer=no_output_layer, + nO=nO, + sparse_linear=SparseLinear_v2(nO=nO, length=length), + ) + + +def _build_bow_text_classifier( + exclusive_classes: bool, + ngram_size: int, + no_output_layer: bool, + sparse_linear: Model[Tuple[ArrayXd, ArrayXd, ArrayXd], ArrayXd], + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + fill_defaults = {"b": 0, "W": 0} + with Model.define_operators({">>": chain}): + output_layer = None + if not no_output_layer: + fill_defaults["b"] = NEG_VALUE + output_layer = softmax_activation() if exclusive_classes else Logistic() + resizable_layer: Model[Floats2d, Floats2d] = resizable( + sparse_linear, + resize_layer=partial(resize_linear_weighted, fill_defaults=fill_defaults), + ) + model = extract_ngrams(ngram_size, attr=ORTH) >> resizable_layer + model = with_cpu(model, model.ops) + if output_layer: + model = model >> with_cpu(output_layer, output_layer.ops) + if nO is not None: + model.set_dim("nO", cast(int, nO)) + model.set_ref("output_layer", sparse_linear) + model.attrs["multi_label"] = not exclusive_classes + model.attrs["resize_output"] = partial( + resize_and_set_ref, resizable_layer=resizable_layer + ) + return model + + +def build_text_classifier_v2( + tok2vec: Model[List[Doc], List[Floats2d]], + linear_model: Model[List[Doc], Floats2d], + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + # TODO: build the model with _build_parametric_attention_with_residual_nonlinear + # in spaCy v4. We don't do this in spaCy v3 to preserve model + # compatibility. + exclusive_classes = not linear_model.attrs["multi_label"] + with Model.define_operators({">>": chain, "|": concatenate}): + width = tok2vec.maybe_get_dim("nO") + attention_layer = ParametricAttention(width) + maxout_layer = Maxout(nO=width, nI=width) + norm_layer = LayerNorm(nI=width) + cnn_model = ( + tok2vec + >> list2ragged() + >> attention_layer + >> reduce_sum() + >> residual(maxout_layer >> norm_layer >> Dropout(0.0)) + ) + + nO_double = nO * 2 if nO else None + if exclusive_classes: + output_layer = Softmax(nO=nO, nI=nO_double) + else: + output_layer = Linear(nO=nO, nI=nO_double) >> Logistic() + model = (linear_model | cnn_model) >> output_layer + model.set_ref("tok2vec", tok2vec) + if model.has_dim("nO") is not False and nO is not None: + model.set_dim("nO", cast(int, nO)) + model.set_ref("output_layer", linear_model.get_ref("output_layer")) + model.set_ref("attention_layer", attention_layer) + model.set_ref("maxout_layer", maxout_layer) + model.set_ref("norm_layer", norm_layer) + model.attrs["multi_label"] = not exclusive_classes + + model.init = init_ensemble_textcat # type: ignore[assignment] + return model + + +def init_ensemble_textcat(model, X, Y) -> Model: + # When tok2vec is lazily initialized, we need to initialize it before + # the rest of the chain to ensure that we can get its width. + tok2vec = model.get_ref("tok2vec") + tok2vec.initialize(X) + + tok2vec_width = get_tok2vec_width(model) + model.get_ref("attention_layer").set_dim("nO", tok2vec_width) + model.get_ref("maxout_layer").set_dim("nO", tok2vec_width) + model.get_ref("maxout_layer").set_dim("nI", tok2vec_width) + model.get_ref("norm_layer").set_dim("nI", tok2vec_width) + model.get_ref("norm_layer").set_dim("nO", tok2vec_width) + init_chain(model, X, Y) + return model + + +def build_text_classifier_lowdata( + width: int, dropout: Optional[float], nO: Optional[int] = None +) -> Model[List[Doc], Floats2d]: + # Don't document this yet, I'm not sure it's right. + # Note, before v.3, this was the default if setting "low_data" and "pretrained_dims" + with Model.define_operators({">>": chain, "**": clone}): + model = ( + StaticVectors(width) + >> list2ragged() + >> ParametricAttention(width) + >> reduce_sum() + >> residual(Relu(width, width)) ** 2 + >> Linear(nO, width) + ) + if dropout: + model = model >> Dropout(dropout) + model = model >> Logistic() + return model + + +def build_textcat_parametric_attention_v1( + tok2vec: Model[List[Doc], List[Floats2d]], + exclusive_classes: bool, + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + width = tok2vec.maybe_get_dim("nO") + parametric_attention = _build_parametric_attention_with_residual_nonlinear( + tok2vec=tok2vec, + nonlinear_layer=Maxout(nI=width, nO=width), + key_transform=Gelu(nI=width, nO=width), + ) + with Model.define_operators({">>": chain}): + if exclusive_classes: + output_layer = Softmax(nO=nO) + else: + output_layer = Linear(nO=nO) >> Logistic() + model = parametric_attention >> output_layer + if model.has_dim("nO") is not False and nO is not None: + model.set_dim("nO", cast(int, nO)) + model.set_ref("output_layer", output_layer) + model.attrs["multi_label"] = not exclusive_classes + + return model + + +def _build_parametric_attention_with_residual_nonlinear( + *, + tok2vec: Model[List[Doc], List[Floats2d]], + nonlinear_layer: Model[Floats2d, Floats2d], + key_transform: Optional[Model[Floats2d, Floats2d]] = None, +) -> Model[List[Doc], Floats2d]: + with Model.define_operators({">>": chain, "|": concatenate}): + width = tok2vec.maybe_get_dim("nO") + attention_layer = ParametricAttention_v2(nO=width, key_transform=key_transform) + norm_layer = LayerNorm(nI=width) + parametric_attention = ( + tok2vec + >> list2ragged() + >> attention_layer + >> reduce_sum() + >> residual(nonlinear_layer >> norm_layer >> Dropout(0.0)) + ) + + parametric_attention.init = _init_parametric_attention_with_residual_nonlinear + + parametric_attention.set_ref("tok2vec", tok2vec) + parametric_attention.set_ref("attention_layer", attention_layer) + parametric_attention.set_ref("key_transform", key_transform) + parametric_attention.set_ref("nonlinear_layer", nonlinear_layer) + parametric_attention.set_ref("norm_layer", norm_layer) + + return parametric_attention + + +def _init_parametric_attention_with_residual_nonlinear(model, X, Y) -> Model: + # When tok2vec is lazily initialized, we need to initialize it before + # the rest of the chain to ensure that we can get its width. + tok2vec = model.get_ref("tok2vec") + tok2vec.initialize(X) + + tok2vec_width = get_tok2vec_width(model) + model.get_ref("attention_layer").set_dim("nO", tok2vec_width) + model.get_ref("key_transform").set_dim("nI", tok2vec_width) + model.get_ref("key_transform").set_dim("nO", tok2vec_width) + model.get_ref("nonlinear_layer").set_dim("nI", tok2vec_width) + model.get_ref("nonlinear_layer").set_dim("nO", tok2vec_width) + model.get_ref("norm_layer").set_dim("nI", tok2vec_width) + model.get_ref("norm_layer").set_dim("nO", tok2vec_width) + init_chain(model, X, Y) + return model + + +def build_reduce_text_classifier( + tok2vec: Model, + exclusive_classes: bool, + use_reduce_first: bool, + use_reduce_last: bool, + use_reduce_max: bool, + use_reduce_mean: bool, + nO: Optional[int] = None, +) -> Model[List[Doc], Floats2d]: + """Build a model that classifies pooled `Doc` representations. + + Pooling is performed using reductions. Reductions are concatenated when + multiple reductions are used. + + tok2vec (Model): the tok2vec layer to pool over. + exclusive_classes (bool): Whether or not classes are mutually exclusive. + use_reduce_first (bool): Pool by using the hidden representation of the + first token of a `Doc`. + use_reduce_last (bool): Pool by using the hidden representation of the + last token of a `Doc`. + use_reduce_max (bool): Pool by taking the maximum values of the hidden + representations of a `Doc`. + use_reduce_mean (bool): Pool by taking the mean of all hidden + representations of a `Doc`. + nO (Optional[int]): Number of classes. + """ + + fill_defaults = {"b": 0, "W": 0} + reductions = [] + if use_reduce_first: + reductions.append(reduce_first()) + if use_reduce_last: + reductions.append(reduce_last()) + if use_reduce_max: + reductions.append(reduce_max()) + if use_reduce_mean: + reductions.append(reduce_mean()) + + if not len(reductions): + raise ValueError(Errors.E1057) + + with Model.define_operators({">>": chain}): + cnn = tok2vec >> list2ragged() >> concatenate(*reductions) + nO_tok2vec = tok2vec.maybe_get_dim("nO") + nI = nO_tok2vec * len(reductions) if nO_tok2vec is not None else None + if exclusive_classes: + output_layer = Softmax(nO=nO, nI=nI) + fill_defaults["b"] = NEG_VALUE + resizable_layer: Model = resizable( + output_layer, + resize_layer=partial( + resize_linear_weighted, fill_defaults=fill_defaults + ), + ) + model = cnn >> resizable_layer + else: + output_layer = Linear(nO=nO, nI=nI) + resizable_layer = resizable( + output_layer, + resize_layer=partial( + resize_linear_weighted, fill_defaults=fill_defaults + ), + ) + model = cnn >> resizable_layer >> Logistic() + model.set_ref("output_layer", output_layer) + model.attrs["resize_output"] = partial( + resize_and_set_ref, + resizable_layer=resizable_layer, + ) + model.set_ref("tok2vec", tok2vec) + if nO is not None: + model.set_dim("nO", cast(int, nO)) + model.attrs["multi_label"] = not exclusive_classes + return model diff --git a/spacy/ml/models/tok2vec.py b/spacy/ml/models/tok2vec.py new file mode 100644 index 0000000..ade8427 --- /dev/null +++ b/spacy/ml/models/tok2vec.py @@ -0,0 +1,342 @@ +from typing import List, Optional, Union, cast + +from thinc.api import ( + HashEmbed, + Maxout, + Mish, + Model, + PyTorchLSTM, + chain, + clone, + concatenate, + expand_window, + list2ragged, + noop, + ragged2list, + residual, + with_array, + with_padded, +) +from thinc.types import Floats2d, Ints2d, Ragged + +from ...attrs import intify_attr +from ...errors import Errors +from ...ml import _character_embed +from ...pipeline.tok2vec import Tok2VecListener +from ...tokens import Doc +from ..featureextractor import FeatureExtractor +from ..staticvectors import StaticVectors + + +def tok2vec_listener_v1(width: int, upstream: str = "*"): + tok2vec = Tok2VecListener(upstream_name=upstream, width=width) + return tok2vec + + +def get_tok2vec_width(model: Model): + nO = None + if model.has_ref("tok2vec"): + tok2vec = model.get_ref("tok2vec") + if tok2vec.has_dim("nO"): + nO = tok2vec.get_dim("nO") + elif tok2vec.has_ref("listener"): + nO = tok2vec.get_ref("listener").get_dim("nO") + return nO + + +def build_hash_embed_cnn_tok2vec( + *, + width: int, + depth: int, + embed_size: int, + window_size: int, + maxout_pieces: int, + subword_features: bool, + pretrained_vectors: Optional[bool], +) -> Model[List[Doc], List[Floats2d]]: + """Build spaCy's 'standard' tok2vec layer, which uses hash embedding + with subword features and a CNN with layer-normalized maxout. + + width (int): The width of the input and output. These are required to be the + same, so that residual connections can be used. Recommended values are + 96, 128 or 300. + depth (int): The number of convolutional layers to use. Recommended values + are between 2 and 8. + window_size (int): The number of tokens on either side to concatenate during + the convolutions. The receptive field of the CNN will be + depth * window_size * 2 + 1, so a 4-layer network with window_size of + 2 will be sensitive to 17 words at a time. Recommended value is 1. + embed_size (int): The number of rows in the hash embedding tables. This can + be surprisingly small, due to the use of the hash embeddings. Recommended + values are between 2000 and 10000. + maxout_pieces (int): The number of pieces to use in the maxout non-linearity. + If 1, the Mish non-linearity is used instead. Recommended values are 1-3. + subword_features (bool): Whether to also embed subword features, specifically + the prefix, suffix and word shape. This is recommended for alphabetic + languages like English, but not if single-character tokens are used for + a language such as Chinese. + pretrained_vectors (bool): Whether to also use static vectors. + """ + if subword_features: + attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] + row_sizes = [embed_size, embed_size // 2, embed_size // 2, embed_size // 2] + else: + attrs = ["NORM"] + row_sizes = [embed_size] + return build_Tok2Vec_model( + embed=MultiHashEmbed( + width=width, + rows=row_sizes, + attrs=attrs, + include_static_vectors=bool(pretrained_vectors), + ), + encode=MaxoutWindowEncoder( + width=width, + depth=depth, + window_size=window_size, + maxout_pieces=maxout_pieces, + ), + ) + + +def build_Tok2Vec_model( + embed: Model[List[Doc], List[Floats2d]], + encode: Model[List[Floats2d], List[Floats2d]], +) -> Model[List[Doc], List[Floats2d]]: + """Construct a tok2vec model out of embedding and encoding subnetworks. + See https://explosion.ai/blog/deep-learning-formula-nlp + + embed (Model[List[Doc], List[Floats2d]]): Embed tokens into context-independent + word vector representations. + encode (Model[List[Floats2d], List[Floats2d]]): Encode context into the + embeddings, using an architecture such as a CNN, BiLSTM or transformer. + """ + tok2vec = chain(embed, encode) + if encode.has_dim("nO"): + tok2vec.set_dim("nO", encode.get_dim("nO")) + tok2vec.set_ref("embed", embed) + tok2vec.set_ref("encode", encode) + return tok2vec + + +def MultiHashEmbed( + width: int, + attrs: Union[List[str], List[int], List[Union[str, int]]], + rows: List[int], + include_static_vectors: bool, +) -> Model[List[Doc], List[Floats2d]]: + """Construct an embedding layer that separately embeds a number of lexical + attributes using hash embedding, concatenates the results, and passes it + through a feed-forward subnetwork to build a mixed representation. + + The features used can be configured with the 'attrs' argument. The suggested + attributes are NORM, PREFIX, SUFFIX and SHAPE. This lets the model take into + account some subword information, without constructing a fully character-based + representation. If pretrained vectors are available, they can be included in + the representation as well, with the vectors table kept static + (i.e. it's not updated). + + The `width` parameter specifies the output width of the layer and the widths + of all embedding tables. If static vectors are included, a learned linear + layer is used to map the vectors to the specified width before concatenating + it with the other embedding outputs. A single Maxout layer is then used to + reduce the concatenated vectors to the final width. + + The `rows` parameter controls the number of rows used by the `HashEmbed` + tables. The HashEmbed layer needs surprisingly few rows, due to its use of + the hashing trick. Generally between 2000 and 10000 rows is sufficient, + even for very large vocabularies. A number of rows must be specified for each + table, so the `rows` list must be of the same length as the `attrs` parameter. + + width (int): The output width. Also used as the width of the embedding tables. + Recommended values are between 64 and 300. + attrs (list of attr IDs): The token attributes to embed. A separate + embedding table will be constructed for each attribute. + rows (List[int]): The number of rows in the embedding tables. Must have the + same length as attrs. + include_static_vectors (bool): Whether to also use static word vectors. + Requires a vectors table to be loaded in the Doc objects' vocab. + """ + if len(rows) != len(attrs): + raise ValueError(f"Mismatched lengths: {len(rows)} vs {len(attrs)}") + seed = 7 + + def make_hash_embed(index): + nonlocal seed + seed += 1 + return HashEmbed(width, rows[index], column=index, seed=seed, dropout=0.0) + + embeddings = [make_hash_embed(i) for i in range(len(attrs))] + concat_size = width * (len(embeddings) + include_static_vectors) + max_out: Model[Ragged, Ragged] = with_array( + Maxout(width, concat_size, nP=3, dropout=0.0, normalize=True) + ) + if include_static_vectors: + feature_extractor: Model[List[Doc], Ragged] = chain( + FeatureExtractor(attrs), + cast(Model[List[Ints2d], Ragged], list2ragged()), + with_array(concatenate(*embeddings)), + ) + model = chain( + concatenate( + feature_extractor, + StaticVectors(width, dropout=0.0), + ), + max_out, + ragged2list(), + ) + else: + model = chain( + FeatureExtractor(attrs), + cast(Model[List[Ints2d], Ragged], list2ragged()), + with_array(concatenate(*embeddings)), + max_out, + ragged2list(), + ) + return model + + +def CharacterEmbed( + width: int, + rows: int, + nM: int, + nC: int, + include_static_vectors: bool, + feature: Union[int, str] = "LOWER", +) -> Model[List[Doc], List[Floats2d]]: + """Construct an embedded representation based on character embeddings, using + a feed-forward network. A fixed number of UTF-8 byte characters are used for + each word, taken from the beginning and end of the word equally. Padding is + used in the centre for words that are too short. + + For instance, let's say nC=4, and the word is "jumping". The characters + used will be jung (two from the start, two from the end). If we had nC=8, + the characters would be "jumpping": 4 from the start, 4 from the end. This + ensures that the final character is always in the last position, instead + of being in an arbitrary position depending on the word length. + + The characters are embedded in a embedding table with a given number of rows, + and the vectors concatenated. A hash-embedded vector of the LOWER of the word is + also concatenated on, and the result is then passed through a feed-forward + network to construct a single vector to represent the information. + + feature (int or str): An attribute to embed, to concatenate with the characters. + width (int): The width of the output vector and the feature embedding. + rows (int): The number of rows in the LOWER hash embedding table. + nM (int): The dimensionality of the character embeddings. Recommended values + are between 16 and 64. + nC (int): The number of UTF-8 bytes to embed per word. Recommended values + are between 3 and 8, although it may depend on the length of words in the + language. + include_static_vectors (bool): Whether to also use static word vectors. + Requires a vectors table to be loaded in the Doc objects' vocab. + """ + feature = intify_attr(feature) + if feature is None: + raise ValueError(Errors.E911.format(feat=feature)) + char_embed = chain( + _character_embed.CharacterEmbed(nM=nM, nC=nC), + cast(Model[List[Floats2d], Ragged], list2ragged()), + ) + feature_extractor: Model[List[Doc], Ragged] = chain( + FeatureExtractor([feature]), + cast(Model[List[Ints2d], Ragged], list2ragged()), + with_array(HashEmbed(nO=width, nV=rows, column=0, seed=5)), # type: ignore[misc] + ) + max_out: Model[Ragged, Ragged] + if include_static_vectors: + max_out = with_array( + Maxout(width, nM * nC + (2 * width), nP=3, normalize=True, dropout=0.0) + ) + model = chain( + concatenate( + char_embed, + feature_extractor, + StaticVectors(width, dropout=0.0), + ), + max_out, + ragged2list(), + ) + else: + max_out = with_array( + Maxout(width, nM * nC + width, nP=3, normalize=True, dropout=0.0) + ) + model = chain( + concatenate( + char_embed, + feature_extractor, + ), + max_out, + ragged2list(), + ) + return model + + +def MaxoutWindowEncoder( + width: int, window_size: int, maxout_pieces: int, depth: int +) -> Model[List[Floats2d], List[Floats2d]]: + """Encode context using convolutions with maxout activation, layer + normalization and residual connections. + + width (int): The input and output width. These are required to be the same, + to allow residual connections. This value will be determined by the + width of the inputs. Recommended values are between 64 and 300. + window_size (int): The number of words to concatenate around each token + to construct the convolution. Recommended value is 1. + maxout_pieces (int): The number of maxout pieces to use. Recommended + values are 2 or 3. + depth (int): The number of convolutional layers. Recommended value is 4. + """ + cnn = chain( + expand_window(window_size=window_size), + Maxout( + nO=width, + nI=width * ((window_size * 2) + 1), + nP=maxout_pieces, + dropout=0.0, + normalize=True, + ), + ) + model = clone(residual(cnn), depth) + model.set_dim("nO", width) + receptive_field = window_size * depth + return with_array(model, pad=receptive_field) + + +def MishWindowEncoder( + width: int, window_size: int, depth: int +) -> Model[List[Floats2d], List[Floats2d]]: + """Encode context using convolutions with mish activation, layer + normalization and residual connections. + + width (int): The input and output width. These are required to be the same, + to allow residual connections. This value will be determined by the + width of the inputs. Recommended values are between 64 and 300. + window_size (int): The number of words to concatenate around each token + to construct the convolution. Recommended value is 1. + depth (int): The number of convolutional layers. Recommended value is 4. + """ + cnn = chain( + expand_window(window_size=window_size), + Mish(nO=width, nI=width * ((window_size * 2) + 1), dropout=0.0, normalize=True), + ) + model = clone(residual(cnn), depth) + model.set_dim("nO", width) + return with_array(model) + + +def BiLSTMEncoder( + width: int, depth: int, dropout: float +) -> Model[List[Floats2d], List[Floats2d]]: + """Encode context using bidirectonal LSTM layers. Requires PyTorch. + + width (int): The input and output width. These are required to be the same, + to allow residual connections. This value will be determined by the + width of the inputs. Recommended values are between 64 and 300. + depth (int): The number of recurrent layers. + dropout (float): Creates a Dropout layer on the outputs of each LSTM layer + except the last layer. Set to 0 to disable this functionality. + """ + if depth == 0: + return noop() + return with_padded(PyTorchLSTM(width, width, bi=True, depth=depth, dropout=dropout)) diff --git a/spacy/ml/parser_model.pxd b/spacy/ml/parser_model.pxd new file mode 100644 index 0000000..4d2d7b3 --- /dev/null +++ b/spacy/ml/parser_model.pxd @@ -0,0 +1,55 @@ +from libc.string cimport memcpy, memset +from thinc.backends.cblas cimport CBlas + +from ..pipeline._parser_internals._state cimport StateC +from ..typedefs cimport hash_t, weight_t + + +cdef struct SizesC: + int states + int classes + int hiddens + int pieces + int feats + int embed_width + + +cdef struct WeightsC: + const float* feat_weights + const float* feat_bias + const float* hidden_bias + const float* hidden_weights + const float* seen_classes + + +cdef struct ActivationsC: + int* token_ids + float* unmaxed + float* scores + float* hiddens + int* is_valid + int _curr_size + int _max_size + + +cdef WeightsC get_c_weights(model) except * + +cdef SizesC get_c_sizes(model, int batch_size) except * + +cdef ActivationsC alloc_activations(SizesC n) nogil + +cdef void free_activations(const ActivationsC* A) nogil + +cdef void predict_states( + CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n +) nogil + +cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) nogil + +cdef void cpu_log_loss( + float* d_scores, + const float* costs, + const int* is_valid, + const float* scores, + int O +) nogil diff --git a/spacy/ml/parser_model.pyx b/spacy/ml/parser_model.pyx new file mode 100644 index 0000000..96f2487 --- /dev/null +++ b/spacy/ml/parser_model.pyx @@ -0,0 +1,533 @@ +# cython: infer_types=True, cdivision=True, boundscheck=False +# cython: profile=False +cimport numpy as np +from libc.math cimport exp +from libc.stdlib cimport calloc, free, realloc +from libc.string cimport memcpy, memset +from thinc.backends.cblas cimport saxpy, sgemm +from thinc.backends.linalg cimport Vec, VecVec + +import numpy +import numpy.random +from thinc.api import CupyOps, Model, NumpyOps + +from .. import util +from ..errors import Errors + +from ..pipeline._parser_internals.stateclass cimport StateClass +from ..typedefs cimport weight_t + + +cdef WeightsC get_c_weights(model) except *: + cdef WeightsC output + cdef precompute_hiddens state2vec = model.state2vec + output.feat_weights = state2vec.get_feat_weights() + output.feat_bias = state2vec.bias.data + cdef np.ndarray vec2scores_W + cdef np.ndarray vec2scores_b + if model.vec2scores is None: + output.hidden_weights = NULL + output.hidden_bias = NULL + else: + vec2scores_W = model.vec2scores.get_param("W") + vec2scores_b = model.vec2scores.get_param("b") + output.hidden_weights = vec2scores_W.data + output.hidden_bias = vec2scores_b.data + cdef np.ndarray class_mask = model._class_mask + output.seen_classes = class_mask.data + return output + + +cdef SizesC get_c_sizes(model, int batch_size) except *: + cdef SizesC output + output.states = batch_size + if model.vec2scores is None: + output.classes = model.state2vec.get_dim("nO") + else: + output.classes = model.vec2scores.get_dim("nO") + output.hiddens = model.state2vec.get_dim("nO") + output.pieces = model.state2vec.get_dim("nP") + output.feats = model.state2vec.get_dim("nF") + output.embed_width = model.tokvecs.shape[1] + return output + + +cdef ActivationsC alloc_activations(SizesC n) noexcept nogil: + cdef ActivationsC A + memset(&A, 0, sizeof(A)) + resize_activations(&A, n) + return A + + +cdef void free_activations(const ActivationsC* A) noexcept nogil: + free(A.token_ids) + free(A.scores) + free(A.unmaxed) + free(A.hiddens) + free(A.is_valid) + + +cdef void resize_activations(ActivationsC* A, SizesC n) noexcept nogil: + if n.states <= A._max_size: + A._curr_size = n.states + return + if A._max_size == 0: + A.token_ids = calloc(n.states * n.feats, sizeof(A.token_ids[0])) + A.scores = calloc(n.states * n.classes, sizeof(A.scores[0])) + A.unmaxed = calloc(n.states * n.hiddens * n.pieces, sizeof(A.unmaxed[0])) + A.hiddens = calloc(n.states * n.hiddens, sizeof(A.hiddens[0])) + A.is_valid = calloc(n.states * n.classes, sizeof(A.is_valid[0])) + A._max_size = n.states + else: + A.token_ids = realloc( + A.token_ids, n.states * n.feats * sizeof(A.token_ids[0]) + ) + A.scores = realloc( + A.scores, n.states * n.classes * sizeof(A.scores[0]) + ) + A.unmaxed = realloc( + A.unmaxed, n.states * n.hiddens * n.pieces * sizeof(A.unmaxed[0]) + ) + A.hiddens = realloc( + A.hiddens, n.states * n.hiddens * sizeof(A.hiddens[0]) + ) + A.is_valid = realloc( + A.is_valid, n.states * n.classes * sizeof(A.is_valid[0]) + ) + A._max_size = n.states + A._curr_size = n.states + + +cdef void predict_states( + CBlas cblas, ActivationsC* A, StateC** states, const WeightsC* W, SizesC n +) noexcept nogil: + resize_activations(A, n) + for i in range(n.states): + states[i].set_context_tokens(&A.token_ids[i*n.feats], n.feats) + memset(A.unmaxed, 0, n.states * n.hiddens * n.pieces * sizeof(float)) + memset(A.hiddens, 0, n.states * n.hiddens * sizeof(float)) + sum_state_features( + cblas, + A.unmaxed, + W.feat_weights, + A.token_ids, + n.states, + n.feats, + n.hiddens * n.pieces + ) + for i in range(n.states): + VecVec.add_i( + &A.unmaxed[i*n.hiddens*n.pieces], + W.feat_bias, 1., + n.hiddens * n.pieces + ) + for j in range(n.hiddens): + index = i * n.hiddens * n.pieces + j * n.pieces + which = Vec.arg_max(&A.unmaxed[index], n.pieces) + A.hiddens[i*n.hiddens + j] = A.unmaxed[index + which] + memset(A.scores, 0, n.states * n.classes * sizeof(float)) + if W.hidden_weights == NULL: + memcpy(A.scores, A.hiddens, n.states * n.classes * sizeof(float)) + else: + # Compute hidden-to-output + sgemm(cblas)( + False, True, n.states, n.classes, n.hiddens, + 1.0, A.hiddens, n.hiddens, + W.hidden_weights, n.hiddens, + 0.0, A.scores, n.classes + ) + # Add bias + for i in range(n.states): + VecVec.add_i(&A.scores[i*n.classes], W.hidden_bias, 1., n.classes) + # Set unseen classes to minimum value + i = 0 + min_ = A.scores[0] + for i in range(1, n.states * n.classes): + if A.scores[i] < min_: + min_ = A.scores[i] + for i in range(n.states): + for j in range(n.classes): + if not W.seen_classes[j]: + A.scores[i*n.classes+j] = min_ + + +cdef void sum_state_features( + CBlas cblas, + float* output, + const float* cached, + const int* token_ids, + int B, + int F, + int O +) noexcept nogil: + cdef int idx, b, f + cdef const float* feature + padding = cached + cached += F * O + cdef int id_stride = F*O + cdef float one = 1. + for b in range(B): + for f in range(F): + if token_ids[f] < 0: + feature = &padding[f*O] + else: + idx = token_ids[f] * id_stride + f*O + feature = &cached[idx] + saxpy(cblas)(O, one, feature, 1, &output[b*O], 1) + token_ids += F + + +cdef void cpu_log_loss( + float* d_scores, + const float* costs, + const int* is_valid, + const float* scores, + int O +) noexcept nogil: + """Do multi-label log loss""" + cdef double max_, gmax, Z, gZ + best = arg_max_if_gold(scores, costs, is_valid, O) + guess = Vec.arg_max(scores, O) + if best == -1 or guess == -1: + # These shouldn't happen, but if they do, we want to make sure we don't + # cause an OOB access. + return + Z = 1e-10 + gZ = 1e-10 + max_ = scores[guess] + gmax = scores[best] + for i in range(O): + Z += exp(scores[i] - max_) + if costs[i] <= costs[best]: + gZ += exp(scores[i] - gmax) + for i in range(O): + if costs[i] <= costs[best]: + d_scores[i] = (exp(scores[i]-max_) / Z) - (exp(scores[i]-gmax)/gZ) + else: + d_scores[i] = exp(scores[i]-max_) / Z + + +cdef int arg_max_if_gold( + const weight_t* scores, const weight_t* costs, const int* is_valid, int n +) noexcept nogil: + # Find minimum cost + cdef float cost = 1 + for i in range(n): + if is_valid[i] and costs[i] < cost: + cost = costs[i] + # Now find best-scoring with that cost + cdef int best = -1 + for i in range(n): + if costs[i] <= cost and is_valid[i]: + if best == -1 or scores[i] > scores[best]: + best = i + return best + + +cdef int arg_max_if_valid(const weight_t* scores, const int* is_valid, int n) noexcept nogil: + cdef int best = -1 + for i in range(n): + if is_valid[i] >= 1: + if best == -1 or scores[i] > scores[best]: + best = i + return best + + +class ParserStepModel(Model): + def __init__( + self, + docs, + layers, + *, + has_upper, + unseen_classes=None, + train=True, + dropout=0.1 + ): + Model.__init__(self, name="parser_step_model", forward=step_forward) + self.attrs["has_upper"] = has_upper + self.attrs["dropout_rate"] = dropout + self.tokvecs, self.bp_tokvecs = layers[0](docs, is_train=train) + if layers[1].get_dim("nP") >= 2: + activation = "maxout" + elif has_upper: + activation = None + else: + activation = "relu" + self.state2vec = precompute_hiddens(len(docs), self.tokvecs, layers[1], + activation=activation, train=train) + if has_upper: + self.vec2scores = layers[-1] + else: + self.vec2scores = None + self.cuda_stream = util.get_cuda_stream(non_blocking=True) + self.backprops = [] + self._class_mask = numpy.zeros((self.nO,), dtype='f') + self._class_mask.fill(1) + if unseen_classes is not None: + for class_ in unseen_classes: + self._class_mask[class_] = 0. + + def clear_memory(self): + del self.tokvecs + del self.bp_tokvecs + del self.state2vec + del self.backprops + del self._class_mask + + @property + def nO(self): + if self.attrs["has_upper"]: + return self.vec2scores.get_dim("nO") + else: + return self.state2vec.get_dim("nO") + + def class_is_unseen(self, class_): + return self._class_mask[class_] + + def mark_class_unseen(self, class_): + self._class_mask[class_] = 0 + + def mark_class_seen(self, class_): + self._class_mask[class_] = 1 + + def get_token_ids(self, states): + cdef StateClass state + states = [state for state in states if not state.is_final()] + cdef np.ndarray ids = numpy.zeros((len(states), self.state2vec.nF), + dtype='i', order='C') + ids.fill(-1) + c_ids = ids.data + for state in states: + state.c.set_context_tokens(c_ids, ids.shape[1]) + c_ids += ids.shape[1] + return ids + + def backprop_step(self, token_ids, d_vector, get_d_tokvecs): + if ( + isinstance(self.state2vec.ops, CupyOps) + and not isinstance(token_ids, self.state2vec.ops.xp.ndarray) + ): + # Move token_ids and d_vector to GPU, asynchronously + self.backprops.append(( + util.get_async(self.cuda_stream, token_ids), + util.get_async(self.cuda_stream, d_vector), + get_d_tokvecs + )) + else: + self.backprops.append((token_ids, d_vector, get_d_tokvecs)) + + def finish_steps(self, golds): + # Add a padding vector to the d_tokvecs gradient, so that missing + # values don't affect the real gradient. + d_tokvecs = self.ops.alloc((self.tokvecs.shape[0]+1, self.tokvecs.shape[1])) + # Tells CUDA to block, so our async copies complete. + if self.cuda_stream is not None: + self.cuda_stream.synchronize() + for ids, d_vector, bp_vector in self.backprops: + d_state_features = bp_vector((d_vector, ids)) + ids = ids.flatten() + d_state_features = d_state_features.reshape( + (ids.size, d_state_features.shape[2])) + self.ops.scatter_add(d_tokvecs, ids, d_state_features) + # Padded -- see update() + self.bp_tokvecs(d_tokvecs[:-1]) + return d_tokvecs + + +NUMPY_OPS = NumpyOps() + + +def step_forward(model: ParserStepModel, states, is_train): + token_ids = model.get_token_ids(states) + vector, get_d_tokvecs = model.state2vec(token_ids, is_train) + mask = None + if model.attrs["has_upper"]: + dropout_rate = model.attrs["dropout_rate"] + if is_train and dropout_rate > 0: + mask = NUMPY_OPS.get_dropout_mask(vector.shape, 0.1) + vector *= mask + scores, get_d_vector = model.vec2scores(vector, is_train) + else: + scores = NumpyOps().asarray(vector) + get_d_vector = lambda d_scores: d_scores # no-cython-lint: E731 + # If the class is unseen, make sure its score is minimum + scores[:, model._class_mask == 0] = numpy.nanmin(scores) + + def backprop_parser_step(d_scores): + # Zero vectors for unseen classes + d_scores *= model._class_mask + d_vector = get_d_vector(d_scores) + if mask is not None: + d_vector *= mask + model.backprop_step(token_ids, d_vector, get_d_tokvecs) + return None + return scores, backprop_parser_step + + +cdef class precompute_hiddens: + """Allow a model to be "primed" by pre-computing input features in bulk. + + This is used for the parser, where we want to take a batch of documents, + and compute vectors for each (token, position) pair. These vectors can then + be reused, especially for beam-search. + + Let's say we're using 12 features for each state, e.g. word at start of + buffer, three words on stack, their children, etc. In the normal arc-eager + system, a document of length N is processed in 2*N states. This means we'll + create 2*N*12 feature vectors --- but if we pre-compute, we only need + N*12 vector computations. The saving for beam-search is much better: + if we have a beam of k, we'll normally make 2*N*12*K computations -- + so we can save the factor k. This also gives a nice CPU/GPU division: + we can do all our hard maths up front, packed into large multiplications, + and do the hard-to-program parsing on the CPU. + """ + cdef readonly int nF, nO, nP + cdef bint _is_synchronized + cdef public object ops + cdef public object numpy_ops + cdef np.ndarray _features + cdef np.ndarray _cached + cdef np.ndarray bias + cdef object _cuda_stream + cdef object _bp_hiddens + cdef object activation + + def __init__(self, batch_size, tokvecs, lower_model, cuda_stream=None, + activation="maxout", train=False): + gpu_cached, bp_features = lower_model(tokvecs, train) + cdef np.ndarray cached + if not isinstance(gpu_cached, numpy.ndarray): + # Note the passing of cuda_stream here: it lets + # cupy make the copy asynchronously. + # We then have to block before first use. + cached = gpu_cached.get(stream=cuda_stream) + else: + cached = gpu_cached + if not isinstance(lower_model.get_param("b"), numpy.ndarray): + self.bias = lower_model.get_param("b").get(stream=cuda_stream) + else: + self.bias = lower_model.get_param("b") + self.nF = cached.shape[1] + if lower_model.has_dim("nP"): + self.nP = lower_model.get_dim("nP") + else: + self.nP = 1 + self.nO = cached.shape[2] + self.ops = lower_model.ops + self.numpy_ops = NumpyOps() + assert activation in (None, "relu", "maxout") + self.activation = activation + self._is_synchronized = False + self._cuda_stream = cuda_stream + self._cached = cached + self._bp_hiddens = bp_features + + cdef const float* get_feat_weights(self) except NULL: + if not self._is_synchronized and self._cuda_stream is not None: + self._cuda_stream.synchronize() + self._is_synchronized = True + return self._cached.data + + def has_dim(self, name): + if name == "nF": + return self.nF if self.nF is not None else True + elif name == "nP": + return self.nP if self.nP is not None else True + elif name == "nO": + return self.nO if self.nO is not None else True + else: + return False + + def get_dim(self, name): + if name == "nF": + return self.nF + elif name == "nP": + return self.nP + elif name == "nO": + return self.nO + else: + raise ValueError(Errors.E1033.format(name=name)) + + def set_dim(self, name, value): + if name == "nF": + self.nF = value + elif name == "nP": + self.nP = value + elif name == "nO": + self.nO = value + else: + raise ValueError(Errors.E1033.format(name=name)) + + def __call__(self, X, bint is_train): + if is_train: + return self.begin_update(X) + else: + return self.predict(X), lambda X: X + + def predict(self, X): + return self.begin_update(X)[0] + + def begin_update(self, token_ids): + cdef np.ndarray state_vector = numpy.zeros( + (token_ids.shape[0], self.nO, self.nP), dtype='f') + # This is tricky, but (assuming GPU available); + # - Input to forward on CPU + # - Output from forward on CPU + # - Input to backward on GPU! + # - Output from backward on GPU + bp_hiddens = self._bp_hiddens + + cdef CBlas cblas + if isinstance(self.ops, CupyOps): + cblas = NUMPY_OPS.cblas() + else: + cblas = self.ops.cblas() + + feat_weights = self.get_feat_weights() + cdef int[:, ::1] ids = token_ids + sum_state_features( + cblas, state_vector.data, + feat_weights, &ids[0, 0], + token_ids.shape[0], self.nF, self.nO*self.nP + ) + state_vector += self.bias + state_vector, bp_nonlinearity = self._nonlinearity(state_vector) + + def backward(d_state_vector_ids): + d_state_vector, token_ids = d_state_vector_ids + d_state_vector = bp_nonlinearity(d_state_vector) + d_tokens = bp_hiddens((d_state_vector, token_ids)) + return d_tokens + return state_vector, backward + + def _nonlinearity(self, state_vector): + if self.activation == "maxout": + return self._maxout_nonlinearity(state_vector) + else: + return self._relu_nonlinearity(state_vector) + + def _maxout_nonlinearity(self, state_vector): + state_vector, mask = self.numpy_ops.maxout(state_vector) + # We're outputting to CPU, but we need this variable on GPU for the + # backward pass. + mask = self.ops.asarray(mask) + + def backprop_maxout(d_best): + return self.ops.backprop_maxout(d_best, mask, self.nP) + + return state_vector, backprop_maxout + + def _relu_nonlinearity(self, state_vector): + state_vector = state_vector.reshape((state_vector.shape[0], -1)) + mask = state_vector >= 0. + state_vector *= mask + # We're outputting to CPU, but we need this variable on GPU for the + # backward pass. + mask = self.ops.asarray(mask) + + def backprop_relu(d_best): + d_best *= mask + return d_best.reshape((d_best.shape + (1,))) + + return state_vector, backprop_relu diff --git a/spacy/ml/staticvectors.py b/spacy/ml/staticvectors.py new file mode 100644 index 0000000..d90acda --- /dev/null +++ b/spacy/ml/staticvectors.py @@ -0,0 +1,124 @@ +import warnings +from typing import Callable, List, Optional, Tuple, cast + +from thinc.api import Model, Ops +from thinc.initializers import glorot_uniform_init +from thinc.types import Floats1d, Floats2d, Ints1d, Ragged +from thinc.util import partial + +from ..attrs import ORTH +from ..errors import Errors, Warnings +from ..tokens import Doc +from ..vectors import Mode, Vectors +from ..vocab import Vocab + + +def StaticVectors( + nO: Optional[int] = None, + nM: Optional[int] = None, + *, + dropout: Optional[float] = None, + init_W: Callable = glorot_uniform_init, + key_attr: str = "ORTH", +) -> Model[List[Doc], Ragged]: + """Embed Doc objects with their vocab's vectors table, applying a learned + linear projection to control the dimensionality. If a dropout rate is + specified, the dropout is applied per dimension over the whole batch. + """ + if key_attr != "ORTH": + warnings.warn(Warnings.W125, DeprecationWarning) + return Model( + "static_vectors", + forward, + init=partial(init, init_W), + params={"W": None}, + attrs={"key_attr": key_attr, "dropout_rate": dropout}, + dims={"nO": nO, "nM": nM}, + ) + + +def forward( + model: Model[List[Doc], Ragged], docs: List[Doc], is_train: bool +) -> Tuple[Ragged, Callable]: + token_count = sum(len(doc) for doc in docs) + if not token_count: + return _handle_empty(model.ops, model.get_dim("nO")) + vocab: Vocab = docs[0].vocab + key_attr: int = getattr(vocab.vectors, "attr", ORTH) + keys = model.ops.flatten([cast(Ints1d, doc.to_array(key_attr)) for doc in docs]) + W = cast(Floats2d, model.ops.as_contig(model.get_param("W"))) + if isinstance(vocab.vectors, Vectors) and vocab.vectors.mode == Mode.default: + V = model.ops.asarray(vocab.vectors.data) + rows = vocab.vectors.find(keys=keys) + V = model.ops.as_contig(V[rows]) + elif isinstance(vocab.vectors, Vectors) and vocab.vectors.mode == Mode.floret: + V = vocab.vectors.get_batch(keys) + V = model.ops.as_contig(V) + elif hasattr(vocab.vectors, "get_batch"): + V = vocab.vectors.get_batch(keys) + V = model.ops.as_contig(V) + else: + raise RuntimeError(Errors.E896) + try: + vectors_data = model.ops.gemm(V, W, trans2=True) + except ValueError: + raise RuntimeError(Errors.E896) + if isinstance(vocab.vectors, Vectors) and vocab.vectors.mode == Mode.default: + # Convert negative indices to 0-vectors + # TODO: more options for UNK tokens + vectors_data[rows < 0] = 0 + output = Ragged(vectors_data, model.ops.asarray1i([len(doc) for doc in docs])) + mask = None + if is_train: + mask = _get_drop_mask(model.ops, W.shape[0], model.attrs.get("dropout_rate")) + if mask is not None: + output.data *= mask + + def backprop(d_output: Ragged) -> List[Doc]: + if mask is not None: + d_output.data *= mask + model.inc_grad( + "W", + model.ops.gemm( + cast(Floats2d, d_output.data), + cast(Floats2d, model.ops.as_contig(V)), + trans1=True, + ), + ) + return [] + + return output, backprop + + +def init( + init_W: Callable, + model: Model[List[Doc], Ragged], + X: Optional[List[Doc]] = None, + Y: Optional[Ragged] = None, +) -> Model[List[Doc], Ragged]: + nM = model.get_dim("nM") if model.has_dim("nM") else None + nO = model.get_dim("nO") if model.has_dim("nO") else None + if X is not None and len(X): + nM = X[0].vocab.vectors.shape[1] + if Y is not None: + nO = Y.data.shape[1] + + if nM is None: + raise ValueError(Errors.E905) + if nO is None: + raise ValueError(Errors.E904) + model.set_dim("nM", nM) + model.set_dim("nO", nO) + model.set_param("W", init_W(model.ops, (nO, nM))) + return model + + +def _handle_empty(ops: Ops, nO: int): + return Ragged(ops.alloc2f(0, nO), ops.alloc1i(0)), lambda d_ragged: [] + + +def _get_drop_mask(ops: Ops, nO: int, rate: Optional[float]) -> Optional[Floats1d]: + if rate is not None: + mask = ops.get_dropout_mask((nO,), rate) + return mask # type: ignore + return None diff --git a/spacy/ml/tb_framework.py b/spacy/ml/tb_framework.py new file mode 100644 index 0000000..e538b9e --- /dev/null +++ b/spacy/ml/tb_framework.py @@ -0,0 +1,49 @@ +from thinc.api import Model, noop + +from .parser_model import ParserStepModel + + +def TransitionModel( + tok2vec, lower, upper, resize_output, dropout=0.2, unseen_classes=set() +): + """Set up a stepwise transition-based model""" + if upper is None: + has_upper = False + upper = noop() + else: + has_upper = True + # don't define nO for this object, because we can't dynamically change it + return Model( + name="parser_model", + forward=forward, + dims={"nI": tok2vec.maybe_get_dim("nI")}, + layers=[tok2vec, lower, upper], + refs={"tok2vec": tok2vec, "lower": lower, "upper": upper}, + init=init, + attrs={ + "has_upper": has_upper, + "unseen_classes": set(unseen_classes), + "resize_output": resize_output, + }, + ) + + +def forward(model, X, is_train): + step_model = ParserStepModel( + X, + model.layers, + unseen_classes=model.attrs["unseen_classes"], + train=is_train, + has_upper=model.attrs["has_upper"], + ) + + return step_model, step_model.finish_steps + + +def init(model, X=None, Y=None): + model.get_ref("tok2vec").initialize(X=X) + lower = model.get_ref("lower") + lower.initialize() + if model.attrs["has_upper"]: + statevecs = model.ops.alloc2f(2, lower.get_dim("nO")) + model.get_ref("upper").initialize(X=statevecs) diff --git a/spacy/morphology.pxd b/spacy/morphology.pxd new file mode 100644 index 0000000..ee43aa4 --- /dev/null +++ b/spacy/morphology.pxd @@ -0,0 +1,27 @@ +cimport numpy as np +from cymem.cymem cimport Pool +from libc.stdint cimport uint64_t +from preshed.maps cimport PreshMap + +from .strings cimport StringStore +from .structs cimport MorphAnalysisC +from .typedefs cimport attr_t, hash_t + + +cdef class Morphology: + cdef readonly Pool mem + cdef readonly StringStore strings + cdef PreshMap tags # Keyed by hash, value is pointer to tag + + cdef MorphAnalysisC create_morph_tag(self, field_feature_pairs) except * + cdef int insert(self, MorphAnalysisC tag) except -1 + + +cdef int check_feature(const MorphAnalysisC* morph, attr_t feature) nogil +cdef list list_features(const MorphAnalysisC* morph) +cdef np.ndarray get_by_field(const MorphAnalysisC* morph, attr_t field) +cdef int get_n_by_field( + attr_t* results, + const MorphAnalysisC* morph, + attr_t field, +) nogil diff --git a/spacy/morphology.pyx b/spacy/morphology.pyx new file mode 100644 index 0000000..6f0cb03 --- /dev/null +++ b/spacy/morphology.pyx @@ -0,0 +1,206 @@ +# cython: infer_types +# cython: profile=False +import warnings + +import numpy + +from .attrs cimport POS + +from . import symbols +from .errors import Warnings +from .parts_of_speech import IDS as POS_IDS + + +cdef class Morphology: + """Store the possible morphological analyses for a language, and index them + by hash. + + To save space on each token, tokens only know the hash of their + morphological analysis, so queries of morphological attributes are delegated + to this class. + """ + FEATURE_SEP = "|" + FIELD_SEP = "=" + VALUE_SEP = "," + # not an empty string so we can distinguish unset morph from empty morph + EMPTY_MORPH = symbols.NAMES[symbols._] + + def __init__(self, StringStore strings): + self.mem = Pool() + self.strings = strings + self.tags = PreshMap() + + def __reduce__(self): + tags = set([self.get(self.strings[s]) for s in self.strings]) + tags -= set([""]) + return (unpickle_morphology, (self.strings, sorted(tags)), None, None) + + def add(self, features): + """Insert a morphological analysis in the morphology table, if not + already present. The morphological analysis may be provided in the UD + FEATS format as a string or in the tag map dict format. + Returns the hash of the new analysis. + """ + cdef MorphAnalysisC* tag_ptr + if isinstance(features, str): + if features == "": + features = self.EMPTY_MORPH + tag_ptr = self.tags.get(self.strings[features]) + if tag_ptr != NULL: + return tag_ptr.key + features = self.feats_to_dict(features) + if not isinstance(features, dict): + warnings.warn(Warnings.W100.format(feature=features)) + features = {} + string_features = {self.strings.as_string(field): self.strings.as_string(values) for field, values in features.items()} + # intified ("Field", "Field=Value") pairs + field_feature_pairs = [] + for field in sorted(string_features): + values = string_features[field] + self.strings.add(field, allow_transient=False), + field_id = self.strings[field] + for value in values.split(self.VALUE_SEP): + field_sep_value = field + self.FIELD_SEP + value + self.strings.add(field_sep_value, allow_transient=False), + field_feature_pairs.append(( + field_id, + self.strings[field_sep_value] + )) + cdef MorphAnalysisC tag = self.create_morph_tag(field_feature_pairs) + # the hash key for the tag is either the hash of the normalized UFEATS + # string or the hash of an empty placeholder + norm_feats_string = self.normalize_features(features) + tag.key = self.strings.add(norm_feats_string, allow_transient=False) + self.insert(tag) + return tag.key + + def normalize_features(self, features): + """Create a normalized FEATS string from a features string or dict. + + features (Union[dict, str]): Features as dict or UFEATS string. + RETURNS (str): Features as normalized UFEATS string. + """ + if isinstance(features, str): + features = self.feats_to_dict(features) + if not isinstance(features, dict): + warnings.warn(Warnings.W100.format(feature=features)) + features = {} + features = self.normalize_attrs(features) + string_features = {self.strings.as_string(field): self.strings.as_string(values) for field, values in features.items()} + # normalized UFEATS string with sorted fields and values + norm_feats_string = self.FEATURE_SEP.join( + sorted( + [self.FIELD_SEP.join([field, values]) for field, values in string_features.items()] + ) + ) + return norm_feats_string or self.EMPTY_MORPH + + def normalize_attrs(self, attrs): + """Convert attrs dict so that POS is always by ID, other features are + by string. Values separated by VALUE_SEP are sorted. + """ + out = {} + attrs = dict(attrs) + for key, value in attrs.items(): + # convert POS value to ID + if key == POS or (isinstance(key, str) and key.upper() == "POS"): + if isinstance(value, str) and value.upper() in POS_IDS: + value = POS_IDS[value.upper()] + elif isinstance(value, int) and value not in POS_IDS.values(): + warnings.warn(Warnings.W100.format(feature={key: value})) + continue + out[POS] = value + # accept any string or ID fields and values and convert to strings + elif isinstance(key, (int, str)) and isinstance(value, (int, str)): + key = self.strings.as_string(key) + value = self.strings.as_string(value) + # sort values + if self.VALUE_SEP in value: + value = self.VALUE_SEP.join(sorted(value.split(self.VALUE_SEP))) + out[key] = value + else: + warnings.warn(Warnings.W100.format(feature={key: value})) + return out + + cdef MorphAnalysisC create_morph_tag(self, field_feature_pairs) except *: + """Creates a MorphAnalysisC from a list of intified + ("Field", "Field=Value") tuples where fields with multiple values have + been split into individual tuples, e.g.: + [("Field1", "Field1=Value1"), ("Field1", "Field1=Value2"), + ("Field2", "Field2=Value3")] + """ + cdef MorphAnalysisC tag + tag.length = len(field_feature_pairs) + if tag.length > 0: + tag.fields = self.mem.alloc(tag.length, sizeof(attr_t)) + tag.features = self.mem.alloc(tag.length, sizeof(attr_t)) + for i, (field, feature) in enumerate(field_feature_pairs): + tag.fields[i] = field + tag.features[i] = feature + return tag + + cdef int insert(self, MorphAnalysisC tag) except -1: + cdef hash_t key = tag.key + if self.tags.get(key) == NULL: + tag_ptr = self.mem.alloc(1, sizeof(MorphAnalysisC)) + tag_ptr[0] = tag + self.tags.set(key, tag_ptr) + + def get(self, hash_t morph): + tag = self.tags.get(morph) + if tag == NULL: + return "" + else: + return self.strings[tag.key] + + @staticmethod + def feats_to_dict(feats): + if not feats or feats == Morphology.EMPTY_MORPH: + return {} + return {field: Morphology.VALUE_SEP.join(sorted(values.split(Morphology.VALUE_SEP))) for field, values in + [feat.split(Morphology.FIELD_SEP) for feat in feats.split(Morphology.FEATURE_SEP)]} + + @staticmethod + def dict_to_feats(feats_dict): + if len(feats_dict) == 0: + return "" + return Morphology.FEATURE_SEP.join(sorted([Morphology.FIELD_SEP.join([field, Morphology.VALUE_SEP.join(sorted(values.split(Morphology.VALUE_SEP)))]) for field, values in feats_dict.items()])) + + +cdef int check_feature(const MorphAnalysisC* morph, attr_t feature) nogil: + cdef int i + for i in range(morph.length): + if morph.features[i] == feature: + return True + return False + + +cdef list list_features(const MorphAnalysisC* morph): + cdef int i + features = [] + for i in range(morph.length): + features.append(morph.features[i]) + return features + + +cdef np.ndarray get_by_field(const MorphAnalysisC* morph, attr_t field): + cdef np.ndarray results = numpy.zeros((morph.length,), dtype="uint64") + n = get_n_by_field(results.data, morph, field) + return results[:n] + + +cdef int get_n_by_field(attr_t* results, const MorphAnalysisC* morph, attr_t field) nogil: + cdef int n_results = 0 + cdef int i + for i in range(morph.length): + if morph.fields[i] == field: + results[n_results] = morph.features[i] + n_results += 1 + return n_results + + +def unpickle_morphology(strings, tags): + cdef Morphology morphology = Morphology(strings) + for tag in tags: + morphology.add(tag) + return morphology diff --git a/spacy/parts_of_speech.pxd b/spacy/parts_of_speech.pxd new file mode 100644 index 0000000..b5423d1 --- /dev/null +++ b/spacy/parts_of_speech.pxd @@ -0,0 +1,25 @@ +from . cimport symbols + + +cpdef enum univ_pos_t: + NO_TAG = 0 + ADJ = symbols.ADJ + ADP + ADV + AUX + CONJ + CCONJ # U20 + DET + INTJ + NOUN + NUM + PART + PRON + PROPN + PUNCT + SCONJ + SYM + VERB + X + EOL + SPACE diff --git a/spacy/parts_of_speech.pyx b/spacy/parts_of_speech.pyx new file mode 100644 index 0000000..1e643c0 --- /dev/null +++ b/spacy/parts_of_speech.pyx @@ -0,0 +1,32 @@ +# cython: profile=False +IDS = { + "": NO_TAG, + "ADJ": ADJ, + "ADP": ADP, + "ADV": ADV, + "AUX": AUX, + "CONJ": CONJ, # U20 + "CCONJ": CCONJ, + "DET": DET, + "INTJ": INTJ, + "NOUN": NOUN, + "NUM": NUM, + "PART": PART, + "PRON": PRON, + "PROPN": PROPN, + "PUNCT": PUNCT, + "SCONJ": SCONJ, + "SYM": SYM, + "VERB": VERB, + "X": X, + "EOL": EOL, + "SPACE": SPACE +} + + +NAMES = {value: key for key, value in IDS.items()} + +# As of Cython 3.1, the global Python namespace no longer has the enum +# contents by default. +globals().update(IDS) + diff --git a/spacy/pipe_analysis.py b/spacy/pipe_analysis.py new file mode 100644 index 0000000..b564b46 --- /dev/null +++ b/spacy/pipe_analysis.py @@ -0,0 +1,138 @@ +from typing import TYPE_CHECKING, Dict, ItemsView, Iterable, List, Set, Union + +from wasabi import msg + +from .errors import Errors +from .tokens import Doc, Span, Token +from .util import dot_to_dict + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language # noqa: F401 + + +DEFAULT_KEYS = ["requires", "assigns", "scores", "retokenizes"] + + +def validate_attrs(values: Iterable[str]) -> Iterable[str]: + """Validate component attributes provided to "assigns", "requires" etc. + Raises error for invalid attributes and formatting. Doesn't check if + custom extension attributes are registered, since this is something the + user might want to do themselves later in the component. + + values (Iterable[str]): The string attributes to check, e.g. `["token.pos"]`. + RETURNS (Iterable[str]): The checked attributes. + """ + data = dot_to_dict(dict.fromkeys(values, True)) + objs = {"doc": Doc, "token": Token, "span": Span} + for obj_key, attrs in data.items(): + if obj_key == "span": + # Support Span only for custom extension attributes + span_attrs = [attr for attr in values if attr.startswith("span.")] + span_attrs = [attr for attr in span_attrs if not attr.startswith("span._.")] + if span_attrs: + raise ValueError(Errors.E180.format(attrs=", ".join(span_attrs))) + if obj_key not in objs: # first element is not doc/token/span + invalid_attrs = ", ".join(a for a in values if a.startswith(obj_key)) + raise ValueError(Errors.E181.format(obj=obj_key, attrs=invalid_attrs)) + if not isinstance(attrs, dict): # attr is something like "doc" + raise ValueError(Errors.E182.format(attr=obj_key)) + for attr, value in attrs.items(): + if attr == "_": + if value is True: # attr is something like "doc._" + raise ValueError(Errors.E182.format(attr="{}._".format(obj_key))) + for ext_attr, ext_value in value.items(): + # We don't check whether the attribute actually exists + if ext_value is not True: # attr is something like doc._.x.y + good = f"{obj_key}._.{ext_attr}" + bad = f"{good}.{'.'.join(ext_value)}" + raise ValueError(Errors.E183.format(attr=bad, solution=good)) + continue # we can't validate those further + if attr.endswith("_"): # attr is something like "token.pos_" + raise ValueError(Errors.E184.format(attr=attr, solution=attr[:-1])) + if value is not True: # attr is something like doc.x.y + good = f"{obj_key}.{attr}" + bad = f"{good}.{'.'.join(value)}" + raise ValueError(Errors.E183.format(attr=bad, solution=good)) + obj = objs[obj_key] + if not hasattr(obj, attr): + raise ValueError(Errors.E185.format(obj=obj_key, attr=attr)) + return values + + +def get_attr_info(nlp: "Language", attr: str) -> Dict[str, List[str]]: + """Check which components in the pipeline assign or require an attribute. + + nlp (Language): The current nlp object. + attr (str): The attribute, e.g. "doc.tensor". + RETURNS (Dict[str, List[str]]): A dict keyed by "assigns" and "requires", + mapped to a list of component names. + """ + result: Dict[str, List[str]] = {"assigns": [], "requires": []} + for pipe_name in nlp.pipe_names: + meta = nlp.get_pipe_meta(pipe_name) + if attr in meta.assigns: + result["assigns"].append(pipe_name) + if attr in meta.requires: + result["requires"].append(pipe_name) + return result + + +def analyze_pipes( + nlp: "Language", *, keys: List[str] = DEFAULT_KEYS +) -> Dict[str, Dict[str, Union[List[str], Dict]]]: + """Print a formatted summary for the current nlp object's pipeline. Shows + a table with the pipeline components and why they assign and require, as + well as any problems if available. + + nlp (Language): The nlp object. + keys (List[str]): The meta keys to show in the table. + RETURNS (dict): A dict with "summary" and "problems". + """ + result: Dict[str, Dict[str, Union[List[str], Dict]]] = { + "summary": {}, + "problems": {}, + } + all_attrs: Set[str] = set() + for i, name in enumerate(nlp.pipe_names): + meta = nlp.get_pipe_meta(name) + all_attrs.update(meta.assigns) + all_attrs.update(meta.requires) + result["summary"][name] = {key: getattr(meta, key, None) for key in keys} + prev_pipes = nlp.pipeline[:i] + requires = dict.fromkeys(meta.requires, False) + if requires: + for prev_name, prev_pipe in prev_pipes: + prev_meta = nlp.get_pipe_meta(prev_name) + for annot in prev_meta.assigns: + requires[annot] = True + result["problems"][name] = [ + annot for annot, fulfilled in requires.items() if not fulfilled + ] + result["attrs"] = {attr: get_attr_info(nlp, attr) for attr in all_attrs} + return result + + +def print_pipe_analysis( + analysis: Dict[str, Dict[str, Union[List[str], Dict]]], + *, + keys: List[str] = DEFAULT_KEYS, +) -> None: + """Print a formatted version of the pipe analysis produced by analyze_pipes. + + analysis (Dict[str, Union[List[str], Dict[str, List[str]]]]): The analysis. + keys (List[str]): The meta keys to show in the table. + """ + msg.divider("Pipeline Overview") + header = ["#", "Component", *[key.capitalize() for key in keys]] + summary: ItemsView = analysis["summary"].items() + body = [[i, n, *[v for v in m.values()]] for i, (n, m) in enumerate(summary)] + msg.table(body, header=header, divider=True, multiline=True) + n_problems = sum(len(p) for p in analysis["problems"].values()) + if any(p for p in analysis["problems"].values()): + msg.divider(f"Problems ({n_problems})") + for name, problem in analysis["problems"].items(): + if problem: + msg.warn(f"'{name}' requirements not met: {', '.join(problem)}") + else: + msg.good("No problems found.") diff --git a/spacy/pipeline/__init__.py b/spacy/pipeline/__init__.py new file mode 100644 index 0000000..2c4a5a8 --- /dev/null +++ b/spacy/pipeline/__init__.py @@ -0,0 +1,45 @@ +from .attributeruler import AttributeRuler +from .dep_parser import DependencyParser +from .edit_tree_lemmatizer import EditTreeLemmatizer +from .entity_linker import EntityLinker +from .entityruler import EntityRuler +from .functions import merge_entities, merge_noun_chunks, merge_subtokens +from .lemmatizer import Lemmatizer +from .morphologizer import Morphologizer +from .ner import EntityRecognizer +from .pipe import Pipe +from .sentencizer import Sentencizer +from .senter import SentenceRecognizer +from .span_finder import SpanFinder +from .span_ruler import SpanRuler +from .spancat import SpanCategorizer +from .tagger import Tagger +from .textcat import TextCategorizer +from .textcat_multilabel import MultiLabel_TextCategorizer +from .tok2vec import Tok2Vec +from .trainable_pipe import TrainablePipe + +__all__ = [ + "AttributeRuler", + "DependencyParser", + "EditTreeLemmatizer", + "EntityLinker", + "EntityRecognizer", + "EntityRuler", + "Morphologizer", + "Lemmatizer", + "MultiLabel_TextCategorizer", + "Pipe", + "SentenceRecognizer", + "Sentencizer", + "SpanCategorizer", + "SpanFinder", + "SpanRuler", + "Tagger", + "TextCategorizer", + "Tok2Vec", + "TrainablePipe", + "merge_entities", + "merge_noun_chunks", + "merge_subtokens", +] diff --git a/spacy/pipeline/_edit_tree_internals/__init__.py b/spacy/pipeline/_edit_tree_internals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/pipeline/_edit_tree_internals/edit_trees.pxd b/spacy/pipeline/_edit_tree_internals/edit_trees.pxd new file mode 100644 index 0000000..41acd2b --- /dev/null +++ b/spacy/pipeline/_edit_tree_internals/edit_trees.pxd @@ -0,0 +1,101 @@ +from libc.stdint cimport uint32_t, uint64_t +from libcpp.unordered_map cimport unordered_map +from libcpp.vector cimport vector + +from ...strings cimport StringStore +from ...typedefs cimport attr_t, hash_t, len_t + + +cdef extern from "" namespace "std" nogil: + void swap[T](T& a, T& b) except + # Only available in Cython 3. + +# An edit tree (Müller et al., 2015) is a tree structure that consists of +# edit operations. The two types of operations are string matches +# and string substitutions. Given an input string s and an output string t, +# subsitution and match nodes should be interpreted as follows: +# +# * Substitution node: consists of an original string and substitute string. +# If s matches the original string, then t is the substitute. Otherwise, +# the node does not apply. +# * Match node: consists of a prefix length, suffix length, prefix edit tree, +# and suffix edit tree. If s is composed of a prefix, middle part, and suffix +# with the given suffix and prefix lengths, then t is the concatenation +# prefix_tree(prefix) + middle + suffix_tree(suffix). +# +# For efficiency, we represent strings in substitution nodes as integers, with +# the actual strings stored in a StringStore. Subtrees in match nodes are stored +# as tree identifiers (rather than pointers) to simplify serialization. + +cdef uint32_t NULL_TREE_ID + +cdef struct MatchNodeC: + len_t prefix_len + len_t suffix_len + uint32_t prefix_tree + uint32_t suffix_tree + +cdef struct SubstNodeC: + attr_t orig + attr_t subst + +cdef union NodeC: + MatchNodeC match_node + SubstNodeC subst_node + +cdef struct EditTreeC: + bint is_match_node + NodeC inner + +cdef inline EditTreeC edittree_new_match( + len_t prefix_len, + len_t suffix_len, + uint32_t prefix_tree, + uint32_t suffix_tree +): + cdef MatchNodeC match_node = MatchNodeC( + prefix_len=prefix_len, + suffix_len=suffix_len, + prefix_tree=prefix_tree, + suffix_tree=suffix_tree + ) + cdef NodeC inner = NodeC(match_node=match_node) + return EditTreeC(is_match_node=True, inner=inner) + +cdef inline EditTreeC edittree_new_subst(attr_t orig, attr_t subst): + cdef EditTreeC node + cdef SubstNodeC subst_node = SubstNodeC(orig=orig, subst=subst) + cdef NodeC inner = NodeC(subst_node=subst_node) + return EditTreeC(is_match_node=False, inner=inner) + +cdef inline uint64_t edittree_hash(EditTreeC tree): + cdef MatchNodeC match_node + cdef SubstNodeC subst_node + + if tree.is_match_node: + match_node = tree.inner.match_node + return hash((match_node.prefix_len, match_node.suffix_len, match_node.prefix_tree, match_node.suffix_tree)) + else: + subst_node = tree.inner.subst_node + return hash((subst_node.orig, subst_node.subst)) + +cdef struct LCS: + int source_begin + int source_end + int target_begin + int target_end + +cdef inline bint lcs_is_empty(LCS lcs): + return lcs.source_begin == 0 and lcs.source_end == 0 and lcs.target_begin == 0 and lcs.target_end == 0 + +cdef class EditTrees: + cdef vector[EditTreeC] trees + cdef unordered_map[hash_t, uint32_t] map + cdef StringStore strings + + cpdef uint32_t add(self, str form, str lemma) + cpdef str apply(self, uint32_t tree_id, str form) + cpdef unicode tree_to_str(self, uint32_t tree_id) + + cdef uint32_t _add(self, str form, str lemma) + cdef _apply(self, uint32_t tree_id, str form_part, list lemma_pieces) + cdef uint32_t _tree_id(self, EditTreeC tree) diff --git a/spacy/pipeline/_edit_tree_internals/edit_trees.pyx b/spacy/pipeline/_edit_tree_internals/edit_trees.pyx new file mode 100644 index 0000000..7abd9f2 --- /dev/null +++ b/spacy/pipeline/_edit_tree_internals/edit_trees.pyx @@ -0,0 +1,302 @@ +# cython: infer_types=True, binding=True +# cython: profile=False +from cython.operator cimport dereference as deref +from libc.stdint cimport UINT32_MAX, uint32_t +from libc.string cimport memset +from libcpp.pair cimport pair +from libcpp.vector cimport vector + +from ...typedefs cimport hash_t + +from ... import util +from ...errors import Errors +from ...strings import StringStore +from .schemas import validate_edit_tree + +NULL_TREE_ID = UINT32_MAX + +cdef LCS find_lcs(str source, str target): + """ + Find the longest common subsequence (LCS) between two strings. If there are + multiple LCSes, only one of them is returned. + + source (str): The first string. + target (str): The second string. + RETURNS (LCS): The spans of the longest common subsequences. + """ + cdef Py_ssize_t target_len = len(target) + cdef size_t longest_align = 0 + cdef int source_idx, target_idx + cdef LCS lcs + cdef Py_UCS4 source_cp, target_cp + + memset(&lcs, 0, sizeof(lcs)) + + cdef vector[size_t] prev_aligns = vector[size_t](target_len) + cdef vector[size_t] cur_aligns = vector[size_t](target_len) + + for (source_idx, source_cp) in enumerate(source): + for (target_idx, target_cp) in enumerate(target): + if source_cp == target_cp: + if source_idx == 0 or target_idx == 0: + cur_aligns[target_idx] = 1 + else: + cur_aligns[target_idx] = prev_aligns[target_idx - 1] + 1 + + # Check if this is the longest alignment and replace previous + # best alignment when this is the case. + if cur_aligns[target_idx] > longest_align: + longest_align = cur_aligns[target_idx] + lcs.source_begin = source_idx - longest_align + 1 + lcs.source_end = source_idx + 1 + lcs.target_begin = target_idx - longest_align + 1 + lcs.target_end = target_idx + 1 + else: + # No match, we start with a zero-length alignment. + cur_aligns[target_idx] = 0 + swap(prev_aligns, cur_aligns) + + return lcs + +cdef class EditTrees: + """Container for constructing and storing edit trees.""" + def __init__(self, strings: StringStore): + """Create a container for edit trees. + + strings (StringStore): the string store to use.""" + self.strings = strings + + cpdef uint32_t add(self, str form, str lemma): + """Add an edit tree that rewrites the given string into the given lemma. + + RETURNS (int): identifier of the edit tree in the container. + """ + # Treat two empty strings as a special case. Generating an edit + # tree for identical strings results in a match node. However, + # since two empty strings have a zero-length LCS, a substitution + # node would be created. Since we do not want to clutter the + # recursive tree construction with logic for this case, handle + # it in this wrapper method. + if len(form) == 0 and len(lemma) == 0: + tree = edittree_new_match(0, 0, NULL_TREE_ID, NULL_TREE_ID) + return self._tree_id(tree) + + return self._add(form, lemma) + + cdef uint32_t _add(self, str form, str lemma): + cdef LCS lcs = find_lcs(form, lemma) + + cdef EditTreeC tree + cdef uint32_t prefix_tree, suffix_tree + if lcs_is_empty(lcs): + tree = edittree_new_subst(self.strings.add(form), self.strings.add(lemma)) + else: + # If we have a non-empty LCS, such as "gooi" in "ge[gooi]d" and "[gooi]en", + # create edit trees for the prefix pair ("ge"/"") and the suffix pair ("d"/"en"). + prefix_tree = NULL_TREE_ID + if lcs.source_begin != 0 or lcs.target_begin != 0: + prefix_tree = self.add(form[:lcs.source_begin], lemma[:lcs.target_begin]) + + suffix_tree = NULL_TREE_ID + if lcs.source_end != len(form) or lcs.target_end != len(lemma): + suffix_tree = self.add(form[lcs.source_end:], lemma[lcs.target_end:]) + + tree = edittree_new_match(lcs.source_begin, len(form) - lcs.source_end, prefix_tree, suffix_tree) + + return self._tree_id(tree) + + cdef uint32_t _tree_id(self, EditTreeC tree): + # If this tree has been constructed before, return its identifier. + cdef hash_t hash = edittree_hash(tree) + cdef unordered_map[hash_t, uint32_t].iterator iter = self.map.find(hash) + if iter != self.map.end(): + return deref(iter).second + + # The tree hasn't been seen before, store it. + cdef uint32_t tree_id = self.trees.size() + self.trees.push_back(tree) + self.map.insert(pair[hash_t, uint32_t](hash, tree_id)) + + return tree_id + + cpdef str apply(self, uint32_t tree_id, str form): + """Apply an edit tree to a form. + + tree_id (uint32_t): the identifier of the edit tree to apply. + form (str): the form to apply the edit tree to. + RETURNS (str): the transformer form or None if the edit tree + could not be applied to the form. + """ + if tree_id >= self.trees.size(): + raise IndexError(Errors.E1030) + + lemma_pieces = [] + try: + self._apply(tree_id, form, lemma_pieces) + except ValueError: + return None + return "".join(lemma_pieces) + + cdef _apply(self, uint32_t tree_id, str form_part, list lemma_pieces): + """Recursively apply an edit tree to a form, adding pieces to + the lemma_pieces list.""" + assert tree_id <= self.trees.size() + + cdef EditTreeC tree = self.trees[tree_id] + cdef MatchNodeC match_node + cdef int suffix_start + + if tree.is_match_node: + match_node = tree.inner.match_node + + if match_node.prefix_len + match_node.suffix_len > len(form_part): + raise ValueError(Errors.E1029) + + suffix_start = len(form_part) - match_node.suffix_len + + if match_node.prefix_tree != NULL_TREE_ID: + self._apply(match_node.prefix_tree, form_part[:match_node.prefix_len], lemma_pieces) + + lemma_pieces.append(form_part[match_node.prefix_len:suffix_start]) + + if match_node.suffix_tree != NULL_TREE_ID: + self._apply(match_node.suffix_tree, form_part[suffix_start:], lemma_pieces) + else: + if form_part == self.strings[tree.inner.subst_node.orig]: + lemma_pieces.append(self.strings[tree.inner.subst_node.subst]) + else: + raise ValueError(Errors.E1029) + + cpdef unicode tree_to_str(self, uint32_t tree_id): + """Return the tree as a string. The tree tree string is formatted + like an S-expression. This is primarily useful for debugging. Match + nodes have the following format: + + (m prefix_len suffix_len prefix_tree suffix_tree) + + Substitution nodes have the following format: + + (s original substitute) + + tree_id (uint32_t): the identifier of the edit tree. + RETURNS (str): the tree as an S-expression. + """ + + if tree_id >= self.trees.size(): + raise IndexError(Errors.E1030) + + cdef EditTreeC tree = self.trees[tree_id] + cdef SubstNodeC subst_node + + if not tree.is_match_node: + subst_node = tree.inner.subst_node + return f"(s '{self.strings[subst_node.orig]}' '{self.strings[subst_node.subst]}')" + + cdef MatchNodeC match_node = tree.inner.match_node + + prefix_tree = "()" + if match_node.prefix_tree != NULL_TREE_ID: + prefix_tree = self.tree_to_str(match_node.prefix_tree) + + suffix_tree = "()" + if match_node.suffix_tree != NULL_TREE_ID: + suffix_tree = self.tree_to_str(match_node.suffix_tree) + + return f"(m {match_node.prefix_len} {match_node.suffix_len} {prefix_tree} {suffix_tree})" + + def from_json(self, trees: list) -> "EditTrees": + self.trees.clear() + + for tree in trees: + tree = _dict2tree(tree) + self.trees.push_back(tree) + + self._rebuild_tree_map() + + def from_bytes(self, bytes_data: bytes, *) -> "EditTrees": + def deserialize_trees(tree_dicts): + cdef EditTreeC c_tree + for tree_dict in tree_dicts: + c_tree = _dict2tree(tree_dict) + self.trees.push_back(c_tree) + + deserializers = {} + deserializers["trees"] = lambda n: deserialize_trees(n) + util.from_bytes(bytes_data, deserializers, []) + + self._rebuild_tree_map() + + return self + + def to_bytes(self, **kwargs) -> bytes: + tree_dicts = [] + for tree in self.trees: + tree = _tree2dict(tree) + tree_dicts.append(tree) + + serializers = {} + serializers["trees"] = lambda: tree_dicts + + return util.to_bytes(serializers, []) + + def to_disk(self, path, **kwargs) -> "EditTrees": + path = util.ensure_path(path) + with path.open("wb") as file_: + file_.write(self.to_bytes()) + + def from_disk(self, path, **kwargs) -> "EditTrees": + path = util.ensure_path(path) + if path.exists(): + with path.open("rb") as file_: + data = file_.read() + return self.from_bytes(data) + + return self + + def __getitem__(self, idx): + return _tree2dict(self.trees[idx]) + + def __len__(self): + return self.trees.size() + + def _rebuild_tree_map(self): + """Rebuild the tree hash -> tree id mapping""" + cdef EditTreeC c_tree + cdef uint32_t tree_id + cdef hash_t tree_hash + + self.map.clear() + + for tree_id in range(self.trees.size()): + c_tree = self.trees[tree_id] + tree_hash = edittree_hash(c_tree) + self.map.insert(pair[hash_t, uint32_t](tree_hash, tree_id)) + + def __reduce__(self): + return (unpickle_edittrees, (self.strings, self.to_bytes())) + + +def unpickle_edittrees(strings, trees_data): + return EditTrees(strings).from_bytes(trees_data) + + +def _tree2dict(tree): + if tree["is_match_node"]: + tree = tree["inner"]["match_node"] + else: + tree = tree["inner"]["subst_node"] + return(dict(tree)) + + +def _dict2tree(tree): + errors = validate_edit_tree(tree) + if errors: + raise ValueError(Errors.E1026.format(errors="\n".join(errors))) + + tree = dict(tree) + if "prefix_len" in tree: + tree = {"is_match_node": True, "inner": {"match_node": tree}} + else: + tree = {"is_match_node": False, "inner": {"subst_node": tree}} + + return tree diff --git a/spacy/pipeline/_edit_tree_internals/schemas.py b/spacy/pipeline/_edit_tree_internals/schemas.py new file mode 100644 index 0000000..ef7a076 --- /dev/null +++ b/spacy/pipeline/_edit_tree_internals/schemas.py @@ -0,0 +1,51 @@ +from collections import defaultdict +from typing import Any, Dict, List, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + RootModel, + StrictBool, + StrictInt, + StrictStr, + ValidationError, +) + + +class MatchNodeSchema(BaseModel): + prefix_len: StrictInt = Field(..., title="Prefix length") + suffix_len: StrictInt = Field(..., title="Suffix length") + prefix_tree: StrictInt = Field(..., title="Prefix tree") + suffix_tree: StrictInt = Field(..., title="Suffix tree") + + model_config = ConfigDict(extra="forbid") + + +class SubstNodeSchema(BaseModel): + orig: Union[int, StrictStr] = Field(..., title="Original substring") + subst: Union[int, StrictStr] = Field(..., title="Replacement substring") + + model_config = ConfigDict(extra="forbid") + + +class EditTreeSchema(RootModel[Union[MatchNodeSchema, SubstNodeSchema]]): + pass + + +def validate_edit_tree(obj: Dict[str, Any]) -> List[str]: + """Validate edit tree. + + obj (Dict[str, Any]): JSON-serializable data to validate. + RETURNS (List[str]): A list of error messages, if available. + """ + try: + EditTreeSchema.model_validate(obj) + return [] + except ValidationError as e: + errors = e.errors() + data = defaultdict(list) + for error in errors: + err_loc = " -> ".join([str(p) for p in error.get("loc", [])]) + data[err_loc].append(error.get("msg")) + return [f"[{loc}] {', '.join(msg)}" for loc, msg in data.items()] # type: ignore[arg-type] diff --git a/spacy/pipeline/_parser_internals/__init__.pxd b/spacy/pipeline/_parser_internals/__init__.pxd new file mode 100644 index 0000000..e69de29 diff --git a/spacy/pipeline/_parser_internals/__init__.py b/spacy/pipeline/_parser_internals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/pipeline/_parser_internals/_beam_utils.pxd b/spacy/pipeline/_parser_internals/_beam_utils.pxd new file mode 100644 index 0000000..596306b --- /dev/null +++ b/spacy/pipeline/_parser_internals/_beam_utils.pxd @@ -0,0 +1,7 @@ +from ...typedefs cimport class_t, hash_t + + +# These are passed as callbacks to thinc.search.Beam +cdef int transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1 + +cdef int check_final_state(void* _state, void* extra_args) except -1 diff --git a/spacy/pipeline/_parser_internals/_beam_utils.pyx b/spacy/pipeline/_parser_internals/_beam_utils.pyx new file mode 100644 index 0000000..ac04be5 --- /dev/null +++ b/spacy/pipeline/_parser_internals/_beam_utils.pyx @@ -0,0 +1,297 @@ +# cython: infer_types=True +import numpy + +from thinc.extra.search cimport Beam + +from thinc.extra.search import MaxViolation + +from thinc.extra.search cimport MaxViolation + +from ...typedefs cimport class_t +from .transition_system cimport Transition, TransitionSystem + +from ...errors import Errors + +from .stateclass cimport StateC, StateClass + + +# These are passed as callbacks to thinc.search.Beam +cdef int transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1: + dest = _dest + src = _src + moves = _moves + dest.clone(src) + moves[clas].do(dest, moves[clas].label) + + +cdef int check_final_state(void* _state, void* extra_args) except -1: + state = _state + return state.is_final() + + +cdef class BeamBatch(object): + cdef public TransitionSystem moves + cdef public object states + cdef public object docs + cdef public object golds + cdef public object beams + + def __init__(self, TransitionSystem moves, states, golds, + int width, float density=0.): + cdef StateClass state + self.moves = moves + self.states = states + self.docs = [state.doc for state in states] + self.golds = golds + self.beams = [] + cdef Beam beam + cdef StateC* st + for state in states: + beam = Beam(self.moves.n_moves, width, min_density=density) + beam.initialize(self.moves.init_beam_state, + self.moves.del_beam_state, state.c.length, + state.c._sent) + for i in range(beam.width): + st = beam.at(i) + st.offset = state.c.offset + beam.check_done(check_final_state, NULL) + self.beams.append(beam) + + @property + def is_done(self): + return all(b.is_done for b in self.beams) + + def __getitem__(self, i): + return self.beams[i] + + def __len__(self): + return len(self.beams) + + def get_states(self): + cdef Beam beam + cdef StateC* state + cdef StateClass stcls + states = [] + for beam, doc in zip(self, self.docs): + for i in range(beam.size): + state = beam.at(i) + stcls = StateClass.borrow(state, doc) + states.append(stcls) + return states + + def get_unfinished_states(self): + return [st for st in self.get_states() if not st.is_final()] + + def advance(self, float[:, ::1] scores, follow_gold=False): + cdef Beam beam + cdef int nr_class = scores.shape[1] + cdef const float* c_scores = &scores[0, 0] + docs = self.docs + for i, beam in enumerate(self): + if not beam.is_done: + nr_state = self._set_scores(beam, c_scores, nr_class) + assert nr_state + if self.golds is not None: + self._set_costs( + beam, + docs[i], + self.golds[i], + follow_gold=follow_gold + ) + c_scores += nr_state * nr_class + beam.advance(transition_state, NULL, self.moves.c) + beam.check_done(check_final_state, NULL) + + cdef int _set_scores(self, Beam beam, const float* scores, int nr_class) except -1: + cdef int nr_state = 0 + for i in range(beam.size): + state = beam.at(i) + if not state.is_final(): + for j in range(nr_class): + beam.scores[i][j] = scores[nr_state * nr_class + j] + self.moves.set_valid(beam.is_valid[i], state) + nr_state += 1 + else: + for j in range(beam.nr_class): + beam.scores[i][j] = 0 + beam.costs[i][j] = 0 + return nr_state + + def _set_costs(self, Beam beam, doc, gold, int follow_gold=False): + cdef const StateC* state + for i in range(beam.size): + state = beam.at(i) + if state.is_final(): + for j in range(beam.nr_class): + beam.is_valid[i][j] = 0 + beam.costs[i][j] = 9000 + else: + self.moves.set_costs(beam.is_valid[i], beam.costs[i], + state, gold) + if follow_gold: + min_cost = 0 + for j in range(beam.nr_class): + if beam.is_valid[i][j] and beam.costs[i][j] < min_cost: + min_cost = beam.costs[i][j] + for j in range(beam.nr_class): + if beam.costs[i][j] > min_cost: + beam.is_valid[i][j] = 0 + + +def update_beam(TransitionSystem moves, states, golds, model, int width, beam_density=0.0): + cdef MaxViolation violn + pbeam = BeamBatch(moves, states, golds, width=width, density=beam_density) + gbeam = BeamBatch(moves, states, golds, width=width, density=0.0) + beam_maps = [] + backprops = [] + violns = [MaxViolation() for _ in range(len(states))] + dones = [False for _ in states] + while not pbeam.is_done or not gbeam.is_done: + # The beam maps let us find the right row in the flattened scores + # array for each state. States are identified by (example id, + # history). We keep a different beam map for each step (since we'll + # have a flat scores array for each step). The beam map will let us + # take the per-state losses, and compute the gradient for each (step, + # state, class). + # Gather all states from the two beams in a list. Some stats may occur + # in both beams. To figure out which beam each state belonged to, + # we keep two lists of indices, p_indices and g_indices + states, p_indices, g_indices, beam_map = get_unique_states(pbeam, gbeam) + beam_maps.append(beam_map) + if not states: + break + # Now that we have our flat list of states, feed them through the model + scores, bp_scores = model.begin_update(states) + assert scores.size != 0 + # Store the callbacks for the backward pass + backprops.append(bp_scores) + # Unpack the scores for the two beams. The indices arrays + # tell us which example and state the scores-row refers to. + # Now advance the states in the beams. The gold beam is constrained to + # to follow only gold analyses. + if not pbeam.is_done: + pbeam.advance(model.ops.as_contig(scores[p_indices])) + if not gbeam.is_done: + gbeam.advance(model.ops.as_contig(scores[g_indices]), follow_gold=True) + # Track the "maximum violation", to use in the update. + for i, violn in enumerate(violns): + if not dones[i]: + violn.check_crf(pbeam[i], gbeam[i]) + if pbeam[i].is_done and gbeam[i].is_done: + dones[i] = True + histories = [] + grads = [] + for violn in violns: + if violn.p_hist: + histories.append(violn.p_hist + violn.g_hist) + d_loss = [d_l * violn.cost for d_l in violn.p_probs + violn.g_probs] + grads.append(d_loss) + else: + histories.append([]) + grads.append([]) + loss = 0.0 + states_d_scores = get_gradient(moves.n_moves, beam_maps, histories, grads) + for i, (d_scores, bp_scores) in enumerate(zip(states_d_scores, backprops)): + loss += (d_scores**2).mean() + bp_scores(d_scores) + return loss + + +def collect_states(beams, docs): + cdef StateClass state + cdef Beam beam + states = [] + for state_or_beam, doc in zip(beams, docs): + if isinstance(state_or_beam, StateClass): + states.append(state_or_beam) + else: + beam = state_or_beam + state = StateClass.borrow(beam.at(0), doc) + states.append(state) + return states + + +def get_unique_states(pbeams, gbeams): + seen = {} + states = [] + p_indices = [] + g_indices = [] + beam_map = {} + docs = pbeams.docs + cdef Beam pbeam, gbeam + if len(pbeams) != len(gbeams): + raise ValueError(Errors.E079.format(pbeams=len(pbeams), gbeams=len(gbeams))) + for eg_id, (pbeam, gbeam, doc) in enumerate(zip(pbeams, gbeams, docs)): + if not pbeam.is_done: + for i in range(pbeam.size): + state = StateClass.borrow(pbeam.at(i), doc) + if not state.is_final(): + key = tuple([eg_id] + pbeam.histories[i]) + if key in seen: + raise ValueError(Errors.E080.format(key=key)) + seen[key] = len(states) + p_indices.append(len(states)) + states.append(state) + beam_map.update(seen) + if not gbeam.is_done: + for i in range(gbeam.size): + state = StateClass.borrow(gbeam.at(i), doc) + if not state.is_final(): + key = tuple([eg_id] + gbeam.histories[i]) + if key in seen: + g_indices.append(seen[key]) + else: + g_indices.append(len(states)) + beam_map[key] = len(states) + states.append(state) + p_indices = numpy.asarray(p_indices, dtype='i') + g_indices = numpy.asarray(g_indices, dtype='i') + return states, p_indices, g_indices, beam_map + + +def get_gradient(nr_class, beam_maps, histories, losses): + """The global model assigns a loss to each parse. The beam scores + are additive, so the same gradient is applied to each action + in the history. This gives the gradient of a single *action* + for a beam state -- so we have "the gradient of loss for taking + action i given history H." + + Histories: Each history is a list of actions + Each candidate has a history + Each beam has multiple candidates + Each batch has multiple beams + So history is list of lists of lists of ints + """ + grads = [] + nr_steps = [] + for eg_id, hists in enumerate(histories): + nr_step = 0 + for loss, hist in zip(losses[eg_id], hists): + assert not numpy.isnan(loss) + if loss != 0.0: + nr_step = max(nr_step, len(hist)) + nr_steps.append(nr_step) + for i in range(max(nr_steps)): + grads.append(numpy.zeros((max(beam_maps[i].values())+1, nr_class), + dtype='f')) + if len(histories) != len(losses): + raise ValueError(Errors.E081.format(n_hist=len(histories), losses=len(losses))) + for eg_id, hists in enumerate(histories): + for loss, hist in zip(losses[eg_id], hists): + assert not numpy.isnan(loss) + if loss == 0.0: + continue + key = tuple([eg_id]) + # Adjust loss for length + # We need to do this because each state in a short path is scored + # multiple times, as we add in the average cost when we run out + # of actions. + avg_loss = loss / len(hist) + loss += avg_loss * (nr_steps[eg_id] - len(hist)) + for step, clas in enumerate(hist): + i = beam_maps[step][key] + # In step j, at state i action clas + # resulted in loss + grads[step][i, clas] += loss + key = key + tuple([clas]) + return grads diff --git a/spacy/pipeline/_parser_internals/_state.pxd b/spacy/pipeline/_parser_internals/_state.pxd new file mode 100644 index 0000000..ea1a787 --- /dev/null +++ b/spacy/pipeline/_parser_internals/_state.pxd @@ -0,0 +1,409 @@ +cimport libcpp +from cpython.exc cimport PyErr_CheckSignals, PyErr_SetFromErrno +from cython.operator cimport dereference as deref +from cython.operator cimport preincrement as incr +from libc.stdint cimport uint32_t, uint64_t +from libc.stdlib cimport calloc, free +from libc.string cimport memcpy, memset +from libcpp.set cimport set +from libcpp.unordered_map cimport unordered_map +from libcpp.vector cimport vector +from murmurhash.mrmr cimport hash64 + +from ...attrs cimport IS_SPACE +from ...lexeme cimport Lexeme +from ...structs cimport SpanC, TokenC +from ...typedefs cimport attr_t +from ...vocab cimport EMPTY_LEXEME + + +cdef inline bint is_space_token(const TokenC* token) noexcept nogil: + return Lexeme.c_check_flag(token.lex, IS_SPACE) + +cdef struct ArcC: + int head + int child + attr_t label + + +cdef cppclass StateC: + int* _heads + const TokenC* _sent + vector[int] _stack + vector[int] _rebuffer + vector[SpanC] _ents + unordered_map[int, vector[ArcC]] _left_arcs + unordered_map[int, vector[ArcC]] _right_arcs + vector[libcpp.bool] _unshiftable + set[int] _sent_starts + TokenC _empty_token + int length + int offset + int _b_i + + inline __init__(const TokenC* sent, int length) noexcept nogil: + this._sent = sent + this._heads = calloc(length, sizeof(int)) + if not (this._sent and this._heads): + with gil: + PyErr_SetFromErrno(MemoryError) + PyErr_CheckSignals() + this.offset = 0 + this.length = length + this._b_i = 0 + for i in range(length): + this._heads[i] = -1 + this._unshiftable.push_back(0) + memset(&this._empty_token, 0, sizeof(TokenC)) + this._empty_token.lex = &EMPTY_LEXEME + + inline __dealloc__(): + free(this._heads) + + inline void set_context_tokens(int* ids, int n) noexcept nogil: + cdef int i, j + if n == 1: + if this.B(0) >= 0: + ids[0] = this.B(0) + else: + ids[0] = -1 + elif n == 2: + ids[0] = this.B(0) + ids[1] = this.S(0) + elif n == 3: + if this.B(0) >= 0: + ids[0] = this.B(0) + else: + ids[0] = -1 + # First word of entity, if any + if this.entity_is_open(): + ids[1] = this.E(0) + else: + ids[1] = -1 + # Last word of entity, if within entity + if ids[0] == -1 or ids[1] == -1: + ids[2] = -1 + else: + ids[2] = ids[0] - 1 + elif n == 8: + ids[0] = this.B(0) + ids[1] = this.B(1) + ids[2] = this.S(0) + ids[3] = this.S(1) + ids[4] = this.S(2) + ids[5] = this.L(this.B(0), 1) + ids[6] = this.L(this.S(0), 1) + ids[7] = this.R(this.S(0), 1) + elif n == 13: + ids[0] = this.B(0) + ids[1] = this.B(1) + ids[2] = this.S(0) + ids[3] = this.S(1) + ids[4] = this.S(2) + ids[5] = this.L(this.S(0), 1) + ids[6] = this.L(this.S(0), 2) + ids[6] = this.R(this.S(0), 1) + ids[7] = this.L(this.B(0), 1) + ids[8] = this.R(this.S(0), 2) + ids[9] = this.L(this.S(1), 1) + ids[10] = this.L(this.S(1), 2) + ids[11] = this.R(this.S(1), 1) + ids[12] = this.R(this.S(1), 2) + elif n == 6: + for i in range(6): + ids[i] = -1 + if this.B(0) >= 0: + ids[0] = this.B(0) + if this.entity_is_open(): + ent = this.get_ent() + j = 1 + for i in range(ent.start, this.B(0)): + ids[j] = i + j += 1 + if j >= 6: + break + else: + # TODO error =/ + pass + for i in range(n): + if ids[i] >= 0: + ids[i] += this.offset + else: + ids[i] = -1 + + inline int S(int i) noexcept nogil const: + if i >= this._stack.size(): + return -1 + elif i < 0: + return -1 + return this._stack.at(this._stack.size() - (i+1)) + + inline int B(int i) noexcept nogil const: + if i < 0: + return -1 + elif i < this._rebuffer.size(): + return this._rebuffer.at(this._rebuffer.size() - (i+1)) + else: + b_i = this._b_i + (i - this._rebuffer.size()) + if b_i >= this.length: + return -1 + else: + return b_i + + inline const TokenC* B_(int i) noexcept nogil const: + return this.safe_get(this.B(i)) + + inline const TokenC* E_(int i) noexcept nogil const: + return this.safe_get(this.E(i)) + + inline const TokenC* safe_get(int i) noexcept nogil const: + if i < 0 or i >= this.length: + return &this._empty_token + else: + return &this._sent[i] + + inline void map_get_arcs(const unordered_map[int, vector[ArcC]] &heads_arcs, vector[ArcC]* out) noexcept nogil const: + cdef const vector[ArcC]* arcs + head_arcs_it = heads_arcs.const_begin() + while head_arcs_it != heads_arcs.const_end(): + arcs = &deref(head_arcs_it).second + arcs_it = arcs.const_begin() + while arcs_it != arcs.const_end(): + arc = deref(arcs_it) + if arc.head != -1 and arc.child != -1: + out.push_back(arc) + incr(arcs_it) + incr(head_arcs_it) + + inline void get_arcs(vector[ArcC]* out) noexcept nogil const: + this.map_get_arcs(this._left_arcs, out) + this.map_get_arcs(this._right_arcs, out) + + inline int H(int child) noexcept nogil const: + if child >= this.length or child < 0: + return -1 + else: + return this._heads[child] + + inline int E(int i) noexcept nogil const: + if this._ents.size() == 0: + return -1 + else: + return this._ents.back().start + + inline int nth_child(const unordered_map[int, vector[ArcC]]& heads_arcs, int head, int idx) noexcept nogil const: + if idx < 1: + return -1 + + head_arcs_it = heads_arcs.const_find(head) + if head_arcs_it == heads_arcs.const_end(): + return -1 + + cdef const vector[ArcC]* arcs = &deref(head_arcs_it).second + + # Work backwards through arcs to find the arc at the + # requested index more quickly. + cdef size_t child_index = 0 + arcs_it = arcs.const_rbegin() + while arcs_it != arcs.const_rend() and child_index != idx: + arc = deref(arcs_it) + if arc.child != -1: + child_index += 1 + if child_index == idx: + return arc.child + incr(arcs_it) + + return -1 + + inline int L(int head, int idx) noexcept nogil const: + return this.nth_child(this._left_arcs, head, idx) + + inline int R(int head, int idx) noexcept nogil const: + return this.nth_child(this._right_arcs, head, idx) + + inline bint empty() noexcept nogil const: + return this._stack.size() == 0 + + inline bint eol() noexcept nogil const: + return this.buffer_length() == 0 + + inline bint is_final() noexcept nogil const: + return this.stack_depth() <= 0 and this.eol() + + inline int cannot_sent_start(int word) noexcept nogil const: + if word < 0 or word >= this.length: + return 0 + elif this._sent[word].sent_start == -1: + return 1 + else: + return 0 + + inline int is_sent_start(int word) noexcept nogil const: + if word < 0 or word >= this.length: + return 0 + elif this._sent[word].sent_start == 1: + return 1 + elif this._sent_starts.count(word) >= 1: + return 1 + else: + return 0 + + inline void set_sent_start(int word, int value) noexcept nogil: + if value >= 1: + this._sent_starts.insert(word) + + inline bint has_head(int child) noexcept nogil const: + return this._heads[child] >= 0 + + inline int l_edge(int word) noexcept nogil const: + return word + + inline int r_edge(int word) noexcept nogil const: + return word + + inline int n_arcs(const unordered_map[int, vector[ArcC]] &heads_arcs, int head) noexcept nogil const: + cdef int n = 0 + head_arcs_it = heads_arcs.const_find(head) + if head_arcs_it == heads_arcs.const_end(): + return n + + cdef const vector[ArcC]* arcs = &deref(head_arcs_it).second + arcs_it = arcs.const_begin() + while arcs_it != arcs.end(): + arc = deref(arcs_it) + if arc.child != -1: + n += 1 + incr(arcs_it) + + return n + + inline int n_L(int head) noexcept nogil const: + return n_arcs(this._left_arcs, head) + + inline int n_R(int head) noexcept nogil const: + return n_arcs(this._right_arcs, head) + + inline bint stack_is_connected() noexcept nogil const: + return False + + inline bint entity_is_open() noexcept nogil const: + if this._ents.size() == 0: + return False + else: + return this._ents.back().end == -1 + + inline int stack_depth() noexcept nogil const: + return this._stack.size() + + inline int buffer_length() noexcept nogil const: + return (this.length - this._b_i) + this._rebuffer.size() + + inline void push() noexcept nogil: + b0 = this.B(0) + if this._rebuffer.size(): + b0 = this._rebuffer.back() + this._rebuffer.pop_back() + else: + b0 = this._b_i + this._b_i += 1 + this._stack.push_back(b0) + + inline void pop() noexcept nogil: + this._stack.pop_back() + + inline void force_final() noexcept nogil: + # This should only be used in desperate situations, as it may leave + # the analysis in an unexpected state. + this._stack.clear() + this._b_i = this.length + + inline void unshift() noexcept nogil: + s0 = this._stack.back() + this._unshiftable[s0] = 1 + this._rebuffer.push_back(s0) + this._stack.pop_back() + + inline int is_unshiftable(int item) noexcept nogil const: + if item >= this._unshiftable.size(): + return 0 + else: + return this._unshiftable.at(item) + + inline void set_reshiftable(int item) noexcept nogil: + if item < this._unshiftable.size(): + this._unshiftable[item] = 0 + + inline void add_arc(int head, int child, attr_t label) noexcept nogil: + if this.has_head(child): + this.del_arc(this.H(child), child) + cdef ArcC arc + arc.head = head + arc.child = child + arc.label = label + if head > child: + this._left_arcs[arc.head].push_back(arc) + else: + this._right_arcs[arc.head].push_back(arc) + this._heads[child] = head + + inline void map_del_arc(unordered_map[int, vector[ArcC]]* heads_arcs, int h_i, int c_i) noexcept nogil: + arcs_it = heads_arcs.find(h_i) + if arcs_it == heads_arcs.end(): + return + + arcs = &deref(arcs_it).second + if arcs.size() == 0: + return + + arc = arcs.back() + if arc.head == h_i and arc.child == c_i: + arcs.pop_back() + else: + for i in range(arcs.size()-1): + arc = arcs.at(i) + if arc.head == h_i and arc.child == c_i: + arc.head = -1 + arc.child = -1 + arc.label = 0 + break + + inline void del_arc(int h_i, int c_i) noexcept nogil: + if h_i > c_i: + this.map_del_arc(&this._left_arcs, h_i, c_i) + else: + this.map_del_arc(&this._right_arcs, h_i, c_i) + + inline SpanC get_ent() noexcept nogil const: + cdef SpanC ent + if this._ents.size() == 0: + ent.start = 0 + ent.end = 0 + ent.label = 0 + return ent + else: + return this._ents.back() + + inline void open_ent(attr_t label) noexcept nogil: + cdef SpanC ent + ent.start = this.B(0) + ent.label = label + ent.end = -1 + this._ents.push_back(ent) + + inline void close_ent() noexcept nogil: + this._ents.back().end = this.B(0)+1 + + inline void clone(const StateC* src) noexcept nogil: + this.length = src.length + this._sent = src._sent + this._stack = src._stack + this._rebuffer = src._rebuffer + this._sent_starts = src._sent_starts + this._unshiftable = src._unshiftable + memcpy(this._heads, src._heads, this.length * sizeof(this._heads[0])) + this._ents = src._ents + this._left_arcs = src._left_arcs + this._right_arcs = src._right_arcs + this._b_i = src._b_i + this.offset = src.offset + this._empty_token = src._empty_token diff --git a/spacy/pipeline/_parser_internals/_state.pyx b/spacy/pipeline/_parser_internals/_state.pyx new file mode 100644 index 0000000..61bf620 --- /dev/null +++ b/spacy/pipeline/_parser_internals/_state.pyx @@ -0,0 +1 @@ +# cython: profile=False diff --git a/spacy/pipeline/_parser_internals/arc_eager.pxd b/spacy/pipeline/_parser_internals/arc_eager.pxd new file mode 100644 index 0000000..2c17e7b --- /dev/null +++ b/spacy/pipeline/_parser_internals/arc_eager.pxd @@ -0,0 +1,7 @@ +from ...typedefs cimport attr_t, weight_t +from ._state cimport StateC +from .transition_system cimport Transition, TransitionSystem + + +cdef class ArcEager(TransitionSystem): + cdef get_arcs(self, StateC* state) diff --git a/spacy/pipeline/_parser_internals/arc_eager.pyx b/spacy/pipeline/_parser_internals/arc_eager.pyx new file mode 100644 index 0000000..cccd51f --- /dev/null +++ b/spacy/pipeline/_parser_internals/arc_eager.pyx @@ -0,0 +1,906 @@ +# cython: cdivision=True, infer_types=True +from cymem.cymem cimport Address, Pool +from libc.stdint cimport int32_t +from libcpp.vector cimport vector + +from collections import Counter, defaultdict + +from ...strings cimport hash_string +from ...structs cimport TokenC +from ...tokens.doc cimport Doc, set_children_from_heads +from ...tokens.token cimport MISSING_DEP +from ...typedefs cimport attr_t + +from ...training import split_bilu_label + +from ...training.example cimport Example +from ._state cimport ArcC, StateC +from .stateclass cimport StateClass + +from ...errors import Errors + +from thinc.extra.search cimport Beam + + +cdef weight_t MIN_SCORE = -90000 +cdef attr_t SUBTOK_LABEL = hash_string('subtok') + +DEF NON_MONOTONIC = True + +cdef enum: + SHIFT + REDUCE + LEFT + RIGHT + + BREAK + + N_MOVES + + +MOVE_NAMES = [None] * N_MOVES +MOVE_NAMES[SHIFT] = 'S' +MOVE_NAMES[REDUCE] = 'D' +MOVE_NAMES[LEFT] = 'L' +MOVE_NAMES[RIGHT] = 'R' +MOVE_NAMES[BREAK] = 'B' + + +cdef enum: + HEAD_IN_STACK = 0 + HEAD_IN_BUFFER + HEAD_UNKNOWN + IS_SENT_START + SENT_START_UNKNOWN + + +cdef struct GoldParseStateC: + char* state_bits + int32_t* n_kids_in_buffer + int32_t* n_kids_in_stack + int32_t* heads + attr_t* labels + int32_t** kids + int32_t* n_kids + int32_t length + int32_t stride + weight_t push_cost + weight_t pop_cost + + +cdef GoldParseStateC create_gold_state( + Pool mem, const StateC* state, heads, labels, sent_starts +) except *: + cdef GoldParseStateC gs + gs.length = len(heads) + gs.stride = 1 + assert gs.length > 0 + gs.labels = mem.alloc(gs.length, sizeof(gs.labels[0])) + gs.heads = mem.alloc(gs.length, sizeof(gs.heads[0])) + gs.n_kids = mem.alloc(gs.length, sizeof(gs.n_kids[0])) + gs.state_bits = mem.alloc(gs.length, sizeof(gs.state_bits[0])) + gs.n_kids_in_buffer = mem.alloc(gs.length, sizeof(gs.n_kids_in_buffer[0])) + gs.n_kids_in_stack = mem.alloc(gs.length, sizeof(gs.n_kids_in_stack[0])) + + for i, is_sent_start in enumerate(sent_starts): + if is_sent_start is True: + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + IS_SENT_START, + 1 + ) + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + SENT_START_UNKNOWN, + 0 + ) + + elif is_sent_start is None: + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + SENT_START_UNKNOWN, + 1 + ) + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + IS_SENT_START, + 0 + ) + else: + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + SENT_START_UNKNOWN, + 0 + ) + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + IS_SENT_START, + 0 + ) + + for i, (head, label) in enumerate(zip(heads, labels)): + if head is not None: + gs.heads[i] = head + gs.labels[i] = label + if i != head: + gs.n_kids[head] += 1 + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + HEAD_UNKNOWN, + 0 + ) + else: + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + HEAD_UNKNOWN, + 1 + ) + # Make an array of pointers, pointing into the gs_kids_flat array. + assert gs.length > 0 + gs.kids = mem.alloc(gs.length, sizeof(int32_t*)) + for i in range(gs.length): + if gs.n_kids[i] != 0: + gs.kids[i] = mem.alloc(gs.n_kids[i], sizeof(int32_t)) + # This is a temporary buffer + js_addr = Address(gs.length, sizeof(int32_t)) + js = js_addr.ptr + for i in range(gs.length): + if not is_head_unknown(&gs, i): + head = gs.heads[i] + if head != i: + gs.kids[head][js[head]] = i + js[head] += 1 + gs.push_cost = push_cost(state, &gs) + gs.pop_cost = pop_cost(state, &gs) + return gs + + +cdef void update_gold_state(GoldParseStateC* gs, const StateC* s) noexcept nogil: + for i in range(gs.length): + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + HEAD_IN_BUFFER, + 0 + ) + gs.state_bits[i] = set_state_flag( + gs.state_bits[i], + HEAD_IN_STACK, + 0 + ) + gs.n_kids_in_stack[i] = 0 + gs.n_kids_in_buffer[i] = 0 + + for i in range(s.stack_depth()): + s_i = s.S(i) + if not is_head_unknown(gs, s_i) and gs.heads[s_i] != s_i: + gs.n_kids_in_stack[gs.heads[s_i]] += 1 + for kid in gs.kids[s_i][:gs.n_kids[s_i]]: + gs.state_bits[kid] = set_state_flag( + gs.state_bits[kid], + HEAD_IN_STACK, + 1 + ) + for i in range(s.buffer_length()): + b_i = s.B(i) + if s.is_sent_start(b_i): + break + if not is_head_unknown(gs, b_i) and gs.heads[b_i] != b_i: + gs.n_kids_in_buffer[gs.heads[b_i]] += 1 + for kid in gs.kids[b_i][:gs.n_kids[b_i]]: + gs.state_bits[kid] = set_state_flag( + gs.state_bits[kid], + HEAD_IN_BUFFER, + 1 + ) + gs.push_cost = push_cost(s, gs) + gs.pop_cost = pop_cost(s, gs) + + +cdef class ArcEagerGold: + cdef GoldParseStateC c + cdef Pool mem + + def __init__(self, ArcEager moves, StateClass stcls, Example example): + self.mem = Pool() + heads, labels = example.get_aligned_parse(projectivize=True) + labels = [example.x.vocab.strings.add(label, allow_transient=False) if label is not None else MISSING_DEP for label in labels] + sent_starts = _get_aligned_sent_starts(example) + assert len(heads) == len(labels) == len(sent_starts), (len(heads), len(labels), len(sent_starts)) + self.c = create_gold_state(self.mem, stcls.c, heads, labels, sent_starts) + + def update(self, StateClass stcls): + update_gold_state(&self.c, stcls.c) + + +def _get_aligned_sent_starts(example): + """Get list of SENT_START attributes aligned to the predicted tokenization. + If the reference has not sentence starts, return a list of None values. + + This function is slightly different from the one on Example, because we also + check whether the reference sentences align across multiple sentences, + and return missing values if they do. This prevents a problem where you have + the start of a sentence merged onto a token that belongs to two sentences. + """ + if example.y.has_annotation("SENT_START"): + align = example.alignment.y2x + sent_starts = [False] * len(example.x) + seen_words = set() + for y_sent in example.y.sents: + x_indices = list(align[y_sent.start : y_sent.end]) + if any(x_idx in seen_words for x_idx in x_indices): + # If there are any tokens in X that align across two sentences, + # regard the sentence annotations as missing, as we can't + # reliably use them. + return [None] * len(example.x) + seen_words.update(x_indices) + sent_starts[x_indices[0]] = True + return sent_starts + else: + return [None] * len(example.x) + + +cdef int check_state_gold(char state_bits, char flag) noexcept nogil: + cdef char one = 1 + return 1 if (state_bits & (one << flag)) else 0 + + +cdef int set_state_flag(char state_bits, char flag, int value) noexcept nogil: + cdef char one = 1 + if value: + return state_bits | (one << flag) + else: + return state_bits & ~(one << flag) + + +cdef int is_head_in_stack(const GoldParseStateC* gold, int i) noexcept nogil: + return check_state_gold(gold.state_bits[i], HEAD_IN_STACK) + + +cdef int is_head_in_buffer(const GoldParseStateC* gold, int i) noexcept nogil: + return check_state_gold(gold.state_bits[i], HEAD_IN_BUFFER) + + +cdef int is_head_unknown(const GoldParseStateC* gold, int i) noexcept nogil: + return check_state_gold(gold.state_bits[i], HEAD_UNKNOWN) + +cdef int is_sent_start(const GoldParseStateC* gold, int i) noexcept nogil: + return check_state_gold(gold.state_bits[i], IS_SENT_START) + +cdef int is_sent_start_unknown(const GoldParseStateC* gold, int i) noexcept nogil: + return check_state_gold(gold.state_bits[i], SENT_START_UNKNOWN) + + +# Helper functions for the arc-eager oracle + +cdef weight_t push_cost(const StateC* state, const GoldParseStateC* gold) noexcept nogil: + cdef weight_t cost = 0 + b0 = state.B(0) + if b0 < 0: + return 9000 + if is_head_in_stack(gold, b0): + cost += 1 + cost += gold.n_kids_in_stack[b0] + if Break.is_valid(state, 0) and is_sent_start(gold, state.B(1)): + cost += 1 + return cost + + +cdef weight_t pop_cost(const StateC* state, const GoldParseStateC* gold) noexcept nogil: + cdef weight_t cost = 0 + s0 = state.S(0) + if s0 < 0: + return 9000 + if is_head_in_buffer(gold, s0): + cost += 1 + cost += gold.n_kids_in_buffer[s0] + return cost + + +cdef bint arc_is_gold(const GoldParseStateC* gold, int head, int child) noexcept nogil: + if is_head_unknown(gold, child): + return True + elif gold.heads[child] == head: + return True + else: + return False + + +cdef bint label_is_gold(const GoldParseStateC* gold, int child, attr_t label) noexcept nogil: + if is_head_unknown(gold, child): + return True + elif label == 0: + return True + elif gold.labels[child] == label: + return True + else: + return False + + +cdef bint _is_gold_root(const GoldParseStateC* gold, int word) noexcept nogil: + return gold.heads[word] == word or is_head_unknown(gold, word) + + +cdef class Shift: + """Move the first word of the buffer onto the stack and mark it as "shifted" + + Validity: + * If stack is empty + * At least two words in sentence + * Word has not been shifted before + + Cost: push_cost + + Action: + * Mark B[0] as 'shifted' + * Push stack + * Advance buffer + """ + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if st.stack_depth() == 0: + return 1 + elif st.buffer_length() < 2: + return 0 + elif st.is_sent_start(st.B(0)): + return 0 + elif st.is_unshiftable(st.B(0)): + return 0 + else: + return 1 + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.push() + + @staticmethod + cdef weight_t cost(const StateC* state, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + return gold.push_cost + + +cdef class Reduce: + """ + Pop from the stack. If it has no head and the stack isn't empty, place + it back on the buffer. + + Validity: + * Stack not empty + * Buffer nt empty + * Stack depth 1 and cannot sent start l_edge(st.B(0)) + + Cost: + * If B[0] is the start of a sentence, cost is 0 + * Arcs between stack and buffer + * If arc has no head, we're saving arcs between S[0] and S[1:], so decrement + cost by those arcs. + """ + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if st.stack_depth() == 0: + return False + elif st.buffer_length() == 0: + return True + elif st.stack_depth() == 1 and st.cannot_sent_start(st.l_edge(st.B(0))): + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + if st.has_head(st.S(0)) or st.stack_depth() == 1: + st.pop() + else: + st.unshift() + + @staticmethod + cdef weight_t cost(const StateC* state, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + if state.is_sent_start(state.B(0)): + return 0 + s0 = state.S(0) + cost = gold.pop_cost + if not state.has_head(s0): + # Decrement cost for the arcs we save, as we'll be putting this + # back to the buffer + if is_head_in_stack(gold, s0): + cost -= 1 + cost -= gold.n_kids_in_stack[s0] + return cost + + +cdef class LeftArc: + """Add an arc between B[0] and S[0], replacing the previous head of S[0] if + one is set. Pop S[0] from the stack. + + Validity: + * len(S) >= 1 + * len(B) >= 1 + * not is_sent_start(B[0]) + + Cost: + pop_cost - Arc(B[0], S[0], label) + (Arc(S[1], S[0]) if H(S[0]) else Arcs(S, S[0])) + """ + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if st.stack_depth() == 0: + return 0 + elif st.buffer_length() == 0: + return 0 + elif st.is_sent_start(st.B(0)): + return 0 + elif label == SUBTOK_LABEL and st.S(0) != (st.B(0)-1): + return 0 + else: + return 1 + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.add_arc(st.B(0), st.S(0), label) + # If we change the stack, it's okay to remove the shifted mark, as + # we can't get in an infinite loop this way. + st.set_reshiftable(st.B(0)) + st.pop() + + @staticmethod + cdef inline weight_t cost(const StateC* state, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cdef weight_t cost = gold.pop_cost + s0 = state.S(0) + s1 = state.S(1) + b0 = state.B(0) + if state.has_head(s0): + # Increment cost if we're clobbering a correct arc + cost += gold.heads[s0] == s1 + else: + # If there's no head, we're losing arcs between S0 and S[1:]. + cost += is_head_in_stack(gold, s0) + cost += gold.n_kids_in_stack[s0] + if b0 != -1 and s0 != -1 and gold.heads[s0] == b0: + cost -= 1 + cost += not label_is_gold(gold, s0, label) + return cost + + +cdef class RightArc: + """ + Add an arc from S[0] to B[0]. Push B[0]. + + Validity: + * len(S) >= 1 + * len(B) >= 1 + * not is_sent_start(B[0]) + + Cost: + push_cost + (not shifted[b0] and Arc(B[1:], B[0])) - Arc(S[0], B[0], label) + """ + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if st.stack_depth() == 0: + return 0 + elif st.buffer_length() == 0: + return 0 + elif st.is_sent_start(st.B(0)): + return 0 + elif label == SUBTOK_LABEL and st.S(0) != (st.B(0)-1): + # If there's (perhaps partial) parse pre-set, don't allow cycle. + return 0 + else: + return 1 + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.add_arc(st.S(0), st.B(0), label) + st.push() + + @staticmethod + cdef inline weight_t cost(const StateC* state, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cost = gold.push_cost + s0 = state.S(0) + b0 = state.B(0) + if s0 != -1 and b0 != -1 and gold.heads[b0] == s0: + cost -= 1 + cost += not label_is_gold(gold, b0, label) + elif is_head_in_buffer(gold, b0) and not state.is_unshiftable(b0): + cost += 1 + return cost + + +cdef class Break: + """Mark the second word of the buffer as the start of a + sentence. + + Validity: + * len(buffer) >= 2 + * B[1] == B[0] + 1 + * not is_sent_start(B[1]) + * not cannot_sent_start(B[1]) + + Action: + * mark_sent_start(B[1]) + + Cost: + * not is_sent_start(B[1]) + * Arcs between B[0] and B[1:] + * Arcs between S and B[1] + """ + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if st.buffer_length() < 2: + return False + elif st.B(1) != st.B(0) + 1: + return False + elif st.is_sent_start(st.B(1)): + return False + elif st.cannot_sent_start(st.B(1)): + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.set_sent_start(st.B(1), 1) + + @staticmethod + cdef weight_t cost(const StateC* state, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cdef int b0 = state.B(0) + cdef int cost = 0 + cdef int si + for i in range(state.stack_depth()): + si = state.S(i) + if is_head_in_buffer(gold, si): + cost += 1 + cost += gold.n_kids_in_buffer[si] + # We need to score into B[1:], so subtract deps that are at b0 + if gold.heads[b0] == si: + cost -= 1 + if gold.heads[si] == b0: + cost -= 1 + if not is_sent_start(gold, state.B(1)) and\ + not is_sent_start_unknown(gold, state.B(1)): + cost += 1 + return cost + + +cdef void* _init_state(Pool mem, int length, void* tokens) except NULL: + st = new StateC(tokens, length) + return st + + +cdef int _del_state(Pool mem, void* state, void* x) except -1: + cdef StateC* st = state + del st + + +cdef class ArcEager(TransitionSystem): + def __init__(self, *args, **kwargs): + TransitionSystem.__init__(self, *args, **kwargs) + self.init_beam_state = _init_state + self.del_beam_state = _del_state + + @classmethod + def get_actions(cls, **kwargs): + min_freq = kwargs.get('min_freq', None) + actions = defaultdict(lambda: Counter()) + actions[SHIFT][''] = 1 + actions[REDUCE][''] = 1 + for label in kwargs.get('left_labels', []): + actions[LEFT][label] = 1 + actions[SHIFT][label] = 1 + for label in kwargs.get('right_labels', []): + actions[RIGHT][label] = 1 + actions[REDUCE][label] = 1 + for example in kwargs.get('examples', []): + # use heads and labels from the reference parse (without regard to + # misalignments between the predicted and reference) + example_gold_preproc = Example(example.reference, example.reference) + heads, labels = example_gold_preproc.get_aligned_parse(projectivize=True) + for child, (head, label) in enumerate(zip(heads, labels)): + if head is None or label is None: + continue + if label.upper() == 'ROOT' : + label = 'ROOT' + if head == child: + actions[BREAK][label] += 1 + if head < child: + actions[RIGHT][label] += 1 + actions[REDUCE][''] += 1 + elif head > child: + actions[LEFT][label] += 1 + actions[SHIFT][''] += 1 + if min_freq is not None: + for action, label_freqs in actions.items(): + for label, freq in label_freqs.copy().items(): + if freq < min_freq: + label_freqs.pop(label) + # Ensure these actions are present + actions[BREAK].setdefault('ROOT', 0) + if kwargs.get("learn_tokens") is True: + actions[RIGHT].setdefault('subtok', 0) + actions[LEFT].setdefault('subtok', 0) + # Used for backoff + actions[RIGHT].setdefault('dep', 0) + actions[LEFT].setdefault('dep', 0) + return actions + + @property + def builtin_labels(self): + return ["ROOT", "dep"] + + @property + def action_types(self): + return (SHIFT, REDUCE, LEFT, RIGHT, BREAK) + + def get_doc_labels(self, doc): + """Get the labels required for a given Doc.""" + labels = set(self.builtin_labels) + for token in doc: + if token.dep_: + labels.add(token.dep_) + return labels + + def transition(self, StateClass state, action): + cdef Transition t = self.lookup_transition(action) + t.do(state.c, t.label) + return state + + def is_gold_parse(self, StateClass state, ArcEagerGold gold): + for i in range(state.c.length): + token = state.c.safe_get(i) + if not arc_is_gold(&gold.c, i, i+token.head): + return False + elif not label_is_gold(&gold.c, i, token.dep): + return False + return True + + def init_gold(self, StateClass state, Example example): + gold = ArcEagerGold(self, state, example) + self._replace_unseen_labels(gold) + return gold + + def init_gold_batch(self, examples): + # TODO: Projectivity? + all_states = self.init_batch([eg.predicted for eg in examples]) + golds = [] + states = [] + for state, eg in zip(all_states, examples): + if self.has_gold(eg) and not state.is_final(): + golds.append(self.init_gold(state, eg)) + states.append(state) + n_steps = sum([len(s.queue) for s in states]) + return states, golds, n_steps + + def _replace_unseen_labels(self, ArcEagerGold gold): + backoff_label = self.strings["dep"] + root_label = self.strings["ROOT"] + left_labels = self.labels[LEFT] + right_labels = self.labels[RIGHT] + break_labels = self.labels[BREAK] + for i in range(gold.c.length): + if not is_head_unknown(&gold.c, i): + head = gold.c.heads[i] + label = self.strings[gold.c.labels[i]] + if head > i and label not in left_labels: + gold.c.labels[i] = backoff_label + elif head < i and label not in right_labels: + gold.c.labels[i] = backoff_label + elif head == i and label not in break_labels: + gold.c.labels[i] = root_label + return gold + + cdef Transition lookup_transition(self, object name_or_id) except *: + if isinstance(name_or_id, int): + return self.c[name_or_id] + name = name_or_id + if '-' in name: + move_str, label_str = split_bilu_label(name) + label = self.strings[label_str] + else: + move_str = name + label = 0 + move = MOVE_NAMES.index(move_str) + for i in range(self.n_moves): + if self.c[i].move == move and self.c[i].label == label: + return self.c[i] + raise KeyError(f"Unknown transition: {name}") + + def move_name(self, int move, attr_t label): + label_str = self.strings[label] + if label_str: + return MOVE_NAMES[move] + '-' + label_str + else: + return MOVE_NAMES[move] + + def class_name(self, int i): + return self.move_name(self.c[i].move, self.c[i].label) + + cdef Transition init_transition(self, int clas, int move, attr_t label) except *: + # TODO: Apparent Cython bug here when we try to use the Transition() + # constructor with the function pointers + cdef Transition t + t.score = 0 + t.clas = clas + t.move = move + t.label = label + if move == SHIFT: + t.is_valid = Shift.is_valid + t.do = Shift.transition + t.get_cost = Shift.cost + elif move == REDUCE: + t.is_valid = Reduce.is_valid + t.do = Reduce.transition + t.get_cost = Reduce.cost + elif move == LEFT: + t.is_valid = LeftArc.is_valid + t.do = LeftArc.transition + t.get_cost = LeftArc.cost + elif move == RIGHT: + t.is_valid = RightArc.is_valid + t.do = RightArc.transition + t.get_cost = RightArc.cost + elif move == BREAK: + t.is_valid = Break.is_valid + t.do = Break.transition + t.get_cost = Break.cost + else: + raise ValueError(Errors.E019.format(action=move, src='arc_eager')) + return t + + def set_annotations(self, StateClass state, Doc doc): + for arc in state.arcs: + doc.c[arc["child"]].head = arc["head"] - arc["child"] + doc.c[arc["child"]].dep = arc["label"] + for i in range(doc.length): + if doc.c[i].head == 0: + doc.c[i].dep = self.root_label + set_children_from_heads(doc.c, 0, doc.length) + + def get_beam_parses(self, Beam beam): + parses = [] + probs = beam.probs + for i in range(beam.size): + state = beam.at(i) + if state.is_final(): + prob = probs[i] + parse = [] + arcs = self.get_arcs(state) + if arcs: + for arc in arcs: + dep = arc["label"] + label = self.strings[dep] + parse.append((arc["head"], arc["child"], label)) + parses.append((prob, parse)) + return parses + + cdef get_arcs(self, StateC* state): + cdef vector[ArcC] arcs + state.get_arcs(&arcs) + return list(arcs) + + def has_gold(self, Example eg, start=0, end=None): + for word in eg.y[start:end]: + if word.dep != 0: + return True + else: + return False + + cdef int set_valid(self, int* output, const StateC* st) noexcept nogil: + cdef int[N_MOVES] is_valid + is_valid[SHIFT] = Shift.is_valid(st, 0) + is_valid[REDUCE] = Reduce.is_valid(st, 0) + is_valid[LEFT] = LeftArc.is_valid(st, 0) + is_valid[RIGHT] = RightArc.is_valid(st, 0) + is_valid[BREAK] = Break.is_valid(st, 0) + cdef int i + for i in range(self.n_moves): + if self.c[i].label == SUBTOK_LABEL: + output[i] = self.c[i].is_valid(st, self.c[i].label) + else: + output[i] = is_valid[self.c[i].move] + + def get_cost(self, StateClass stcls, gold, int i): + if not isinstance(gold, ArcEagerGold): + raise TypeError(Errors.E909.format(name="ArcEagerGold")) + cdef ArcEagerGold gold_ = gold + gold_state = gold_.c + if self.c[i].is_valid(stcls.c, self.c[i].label): + cost = self.c[i].get_cost(stcls.c, &gold_state, self.c[i].label) + else: + cost = 9000 + return cost + + cdef int set_costs(self, int* is_valid, weight_t* costs, + const StateC* state, gold) except -1: + if not isinstance(gold, ArcEagerGold): + raise TypeError(Errors.E909.format(name="ArcEagerGold")) + cdef ArcEagerGold gold_ = gold + gold_state = gold_.c + update_gold_state(&gold_state, state) + self.set_valid(is_valid, state) + cdef int n_gold = 0 + for i in range(self.n_moves): + if is_valid[i]: + costs[i] = self.c[i].get_cost(state, &gold_state, self.c[i].label) + if costs[i] <= 0: + n_gold += 1 + else: + costs[i] = 9000 + if n_gold < 1: + for i in range(self.n_moves): + print(self.get_class_name(i), is_valid[i], costs[i]) + print("Gold sent starts?", is_sent_start(&gold_state, state.B(0)), is_sent_start(&gold_state, state.B(1))) + raise ValueError(Errors.E1031) + + def get_oracle_sequence_from_state(self, StateClass state, ArcEagerGold gold, _debug=None): + cdef int i + cdef Pool mem = Pool() + # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc + assert self.n_moves > 0 + costs = mem.alloc(self.n_moves, sizeof(float)) + is_valid = mem.alloc(self.n_moves, sizeof(int)) + + history = [] + debug_log = [] + failed = False + while not state.is_final(): + try: + self.set_costs(is_valid, costs, state.c, gold) + except ValueError: + failed = True + break + min_cost = min(costs[i] for i in range(self.n_moves)) + for i in range(self.n_moves): + if is_valid[i] and costs[i] <= min_cost: + action = self.c[i] + history.append(i) + s0 = state.S(0) + b0 = state.B(0) + if _debug: + example = _debug + debug_log.append(" ".join(( + self.get_class_name(i), + state.print_state() + ))) + action.do(state.c, action.label) + break + else: + failed = False + break + if failed and _debug not in (False, None): + example = _debug + print("Actions") + for i in range(self.n_moves): + print(self.get_class_name(i)) + print("Gold") + for token in example.y: + print(token.i, token.text, token.dep_, token.head.text) + aligned_heads, _aligned_labels = example.get_aligned_parse() + print("Aligned heads") + for i, head in enumerate(aligned_heads): + print(example.x[i], example.x[head] if head is not None else "__") + print("Aligned sent starts") + print(example.get_aligned_sent_starts()) + + print("Predicted tokens") + print([(w.i, w.text) for w in example.x]) + s0 = state.S(0) + b0 = state.B(0) + debug_log.append(" ".join(( + "?", + "S0=", (example.x[s0].text if s0 >= 0 else "-"), + "B0=", (example.x[b0].text if b0 >= 0 else "-"), + "S0 head?", str(state.has_head(state.S(0))), + ))) + s0 = state.S(0) + b0 = state.B(0) + print("\n".join(debug_log)) + print("Arc is gold B0, S0?", arc_is_gold(&gold.c, b0, s0)) + print("Arc is gold S0, B0?", arc_is_gold(&gold.c, s0, b0)) + print("is_head_unknown(s0)", is_head_unknown(&gold.c, s0)) + print("is_head_unknown(b0)", is_head_unknown(&gold.c, b0)) + print("b0", b0, "gold.heads[s0]", gold.c.heads[s0]) + print("Stack", [example.x[i] for i in state.stack]) + print("Buffer", [example.x[i] for i in state.queue]) + raise ValueError(Errors.E024) + return history diff --git a/spacy/pipeline/_parser_internals/ner.pxd b/spacy/pipeline/_parser_internals/ner.pxd new file mode 100644 index 0000000..2264a15 --- /dev/null +++ b/spacy/pipeline/_parser_internals/ner.pxd @@ -0,0 +1,5 @@ +from .transition_system cimport TransitionSystem + + +cdef class BiluoPushDown(TransitionSystem): + pass diff --git a/spacy/pipeline/_parser_internals/ner.pyx b/spacy/pipeline/_parser_internals/ner.pyx new file mode 100644 index 0000000..84d8ed2 --- /dev/null +++ b/spacy/pipeline/_parser_internals/ner.pyx @@ -0,0 +1,689 @@ +# cython: profile=False +from cymem.cymem cimport Pool +from libc.stdint cimport int32_t + +from collections import Counter + +from thinc.extra.search cimport Beam + +from ...tokens.doc cimport Doc + +from ...tokens.span import Span + +from ...attrs cimport IS_SPACE +from ...lexeme cimport Lexeme +from ...structs cimport SpanC +from ...tokens.span cimport Span +from ...typedefs cimport attr_t, weight_t + +from ...training import split_bilu_label + +from ...training.example cimport Example +from ._state cimport StateC +from .stateclass cimport StateClass +from .transition_system cimport Transition, do_func_t + +from ...errors import Errors + + +cdef enum: + MISSING + BEGIN + IN + LAST + UNIT + OUT + N_MOVES + + +MOVE_NAMES = [None] * N_MOVES +MOVE_NAMES[MISSING] = 'M' +MOVE_NAMES[BEGIN] = 'B' +MOVE_NAMES[IN] = 'I' +MOVE_NAMES[LAST] = 'L' +MOVE_NAMES[UNIT] = 'U' +MOVE_NAMES[OUT] = 'O' + + +cdef struct GoldNERStateC: + Transition* ner + SpanC* negs + int32_t length + int32_t nr_neg + + +cdef class BiluoGold: + cdef Pool mem + cdef GoldNERStateC c + + def __init__(self, BiluoPushDown moves, StateClass stcls, Example example, neg_key): + self.mem = Pool() + self.c = create_gold_state(self.mem, moves, stcls.c, example, neg_key) + + def update(self, StateClass stcls): + update_gold_state(&self.c, stcls.c) + + +cdef GoldNERStateC create_gold_state( + Pool mem, + BiluoPushDown moves, + const StateC* stcls, + Example example, + neg_key +) except *: + cdef GoldNERStateC gs + cdef Span neg + if neg_key is not None: + negs = example.get_aligned_spans_y2x( + example.y.spans.get(neg_key, []), + allow_overlap=True + ) + else: + negs = [] + assert example.x.length > 0 + gs.ner = mem.alloc(example.x.length, sizeof(Transition)) + gs.negs = mem.alloc(len(negs), sizeof(SpanC)) + gs.nr_neg = len(negs) + ner_ents, ner_tags = example.get_aligned_ents_and_ner() + for i, ner_tag in enumerate(ner_tags): + gs.ner[i] = moves.lookup_transition(ner_tag) + + # Prevent conflicting spans in the data. For NER, spans are equal if they have the same offsets and label. + neg_span_triples = {(neg_ent.start_char, neg_ent.end_char, neg_ent.label) for neg_ent in negs} + for pos_span in ner_ents: + if (pos_span.start_char, pos_span.end_char, pos_span.label) in neg_span_triples: + raise ValueError(Errors.E868.format(span=(pos_span.start_char, pos_span.end_char, pos_span.label_))) + + # In order to handle negative samples, we need to maintain the full + # (start, end, label) triple. If we break it down to the 'isnt B-LOC' + # thing, we'll get blocked if there's an incorrect prefix. + for i, neg in enumerate(negs): + gs.negs[i] = neg.c + return gs + + +cdef void update_gold_state(GoldNERStateC* gs, const StateC* state) except *: + # We don't need to update each time, unlike the parser. + pass + + +cdef do_func_t[N_MOVES] do_funcs + + +cdef bint _entity_is_sunk(const StateC* state, Transition* golds) noexcept nogil: + if not state.entity_is_open(): + return False + + cdef const Transition* gold = &golds[state.E(0)] + ent = state.get_ent() + if gold.move != BEGIN and gold.move != UNIT: + return True + elif gold.label != ent.label: + return True + else: + return False + + +cdef class BiluoPushDown(TransitionSystem): + def __init__(self, *args, **kwargs): + TransitionSystem.__init__(self, *args, **kwargs) + + @classmethod + def get_actions(cls, **kwargs): + actions = { + MISSING: Counter(), + BEGIN: Counter(), + IN: Counter(), + LAST: Counter(), + UNIT: Counter(), + OUT: Counter() + } + actions[OUT][''] = 1 # Represents a token predicted to be outside of any entity + actions[UNIT][''] = 1 # Represents a token prohibited to be in an entity + for entity_type in kwargs.get('entity_types', []): + for action in (BEGIN, IN, LAST, UNIT): + actions[action][entity_type] = 1 + for example in kwargs.get('examples', []): + for token in example.y: + ent_type = token.ent_type_ + if ent_type: + for action in (BEGIN, IN, LAST, UNIT): + actions[action][ent_type] += 1 + return actions + + @property + def action_types(self): + return (BEGIN, IN, LAST, UNIT, OUT) + + def get_doc_labels(self, doc): + labels = set() + for token in doc: + if token.ent_type: + labels.add(token.ent_type_) + return labels + + def move_name(self, int move, attr_t label): + if move == OUT: + return 'O' + elif move == MISSING: + return 'M' + else: + return MOVE_NAMES[move] + '-' + self.strings[label] + + def init_gold_batch(self, examples): + all_states = self.init_batch([eg.predicted for eg in examples]) + golds = [] + states = [] + for state, eg in zip(all_states, examples): + if self.has_gold(eg) and not state.is_final(): + golds.append(self.init_gold(state, eg)) + states.append(state) + n_steps = sum([len(s.queue) for s in states]) + return states, golds, n_steps + + cdef Transition lookup_transition(self, object name) except *: + cdef attr_t label + if name == '-' or name == '' or name is None: + return Transition(clas=0, move=MISSING, label=0, score=0) + elif '-' in name: + move_str, label_str = split_bilu_label(name) + # Deprecated, hacky way to denote 'not this entity' + if label_str.startswith('!'): + raise ValueError(Errors.E869.format(label=name)) + label = self.strings.add(label_str) + else: + move_str = name + label = 0 + move = MOVE_NAMES.index(move_str) + for i in range(self.n_moves): + if self.c[i].move == move and self.c[i].label == label: + return self.c[i] + raise KeyError(Errors.E022.format(name=name)) + + cdef Transition init_transition(self, int clas, int move, attr_t label) except *: + # TODO: Apparent Cython bug here when we try to use the Transition() + # constructor with the function pointers + cdef Transition t + t.score = 0 + t.clas = clas + t.move = move + t.label = label + if move == MISSING: + t.is_valid = Missing.is_valid + t.do = Missing.transition + t.get_cost = Missing.cost + elif move == BEGIN: + t.is_valid = Begin.is_valid + t.do = Begin.transition + t.get_cost = Begin.cost + elif move == IN: + t.is_valid = In.is_valid + t.do = In.transition + t.get_cost = In.cost + elif move == LAST: + t.is_valid = Last.is_valid + t.do = Last.transition + t.get_cost = Last.cost + elif move == UNIT: + t.is_valid = Unit.is_valid + t.do = Unit.transition + t.get_cost = Unit.cost + elif move == OUT: + t.is_valid = Out.is_valid + t.do = Out.transition + t.get_cost = Out.cost + else: + raise ValueError(Errors.E019.format(action=move, src='ner')) + return t + + def add_action(self, int action, label_name, freq=None): + cdef attr_t label_id + if not isinstance(label_name, int): + label_id = self.strings.add(label_name) + else: + label_id = label_name + if action == OUT and label_id != 0: + return None + if action == MISSING: + return None + # Check we're not creating a move we already have, so that this is + # idempotent + for trans in self.c[:self.n_moves]: + if trans.move == action and trans.label == label_id: + return 0 + if self.n_moves >= self._size: + self._size = self.n_moves + self._size *= 2 + self.c = self.mem.realloc(self.c, self._size * sizeof(self.c[0])) + self.c[self.n_moves] = self.init_transition(self.n_moves, action, label_id) + self.n_moves += 1 + if self.labels.get(action, []): + freq = min(0, min(self.labels[action].values())) + self.labels[action][label_name] = freq-1 + else: + self.labels[action] = Counter() + self.labels[action][label_name] = -1 + return 1 + + def set_annotations(self, StateClass state, Doc doc): + cdef int i + ents = [] + for i in range(state.c._ents.size()): + ent = state.c._ents.at(i) + if ent.start != -1 and ent.end != -1: + ents.append(Span(doc, ent.start, ent.end, label=ent.label, kb_id=doc.c[ent.start].ent_kb_id)) + doc.set_ents(ents, default="unmodified") + # Set non-blocked tokens to O + for i in range(doc.length): + if doc.c[i].ent_iob == 0: + doc.c[i].ent_iob = 2 + + def get_beam_parses(self, Beam beam): + parses = [] + probs = beam.probs + for i in range(beam.size): + state = beam.at(i) + if state.is_final(): + prob = probs[i] + parse = [] + for j in range(state._ents.size()): + ent = state._ents.at(j) + if ent.start != -1 and ent.end != -1: + parse.append((ent.start, ent.end, self.strings[ent.label])) + parses.append((prob, parse)) + return parses + + def init_gold(self, StateClass state, Example example): + return BiluoGold(self, state, example, self.neg_key) + + def has_gold(self, Example eg, start=0, end=None): + # We get x and y referring to X, we want to check relative to Y, + # the reference + y_spans = eg.get_aligned_spans_x2y([eg.x[start:end]]) + if not y_spans: + y_spans = [eg.y[:]] + y_span = y_spans[0] + start = y_span.start + end = y_span.end + neg_key = self.neg_key + if neg_key is not None: + # If we have any negative samples, count that as having annotation. + for span in eg.y.spans.get(neg_key, []): + if span.start >= start and span.end <= end: + return True + for word in eg.y[start:end]: + if word.ent_iob != 0: + return True + else: + return False + + def get_cost(self, StateClass stcls, gold, int i): + if not isinstance(gold, BiluoGold): + raise TypeError(Errors.E909.format(name="BiluoGold")) + cdef BiluoGold gold_ = gold + gold_state = gold_.c + if self.c[i].is_valid(stcls.c, self.c[i].label): + cost = self.c[i].get_cost(stcls.c, &gold_state, self.c[i].label) + else: + cost = 9000 + return cost + + cdef int set_costs(self, int* is_valid, weight_t* costs, + const StateC* state, gold) except -1: + if not isinstance(gold, BiluoGold): + raise TypeError(Errors.E909.format(name="BiluoGold")) + cdef BiluoGold gold_ = gold + gold_state = gold_.c + update_gold_state(&gold_state, state) + n_gold = 0 + self.set_valid(is_valid, state) + for i in range(self.n_moves): + if is_valid[i]: + costs[i] = self.c[i].get_cost(state, &gold_state, self.c[i].label) + n_gold += costs[i] <= 0 + else: + costs[i] = 9000 + + +cdef class Missing: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + return False + + @staticmethod + cdef int transition(StateC* s, attr_t label) noexcept nogil: + pass + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + return 9000 + + +cdef class Begin: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + cdef int preset_ent_iob = st.B_(0).ent_iob + cdef attr_t preset_ent_label = st.B_(0).ent_type + if st.entity_is_open(): + return False + if st.buffer_length() < 2: + # If we're the last token of the input, we can't B -- must U or O. + return False + elif label == 0: + return False + elif preset_ent_iob == 1: + # Ensure we don't clobber preset entities. If no entity preset, + # ent_iob is 0 + return False + elif preset_ent_iob == 3: + # Okay, we're in a preset entity. + if label != preset_ent_label: + # If label isn't right, reject + return False + elif st.B_(1).ent_iob != 1: + # If next token isn't marked I, we need to make U, not B. + return False + else: + # Otherwise, force acceptance, even if we're across a sentence + # boundary or the token is whitespace. + return True + elif st.B_(1).ent_iob == 3: + # If the next word is B, we can't B now + return False + elif st.B_(1).sent_start == 1: + # Don't allow entities to extend across sentence boundaries + return False + # Don't allow entities to start on whitespace + elif Lexeme.get_struct_attr(st.B_(0).lex, IS_SPACE): + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.open_ent(label) + st.push() + st.pop() + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + b0 = s.B(0) + cdef int cost = 0 + cdef int g_act = gold.ner[b0].move + cdef attr_t g_tag = gold.ner[b0].label + + if g_act == MISSING: + pass + elif g_act == BEGIN: + # B, Gold B --> Label match + cost += label != g_tag + else: + # B, Gold I --> False (P) + # B, Gold L --> False (P) + # B, Gold O --> False (P) + # B, Gold U --> False (P) + cost += 1 + if s.buffer_length() < 3: + # Handle negatives. In general we can't really do much to block + # B, because we don't know whether the whole entity is going to + # be correct or not. However, we can at least tell whether we're + # going to be opening an entity where there's only one possible + # L. + for span in gold.negs[:gold.nr_neg]: + if span.label == label and span.start == b0: + cost += 1 + break + return cost + + +cdef class In: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + if not st.entity_is_open(): + return False + if st.buffer_length() < 2: + # If we're at the end, we can't I. + return False + ent = st.get_ent() + cdef int preset_ent_iob = st.B_(0).ent_iob + cdef attr_t preset_ent_label = st.B_(0).ent_type + if label == 0: + return False + elif ent.label != label: + return False + elif preset_ent_iob == 3: + return False + elif st.B_(1).ent_iob == 3: + # If we know the next word is B, we can't be I (must be L) + return False + elif preset_ent_iob == 1: + if st.B_(1).ent_iob in (0, 2): + # if next preset is missing or O, this can't be I (must be L) + return False + elif label != preset_ent_label: + # If label isn't right, reject + return False + else: + # Otherwise, force acceptance, even if we're across a sentence + # boundary or the token is whitespace. + return True + elif st.B(1) != -1 and st.B_(1).sent_start == 1: + # Don't allow entities to extend across sentence boundaries + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.push() + st.pop() + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cdef int next_act = gold.ner[s.B(1)].move if s.B(1) >= 0 else OUT + cdef int g_act = gold.ner[s.B(0)].move + cdef bint is_sunk = _entity_is_sunk(s, gold.ner) + + if g_act == MISSING: + return 0 + elif g_act == BEGIN: + # I, Gold B --> True + # (P of bad open entity sunk, R of this entity sunk) + return 0 + elif g_act == IN: + # I, Gold I --> True + # (label forced by prev, if mismatch, P and R both sunk) + return 0 + elif g_act == LAST: + # I, Gold L --> True iff this entity sunk and next tag == O + return not (is_sunk and (next_act == OUT or next_act == MISSING)) + elif g_act == OUT: + # I, Gold O --> True iff next tag == O + return not (next_act == OUT or next_act == MISSING) + elif g_act == UNIT: + # I, Gold U --> True iff next tag == O + return next_act != OUT + else: + return 1 + +cdef class Last: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + cdef int preset_ent_iob = st.B_(0).ent_iob + cdef attr_t preset_ent_label = st.B_(0).ent_type + if label == 0: + return False + elif not st.entity_is_open(): + return False + elif preset_ent_iob == 1 and st.B_(1).ent_iob != 1: + # If a preset entity has I followed by not-I, is L + if label != preset_ent_label: + # If label isn't right, reject + return False + else: + # Otherwise, force acceptance, even if we're across a sentence + # boundary or the token is whitespace. + return True + elif st.get_ent().label != label: + return False + elif st.B_(1).ent_iob == 1: + # If a preset entity has I next, we can't L here. + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.close_ent() + st.push() + st.pop() + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + b0 = s.B(0) + ent_start = s.E(0) + + cdef int g_act = gold.ner[b0].move + + cdef int cost = 0 + + if g_act == MISSING: + pass + elif g_act == BEGIN: + # L, Gold B --> True + pass + elif g_act == IN: + # L, Gold I --> True iff this entity sunk + cost += not _entity_is_sunk(s, gold.ner) + elif g_act == LAST: + # L, Gold L --> True + pass + elif g_act == OUT: + # L, Gold O --> True + pass + elif g_act == UNIT: + # L, Gold U --> True + pass + else: + cost += 1 + # If we have negative-example entities, integrate them into the objective, + # by marking actions that close an entity that we know is incorrect + # as costly. + for span in gold.negs[:gold.nr_neg]: + if span.label == label and (span.end-1) == b0 and span.start == ent_start: + cost += 1 + break + return cost + + +cdef class Unit: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + cdef int preset_ent_iob = st.B_(0).ent_iob + cdef attr_t preset_ent_label = st.B_(0).ent_type + if label == 0: + # this is only allowed if it's a preset blocked annotation + if preset_ent_label == 0 and preset_ent_iob == 3: + return True + else: + return False + elif st.entity_is_open(): + return False + elif st.B_(1).ent_iob == 1: + # If next token is In, we can't be Unit -- must be Begin + return False + elif preset_ent_iob == 3: + # Okay, there's a preset entity here + if label != preset_ent_label: + # Require labels to match + return False + else: + # Otherwise return True, ignoring the whitespace constraint. + return True + elif Lexeme.get_struct_attr(st.B_(0).lex, IS_SPACE): + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.open_ent(label) + st.close_ent() + st.push() + st.pop() + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cdef int g_act = gold.ner[s.B(0)].move + cdef attr_t g_tag = gold.ner[s.B(0)].label + cdef int cost = 0 + + if g_act == MISSING: + pass + elif g_act == UNIT: + # U, Gold U --> True iff tag match + cost += label != g_tag + else: + # U, Gold B --> False + # U, Gold I --> False + # U, Gold L --> False + # U, Gold O --> False + cost += 1 + # If we have negative-example entities, integrate them into the objective. + # This is fairly straight-forward for U- entities, as we have a single + # action + cdef int b0 = s.B(0) + for span in gold.negs[:gold.nr_neg]: + if span.label == label and span.start == b0 and span.end == (b0+1): + cost += 1 + break + return cost + + +cdef class Out: + @staticmethod + cdef bint is_valid(const StateC* st, attr_t label) noexcept nogil: + cdef int preset_ent_iob = st.B_(0).ent_iob + if st.entity_is_open(): + return False + elif preset_ent_iob == 3: + return False + elif preset_ent_iob == 1: + return False + else: + return True + + @staticmethod + cdef int transition(StateC* st, attr_t label) noexcept nogil: + st.push() + st.pop() + + @staticmethod + cdef weight_t cost(const StateC* s, const void* _gold, attr_t label) noexcept nogil: + gold = _gold + cdef int g_act = gold.ner[s.B(0)].move + cdef weight_t cost = 0 + if g_act == MISSING: + pass + elif g_act == BEGIN: + # O, Gold B --> False + cost += 1 + elif g_act == IN: + # O, Gold I --> True + pass + elif g_act == LAST: + # O, Gold L --> True + pass + elif g_act == OUT: + # O, Gold O --> True + pass + elif g_act == UNIT: + # O, Gold U --> False + cost += 1 + else: + cost += 1 + return cost diff --git a/spacy/pipeline/_parser_internals/nonproj.hh b/spacy/pipeline/_parser_internals/nonproj.hh new file mode 100644 index 0000000..071fd57 --- /dev/null +++ b/spacy/pipeline/_parser_internals/nonproj.hh @@ -0,0 +1,11 @@ +#ifndef NONPROJ_HH +#define NONPROJ_HH + +#include +#include + +void raise_domain_error(std::string const &msg) { + throw std::domain_error(msg); +} + +#endif // NONPROJ_HH diff --git a/spacy/pipeline/_parser_internals/nonproj.pxd b/spacy/pipeline/_parser_internals/nonproj.pxd new file mode 100644 index 0000000..1a349d5 --- /dev/null +++ b/spacy/pipeline/_parser_internals/nonproj.pxd @@ -0,0 +1,5 @@ +from libcpp.string cimport string + + +cdef extern from "nonproj.hh": + cdef void raise_domain_error(const string& msg) nogil except + diff --git a/spacy/pipeline/_parser_internals/nonproj.pyx b/spacy/pipeline/_parser_internals/nonproj.pyx new file mode 100644 index 0000000..016b8b4 --- /dev/null +++ b/spacy/pipeline/_parser_internals/nonproj.pyx @@ -0,0 +1,258 @@ +# cython: infer_types=True +"""Implements the projectivize/deprojectivize mechanism in Nivre & Nilsson 2005 +for doing pseudo-projective parsing implementation uses the HEAD decoration +scheme. +""" +from copy import copy + +from cython.operator cimport dereference as deref +from cython.operator cimport preincrement as incr +from libc.limits cimport INT_MAX +from libc.stdlib cimport abs +from libcpp cimport bool +from libcpp.string cimport string, to_string +from libcpp.unordered_set cimport unordered_set +from libcpp.vector cimport vector + +from ...tokens.doc cimport Doc, set_children_from_heads + +from ...errors import Errors + +DELIMITER = '||' + + +def ancestors(tokenid, heads): + # Returns all words going from the word up the path to the root. The path + # to root cannot be longer than the number of words in the sentence. This + # function ends after at most len(heads) steps, because it would otherwise + # loop indefinitely on cycles. + head = tokenid + cnt = 0 + while heads[head] != head and cnt < len(heads): + head = heads[head] + cnt += 1 + yield head + if head is None: + break + + +def contains_cycle(heads): + # in an acyclic tree, the path from each word following the head relation + # upwards always ends at the root node + for tokenid in range(len(heads)): + seen = set([tokenid]) + for ancestor in ancestors(tokenid, heads): + if ancestor in seen: + return seen + seen.add(ancestor) + return None + + +def is_nonproj_arc(tokenid, heads): + cdef vector[int] c_heads = _heads_to_c(heads) + return _is_nonproj_arc(tokenid, c_heads) + + +cdef bool _is_nonproj_arc(int tokenid, const vector[int]& heads) nogil except *: + # definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective + # if there is a token k, h < k < d such that h is not + # an ancestor of k. Same for h -> d, h > d + head = heads[tokenid] + if head == tokenid: # root arcs cannot be non-projective + return False + elif head < 0: # unattached tokens cannot be non-projective + return False + + cdef int start, end + if head < tokenid: + start, end = (head+1, tokenid) + else: + start, end = (tokenid+1, head) + for k in range(start, end): + if not _has_head_as_ancestor(k, head, heads): + return True + return False + + +cdef bool _has_head_as_ancestor(int tokenid, int head, const vector[int]& heads) nogil except *: + ancestor = tokenid + cdef unordered_set[int] seen_tokens + seen_tokens.insert(ancestor) + while True: + # Reached the head or a disconnected node + if heads[ancestor] == head or heads[ancestor] < 0: + return True + # Reached the root + if heads[ancestor] == ancestor: + return False + ancestor = heads[ancestor] + result = seen_tokens.insert(ancestor) + # Found cycle + if not result.second: + raise_domain_error(heads_to_string(heads)) + + return False + + +cdef string heads_to_string(const vector[int]& heads) noexcept nogil: + cdef vector[int].const_iterator citer + cdef string cycle_str + + cycle_str.append("Found cycle in dependency graph: [") + + # FIXME: Rewrite using ostringstream when available in Cython. + citer = heads.const_begin() + while citer != heads.const_end(): + if citer != heads.const_begin(): + cycle_str.append(", ") + cycle_str.append(to_string(deref(citer))) + incr(citer) + cycle_str.append("]") + + return cycle_str + + +def is_nonproj_tree(heads): + cdef vector[int] c_heads = _heads_to_c(heads) + # a tree is non-projective if at least one arc is non-projective + return any(_is_nonproj_arc(word, c_heads) for word in range(len(heads))) + + +def decompose(label): + return label.partition(DELIMITER)[::2] + + +def is_decorated(label): + return DELIMITER in label + + +def count_decorated_labels(gold_data): + freqs = {} + for example in gold_data: + proj_heads, deco_deps = projectivize(example.get_aligned("HEAD"), + example.get_aligned("DEP")) + # set the label to ROOT for each root dependent + deco_deps = [ + 'ROOT' if head == i else deco_deps[i] + for i, head in enumerate(proj_heads) + ] + # count label frequencies + for label in deco_deps: + if is_decorated(label): + freqs[label] = freqs.get(label, 0) + 1 + return freqs + + +def projectivize(heads, labels): + # Use the algorithm by Nivre & Nilsson 2005. Assumes heads to be a proper + # tree, i.e. connected and cycle-free. Returns a new pair (heads, labels) + # which encode a projective and decorated tree. + proj_heads = copy(heads) + + cdef int new_head + cdef vector[int] c_proj_heads = _heads_to_c(proj_heads) + cdef int smallest_np_arc = _get_smallest_nonproj_arc(c_proj_heads) + if smallest_np_arc == -1: # this sentence is already projective + return proj_heads, copy(labels) + while smallest_np_arc != -1: + new_head = _lift(smallest_np_arc, proj_heads) + c_proj_heads[smallest_np_arc] = new_head + smallest_np_arc = _get_smallest_nonproj_arc(c_proj_heads) + deco_labels = _decorate(heads, proj_heads, labels) + return proj_heads, deco_labels + + +cdef vector[int] _heads_to_c(heads): + cdef vector[int] c_heads + for head in heads: + if head is None: + c_heads.push_back(-1) + else: + assert head < len(heads) + c_heads.push_back(head) + return c_heads + + +cpdef deprojectivize(Doc doc): + # Reattach arcs with decorated labels (following HEAD scheme). For each + # decorated arc X||Y, search top-down, left-to-right, breadth-first until + # hitting a Y then make this the new head. + for i in range(doc.length): + label = doc.vocab.strings[doc.c[i].dep] + if DELIMITER in label: + new_label, head_label = label.split(DELIMITER) + new_head = _find_new_head(doc[i], head_label) + doc.c[i].head = new_head.i - i + doc.c[i].dep = doc.vocab.strings.add(new_label, allow_transient=False) + set_children_from_heads(doc.c, 0, doc.length) + return doc + + +def _decorate(heads, proj_heads, labels): + # uses decoration scheme HEAD from Nivre & Nilsson 2005 + if (len(heads) != len(proj_heads)) or (len(proj_heads) != len(labels)): + raise ValueError(Errors.E082.format(n_heads=len(heads), + n_proj_heads=len(proj_heads), + n_labels=len(labels))) + deco_labels = [] + for tokenid, head in enumerate(heads): + if head != proj_heads[tokenid]: + deco_labels.append(f"{labels[tokenid]}{DELIMITER}{labels[head]}") + else: + deco_labels.append(labels[tokenid]) + return deco_labels + + +def get_smallest_nonproj_arc_slow(heads): + cdef vector[int] c_heads = _heads_to_c(heads) + return _get_smallest_nonproj_arc(c_heads) + + +cdef int _get_smallest_nonproj_arc(const vector[int]& heads) nogil except -2: + # return the smallest non-proj arc or None + # where size is defined as the distance between dep and head + # and ties are broken left to right + cdef int smallest_size = INT_MAX + # -1 means its already projective. + cdef int smallest_np_arc = -1 + cdef int size + cdef int tokenid + cdef int head + + for tokenid in range(heads.size()): + head = heads[tokenid] + size = abs(tokenid-head) + if size < smallest_size and _is_nonproj_arc(tokenid, heads): + smallest_size = size + smallest_np_arc = tokenid + return smallest_np_arc + + +cpdef int _lift(tokenid, heads): + # reattaches a word to it's grandfather + head = heads[tokenid] + ghead = heads[head] + cdef int new_head = ghead if head != ghead else tokenid + # attach to ghead if head isn't attached to root else attach to root + heads[tokenid] = new_head + return new_head + + +def _find_new_head(token, headlabel): + # search through the tree starting from the head of the given token + # returns the id of the first descendant with the given label + # if there is none, return the current head (no change) + queue = [token.head] + while queue: + next_queue = [] + for qtoken in queue: + for child in qtoken.children: + if child.is_space: + continue + if child == token: + continue + if child.dep_ == headlabel: + return child + next_queue.append(child) + queue = next_queue + return token.head diff --git a/spacy/pipeline/_parser_internals/stateclass.pxd b/spacy/pipeline/_parser_internals/stateclass.pxd new file mode 100644 index 0000000..b8ecc1b --- /dev/null +++ b/spacy/pipeline/_parser_internals/stateclass.pxd @@ -0,0 +1,28 @@ +from cymem.cymem cimport Pool + +from ...structs cimport SpanC, TokenC +from ...tokens.doc cimport Doc +from ...typedefs cimport attr_t +from ._state cimport StateC + + +cdef class StateClass: + cdef StateC* c + cdef readonly Doc doc + cdef int _borrowed + + @staticmethod + cdef inline StateClass borrow(StateC* ptr, Doc doc): + cdef StateClass self = StateClass() + self.c = ptr + self._borrowed = 1 + self.doc = doc + return self + + @staticmethod + cdef inline StateClass init_offset(const TokenC* sent, int length, int + offset): + cdef StateClass self = StateClass() + self.c = new StateC(sent, length) + self.c.offset = offset + return self diff --git a/spacy/pipeline/_parser_internals/stateclass.pyx b/spacy/pipeline/_parser_internals/stateclass.pyx new file mode 100644 index 0000000..24b9f1a --- /dev/null +++ b/spacy/pipeline/_parser_internals/stateclass.pyx @@ -0,0 +1,178 @@ +# cython: infer_types=True +# cython: profile=False +from libcpp.vector cimport vector + +from ...tokens.doc cimport Doc +from ._state cimport ArcC + + +cdef class StateClass: + def __init__(self, Doc doc=None, int offset=0): + self._borrowed = 0 + if doc is not None: + self.c = new StateC(doc.c, doc.length) + self.c.offset = offset + self.doc = doc + else: + self.doc = None + + def __dealloc__(self): + if self._borrowed != 1: + del self.c + + @property + def stack(self): + return [self.S(i) for i in range(self.c.stack_depth())] + + @property + def queue(self): + return [self.B(i) for i in range(self.c.buffer_length())] + + @property + def token_vector_length(self): + return self.doc.tensor.shape[1] + + @property + def arcs(self): + cdef vector[ArcC] arcs + self.c.get_arcs(&arcs) + return list(arcs) + # py_arcs = [] + # for arc in arcs: + # if arc.head != -1 and arc.child != -1: + # py_arcs.append((arc.head, arc.child, arc.label)) + # return arcs + + def add_arc(self, int head, int child, int label): + self.c.add_arc(head, child, label) + + def del_arc(self, int head, int child): + self.c.del_arc(head, child) + + def H(self, int child): + return self.c.H(child) + + def L(self, int head, int idx): + return self.c.L(head, idx) + + def R(self, int head, int idx): + return self.c.R(head, idx) + + @property + def _b_i(self): + return self.c._b_i + + @property + def length(self): + return self.c.length + + def is_final(self): + return self.c.is_final() + + def copy(self): + cdef StateClass new_state = StateClass(doc=self.doc, offset=self.c.offset) + new_state.c.clone(self.c) + return new_state + + def print_state(self): + words = [token.text for token in self.doc] + words = list(words) + ['_'] + bools = ["F", "T"] + sent_starts = [bools[self.c.is_sent_start(i)] for i in range(len(self.doc))] + shifted = [1 if self.c.is_unshiftable(i) else 0 for i in range(self.c.length)] + shifted.append("") + sent_starts.append("") + top = f"{self.S(0)}{words[self.S(0)]}_{words[self.H(self.S(0))]}_{shifted[self.S(0)]}" + second = f"{self.S(1)}{words[self.S(1)]}_{words[self.H(self.S(1))]}_{shifted[self.S(1)]}" + third = f"{self.S(2)}{words[self.S(2)]}_{words[self.H(self.S(2))]}_{shifted[self.S(2)]}" + n0 = f"{self.B(0)}{words[self.B(0)]}_{sent_starts[self.B(0)]}_{shifted[self.B(0)]}" + n1 = f"{self.B(1)}{words[self.B(1)]}_{sent_starts[self.B(1)]}_{shifted[self.B(1)]}" + return ' '.join((str(self.stack_depth()), str(self.buffer_length()), third, second, top, '|', n0, n1)) + + def S(self, int i): + return self.c.S(i) + + def B(self, int i): + return self.c.B(i) + + def H(self, int i): + return self.c.H(i) + + def E(self, int i): + return self.c.E(i) + + def L(self, int i, int idx): + return self.c.L(i, idx) + + def R(self, int i, int idx): + return self.c.R(i, idx) + + def S_(self, int i): + return self.doc[self.c.S(i)] + + def B_(self, int i): + return self.doc[self.c.B(i)] + + def H_(self, int i): + return self.doc[self.c.H(i)] + + def E_(self, int i): + return self.doc[self.c.E(i)] + + def L_(self, int i, int idx): + return self.doc[self.c.L(i, idx)] + + def R_(self, int i, int idx): + return self.doc[self.c.R(i, idx)] + + def empty(self): + return self.c.empty() + + def eol(self): + return self.c.eol() + + def at_break(self): + return False + # return self.c.at_break() + + def has_head(self, int i): + return self.c.has_head(i) + + def n_L(self, int i): + return self.c.n_L(i) + + def n_R(self, int i): + return self.c.n_R(i) + + def entity_is_open(self): + return self.c.entity_is_open() + + def stack_depth(self): + return self.c.stack_depth() + + def buffer_length(self): + return self.c.buffer_length() + + def push(self): + self.c.push() + + def pop(self): + self.c.pop() + + def unshift(self): + self.c.unshift() + + def add_arc(self, int head, int child, attr_t label): + self.c.add_arc(head, child, label) + + def del_arc(self, int head, int child): + self.c.del_arc(head, child) + + def open_ent(self, attr_t label): + self.c.open_ent(label) + + def close_ent(self): + self.c.close_ent() + + def clone(self, StateClass src): + self.c.clone(src.c) diff --git a/spacy/pipeline/_parser_internals/transition_system.pxd b/spacy/pipeline/_parser_internals/transition_system.pxd new file mode 100644 index 0000000..74fd489 --- /dev/null +++ b/spacy/pipeline/_parser_internals/transition_system.pxd @@ -0,0 +1,59 @@ +from cymem.cymem cimport Pool + +from ...strings cimport StringStore +from ...structs cimport TokenC +from ...training.example cimport Example +from ...typedefs cimport attr_t, weight_t +from ._state cimport StateC +from .stateclass cimport StateClass + + +cdef struct Transition: + int clas + int move + attr_t label + + weight_t score + + bint (*is_valid)(const StateC* state, attr_t label) noexcept nogil + weight_t (*get_cost)(const StateC* state, const void* gold, attr_t label) noexcept nogil + int (*do)(StateC* state, attr_t label) noexcept nogil + + +ctypedef weight_t (*get_cost_func_t)( + const StateC* state, const void* gold, attr_tlabel +) noexcept nogil +ctypedef weight_t (*move_cost_func_t)( + const StateC* state, const void* gold +) noexcept nogil +ctypedef weight_t (*label_cost_func_t)( + const StateC* state, const void* gold, attr_t label +) noexcept nogil + +ctypedef int (*do_func_t)(StateC* state, attr_t label) noexcept nogil + +ctypedef void* (*init_state_t)(Pool mem, int length, void* tokens) except NULL + +ctypedef int (*del_state_t)(Pool mem, void* state, void* extra_args) except -1 + +cdef class TransitionSystem: + cdef Pool mem + cdef StringStore strings + cdef Transition* c + cdef readonly int n_moves + cdef int _size + cdef public attr_t root_label + cdef public freqs + cdef public object labels + cdef public object cfg + cdef init_state_t init_beam_state + cdef del_state_t del_beam_state + + cdef Transition lookup_transition(self, object name) except * + + cdef Transition init_transition(self, int clas, int move, attr_t label) except * + + cdef int set_valid(self, int* output, const StateC* st) noexcept nogil + + cdef int set_costs(self, int* is_valid, weight_t* costs, + const StateC* state, gold) except -1 diff --git a/spacy/pipeline/_parser_internals/transition_system.pyx b/spacy/pipeline/_parser_internals/transition_system.pyx new file mode 100644 index 0000000..c859135 --- /dev/null +++ b/spacy/pipeline/_parser_internals/transition_system.pyx @@ -0,0 +1,251 @@ +# cython: infer_types=True +# cython: profile=False +from __future__ import print_function + +from cymem.cymem cimport Pool + +from collections import Counter + +import srsly + +from ...structs cimport TokenC +from ...typedefs cimport attr_t, weight_t +from .stateclass cimport StateClass + +from ... import util +from ...errors import Errors + + +cdef weight_t MIN_SCORE = -90000 + + +class OracleError(Exception): + pass + + +cdef void* _init_state(Pool mem, int length, void* tokens) except NULL: + cdef StateC* st = new StateC(tokens, length) + return st + + +cdef int _del_state(Pool mem, void* state, void* x) except -1: + cdef StateC* st = state + del st + + +cdef class TransitionSystem: + def __init__( + self, + StringStore string_table, + labels_by_action=None, + min_freq=None, + incorrect_spans_key=None + ): + self.cfg = {"neg_key": incorrect_spans_key} + self.mem = Pool() + self.strings = string_table + self.n_moves = 0 + self._size = 100 + + self.c = self.mem.alloc(self._size, sizeof(Transition)) + + self.labels = {} + if labels_by_action: + self.initialize_actions(labels_by_action, min_freq=min_freq) + self.root_label = self.strings.add('ROOT') + self.init_beam_state = _init_state + self.del_beam_state = _del_state + + def __reduce__(self): + # TODO: This loses the 'cfg' + return (self.__class__, (self.strings, self.labels), None, None) + + @property + def neg_key(self): + return self.cfg.get("neg_key") + + def init_batch(self, docs): + cdef StateClass state + states = [] + offset = 0 + for doc in docs: + state = StateClass(doc, offset=offset) + states.append(state) + offset += len(doc) + return states + + def get_oracle_sequence(self, Example example, _debug=False): + states, golds, _ = self.init_gold_batch([example]) + if not states: + return [] + state = states[0] + gold = golds[0] + if _debug: + return self.get_oracle_sequence_from_state(state, gold, _debug=example) + else: + return self.get_oracle_sequence_from_state(state, gold) + + def get_oracle_sequence_from_state(self, StateClass state, gold, _debug=None): + cdef Pool mem = Pool() + # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc + assert self.n_moves > 0 + costs = mem.alloc(self.n_moves, sizeof(float)) + is_valid = mem.alloc(self.n_moves, sizeof(int)) + + history = [] + debug_log = [] + while not state.is_final(): + self.set_costs(is_valid, costs, state.c, gold) + for i in range(self.n_moves): + if is_valid[i] and costs[i] <= 0: + action = self.c[i] + history.append(i) + if _debug: + s0 = state.S(0) + b0 = state.B(0) + example = _debug + debug_log.append(" ".join(( + self.get_class_name(i), + "S0=", (example.x[s0].text if s0 >= 0 else "__"), + "B0=", (example.x[b0].text if b0 >= 0 else "__"), + "S0 head?", str(state.has_head(state.S(0))), + ))) + action.do(state.c, action.label) + break + else: + if _debug: + example = _debug + print("Actions") + for i in range(self.n_moves): + print(self.get_class_name(i)) + print("Gold") + for token in example.y: + print(token.text, token.dep_, token.head.text) + s0 = state.S(0) + b0 = state.B(0) + debug_log.append(" ".join(( + "?", + "S0=", (example.x[s0].text if s0 >= 0 else "-"), + "B0=", (example.x[b0].text if b0 >= 0 else "-"), + "S0 head?", str(state.has_head(state.S(0))), + ))) + print("\n".join(debug_log)) + raise ValueError(Errors.E024) + return history + + def apply_transition(self, StateClass state, name): + if not self.is_valid(state, name): + raise ValueError(Errors.E170.format(name=name)) + action = self.lookup_transition(name) + action.do(state.c, action.label) + + cdef Transition lookup_transition(self, object name) except *: + raise NotImplementedError + + cdef Transition init_transition(self, int clas, int move, attr_t label) except *: + raise NotImplementedError + + def is_valid(self, StateClass stcls, move_name): + action = self.lookup_transition(move_name) + return action.is_valid(stcls.c, action.label) + + cdef int set_valid(self, int* is_valid, const StateC* st) noexcept nogil: + cdef int i + for i in range(self.n_moves): + is_valid[i] = self.c[i].is_valid(st, self.c[i].label) + + cdef int set_costs(self, int* is_valid, weight_t* costs, + const StateC* state, gold) except -1: + raise NotImplementedError + + def get_class_name(self, int clas): + act = self.c[clas] + return self.move_name(act.move, act.label) + + def initialize_actions(self, labels_by_action, min_freq=None): + self.labels = {} + self.n_moves = 0 + added_labels = [] + added_actions = {} + for action, label_freqs in sorted(labels_by_action.items()): + action = int(action) + # Make sure we take a copy here, and that we get a Counter + self.labels[action] = Counter() + # Have to be careful here: Sorting must be stable, or our model + # won't be read back in correctly. + sorted_labels = [(f, L) for L, f in label_freqs.items()] + sorted_labels.sort() + sorted_labels.reverse() + for freq, label_str in sorted_labels: + if freq < 0: + added_labels.append((freq, label_str)) + added_actions.setdefault(label_str, []).append(action) + else: + self.add_action(int(action), label_str) + self.labels[action][label_str] = freq + added_labels.sort(reverse=True) + for freq, label_str in added_labels: + for action in added_actions[label_str]: + self.add_action(int(action), label_str) + self.labels[action][label_str] = freq + + def add_action(self, int action, label_name): + cdef attr_t label_id + if not isinstance(label_name, int): + label_id = self.strings.add(label_name) + else: + label_id = label_name + # Check we're not creating a move we already have, so that this is + # idempotent + for trans in self.c[:self.n_moves]: + if trans.move == action and trans.label == label_id: + return 0 + if self.n_moves >= self._size: + self._size *= 2 + self.c = self.mem.realloc(self.c, self._size * sizeof(self.c[0])) + self.c[self.n_moves] = self.init_transition(self.n_moves, action, label_id) + self.n_moves += 1 + # Add the new (action, label) pair, making up a frequency for it if + # necessary. To preserve sort order, the frequency needs to be lower + # than previous frequencies. + if self.labels.get(action, []): + new_freq = min(self.labels[action].values()) + else: + self.labels[action] = Counter() + new_freq = -1 + if new_freq > 0: + new_freq = 0 + self.labels[action][label_name] = new_freq-1 + return 1 + + def to_disk(self, path, **kwargs): + with path.open('wb') as file_: + file_.write(self.to_bytes(**kwargs)) + + def from_disk(self, path, **kwargs): + with path.open('rb') as file_: + byte_data = file_.read() + self.from_bytes(byte_data, **kwargs) + return self + + def to_bytes(self, exclude=tuple()): + serializers = { + 'moves': lambda: srsly.json_dumps(self.labels), + 'strings': lambda: self.strings.to_bytes(), + 'cfg': lambda: self.cfg + } + return util.to_bytes(serializers, exclude) + + def from_bytes(self, bytes_data, exclude=tuple()): + # We're adding a new field, 'cfg', here and we don't want to break + # previous models that don't have it. + msg = srsly.msgpack_loads(bytes_data) + labels = {} + if 'moves' not in exclude: + labels.update(srsly.json_loads(msg['moves'])) + if 'strings' not in exclude: + self.strings.from_bytes(msg['strings']) + if 'cfg' not in exclude and 'cfg' in msg: + self.cfg.update(msg['cfg']) + self.initialize_actions(labels) + return self diff --git a/spacy/pipeline/attributeruler.py b/spacy/pipeline/attributeruler.py new file mode 100644 index 0000000..d4f96ec --- /dev/null +++ b/spacy/pipeline/attributeruler.py @@ -0,0 +1,355 @@ +import importlib +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + +import srsly + +from .. import util +from ..errors import Errors +from ..language import Language +from ..matcher import Matcher +from ..scorer import Scorer +from ..symbols import IDS +from ..tokens import Doc, Span +from ..tokens._retokenize import normalize_token_attrs, set_token_attrs +from ..training import Example +from ..util import SimpleFrozenList +from ..vocab import Vocab +from .pipe import Pipe + +MatcherPatternType = List[Dict[Union[int, str], Any]] +AttributeRulerPatternType = Dict[ + str, Union[List[MatcherPatternType], MatcherPatternType, Dict, int] +] +TagMapType = Dict[str, Dict[Union[int, str], Union[int, str]]] +MorphRulesType = Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]] + + +def attribute_ruler_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + def morph_key_getter(token, attr): + return getattr(token, attr).key + + results = {} + results.update(Scorer.score_token_attr(examples, "tag", **kwargs)) + results.update(Scorer.score_token_attr(examples, "pos", **kwargs)) + results.update( + Scorer.score_token_attr(examples, "morph", getter=morph_key_getter, **kwargs) + ) + results.update( + Scorer.score_token_attr_per_feat( + examples, "morph", getter=morph_key_getter, **kwargs + ) + ) + results.update(Scorer.score_token_attr(examples, "lemma", **kwargs)) + return results + + +def make_attribute_ruler_scorer(): + return attribute_ruler_score + + +class AttributeRuler(Pipe): + """Set token-level attributes for tokens matched by Matcher patterns. + Additionally supports importing patterns from tag maps and morph rules. + + DOCS: https://spacy.io/api/attributeruler + """ + + def __init__( + self, + vocab: Vocab, + name: str = "attribute_ruler", + *, + validate: bool = False, + scorer: Optional[Callable] = attribute_ruler_score, + ) -> None: + """Create the AttributeRuler. After creation, you can add patterns + with the `.initialize()` or `.add_patterns()` methods, or load patterns + with `.from_bytes()` or `.from_disk()`. Loading patterns will remove + any patterns you've added previously. + + vocab (Vocab): The vocab. + name (str): The pipe name. Defaults to "attribute_ruler". + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_token_attr for the attributes "tag", "pos", "morph" and + "lemma" and Scorer.score_token_attr_per_feat for the attribute + "morph". + + RETURNS (AttributeRuler): The AttributeRuler component. + + DOCS: https://spacy.io/api/attributeruler#init + """ + self.name = name + self.vocab = vocab + self.matcher = Matcher(self.vocab, validate=validate) + self.validate = validate + self.attrs: List[Dict] = [] + self._attrs_unnormed: List[Dict] = [] # store for reference + self.indices: List[int] = [] + self.scorer = scorer + + def clear(self) -> None: + """Reset all patterns.""" + self.matcher = Matcher(self.vocab, validate=self.validate) + self.attrs = [] + self._attrs_unnormed = [] + self.indices = [] + + def initialize( + self, + get_examples: Optional[Callable[[], Iterable[Example]]], + *, + nlp: Optional[Language] = None, + patterns: Optional[Iterable[AttributeRulerPatternType]] = None, + tag_map: Optional[TagMapType] = None, + morph_rules: Optional[MorphRulesType] = None, + ) -> None: + """Initialize the attribute ruler by adding zero or more patterns. + + Rules can be specified as a sequence of dicts using the `patterns` + keyword argument. You can also provide rules using the "tag map" or + "morph rules" formats supported by spaCy prior to v3. + """ + self.clear() + if patterns: + self.add_patterns(patterns) + if tag_map: + self.load_from_tag_map(tag_map) + if morph_rules: + self.load_from_morph_rules(morph_rules) + + def __call__(self, doc: Doc) -> Doc: + """Apply the AttributeRuler to a Doc and set all attribute exceptions. + + doc (Doc): The document to process. + RETURNS (Doc): The processed Doc. + + DOCS: https://spacy.io/api/attributeruler#call + """ + error_handler = self.get_error_handler() + try: + matches = self.match(doc) + self.set_annotations(doc, matches) + return doc + except Exception as e: + return error_handler(self.name, self, [doc], e) + + def match(self, doc: Doc): + matches = self.matcher(doc, allow_missing=True, as_spans=False) + # Sort by the attribute ID, so that later rules have precedence + matches = [ + (int(self.vocab.strings[m_id]), m_id, s, e) + for m_id, s, e in matches # type: ignore + ] + matches.sort() + return matches + + def set_annotations(self, doc, matches): + """Modify the document in place""" + for attr_id, match_id, start, end in matches: + span = Span(doc, start, end, label=match_id) + attrs = self.attrs[attr_id] + index = self.indices[attr_id] + try: + # The index can be negative, which makes it annoying to do + # the boundscheck. Let Span do it instead. + token = span[index] # noqa: F841 + except IndexError: + # The original exception is just our conditional logic, so we + # raise from. + raise ValueError( + Errors.E1001.format( + patterns=self.matcher.get(span.label), + span=[t.text for t in span], + index=index, + ) + ) from None + set_token_attrs(span[index], attrs) + + def load_from_tag_map( + self, tag_map: Dict[str, Dict[Union[int, str], Union[int, str]]] + ) -> None: + """Load attribute ruler patterns from a tag map. + + tag_map (dict): The tag map that maps fine-grained tags to + coarse-grained tags and morphological features. + + DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules + """ + for tag, attrs in tag_map.items(): + pattern = [{"TAG": tag}] + attrs, morph_attrs = _split_morph_attrs(attrs) + if "MORPH" not in attrs: + morph = self.vocab.morphology.add(morph_attrs) + attrs["MORPH"] = self.vocab.strings[morph] + else: + morph = self.vocab.morphology.add(attrs["MORPH"]) + attrs["MORPH"] = self.vocab.strings[morph] + self.add([pattern], attrs) # type: ignore[list-item] + + def load_from_morph_rules( + self, morph_rules: Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]] + ) -> None: + """Load attribute ruler patterns from morph rules. + + morph_rules (dict): The morph rules that map token text and + fine-grained tags to coarse-grained tags, lemmas and morphological + features. + + DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules + """ + for tag in morph_rules: + for word in morph_rules[tag]: + pattern = [{"ORTH": word, "TAG": tag}] + attrs = morph_rules[tag][word] + attrs, morph_attrs = _split_morph_attrs(attrs) + if "MORPH" in attrs: + morph = self.vocab.morphology.add(attrs["MORPH"]) + attrs["MORPH"] = self.vocab.strings[morph] + elif morph_attrs: + morph = self.vocab.morphology.add(morph_attrs) + attrs["MORPH"] = self.vocab.strings[morph] + self.add([pattern], attrs) # type: ignore[list-item] + + def add( + self, patterns: Iterable[MatcherPatternType], attrs: Dict, index: int = 0 + ) -> None: + """Add Matcher patterns for tokens that should be modified with the + provided attributes. The token at the specified index within the + matched span will be assigned the attributes. + + patterns (Iterable[List[Dict]]): A list of Matcher patterns. + attrs (Dict): The attributes to assign to the target token in the + matched span. + index (int): The index of the token in the matched span to modify. May + be negative to index from the end of the span. Defaults to 0. + + DOCS: https://spacy.io/api/attributeruler#add + """ + # We need to make a string here, because otherwise the ID we pass back + # will be interpreted as the hash of a string, rather than an ordinal. + key = str(len(self.attrs)) + self.matcher.add(self.vocab.strings.add(key), patterns) # type: ignore[arg-type] + self._attrs_unnormed.append(attrs) + attrs = normalize_token_attrs(self.vocab, attrs) + self.attrs.append(attrs) + self.indices.append(index) + + def add_patterns(self, patterns: Iterable[AttributeRulerPatternType]) -> None: + """Add patterns from a list of pattern dicts with the keys as the + arguments to AttributeRuler.add. + patterns (Iterable[dict]): A list of pattern dicts with the keys + as the arguments to AttributeRuler.add (patterns/attrs/index) to + add as patterns. + + DOCS: https://spacy.io/api/attributeruler#add_patterns + """ + for p in patterns: + self.add(**p) # type: ignore[arg-type] + + @property + def patterns(self) -> List[AttributeRulerPatternType]: + """All the added patterns.""" + all_patterns = [] + for i in range(len(self.attrs)): + p = {} + p["patterns"] = self.matcher.get(str(i))[1] + p["attrs"] = self._attrs_unnormed[i] # type: ignore + p["index"] = self.indices[i] # type: ignore + all_patterns.append(p) + return all_patterns # type: ignore[return-value] + + def to_bytes(self, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the AttributeRuler to a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/attributeruler#to_bytes + """ + serialize = {} + serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serialize["patterns"] = lambda: srsly.msgpack_dumps(self.patterns) + return util.to_bytes(serialize, exclude) + + def from_bytes( + self, bytes_data: bytes, exclude: Iterable[str] = SimpleFrozenList() + ) -> "AttributeRuler": + """Load the AttributeRuler from a bytestring. + + bytes_data (bytes): The data to load. + exclude (Iterable[str]): String names of serialization fields to exclude. + returns (AttributeRuler): The loaded object. + + DOCS: https://spacy.io/api/attributeruler#from_bytes + """ + + def load_patterns(b): + self.add_patterns(srsly.msgpack_loads(b)) + + deserialize = { + "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude), + "patterns": load_patterns, + } + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk( + self, path: Union[Path, str], exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Serialize the AttributeRuler to disk. + + path (Union[Path, str]): A path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/attributeruler#to_disk + """ + serialize = { + "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude), + "patterns": lambda p: srsly.write_msgpack(p, self.patterns), + } + util.to_disk(path, serialize, exclude) + + def from_disk( + self, path: Union[Path, str], exclude: Iterable[str] = SimpleFrozenList() + ) -> "AttributeRuler": + """Load the AttributeRuler from disk. + + path (Union[Path, str]): A path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (AttributeRuler): The loaded object. + + DOCS: https://spacy.io/api/attributeruler#from_disk + """ + + def load_patterns(p): + self.add_patterns(srsly.read_msgpack(p)) + + deserialize = { + "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude), + "patterns": load_patterns, + } + util.from_disk(path, deserialize, exclude) + return self + + +def _split_morph_attrs(attrs: dict) -> Tuple[dict, dict]: + """Split entries from a tag map or morph rules dict into to two dicts, one + with the token-level features (POS, LEMMA) and one with the remaining + features, which are presumed to be individual MORPH features.""" + other_attrs = {} + morph_attrs = {} + for k, v in attrs.items(): + if k in "_" or k in IDS.keys() or k in IDS.values(): + other_attrs[k] = v + else: + morph_attrs[k] = v + return other_attrs, morph_attrs + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_attribute_ruler": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_attribute_ruler + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/dep_parser.pyx b/spacy/pipeline/dep_parser.pyx new file mode 100644 index 0000000..881ec2d --- /dev/null +++ b/spacy/pipeline/dep_parser.pyx @@ -0,0 +1,178 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from collections import defaultdict +from typing import Callable, Optional + +from thinc.api import Config, Model + +from ._parser_internals.transition_system import TransitionSystem + +from ._parser_internals.arc_eager cimport ArcEager +from .transition_parser cimport Parser + +from ..language import Language +from ..scorer import Scorer +from ..training import remove_bilu_prefix +from ..util import registry +from ._parser_internals import nonproj +from ._parser_internals.nonproj import DELIMITER +from .functions import merge_subtokens + +default_model_config = """ +[model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = true + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_PARSER_MODEL = Config().from_str(default_model_config)["model"] + + +def parser_score(examples, **kwargs): + """Score a batch of examples. + + examples (Iterable[Example]): The examples to score. + RETURNS (Dict[str, Any]): The scores, produced by Scorer.score_spans + and Scorer.score_deps. + + DOCS: https://spacy.io/api/dependencyparser#score + """ + def has_sents(doc): + return doc.has_annotation("SENT_START") + + def dep_getter(token, attr): + dep = getattr(token, attr) + dep = token.vocab.strings.as_string(dep).lower() + return dep + results = {} + results.update(Scorer.score_spans(examples, "sents", has_annotation=has_sents, **kwargs)) + kwargs.setdefault("getter", dep_getter) + kwargs.setdefault("ignore_labels", ("p", "punct")) + results.update(Scorer.score_deps(examples, "dep", **kwargs)) + del results["sents_per_type"] + return results + + +def make_parser_scorer(): + return parser_score + + +cdef class DependencyParser(Parser): + """Pipeline component for dependency parsing. + + DOCS: https://spacy.io/api/dependencyparser + """ + TransitionSystem = ArcEager + + def __init__( + self, + vocab, + model, + name="parser", + moves=None, + *, + update_with_oracle_cut_size=100, + min_action_freq=30, + learn_tokens=False, + beam_width=1, + beam_density=0.0, + beam_update_prob=0.0, + multitasks=tuple(), + incorrect_spans_key=None, + scorer=parser_score, + ): + """Create a DependencyParser. + """ + super().__init__( + vocab, + model, + name, + moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + min_action_freq=min_action_freq, + learn_tokens=learn_tokens, + beam_width=beam_width, + beam_density=beam_density, + beam_update_prob=beam_update_prob, + multitasks=multitasks, + incorrect_spans_key=incorrect_spans_key, + scorer=scorer, + ) + + @property + def postprocesses(self): + output = [nonproj.deprojectivize] + if self.cfg.get("learn_tokens") is True: + output.append(merge_subtokens) + return tuple(output) + + def add_multitask_objective(self, mt_component): + self._multitasks.append(mt_component) + + def init_multitask_objectives(self, get_examples, nlp=None, **cfg): + # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? + for labeller in self._multitasks: + labeller.model.set_dim("nO", len(self.labels)) + if labeller.model.has_ref("output_layer"): + labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) + labeller.initialize(get_examples, nlp=nlp) + + @property + def labels(self): + labels = set() + # Get the labels from the model by looking at the available moves + for move in self.move_names: + if "-" in move: + label = remove_bilu_prefix(move) + if DELIMITER in label: + label = label.split(DELIMITER)[1] + labels.add(label) + return tuple(sorted(labels)) + + def scored_parses(self, beams): + """Return two dictionaries with scores for each beam/doc that was processed: + one containing (i, head) keys, and another containing (i, label) keys. + """ + head_scores = [] + label_scores = [] + for beam in beams: + score_head_dict = defaultdict(float) + score_label_dict = defaultdict(float) + for score, parses in self.moves.get_beam_parses(beam): + for head, i, label in parses: + score_head_dict[(i, head)] += score + score_label_dict[(i, label)] += score + head_scores.append(score_head_dict) + label_scores.append(score_label_dict) + return head_scores, label_scores + + def _ensure_labels_are_added(self, docs): + # This gives the parser a chance to add labels it's missing for a batch + # of documents. However, this isn't desirable for the dependency parser, + # because we instead have a label frequency cut-off and back off rare + # labels to 'dep'. + pass + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_parser": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_parser + elif name == "make_beam_parser": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_beam_parser + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py new file mode 100644 index 0000000..77f033b --- /dev/null +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -0,0 +1,395 @@ +import importlib +from collections import Counter +from itertools import islice +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, cast + +import numpy as np +import srsly +from thinc.api import Config, Model, NumpyOps, SequenceCategoricalCrossentropy +from thinc.types import Floats2d, Ints2d + +from .. import util +from ..errors import Errors +from ..language import Language +from ..tokens import Doc +from ..training import Example, validate_examples, validate_get_examples +from ..vocab import Vocab +from ._edit_tree_internals.edit_trees import EditTrees +from ._edit_tree_internals.schemas import validate_edit_tree +from .lemmatizer import lemmatizer_score +from .trainable_pipe import TrainablePipe + +# The cutoff value of *top_k* above which an alternative method is used to process guesses. +TOP_K_GUARDRAIL = 20 + + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v2" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_EDIT_TREE_LEMMATIZER_MODEL = Config().from_str(default_model_config)["model"] + + +class EditTreeLemmatizer(TrainablePipe): + """ + Lemmatizer that lemmatizes each word using a predicted edit tree. + """ + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "trainable_lemmatizer", + *, + backoff: Optional[str] = "orth", + min_tree_freq: int = 3, + overwrite: bool = False, + top_k: int = 1, + scorer: Optional[Callable] = lemmatizer_score, + ): + """ + Construct an edit tree lemmatizer. + + backoff (Optional[str]): backoff to use when the predicted edit trees + are not applicable. Must be an attribute of Token or None (leave the + lemma unset). + min_tree_freq (int): prune trees that are applied less than this + frequency in the training data. + overwrite (bool): overwrite existing lemma annotations. + top_k (int): try to apply at most the k most probable edit trees. + """ + self.vocab = vocab + self.model = model + self.name = name + self.backoff = backoff + self.min_tree_freq = min_tree_freq + self.overwrite = overwrite + self.top_k = top_k + + self.trees = EditTrees(self.vocab.strings) + self.tree2label: Dict[int, int] = {} + + self.cfg: Dict[str, Any] = {"labels": []} + self.scorer = scorer + self.numpy_ops = NumpyOps() + + def get_loss( + self, examples: Iterable[Example], scores: List[Floats2d] + ) -> Tuple[float, List[Floats2d]]: + validate_examples(examples, "EditTreeLemmatizer.get_loss") + loss_func = SequenceCategoricalCrossentropy(normalize=False, missing_value=-1) + + truths = [] + for eg in examples: + eg_truths = [] + for predicted, gold_lemma in zip( + eg.predicted, eg.get_aligned("LEMMA", as_string=True) + ): + if gold_lemma is None or gold_lemma == "": + label = -1 + else: + tree_id = self.trees.add(predicted.text, gold_lemma) + label = self.tree2label.get(tree_id, 0) + eg_truths.append(label) + + truths.append(eg_truths) + + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError(Errors.E910.format(name=self.name)) + + return float(loss), d_scores + + def predict(self, docs: Iterable[Doc]) -> List[Ints2d]: + if self.top_k == 1: + scores2guesses = self._scores2guesses_top_k_equals_1 + elif self.top_k <= TOP_K_GUARDRAIL: + scores2guesses = self._scores2guesses_top_k_greater_1 + else: + scores2guesses = self._scores2guesses_top_k_guardrail + # The behaviour of *_scores2guesses_top_k_greater_1()* is efficient for values + # of *top_k>1* that are likely to be useful when the edit tree lemmatizer is used + # for its principal purpose of lemmatizing tokens. However, the code could also + # be used for other purposes, and with very large values of *top_k* the method + # becomes inefficient. In such cases, *_scores2guesses_top_k_guardrail()* is used + # instead. + n_docs = len(list(docs)) + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + n_labels = len(self.cfg["labels"]) + guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs] + assert len(guesses) == n_docs + return guesses + scores = self.model.predict(docs) + assert len(scores) == n_docs + guesses = scores2guesses(docs, scores) + assert len(guesses) == n_docs + return guesses + + def _scores2guesses_top_k_equals_1(self, docs, scores): + guesses = [] + for doc, doc_scores in zip(docs, scores): + doc_guesses = doc_scores.argmax(axis=1) + doc_guesses = self.numpy_ops.asarray(doc_guesses) + + doc_compat_guesses = [] + for i, token in enumerate(doc): + tree_id = self.cfg["labels"][doc_guesses[i]] + if self.trees.apply(tree_id, token.text) is not None: + doc_compat_guesses.append(tree_id) + else: + doc_compat_guesses.append(-1) + guesses.append(np.array(doc_compat_guesses)) + + return guesses + + def _scores2guesses_top_k_greater_1(self, docs, scores): + guesses = [] + top_k = min(self.top_k, len(self.labels)) + for doc, doc_scores in zip(docs, scores): + doc_scores = self.numpy_ops.asarray(doc_scores) + doc_compat_guesses = [] + for i, token in enumerate(doc): + for _ in range(top_k): + candidate = int(doc_scores[i].argmax()) + candidate_tree_id = self.cfg["labels"][candidate] + if self.trees.apply(candidate_tree_id, token.text) is not None: + doc_compat_guesses.append(candidate_tree_id) + break + doc_scores[i, candidate] = np.finfo(np.float32).min + else: + doc_compat_guesses.append(-1) + guesses.append(np.array(doc_compat_guesses)) + + return guesses + + def _scores2guesses_top_k_guardrail(self, docs, scores): + guesses = [] + for doc, doc_scores in zip(docs, scores): + doc_guesses = np.argsort(doc_scores)[..., : -self.top_k - 1 : -1] + doc_guesses = self.numpy_ops.asarray(doc_guesses) + + doc_compat_guesses = [] + for token, candidates in zip(doc, doc_guesses): + tree_id = -1 + for candidate in candidates: + candidate_tree_id = self.cfg["labels"][candidate] + + if self.trees.apply(candidate_tree_id, token.text) is not None: + tree_id = candidate_tree_id + break + doc_compat_guesses.append(tree_id) + + guesses.append(np.array(doc_compat_guesses)) + + return guesses + + def set_annotations(self, docs: Iterable[Doc], batch_tree_ids): + for i, doc in enumerate(docs): + doc_tree_ids = batch_tree_ids[i] + if hasattr(doc_tree_ids, "get"): + doc_tree_ids = doc_tree_ids.get() + for j, tree_id in enumerate(doc_tree_ids): + if self.overwrite or doc[j].lemma == 0: + # If no applicable tree could be found during prediction, + # the special identifier -1 is used. Otherwise the tree + # is guaranteed to be applicable. + if tree_id == -1: + if self.backoff is not None: + doc[j].lemma = getattr(doc[j], self.backoff) + else: + lemma = self.trees.apply(tree_id, doc[j].text) + doc[j].lemma_ = lemma + + @property + def labels(self) -> Tuple[int, ...]: + """Returns the labels currently added to the component.""" + return tuple(self.cfg["labels"]) + + @property + def hide_labels(self) -> bool: + return True + + @property + def label_data(self) -> Dict: + trees = [] + for tree_id in range(len(self.trees)): + tree = self.trees[tree_id] + if "orig" in tree: + tree["orig"] = self.vocab.strings[tree["orig"]] + if "subst" in tree: + tree["subst"] = self.vocab.strings[tree["subst"]] + trees.append(tree) + return dict(trees=trees, labels=tuple(self.cfg["labels"])) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + labels: Optional[Dict] = None, + ): + validate_get_examples(get_examples, "EditTreeLemmatizer.initialize") + + if labels is None: + self._labels_from_data(get_examples) + else: + self._add_labels(labels) + + # Sample for the model. + doc_sample = [] + label_sample = [] + for example in islice(get_examples(), 10): + doc_sample.append(example.x) + gold_labels: List[List[float]] = [] + for token in example.reference: + if token.lemma == 0: + gold_label = None + else: + gold_label = self._pair2label(token.text, token.lemma_) + + gold_labels.append( + [ + 1.0 if label == gold_label else 0.0 + for label in self.cfg["labels"] + ] + ) + + gold_labels = cast(Floats2d, gold_labels) + label_sample.append(self.model.ops.asarray(gold_labels, dtype="float32")) + + self._require_labels() + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + + self.model.initialize(X=doc_sample, Y=label_sample) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + deserializers = { + "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), + "model": lambda b: self.model.from_bytes(b), + "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude), + "trees": lambda b: self.trees.from_bytes(b), + } + + util.from_bytes(bytes_data, deserializers, exclude) + + return self + + def to_bytes(self, *, exclude=tuple()): + serializers = { + "cfg": lambda: srsly.json_dumps(self.cfg), + "model": lambda: self.model.to_bytes(), + "vocab": lambda: self.vocab.to_bytes(exclude=exclude), + "trees": lambda: self.trees.to_bytes(), + } + + return util.to_bytes(serializers, exclude) + + def to_disk(self, path, exclude=tuple()): + path = util.ensure_path(path) + serializers = { + "cfg": lambda p: srsly.write_json(p, self.cfg), + "model": lambda p: self.model.to_disk(p), + "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude), + "trees": lambda p: self.trees.to_disk(p), + } + util.to_disk(path, serializers, exclude) + + def from_disk(self, path, exclude=tuple()): + def load_model(p): + try: + with open(p, "rb") as mfile: + self.model.from_bytes(mfile.read()) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserializers = { + "cfg": lambda p: self.cfg.update(srsly.read_json(p)), + "model": load_model, + "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude), + "trees": lambda p: self.trees.from_disk(p), + } + + util.from_disk(path, deserializers, exclude) + return self + + def _add_labels(self, labels: Dict): + if "labels" not in labels: + raise ValueError(Errors.E857.format(name="labels")) + if "trees" not in labels: + raise ValueError(Errors.E857.format(name="trees")) + + self.cfg["labels"] = list(labels["labels"]) + trees = [] + for tree in labels["trees"]: + errors = validate_edit_tree(tree) + if errors: + raise ValueError(Errors.E1026.format(errors="\n".join(errors))) + + tree = dict(tree) + if "orig" in tree: + tree["orig"] = self.vocab.strings.add(tree["orig"]) + if "orig" in tree: + tree["subst"] = self.vocab.strings.add(tree["subst"]) + + trees.append(tree) + + self.trees.from_json(trees) + + for label, tree in enumerate(self.labels): + self.tree2label[tree] = label + + def _labels_from_data(self, get_examples: Callable[[], Iterable[Example]]): + # Count corpus tree frequencies in ad-hoc storage to avoid cluttering + # the final pipe/string store. + vocab = Vocab() + trees = EditTrees(vocab.strings) + tree_freqs: Counter = Counter() + repr_pairs: Dict = {} + for example in get_examples(): + for token in example.reference: + if token.lemma != 0: + tree_id = trees.add(token.text, token.lemma_) + tree_freqs[tree_id] += 1 + repr_pairs[tree_id] = (token.text, token.lemma_) + + # Construct trees that make the frequency cut-off using representative + # form - token pairs. + for tree_id, freq in tree_freqs.items(): + if freq >= self.min_tree_freq: + form, lemma = repr_pairs[tree_id] + self._pair2label(form, lemma, add_label=True) + + def _pair2label(self, form, lemma, add_label=False): + """ + Look up the edit tree identifier for a form/label pair. If the edit + tree is unknown and "add_label" is set, the edit tree will be added to + the labels. + """ + tree_id = self.trees.add(form, lemma) + if tree_id not in self.tree2label: + if not add_label: + return None + + self.tree2label[tree_id] = len(self.cfg["labels"]) + self.cfg["labels"].append(tree_id) + return self.tree2label[tree_id] + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_edit_tree_lemmatizer": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_edit_tree_lemmatizer + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py new file mode 100644 index 0000000..4b23fee --- /dev/null +++ b/spacy/pipeline/entity_linker.py @@ -0,0 +1,579 @@ +import importlib +import random +from itertools import islice +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + +import srsly +from thinc.api import Config, CosineDistance, Model, Optimizer, set_dropout_rate +from thinc.types import Floats2d + +from .. import util +from ..errors import Errors +from ..kb import Candidate, KnowledgeBase +from ..language import Language +from ..scorer import Scorer +from ..tokens import Doc, Span +from ..training import Example, validate_examples, validate_get_examples +from ..util import SimpleFrozenList +from ..vocab import Vocab +from .pipe import deserialize_config +from .trainable_pipe import TrainablePipe + +# See #9050 +BACKWARD_OVERWRITE = True + +default_model_config = """ +[model] +@architectures = "spacy.EntityLinker.v2" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 2 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"] + + +def entity_linker_score(examples, **kwargs): + return Scorer.score_links(examples, negative_labels=[EntityLinker.NIL], **kwargs) + + +def make_entity_linker_scorer(): + return entity_linker_score + + +class EntityLinker(TrainablePipe): + """Pipeline component for named entity linking. + + DOCS: https://spacy.io/api/entitylinker + """ + + NIL = "NIL" # string used to refer to a non-existing link + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "entity_linker", + *, + labels_discard: Iterable[str], + n_sents: int, + incl_prior: bool, + incl_context: bool, + entity_vector_length: int, + get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], + generate_empty_kb: Callable[[Vocab, int], KnowledgeBase], + overwrite: bool = BACKWARD_OVERWRITE, + scorer: Optional[Callable] = entity_linker_score, + use_gold_ents: bool, + candidates_batch_size: int, + threshold: Optional[float] = None, + ) -> None: + """Initialize an entity linker. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + labels_discard (Iterable[str]): NER labels that will automatically get a "NIL" prediction. + n_sents (int): The number of neighbouring sentences to take into account. + incl_prior (bool): Whether or not to include prior probabilities from the KB in the model. + incl_context (bool): Whether or not to include the local context in the model. + entity_vector_length (int): Size of encoding vectors in the KB. + get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that + produces a list of candidates, given a certain knowledge base and a textual mention. + get_candidates_batch ( + Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], + Iterable[Candidate]] + ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. + generate_empty_kb (Callable[[Vocab, int], KnowledgeBase]): Callable returning empty KnowledgeBase. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. + use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another + component must provide entity annotations. + candidates_batch_size (int): Size of batches for entity candidate generation. + threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the + threshold, prediction is discarded. If None, predictions are not filtered by any threshold. + DOCS: https://spacy.io/api/entitylinker#init + """ + + if threshold is not None and not (0 <= threshold <= 1): + raise ValueError( + Errors.E1043.format( + range_start=0, + range_end=1, + value=threshold, + ) + ) + + self.vocab = vocab + self.model = model + self.name = name + self.labels_discard = list(labels_discard) + # how many neighbour sentences to take into account + self.n_sents = n_sents + self.incl_prior = incl_prior + self.incl_context = incl_context + self.get_candidates = get_candidates + self.get_candidates_batch = get_candidates_batch + self.cfg: Dict[str, Any] = {"overwrite": overwrite} + self.distance = CosineDistance(normalize=False) + self.kb = generate_empty_kb(self.vocab, entity_vector_length) + self.use_gold_ents = use_gold_ents + self.candidates_batch_size = candidates_batch_size + self.threshold = threshold + + if candidates_batch_size < 1: + raise ValueError(Errors.E1044) + + def _score_with_ents_set(examples: Iterable[Example], **kwargs): + # Because of how spaCy works, we can't just score immediately, because Language.evaluate + # calls pipe() on the predicted docs, which won't have entities if there is no NER in the pipeline. + if not scorer: + return scorer + if not self.use_gold_ents: + return scorer(examples, **kwargs) + else: + examples = self._ensure_ents(examples) + docs = self.pipe( + (eg.predicted for eg in examples), + ) + for eg, doc in zip(examples, docs): + eg.predicted = doc + return scorer(examples, **kwargs) + + self.scorer = _score_with_ents_set + + def _ensure_ents(self, examples: Iterable[Example]) -> Iterable[Example]: + """If use_gold_ents is true, set the gold entities to (a copy of) eg.predicted.""" + if not self.use_gold_ents: + return examples + + new_examples = [] + for eg in examples: + ents, _ = eg.get_aligned_ents_and_ner() + new_eg = eg.copy() + new_eg.predicted.ents = ents + new_examples.append(new_eg) + return new_examples + + def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]): + """Define the KB of this pipe by providing a function that will + create it using this object's vocab.""" + if not callable(kb_loader): + raise ValueError(Errors.E885.format(arg_type=type(kb_loader))) + + self.kb = kb_loader(self.vocab) # type: ignore + + def validate_kb(self) -> None: + # Raise an error if the knowledge base is not initialized. + if self.kb is None: + raise ValueError(Errors.E1018.format(name=self.name)) + if hasattr(self.kb, "is_empty") and self.kb.is_empty(): + raise ValueError(Errors.E139.format(name=self.name)) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + kb_loader: Optional[Callable[[Vocab], KnowledgeBase]] = None, + ): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab + instance. Note that providing this argument will overwrite all data accumulated in the current KB. + Use this only when loading a KB as-such from file. + + DOCS: https://spacy.io/api/entitylinker#initialize + """ + validate_get_examples(get_examples, "EntityLinker.initialize") + if kb_loader is not None: + self.set_kb(kb_loader) + self.validate_kb() + nO = self.kb.entity_vector_length + doc_sample = [] + vector_sample = [] + examples = self._ensure_ents(islice(get_examples(), 10)) + for eg in examples: + doc = eg.x + doc_sample.append(doc) + vector_sample.append(self.model.ops.alloc1f(nO)) + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(vector_sample) > 0, Errors.E923.format(name=self.name) + + # XXX In order for size estimation to work, there has to be at least + # one entity. It's not used for training so it doesn't have to be real, + # so we add a fake one if none are present. + # We can't use Doc.has_annotation here because it can be True for docs + # that have been through an NER component but got no entities. + has_annotations = any([doc.ents for doc in doc_sample]) + if not has_annotations: + doc = doc_sample[0] + ent = doc[0:1] + ent.label_ = "XXX" + doc.ents = (ent,) + + self.model.initialize( + X=doc_sample, Y=self.model.ops.asarray(vector_sample, dtype="float32") + ) + + if not has_annotations: + # Clean up dummy annotation + doc.ents = [] + + def batch_has_learnable_example(self, examples): + """Check if a batch contains a learnable example. + + If one isn't present, then the update step needs to be skipped. + """ + + for eg in examples: + for ent in eg.predicted.ents: + candidates = list(self.get_candidates(self.kb, ent)) + if candidates: + return True + + return False + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/entitylinker#update + """ + self.validate_kb() + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + if not examples: + return losses + examples = self._ensure_ents(examples) + validate_examples(examples, "EntityLinker.update") + + # make sure we have something to learn from, if not, short-circuit + if not self.batch_has_learnable_example(examples): + return losses + + set_dropout_rate(self.model, drop) + docs = [eg.predicted for eg in examples] + sentence_encodings, bp_context = self.model.begin_update(docs) + + loss, d_scores = self.get_loss( + sentence_encodings=sentence_encodings, examples=examples + ) + bp_context(d_scores) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss + + return losses + + def get_loss(self, examples: Iterable[Example], sentence_encodings: Floats2d): + validate_examples(examples, "EntityLinker.get_loss") + entity_encodings = [] + # We assume that get_loss is called with gold ents set in the examples if need be + eidx = 0 # indices in gold entities to keep + keep_ents = [] # indices in sentence_encodings to keep + + for eg in examples: + kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) + + for ent in eg.get_matching_ents(): + kb_id = kb_ids[ent.start] + if kb_id: + entity_encoding = self.kb.get_vector(kb_id) + entity_encodings.append(entity_encoding) + keep_ents.append(eidx) + + eidx += 1 + entity_encodings = self.model.ops.asarray2f(entity_encodings, dtype="float32") + selected_encodings = sentence_encodings[keep_ents] + + # if there are no matches, short circuit + if not keep_ents: + out = self.model.ops.alloc2f(*sentence_encodings.shape) + return 0, out + + if selected_encodings.shape != entity_encodings.shape: + err = Errors.E147.format( + method="get_loss", msg="gold entities do not match up" + ) + raise RuntimeError(err) + gradients = self.distance.get_grad(selected_encodings, entity_encodings) + # to match the input size, we need to give a zero gradient for items not in the kb + out = self.model.ops.alloc2f(*sentence_encodings.shape) + out[keep_ents] = gradients + + loss = self.distance.get_loss(selected_encodings, entity_encodings) + loss = loss / len(entity_encodings) + return float(loss), out + + def predict(self, docs: Iterable[Doc]) -> List[str]: + """Apply the pipeline's model to a batch of docs, without modifying them. + Returns the KB IDs for each entity in each doc, including NIL if there is + no prediction. + + docs (Iterable[Doc]): The documents to predict. + RETURNS (List[str]): The models prediction for each document. + + DOCS: https://spacy.io/api/entitylinker#predict + """ + self.validate_kb() + entity_count = 0 + final_kb_ids: List[str] = [] + xp = self.model.ops.xp + if not docs: + return final_kb_ids + if isinstance(docs, Doc): + docs = [docs] + for i, doc in enumerate(docs): + if len(doc) == 0: + continue + sentences = [s for s in doc.sents] + + # Loop over entities in batches. + for ent_idx in range(0, len(doc.ents), self.candidates_batch_size): + ent_batch = doc.ents[ent_idx : ent_idx + self.candidates_batch_size] + + # Look up candidate entities. + valid_ent_idx = [ + idx + for idx in range(len(ent_batch)) + if ent_batch[idx].label_ not in self.labels_discard + ] + + batch_candidates = list( + self.get_candidates_batch( + self.kb, [ent_batch[idx] for idx in valid_ent_idx] + ) + if self.candidates_batch_size > 1 + else [ + self.get_candidates(self.kb, ent_batch[idx]) + for idx in valid_ent_idx + ] + ) + + # Looping through each entity in batch (TODO: rewrite) + for j, ent in enumerate(ent_batch): + assert hasattr(ent, "sents") + sents = list(ent.sents) + sent_indices = ( + sentences.index(sents[0]), + sentences.index(sents[-1]), + ) + assert sent_indices[1] >= sent_indices[0] >= 0 + + if self.incl_context: + # get n_neighbour sentences, clipped to the length of the document + start_sentence = max(0, sent_indices[0] - self.n_sents) + end_sentence = min( + len(sentences) - 1, sent_indices[1] + self.n_sents + ) + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + sent_doc = doc[start_token:end_token].as_doc() + + # currently, the context is the same for each entity in a sentence (should be refined) + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(batch_candidates[j]) + if not candidates: + # no prediction possible for this entity - setting to NIL + final_kb_ids.append(self.NIL) + elif len(candidates) == 1 and self.threshold is None: + # shortcut for efficiency reasons: take the 1 candidate + final_kb_ids.append(candidates[0].entity_) + else: + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", + ) + ) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + final_kb_ids.append( + candidates[scores.argmax().item()].entity_ + if self.threshold is None + or scores.max() >= self.threshold + else EntityLinker.NIL + ) + + if not (len(final_kb_ids) == entity_count): + err = Errors.E147.format( + method="predict", msg="result variables not of equal length" + ) + raise RuntimeError(err) + return final_kb_ids + + def set_annotations(self, docs: Iterable[Doc], kb_ids: List[str]) -> None: + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + kb_ids (List[str]): The IDs to set, produced by EntityLinker.predict. + + DOCS: https://spacy.io/api/entitylinker#set_annotations + """ + count_ents = len([ent for doc in docs for ent in doc.ents]) + if count_ents != len(kb_ids): + raise ValueError(Errors.E148.format(ents=count_ents, ids=len(kb_ids))) + i = 0 + overwrite = self.cfg["overwrite"] + for doc in docs: + for ent in doc.ents: + kb_id = kb_ids[i] + i += 1 + for token in ent: + if token.ent_kb_id == 0 or overwrite: + token.ent_kb_id_ = kb_id + + def to_bytes(self, *, exclude=tuple()): + """Serialize the pipe to a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/entitylinker#to_bytes + """ + self._validate_serialization_attrs() + serialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serialize["kb"] = self.kb.to_bytes + serialize["model"] = self.model.to_bytes + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load the pipe from a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (TrainablePipe): The loaded object. + + DOCS: https://spacy.io/api/entitylinker#from_bytes + """ + self._validate_serialization_attrs() + + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b)) + deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) + deserialize["kb"] = lambda b: self.kb.from_bytes(b) + deserialize["model"] = load_model + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Serialize the pipe to disk. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/entitylinker#to_disk + """ + serialize = {} + serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) + serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) + serialize["kb"] = lambda p: self.kb.to_disk(p) + serialize["model"] = lambda p: self.model.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "EntityLinker": + """Load the pipe from disk. Modifies the object in place and returns it. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (EntityLinker): The modified EntityLinker object. + + DOCS: https://spacy.io/api/entitylinker#from_disk + """ + + def load_model(p): + try: + with p.open("rb") as infile: + self.model.from_bytes(infile.read()) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize: Dict[str, Callable[[Any], Any]] = {} + deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p)) + deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude) + deserialize["kb"] = lambda p: self.kb.from_disk(p) + deserialize["model"] = load_model + util.from_disk(path, deserialize, exclude) + return self + + def rehearse(self, examples, *, sgd=None, losses=None, **config): + raise NotImplementedError + + def add_label(self, label): + raise NotImplementedError + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_entity_linker": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_entity_linker + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py new file mode 100644 index 0000000..0728c3f --- /dev/null +++ b/spacy/pipeline/entityruler.py @@ -0,0 +1,509 @@ +import importlib +import warnings +from collections import defaultdict +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union + +import srsly + +from ..errors import Errors, Warnings +from ..language import Language +from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare +from ..scorer import get_ner_prf +from ..tokens import Doc, Span +from ..training import Example +from ..util import SimpleFrozenList, ensure_path, from_disk, to_disk +from .pipe import Pipe + +DEFAULT_ENT_ID_SEP = "||" +PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] + + +def entity_ruler_score(examples, **kwargs): + return get_ner_prf(examples) + + +def make_entity_ruler_scorer(): + return entity_ruler_score + + +class EntityRuler(Pipe): + """The EntityRuler lets you add spans to the `Doc.ents` using token-based + rules or exact phrase matches. It can be combined with the statistical + `EntityRecognizer` to boost accuracy, or used on its own to implement a + purely rule-based entity recognition system. After initialization, the + component is typically added to the pipeline using `nlp.add_pipe`. + + DOCS: https://spacy.io/api/entityruler + USAGE: https://spacy.io/usage/rule-based-matching#entityruler + """ + + def __init__( + self, + nlp: Language, + name: str = "entity_ruler", + *, + phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, + validate: bool = False, + overwrite_ents: bool = False, + ent_id_sep: str = DEFAULT_ENT_ID_SEP, + patterns: Optional[List[PatternType]] = None, + scorer: Optional[Callable] = entity_ruler_score, + ) -> None: + """Initialize the entity ruler. If patterns are supplied here, they + need to be a list of dictionaries with a `"label"` and `"pattern"` + key. A pattern can either be a token pattern (list) or a phrase pattern + (string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`. + + nlp (Language): The shared nlp object to pass the vocab to the matchers + and process phrase patterns. + name (str): Instance name of the current pipeline component. Typically + passed in automatically from the factory when the component is + added. Used to disable the current entity ruler while creating + phrase patterns with the nlp object. + phrase_matcher_attr (int / str): Token attribute to match on, passed + to the internal PhraseMatcher as `attr`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. + validate (bool): Whether patterns should be validated, passed to + Matcher and PhraseMatcher as `validate` + patterns (iterable): Optional patterns to load in. + overwrite_ents (bool): If existing entities are present, e.g. entities + added by the model, overwrite them by matches if necessary. + ent_id_sep (str): Separator used internally for entity IDs. + scorer (Optional[Callable]): The scoring method. Defaults to + spacy.scorer.get_ner_prf. + + DOCS: https://spacy.io/api/entityruler#init + """ + self.nlp = nlp + self.name = name + self.overwrite = overwrite_ents + self.token_patterns = defaultdict(list) # type: ignore + self.phrase_patterns = defaultdict(list) # type: ignore + self._validate = validate + self.matcher_fuzzy_compare = matcher_fuzzy_compare + self.matcher = Matcher( + nlp.vocab, validate=validate, fuzzy_compare=self.matcher_fuzzy_compare + ) + self.phrase_matcher_attr = phrase_matcher_attr + self.phrase_matcher = PhraseMatcher( + nlp.vocab, attr=self.phrase_matcher_attr, validate=validate + ) + self.ent_id_sep = ent_id_sep + self._ent_ids = defaultdict(tuple) # type: ignore + if patterns is not None: + self.add_patterns(patterns) + self.scorer = scorer + + def __len__(self) -> int: + """The number of all patterns added to the entity ruler.""" + n_token_patterns = sum(len(p) for p in self.token_patterns.values()) + n_phrase_patterns = sum(len(p) for p in self.phrase_patterns.values()) + return n_token_patterns + n_phrase_patterns + + def __contains__(self, label: str) -> bool: + """Whether a label is present in the patterns.""" + return label in self.token_patterns or label in self.phrase_patterns + + def __call__(self, doc: Doc) -> Doc: + """Find matches in document and add them as entities. + + doc (Doc): The Doc object in the pipeline. + RETURNS (Doc): The Doc with added entities, if available. + + DOCS: https://spacy.io/api/entityruler#call + """ + error_handler = self.get_error_handler() + try: + matches = self.match(doc) + self.set_annotations(doc, matches) + return doc + except Exception as e: + return error_handler(self.name, self, [doc], e) + + def match(self, doc: Doc): + self._require_patterns() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W036") + matches = list(self.matcher(doc)) + list(self.phrase_matcher(doc)) + + final_matches = set( + [(m_id, start, end) for m_id, start, end in matches if start != end] + ) + get_sort_key = lambda m: (m[2] - m[1], -m[1]) + final_matches = sorted(final_matches, key=get_sort_key, reverse=True) + return final_matches + + def set_annotations(self, doc, matches): + """Modify the document in place""" + entities = list(doc.ents) + new_entities = [] + seen_tokens = set() + for match_id, start, end in matches: + if any(t.ent_type for t in doc[start:end]) and not self.overwrite: + continue + # check for end - 1 here because boundaries are inclusive + if start not in seen_tokens and end - 1 not in seen_tokens: + if match_id in self._ent_ids: + label, ent_id = self._ent_ids[match_id] + span = Span(doc, start, end, label=label, span_id=ent_id) + else: + span = Span(doc, start, end, label=match_id) + new_entities.append(span) + entities = [ + e for e in entities if not (e.start < end and e.end > start) + ] + seen_tokens.update(range(start, end)) + doc.ents = entities + new_entities + + @property + def labels(self) -> Tuple[str, ...]: + """All labels present in the match patterns. + + RETURNS (set): The string labels. + + DOCS: https://spacy.io/api/entityruler#labels + """ + keys = set(self.token_patterns.keys()) + keys.update(self.phrase_patterns.keys()) + all_labels = set() + + for l in keys: + if self.ent_id_sep in l: + label, _ = self._split_label(l) + all_labels.add(label) + else: + all_labels.add(l) + return tuple(sorted(all_labels)) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + patterns: Optional[Sequence[PatternType]] = None, + ): + """Initialize the pipe for training. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + patterns Optional[Iterable[PatternType]]: The list of patterns. + + DOCS: https://spacy.io/api/entityruler#initialize + """ + self.clear() + if patterns: + self.add_patterns(patterns) # type: ignore[arg-type] + + @property + def ent_ids(self) -> Tuple[Optional[str], ...]: + """All entity ids present in the match patterns `id` properties + + RETURNS (set): The string entity ids. + + DOCS: https://spacy.io/api/entityruler#ent_ids + """ + keys = set(self.token_patterns.keys()) + keys.update(self.phrase_patterns.keys()) + all_ent_ids = set() + + for l in keys: + if self.ent_id_sep in l: + _, ent_id = self._split_label(l) + all_ent_ids.add(ent_id) + return tuple(all_ent_ids) + + @property + def patterns(self) -> List[PatternType]: + """Get all patterns that were added to the entity ruler. + + RETURNS (list): The original patterns, one dictionary per pattern. + + DOCS: https://spacy.io/api/entityruler#patterns + """ + all_patterns = [] + for label, patterns in self.token_patterns.items(): + for pattern in patterns: + ent_label, ent_id = self._split_label(label) + p = {"label": ent_label, "pattern": pattern} + if ent_id: + p["id"] = ent_id + all_patterns.append(p) + for label, patterns in self.phrase_patterns.items(): + for pattern in patterns: + ent_label, ent_id = self._split_label(label) + p = {"label": ent_label, "pattern": pattern.text} + if ent_id: + p["id"] = ent_id + all_patterns.append(p) + return all_patterns + + def add_patterns(self, patterns: List[PatternType]) -> None: + """Add patterns to the entity ruler. A pattern can either be a token + pattern (list of dicts) or a phrase pattern (string). For example: + {'label': 'ORG', 'pattern': 'Apple'} + {'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]} + + patterns (list): The patterns to add. + + DOCS: https://spacy.io/api/entityruler#add_patterns + """ + + # disable the nlp components after this one in case they hadn't been initialized / deserialised yet + try: + current_index = -1 + for i, (name, pipe) in enumerate(self.nlp.pipeline): + if self == pipe: + current_index = i + break + subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]] + except ValueError: + subsequent_pipes = [] + with self.nlp.select_pipes(disable=subsequent_pipes): + token_patterns = [] + phrase_pattern_labels = [] + phrase_pattern_texts = [] + phrase_pattern_ids = [] + for entry in patterns: + if isinstance(entry["pattern"], str): + phrase_pattern_labels.append(entry["label"]) + phrase_pattern_texts.append(entry["pattern"]) + phrase_pattern_ids.append(entry.get("id")) + elif isinstance(entry["pattern"], list): + token_patterns.append(entry) + phrase_patterns = [] + for label, pattern, ent_id in zip( + phrase_pattern_labels, + self.nlp.pipe(phrase_pattern_texts), + phrase_pattern_ids, + ): + phrase_pattern = {"label": label, "pattern": pattern} + if ent_id: + phrase_pattern["id"] = ent_id + phrase_patterns.append(phrase_pattern) + for entry in token_patterns + phrase_patterns: # type: ignore[operator] + label = entry["label"] # type: ignore + if "id" in entry: + ent_label = label + label = self._create_label(label, entry["id"]) + key = self.matcher._normalize_key(label) + self._ent_ids[key] = (ent_label, entry["id"]) + pattern = entry["pattern"] # type: ignore + if isinstance(pattern, Doc): + self.phrase_patterns[label].append(pattern) + self.phrase_matcher.add(label, [pattern]) # type: ignore + elif isinstance(pattern, list): + self.token_patterns[label].append(pattern) + self.matcher.add(label, [pattern]) + else: + raise ValueError(Errors.E097.format(pattern=pattern)) + + def clear(self) -> None: + """Reset all patterns.""" + self.token_patterns = defaultdict(list) + self.phrase_patterns = defaultdict(list) + self._ent_ids = defaultdict(tuple) + self.matcher = Matcher( + self.nlp.vocab, + validate=self._validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) + self.phrase_matcher = PhraseMatcher( + self.nlp.vocab, attr=self.phrase_matcher_attr, validate=self._validate + ) + + def remove(self, ent_id: str) -> None: + """Remove a pattern by its ent_id if a pattern with this ent_id was added before + + ent_id (str): id of the pattern to be removed + RETURNS: None + DOCS: https://spacy.io/api/entityruler#remove + """ + label_id_pairs = [ + (label, eid) for (label, eid) in self._ent_ids.values() if eid == ent_id + ] + if not label_id_pairs: + raise ValueError( + Errors.E1024.format(attr_type="ID", label=ent_id, component=self.name) + ) + created_labels = [ + self._create_label(label, eid) for (label, eid) in label_id_pairs + ] + # remove the patterns from self.phrase_patterns + self.phrase_patterns = defaultdict( + list, + { + label: val + for (label, val) in self.phrase_patterns.items() + if label not in created_labels + }, + ) + # remove the patterns from self.token_pattern + self.token_patterns = defaultdict( + list, + { + label: val + for (label, val) in self.token_patterns.items() + if label not in created_labels + }, + ) + # remove the patterns from self.token_pattern + for label in created_labels: + if label in self.phrase_matcher: + self.phrase_matcher.remove(label) + else: + self.matcher.remove(label) + + def _require_patterns(self) -> None: + """Raise a warning if this component has no patterns defined.""" + if len(self) == 0: + warnings.warn(Warnings.W036.format(name=self.name)) + + def _split_label(self, label: str) -> Tuple[str, Optional[str]]: + """Split Entity label into ent_label and ent_id if it contains self.ent_id_sep + + label (str): The value of label in a pattern entry + RETURNS (tuple): ent_label, ent_id + """ + if self.ent_id_sep in label: + ent_label, ent_id = label.rsplit(self.ent_id_sep, 1) + else: + ent_label = label + ent_id = None # type: ignore + return ent_label, ent_id + + def _create_label(self, label: Any, ent_id: Any) -> str: + """Join Entity label with ent_id if the pattern has an `id` attribute + If ent_id is not a string, the label is returned as is. + + label (str): The label to set for ent.label_ + ent_id (str): The label + RETURNS (str): The ent_label joined with configured `ent_id_sep` + """ + if isinstance(ent_id, str): + label = f"{label}{self.ent_id_sep}{ent_id}" + return label + + def from_bytes( + self, patterns_bytes: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "EntityRuler": + """Load the entity ruler from a bytestring. + + patterns_bytes (bytes): The bytestring to load. + RETURNS (EntityRuler): The loaded entity ruler. + + DOCS: https://spacy.io/api/entityruler#from_bytes + """ + cfg = srsly.msgpack_loads(patterns_bytes) + self.clear() + if isinstance(cfg, dict): + self.add_patterns(cfg.get("patterns", cfg)) + self.overwrite = cfg.get("overwrite", False) + self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None) + self.phrase_matcher = PhraseMatcher( + self.nlp.vocab, + attr=self.phrase_matcher_attr, + ) + self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) + else: + self.add_patterns(cfg) + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the entity ruler patterns to a bytestring. + + RETURNS (bytes): The serialized patterns. + + DOCS: https://spacy.io/api/entityruler#to_bytes + """ + serial = { + "overwrite": self.overwrite, + "ent_id_sep": self.ent_id_sep, + "phrase_matcher_attr": self.phrase_matcher_attr, + "patterns": self.patterns, + } + return srsly.msgpack_dumps(serial) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "EntityRuler": + """Load the entity ruler from a file. Expects a file containing + newline-delimited JSON (JSONL) with one entry per line. + + path (str / Path): The JSONL file to load. + RETURNS (EntityRuler): The loaded entity ruler. + + DOCS: https://spacy.io/api/entityruler#from_disk + """ + path = ensure_path(path) + self.clear() + depr_patterns_path = path.with_suffix(".jsonl") + if path.suffix == ".jsonl": # user provides a jsonl + if path.is_file: + patterns = srsly.read_jsonl(path) + self.add_patterns(patterns) + else: + raise ValueError(Errors.E1023.format(path=path)) + elif depr_patterns_path.is_file(): + patterns = srsly.read_jsonl(depr_patterns_path) + self.add_patterns(patterns) + elif path.is_dir(): # path is a valid directory + cfg = {} + deserializers_patterns = { + "patterns": lambda p: self.add_patterns( + srsly.read_jsonl(p.with_suffix(".jsonl")) + ) + } + deserializers_cfg = {"cfg": lambda p: cfg.update(srsly.read_json(p))} + from_disk(path, deserializers_cfg, {}) + self.overwrite = cfg.get("overwrite", False) + self.phrase_matcher_attr = cfg.get("phrase_matcher_attr") + self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) + + self.phrase_matcher = PhraseMatcher( + self.nlp.vocab, attr=self.phrase_matcher_attr + ) + from_disk(path, deserializers_patterns, {}) + else: # path is not a valid directory or file + raise ValueError(Errors.E146.format(path=path)) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Save the entity ruler patterns to a directory. The patterns will be + saved as newline-delimited JSON (JSONL). + + path (str / Path): The JSONL file to save. + + DOCS: https://spacy.io/api/entityruler#to_disk + """ + path = ensure_path(path) + cfg = { + "overwrite": self.overwrite, + "phrase_matcher_attr": self.phrase_matcher_attr, + "ent_id_sep": self.ent_id_sep, + } + serializers = { + "patterns": lambda p: srsly.write_jsonl( + p.with_suffix(".jsonl"), self.patterns + ), + "cfg": lambda p: srsly.write_json(p, cfg), + } + if path.suffix == ".jsonl": # user wants to save only JSONL + srsly.write_jsonl(path, self.patterns) + else: + to_disk(path, serializers, {}) + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_entity_ruler": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_entity_ruler + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/factories.py b/spacy/pipeline/factories.py new file mode 100644 index 0000000..8c71067 --- /dev/null +++ b/spacy/pipeline/factories.py @@ -0,0 +1,930 @@ +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + +from thinc.api import Model +from thinc.types import Floats2d, Ragged + +from ..kb import Candidate, KnowledgeBase +from ..language import Language +from ..pipeline._parser_internals.transition_system import TransitionSystem +from ..pipeline.attributeruler import AttributeRuler +from ..pipeline.dep_parser import DEFAULT_PARSER_MODEL, DependencyParser +from ..pipeline.edit_tree_lemmatizer import ( + DEFAULT_EDIT_TREE_LEMMATIZER_MODEL, + EditTreeLemmatizer, +) + +# Import factory default configurations +from ..pipeline.entity_linker import DEFAULT_NEL_MODEL, EntityLinker +from ..pipeline.entityruler import DEFAULT_ENT_ID_SEP, EntityRuler +from ..pipeline.functions import DocCleaner, TokenSplitter +from ..pipeline.legacy import EntityLinker_v1 +from ..pipeline.lemmatizer import Lemmatizer +from ..pipeline.morphologizer import DEFAULT_MORPH_MODEL, Morphologizer +from ..pipeline.multitask import DEFAULT_MT_MODEL, MultitaskObjective +from ..pipeline.ner import DEFAULT_NER_MODEL, EntityRecognizer +from ..pipeline.sentencizer import Sentencizer +from ..pipeline.senter import DEFAULT_SENTER_MODEL, SentenceRecognizer +from ..pipeline.span_finder import DEFAULT_SPAN_FINDER_MODEL, SpanFinder +from ..pipeline.span_ruler import ( + DEFAULT_SPANS_KEY as SPAN_RULER_DEFAULT_SPANS_KEY, + SpanRuler, + prioritize_existing_ents_filter, + prioritize_new_ents_filter, +) +from ..pipeline.spancat import ( + DEFAULT_SPANCAT_MODEL, + DEFAULT_SPANCAT_SINGLELABEL_MODEL, + DEFAULT_SPANS_KEY, + SpanCategorizer, + Suggester, +) +from ..pipeline.tagger import DEFAULT_TAGGER_MODEL, Tagger +from ..pipeline.textcat import DEFAULT_SINGLE_TEXTCAT_MODEL, TextCategorizer +from ..pipeline.textcat_multilabel import ( + DEFAULT_MULTI_TEXTCAT_MODEL, + MultiLabel_TextCategorizer, +) +from ..pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL, Tok2Vec +from ..tokens.doc import Doc +from ..tokens.span import Span +from ..vocab import Vocab + +# Global flag to track if factories have been registered +FACTORIES_REGISTERED = False + + +def register_factories() -> None: + """Register all factories with the registry. + + This function registers all pipeline component factories, centralizing + the registrations that were previously done with @Language.factory decorators. + """ + global FACTORIES_REGISTERED + + if FACTORIES_REGISTERED: + return + + # Register factories using the same pattern as Language.factory decorator + # We use Language.factory()() pattern which exactly mimics the decorator + + # attributeruler + Language.factory( + "attribute_ruler", + default_config={ + "validate": False, + "scorer": {"@scorers": "spacy.attribute_ruler_scorer.v1"}, + }, + )(make_attribute_ruler) + + # entity_linker + Language.factory( + "entity_linker", + requires=["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"], + assigns=["token.ent_kb_id"], + default_config={ + "model": DEFAULT_NEL_MODEL, + "labels_discard": [], + "n_sents": 0, + "incl_prior": True, + "incl_context": True, + "entity_vector_length": 64, + "get_candidates": {"@misc": "spacy.CandidateGenerator.v1"}, + "get_candidates_batch": {"@misc": "spacy.CandidateBatchGenerator.v1"}, + "generate_empty_kb": {"@misc": "spacy.EmptyKB.v2"}, + "overwrite": True, + "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"}, + "use_gold_ents": True, + "candidates_batch_size": 1, + "threshold": None, + }, + default_score_weights={ + "nel_micro_f": 1.0, + "nel_micro_r": None, + "nel_micro_p": None, + }, + )(make_entity_linker) + + # entity_ruler + Language.factory( + "entity_ruler", + assigns=["doc.ents", "token.ent_type", "token.ent_iob"], + default_config={ + "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, + "validate": False, + "overwrite_ents": False, + "ent_id_sep": DEFAULT_ENT_ID_SEP, + "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, + )(make_entity_ruler) + + # lemmatizer + Language.factory( + "lemmatizer", + assigns=["token.lemma"], + default_config={ + "model": None, + "mode": "lookup", + "overwrite": False, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, + )(make_lemmatizer) + + # textcat + Language.factory( + "textcat", + assigns=["doc.cats"], + default_config={ + "threshold": 0.0, + "model": DEFAULT_SINGLE_TEXTCAT_MODEL, + "scorer": {"@scorers": "spacy.textcat_scorer.v2"}, + }, + default_score_weights={ + "cats_score": 1.0, + "cats_score_desc": None, + "cats_micro_p": None, + "cats_micro_r": None, + "cats_micro_f": None, + "cats_macro_p": None, + "cats_macro_r": None, + "cats_macro_f": None, + "cats_macro_auc": None, + "cats_f_per_type": None, + }, + )(make_textcat) + + # token_splitter + Language.factory( + "token_splitter", + default_config={"min_length": 25, "split_length": 10}, + retokenizes=True, + )(make_token_splitter) + + # doc_cleaner + Language.factory( + "doc_cleaner", + default_config={"attrs": {"tensor": None, "_.trf_data": None}, "silent": True}, + )(make_doc_cleaner) + + # tok2vec + Language.factory( + "tok2vec", + assigns=["doc.tensor"], + default_config={"model": DEFAULT_TOK2VEC_MODEL}, + )(make_tok2vec) + + # senter + Language.factory( + "senter", + assigns=["token.is_sent_start"], + default_config={ + "model": DEFAULT_SENTER_MODEL, + "overwrite": False, + "scorer": {"@scorers": "spacy.senter_scorer.v1"}, + }, + default_score_weights={"sents_f": 1.0, "sents_p": 0.0, "sents_r": 0.0}, + )(make_senter) + + # morphologizer + Language.factory( + "morphologizer", + assigns=["token.morph", "token.pos"], + default_config={ + "model": DEFAULT_MORPH_MODEL, + "overwrite": True, + "extend": False, + "scorer": {"@scorers": "spacy.morphologizer_scorer.v1"}, + "label_smoothing": 0.0, + }, + default_score_weights={ + "pos_acc": 0.5, + "morph_acc": 0.5, + "morph_per_feat": None, + }, + )(make_morphologizer) + + # spancat + Language.factory( + "spancat", + assigns=["doc.spans"], + default_config={ + "threshold": 0.5, + "spans_key": DEFAULT_SPANS_KEY, + "max_positive": None, + "model": DEFAULT_SPANCAT_MODEL, + "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, + "scorer": {"@scorers": "spacy.spancat_scorer.v1"}, + }, + default_score_weights={"spans_sc_f": 1.0, "spans_sc_p": 0.0, "spans_sc_r": 0.0}, + )(make_spancat) + + # spancat_singlelabel + Language.factory( + "spancat_singlelabel", + assigns=["doc.spans"], + default_config={ + "spans_key": DEFAULT_SPANS_KEY, + "model": DEFAULT_SPANCAT_SINGLELABEL_MODEL, + "negative_weight": 1.0, + "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, + "scorer": {"@scorers": "spacy.spancat_scorer.v1"}, + "allow_overlap": True, + }, + default_score_weights={"spans_sc_f": 1.0, "spans_sc_p": 0.0, "spans_sc_r": 0.0}, + )(make_spancat_singlelabel) + + # future_entity_ruler + Language.factory( + "future_entity_ruler", + assigns=["doc.ents"], + default_config={ + "phrase_matcher_attr": None, + "validate": False, + "overwrite_ents": False, + "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, + "ent_id_sep": "__unused__", + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, + )(make_future_entity_ruler) + + # span_ruler + Language.factory( + "span_ruler", + assigns=["doc.spans"], + default_config={ + "spans_key": SPAN_RULER_DEFAULT_SPANS_KEY, + "spans_filter": None, + "annotate_ents": False, + "ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}, + "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, + "validate": False, + "overwrite": True, + "scorer": { + "@scorers": "spacy.overlapping_labeled_spans_scorer.v1", + "spans_key": SPAN_RULER_DEFAULT_SPANS_KEY, + }, + }, + default_score_weights={ + f"spans_{SPAN_RULER_DEFAULT_SPANS_KEY}_f": 1.0, + f"spans_{SPAN_RULER_DEFAULT_SPANS_KEY}_p": 0.0, + f"spans_{SPAN_RULER_DEFAULT_SPANS_KEY}_r": 0.0, + f"spans_{SPAN_RULER_DEFAULT_SPANS_KEY}_per_type": None, + }, + )(make_span_ruler) + + # trainable_lemmatizer + Language.factory( + "trainable_lemmatizer", + assigns=["token.lemma"], + requires=[], + default_config={ + "model": DEFAULT_EDIT_TREE_LEMMATIZER_MODEL, + "backoff": "orth", + "min_tree_freq": 3, + "overwrite": False, + "top_k": 1, + "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, + }, + default_score_weights={"lemma_acc": 1.0}, + )(make_edit_tree_lemmatizer) + + # textcat_multilabel + Language.factory( + "textcat_multilabel", + assigns=["doc.cats"], + default_config={ + "threshold": 0.5, + "model": DEFAULT_MULTI_TEXTCAT_MODEL, + "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v2"}, + }, + default_score_weights={ + "cats_score": 1.0, + "cats_score_desc": None, + "cats_micro_p": None, + "cats_micro_r": None, + "cats_micro_f": None, + "cats_macro_p": None, + "cats_macro_r": None, + "cats_macro_f": None, + "cats_macro_auc": None, + "cats_f_per_type": None, + }, + )(make_multilabel_textcat) + + # span_finder + Language.factory( + "span_finder", + assigns=["doc.spans"], + default_config={ + "threshold": 0.5, + "model": DEFAULT_SPAN_FINDER_MODEL, + "spans_key": DEFAULT_SPANS_KEY, + "max_length": 25, + "min_length": None, + "scorer": {"@scorers": "spacy.span_finder_scorer.v1"}, + }, + default_score_weights={ + f"spans_{DEFAULT_SPANS_KEY}_f": 1.0, + f"spans_{DEFAULT_SPANS_KEY}_p": 0.0, + f"spans_{DEFAULT_SPANS_KEY}_r": 0.0, + }, + )(make_span_finder) + + # ner + Language.factory( + "ner", + assigns=["doc.ents", "token.ent_iob", "token.ent_type"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "model": DEFAULT_NER_MODEL, + "incorrect_spans_key": None, + "scorer": {"@scorers": "spacy.ner_scorer.v1"}, + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, + )(make_ner) + + # beam_ner + Language.factory( + "beam_ner", + assigns=["doc.ents", "token.ent_iob", "token.ent_type"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "model": DEFAULT_NER_MODEL, + "beam_density": 0.01, + "beam_update_prob": 0.5, + "beam_width": 32, + "incorrect_spans_key": None, + "scorer": {"@scorers": "spacy.ner_scorer.v1"}, + }, + default_score_weights={ + "ents_f": 1.0, + "ents_p": 0.0, + "ents_r": 0.0, + "ents_per_type": None, + }, + )(make_beam_ner) + + # parser + Language.factory( + "parser", + assigns=["token.dep", "token.head", "token.is_sent_start", "doc.sents"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "learn_tokens": False, + "min_action_freq": 30, + "model": DEFAULT_PARSER_MODEL, + "scorer": {"@scorers": "spacy.parser_scorer.v1"}, + }, + default_score_weights={ + "dep_uas": 0.5, + "dep_las": 0.5, + "dep_las_per_type": None, + "sents_p": None, + "sents_r": None, + "sents_f": 0.0, + }, + )(make_parser) + + # beam_parser + Language.factory( + "beam_parser", + assigns=["token.dep", "token.head", "token.is_sent_start", "doc.sents"], + default_config={ + "moves": None, + "update_with_oracle_cut_size": 100, + "learn_tokens": False, + "min_action_freq": 30, + "beam_width": 8, + "beam_density": 0.0001, + "beam_update_prob": 0.5, + "model": DEFAULT_PARSER_MODEL, + "scorer": {"@scorers": "spacy.parser_scorer.v1"}, + }, + default_score_weights={ + "dep_uas": 0.5, + "dep_las": 0.5, + "dep_las_per_type": None, + "sents_p": None, + "sents_r": None, + "sents_f": 0.0, + }, + )(make_beam_parser) + + # tagger + Language.factory( + "tagger", + assigns=["token.tag"], + default_config={ + "model": DEFAULT_TAGGER_MODEL, + "overwrite": False, + "scorer": {"@scorers": "spacy.tagger_scorer.v1"}, + "neg_prefix": "!", + "label_smoothing": 0.0, + }, + default_score_weights={ + "tag_acc": 1.0, + "pos_acc": 0.0, + "tag_micro_p": None, + "tag_micro_r": None, + "tag_micro_f": None, + }, + )(make_tagger) + + # nn_labeller + Language.factory( + "nn_labeller", + default_config={ + "labels": None, + "target": "dep_tag_offset", + "model": DEFAULT_MT_MODEL, + }, + )(make_nn_labeller) + + # sentencizer + Language.factory( + "sentencizer", + assigns=["token.is_sent_start", "doc.sents"], + default_config={ + "punct_chars": None, + "overwrite": False, + "scorer": {"@scorers": "spacy.senter_scorer.v1"}, + }, + default_score_weights={"sents_f": 1.0, "sents_p": 0.0, "sents_r": 0.0}, + )(make_sentencizer) + + # Set the flag to indicate that all factories have been registered + FACTORIES_REGISTERED = True + + +# We can't have function implementations for these factories in Cython, because +# we need to build a Pydantic model for them dynamically, reading their argument +# structure from the signature. In Cython 3, this doesn't work because the +# from __future__ import annotations semantics are used, which means the types +# are stored as strings. +def make_sentencizer( + nlp: Language, + name: str, + punct_chars: Optional[List[str]], + overwrite: bool, + scorer: Optional[Callable], +): + return Sentencizer( + name, punct_chars=punct_chars, overwrite=overwrite, scorer=scorer + ) + + +def make_attribute_ruler( + nlp: Language, name: str, validate: bool, scorer: Optional[Callable] +): + return AttributeRuler(nlp.vocab, name, validate=validate, scorer=scorer) + + +def make_entity_linker( + nlp: Language, + name: str, + model: Model, + *, + labels_discard: Iterable[str], + n_sents: int, + incl_prior: bool, + incl_context: bool, + entity_vector_length: int, + get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], + generate_empty_kb: Callable[[Vocab, int], KnowledgeBase], + overwrite: bool, + scorer: Optional[Callable], + use_gold_ents: bool, + candidates_batch_size: int, + threshold: Optional[float] = None, +): + + if not model.attrs.get("include_span_maker", False): + # The only difference in arguments here is that use_gold_ents and threshold aren't available. + return EntityLinker_v1( + nlp.vocab, + model, + name, + labels_discard=labels_discard, + n_sents=n_sents, + incl_prior=incl_prior, + incl_context=incl_context, + entity_vector_length=entity_vector_length, + get_candidates=get_candidates, + overwrite=overwrite, + scorer=scorer, + ) + return EntityLinker( + nlp.vocab, + model, + name, + labels_discard=labels_discard, + n_sents=n_sents, + incl_prior=incl_prior, + incl_context=incl_context, + entity_vector_length=entity_vector_length, + get_candidates=get_candidates, + get_candidates_batch=get_candidates_batch, + generate_empty_kb=generate_empty_kb, + overwrite=overwrite, + scorer=scorer, + use_gold_ents=use_gold_ents, + candidates_batch_size=candidates_batch_size, + threshold=threshold, + ) + + +def make_lemmatizer( + nlp: Language, + model: Optional[Model], + name: str, + mode: str, + overwrite: bool, + scorer: Optional[Callable], +): + return Lemmatizer( + nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer + ) + + +def make_textcat( + nlp: Language, + name: str, + model: Model[List[Doc], List[Floats2d]], + threshold: float, + scorer: Optional[Callable], +) -> TextCategorizer: + return TextCategorizer(nlp.vocab, model, name, threshold=threshold, scorer=scorer) + + +def make_token_splitter( + nlp: Language, name: str, *, min_length: int = 0, split_length: int = 0 +): + return TokenSplitter(min_length=min_length, split_length=split_length) + + +def make_doc_cleaner(nlp: Language, name: str, *, attrs: Dict[str, Any], silent: bool): + return DocCleaner(attrs, silent=silent) + + +def make_tok2vec(nlp: Language, name: str, model: Model) -> Tok2Vec: + return Tok2Vec(nlp.vocab, model, name) + + +def make_spancat( + nlp: Language, + name: str, + suggester: Suggester, + model: Model[Tuple[List[Doc], Ragged], Floats2d], + spans_key: str, + scorer: Optional[Callable], + threshold: float, + max_positive: Optional[int], +) -> SpanCategorizer: + return SpanCategorizer( + nlp.vocab, + model=model, + suggester=suggester, + name=name, + spans_key=spans_key, + negative_weight=None, + allow_overlap=True, + max_positive=max_positive, + threshold=threshold, + scorer=scorer, + add_negative_label=False, + ) + + +def make_spancat_singlelabel( + nlp: Language, + name: str, + suggester: Suggester, + model: Model[Tuple[List[Doc], Ragged], Floats2d], + spans_key: str, + negative_weight: float, + allow_overlap: bool, + scorer: Optional[Callable], +) -> SpanCategorizer: + return SpanCategorizer( + nlp.vocab, + model=model, + suggester=suggester, + name=name, + spans_key=spans_key, + negative_weight=negative_weight, + allow_overlap=allow_overlap, + max_positive=1, + add_negative_label=True, + threshold=None, + scorer=scorer, + ) + + +def make_future_entity_ruler( + nlp: Language, + name: str, + phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, + validate: bool, + overwrite_ents: bool, + scorer: Optional[Callable], + ent_id_sep: str, +): + if overwrite_ents: + ents_filter = prioritize_new_ents_filter + else: + ents_filter = prioritize_existing_ents_filter + return SpanRuler( + nlp, + name, + spans_key=None, + spans_filter=None, + annotate_ents=True, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, + validate=validate, + overwrite=False, + scorer=scorer, + ) + + +def make_entity_ruler( + nlp: Language, + name: str, + phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, + validate: bool, + overwrite_ents: bool, + ent_id_sep: str, + scorer: Optional[Callable], +): + return EntityRuler( + nlp, + name, + phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, + validate=validate, + overwrite_ents=overwrite_ents, + ent_id_sep=ent_id_sep, + scorer=scorer, + ) + + +def make_span_ruler( + nlp: Language, + name: str, + spans_key: Optional[str], + spans_filter: Optional[Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]]], + annotate_ents: bool, + ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]], + phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, + validate: bool, + overwrite: bool, + scorer: Optional[Callable], +): + return SpanRuler( + nlp, + name, + spans_key=spans_key, + spans_filter=spans_filter, + annotate_ents=annotate_ents, + ents_filter=ents_filter, + phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, + validate=validate, + overwrite=overwrite, + scorer=scorer, + ) + + +def make_edit_tree_lemmatizer( + nlp: Language, + name: str, + model: Model, + backoff: Optional[str], + min_tree_freq: int, + overwrite: bool, + top_k: int, + scorer: Optional[Callable], +): + return EditTreeLemmatizer( + nlp.vocab, + model, + name, + backoff=backoff, + min_tree_freq=min_tree_freq, + overwrite=overwrite, + top_k=top_k, + scorer=scorer, + ) + + +def make_multilabel_textcat( + nlp: Language, + name: str, + model: Model[List[Doc], List[Floats2d]], + threshold: float, + scorer: Optional[Callable], +) -> MultiLabel_TextCategorizer: + return MultiLabel_TextCategorizer( + nlp.vocab, model, name, threshold=threshold, scorer=scorer + ) + + +def make_span_finder( + nlp: Language, + name: str, + model: Model[Iterable[Doc], Floats2d], + spans_key: str, + threshold: float, + max_length: Optional[int], + min_length: Optional[int], + scorer: Optional[Callable], +) -> SpanFinder: + return SpanFinder( + nlp, + model=model, + threshold=threshold, + name=name, + scorer=scorer, + max_length=max_length, + min_length=min_length, + spans_key=spans_key, + ) + + +def make_ner( + nlp: Language, + name: str, + model: Model, + moves: Optional[TransitionSystem], + update_with_oracle_cut_size: int, + incorrect_spans_key: Optional[str], + scorer: Optional[Callable], +): + return EntityRecognizer( + nlp.vocab, + model, + name=name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + incorrect_spans_key=incorrect_spans_key, + scorer=scorer, + ) + + +def make_beam_ner( + nlp: Language, + name: str, + model: Model, + moves: Optional[TransitionSystem], + update_with_oracle_cut_size: int, + beam_width: int, + beam_density: float, + beam_update_prob: float, + incorrect_spans_key: Optional[str], + scorer: Optional[Callable], +): + return EntityRecognizer( + nlp.vocab, + model, + name=name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + beam_width=beam_width, + beam_density=beam_density, + beam_update_prob=beam_update_prob, + incorrect_spans_key=incorrect_spans_key, + scorer=scorer, + ) + + +def make_parser( + nlp: Language, + name: str, + model: Model, + moves: Optional[TransitionSystem], + update_with_oracle_cut_size: int, + learn_tokens: bool, + min_action_freq: int, + scorer: Optional[Callable], +): + return DependencyParser( + nlp.vocab, + model, + name=name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + learn_tokens=learn_tokens, + min_action_freq=min_action_freq, + scorer=scorer, + ) + + +def make_beam_parser( + nlp: Language, + name: str, + model: Model, + moves: Optional[TransitionSystem], + update_with_oracle_cut_size: int, + learn_tokens: bool, + min_action_freq: int, + beam_width: int, + beam_density: float, + beam_update_prob: float, + scorer: Optional[Callable], +): + return DependencyParser( + nlp.vocab, + model, + name=name, + moves=moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + learn_tokens=learn_tokens, + min_action_freq=min_action_freq, + beam_width=beam_width, + beam_density=beam_density, + beam_update_prob=beam_update_prob, + scorer=scorer, + ) + + +def make_tagger( + nlp: Language, + name: str, + model: Model, + overwrite: bool, + scorer: Optional[Callable], + neg_prefix: str, + label_smoothing: float, +): + return Tagger( + nlp.vocab, + model, + name=name, + overwrite=overwrite, + scorer=scorer, + neg_prefix=neg_prefix, + label_smoothing=label_smoothing, + ) + + +def make_nn_labeller( + nlp: Language, name: str, model: Model, labels: Optional[dict], target: str +): + return MultitaskObjective(nlp.vocab, model, name, target=target) + + +def make_morphologizer( + nlp: Language, + model: Model, + name: str, + overwrite: bool, + extend: bool, + label_smoothing: float, + scorer: Optional[Callable], +): + return Morphologizer( + nlp.vocab, + model, + name, + overwrite=overwrite, + extend=extend, + label_smoothing=label_smoothing, + scorer=scorer, + ) + + +def make_senter( + nlp: Language, name: str, model: Model, overwrite: bool, scorer: Optional[Callable] +): + return SentenceRecognizer( + nlp.vocab, model, name, overwrite=overwrite, scorer=scorer + ) diff --git a/spacy/pipeline/functions.py b/spacy/pipeline/functions.py new file mode 100644 index 0000000..b2aa8b7 --- /dev/null +++ b/spacy/pipeline/functions.py @@ -0,0 +1,196 @@ +import importlib +import warnings +from typing import Any, Dict + +import srsly + +from .. import util +from ..errors import Warnings +from ..language import Language +from ..matcher import Matcher +from ..tokens import Doc + + +@Language.component( + "merge_noun_chunks", + requires=["token.dep", "token.tag", "token.pos"], + retokenizes=True, +) +def merge_noun_chunks(doc: Doc) -> Doc: + """Merge noun chunks into a single token. + + doc (Doc): The Doc object. + RETURNS (Doc): The Doc object with merged noun chunks. + + DOCS: https://spacy.io/api/pipeline-functions#merge_noun_chunks + """ + if not doc.has_annotation("DEP"): + return doc + with doc.retokenize() as retokenizer: + for np in doc.noun_chunks: + attrs = {"tag": np.root.tag, "dep": np.root.dep} + retokenizer.merge(np, attrs=attrs) # type: ignore[arg-type] + return doc + + +@Language.component( + "merge_entities", + requires=["doc.ents", "token.ent_iob", "token.ent_type"], + retokenizes=True, +) +def merge_entities(doc: Doc): + """Merge entities into a single token. + + doc (Doc): The Doc object. + RETURNS (Doc): The Doc object with merged entities. + + DOCS: https://spacy.io/api/pipeline-functions#merge_entities + """ + with doc.retokenize() as retokenizer: + for ent in doc.ents: + attrs = {"tag": ent.root.tag, "dep": ent.root.dep, "ent_type": ent.label} + retokenizer.merge(ent, attrs=attrs) # type: ignore[arg-type] + return doc + + +@Language.component("merge_subtokens", requires=["token.dep"], retokenizes=True) +def merge_subtokens(doc: Doc, label: str = "subtok") -> Doc: + """Merge subtokens into a single token. + + doc (Doc): The Doc object. + label (str): The subtoken dependency label. + RETURNS (Doc): The Doc object with merged subtokens. + + DOCS: https://spacy.io/api/pipeline-functions#merge_subtokens + """ + # TODO: make stateful component with "label" config + merger = Matcher(doc.vocab) + merger.add("SUBTOK", [[{"DEP": label, "op": "+"}]]) + matches = merger(doc) + spans = util.filter_spans([doc[start : end + 1] for _, start, end in matches]) # type: ignore[misc, operator] + with doc.retokenize() as retokenizer: + for span in spans: + retokenizer.merge(span) + return doc + + +class TokenSplitter: + def __init__(self, min_length: int = 0, split_length: int = 0): + self.min_length = min_length + self.split_length = split_length + + def __call__(self, doc: Doc) -> Doc: + if self.min_length > 0 and self.split_length > 0: + with doc.retokenize() as retokenizer: + for t in doc: + if len(t.text) >= self.min_length: + orths = [] + heads = [] + attrs = {} # type: ignore[var-annotated] + for i in range(0, len(t.text), self.split_length): + orths.append(t.text[i : i + self.split_length]) + heads.append((t, i / self.split_length)) + retokenizer.split(t, orths, heads, attrs) # type: ignore[arg-type] + return doc + + def _get_config(self) -> Dict[str, Any]: + return { + "min_length": self.min_length, + "split_length": self.split_length, + } + + def _set_config(self, config: Dict[str, Any] = {}) -> None: + self.min_length = config.get("min_length", 0) + self.split_length = config.get("split_length", 0) + + def to_bytes(self, **kwargs): + serializers = { + "cfg": lambda: srsly.json_dumps(self._get_config()), + } + return util.to_bytes(serializers, []) + + def from_bytes(self, data, **kwargs): + deserializers = { + "cfg": lambda b: self._set_config(srsly.json_loads(b)), + } + util.from_bytes(data, deserializers, []) + return self + + def to_disk(self, path, **kwargs): + path = util.ensure_path(path) + serializers = { + "cfg": lambda p: srsly.write_json(p, self._get_config()), + } + return util.to_disk(path, serializers, []) + + def from_disk(self, path, **kwargs): + path = util.ensure_path(path) + serializers = { + "cfg": lambda p: self._set_config(srsly.read_json(p)), + } + util.from_disk(path, serializers, []) + + +class DocCleaner: + def __init__(self, attrs: Dict[str, Any], *, silent: bool = True): + self.cfg: Dict[str, Any] = {"attrs": dict(attrs), "silent": silent} + + def __call__(self, doc: Doc) -> Doc: + attrs: dict = self.cfg["attrs"] + silent: bool = self.cfg["silent"] + for attr, value in attrs.items(): + obj = doc + parts = attr.split(".") + skip = False + for part in parts[:-1]: + if hasattr(obj, part): + obj = getattr(obj, part) + else: + skip = True + if not silent: + warnings.warn(Warnings.W116.format(attr=attr)) + if not skip: + if hasattr(obj, parts[-1]): + setattr(obj, parts[-1], value) + else: + if not silent: + warnings.warn(Warnings.W116.format(attr=attr)) + return doc + + def to_bytes(self, **kwargs): + serializers = { + "cfg": lambda: srsly.json_dumps(self.cfg), + } + return util.to_bytes(serializers, []) + + def from_bytes(self, data, **kwargs): + deserializers = { + "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), + } + util.from_bytes(data, deserializers, []) + return self + + def to_disk(self, path, **kwargs): + path = util.ensure_path(path) + serializers = { + "cfg": lambda p: srsly.write_json(p, self.cfg), + } + return util.to_disk(path, serializers, []) + + def from_disk(self, path, **kwargs): + path = util.ensure_path(path) + serializers = { + "cfg": lambda p: self.cfg.update(srsly.read_json(p)), + } + util.from_disk(path, serializers, []) + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_doc_cleaner": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_doc_cleaner + elif name == "make_token_splitter": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_token_splitter + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/legacy/__init__.py b/spacy/pipeline/legacy/__init__.py new file mode 100644 index 0000000..f216840 --- /dev/null +++ b/spacy/pipeline/legacy/__init__.py @@ -0,0 +1,3 @@ +from .entity_linker import EntityLinker_v1 + +__all__ = ["EntityLinker_v1"] diff --git a/spacy/pipeline/legacy/entity_linker.py b/spacy/pipeline/legacy/entity_linker.py new file mode 100644 index 0000000..1e46db0 --- /dev/null +++ b/spacy/pipeline/legacy/entity_linker.py @@ -0,0 +1,422 @@ +# This file is present to provide a prior version of the EntityLinker component +# for backwards compatability. For details see #9669. + +import random +import warnings +from itertools import islice +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + +import srsly +from thinc.api import CosineDistance, Model, Optimizer, set_dropout_rate +from thinc.types import Floats2d + +from ... import util +from ...errors import Errors, Warnings +from ...kb import Candidate, KnowledgeBase +from ...language import Language +from ...ml import empty_kb +from ...scorer import Scorer +from ...tokens import Doc, Span +from ...training import Example, validate_examples, validate_get_examples +from ...util import SimpleFrozenList +from ...vocab import Vocab +from ..pipe import deserialize_config +from ..trainable_pipe import TrainablePipe + +# See #9050 +BACKWARD_OVERWRITE = True + + +def entity_linker_score(examples, **kwargs): + return Scorer.score_links(examples, negative_labels=[EntityLinker_v1.NIL], **kwargs) + + +class EntityLinker_v1(TrainablePipe): + """Pipeline component for named entity linking. + + DOCS: https://spacy.io/api/entitylinker + """ + + NIL = "NIL" # string used to refer to a non-existing link + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "entity_linker", + *, + labels_discard: Iterable[str], + n_sents: int, + incl_prior: bool, + incl_context: bool, + entity_vector_length: int, + get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + overwrite: bool = BACKWARD_OVERWRITE, + scorer: Optional[Callable] = entity_linker_score, + ) -> None: + """Initialize an entity linker. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + labels_discard (Iterable[str]): NER labels that will automatically get a "NIL" prediction. + n_sents (int): The number of neighbouring sentences to take into account. + incl_prior (bool): Whether or not to include prior probabilities from the KB in the model. + incl_context (bool): Whether or not to include the local context in the model. + entity_vector_length (int): Size of encoding vectors in the KB. + get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that + produces a list of candidates, given a certain knowledge base and a textual mention. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. + DOCS: https://spacy.io/api/entitylinker#init + """ + self.vocab = vocab + self.model = model + self.name = name + self.labels_discard = list(labels_discard) + self.n_sents = n_sents + self.incl_prior = incl_prior + self.incl_context = incl_context + self.get_candidates = get_candidates + self.cfg: Dict[str, Any] = {"overwrite": overwrite} + self.distance = CosineDistance(normalize=False) + # how many neighbour sentences to take into account + # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'. + self.kb = empty_kb(entity_vector_length)(self.vocab) + self.scorer = scorer + + def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]): + """Define the KB of this pipe by providing a function that will + create it using this object's vocab.""" + if not callable(kb_loader): + raise ValueError(Errors.E885.format(arg_type=type(kb_loader))) + + self.kb = kb_loader(self.vocab) + + def validate_kb(self) -> None: + # Raise an error if the knowledge base is not initialized. + if self.kb is None: + raise ValueError(Errors.E1018.format(name=self.name)) + if len(self.kb) == 0: + raise ValueError(Errors.E139.format(name=self.name)) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + kb_loader: Optional[Callable[[Vocab], KnowledgeBase]] = None, + ): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates an InMemoryLookupKB from a Vocab instance. + Note that providing this argument, will overwrite all data accumulated in the current KB. + Use this only when loading a KB as-such from file. + + DOCS: https://spacy.io/api/entitylinker#initialize + """ + validate_get_examples(get_examples, "EntityLinker_v1.initialize") + if kb_loader is not None: + self.set_kb(kb_loader) + self.validate_kb() + nO = self.kb.entity_vector_length + doc_sample = [] + vector_sample = [] + for example in islice(get_examples(), 10): + doc_sample.append(example.x) + vector_sample.append(self.model.ops.alloc1f(nO)) + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(vector_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize( + X=doc_sample, Y=self.model.ops.asarray(vector_sample, dtype="float32") + ) + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/entitylinker#update + """ + self.validate_kb() + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + if not examples: + return losses + validate_examples(examples, "EntityLinker_v1.update") + sentence_docs = [] + for eg in examples: + sentences = [s for s in eg.reference.sents] + kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) + for ent in eg.reference.ents: + # KB ID of the first token is the same as the whole span + kb_id = kb_ids[ent.start] + if kb_id: + try: + # find the sentence in the list of sentences. + sent_index = sentences.index(ent.sent) + except AttributeError: + # Catch the exception when ent.sent is None and provide a user-friendly warning + raise RuntimeError(Errors.E030) from None + # get n previous sentences, if there are any + start_sentence = max(0, sent_index - self.n_sents) + # get n posterior sentences, or as many < n as there are + end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) + # get token positions + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + # append that span as a doc to training + sent_doc = eg.predicted[start_token:end_token].as_doc() + sentence_docs.append(sent_doc) + set_dropout_rate(self.model, drop) + if not sentence_docs: + warnings.warn(Warnings.W093.format(name="Entity Linker")) + return losses + sentence_encodings, bp_context = self.model.begin_update(sentence_docs) + loss, d_scores = self.get_loss( + sentence_encodings=sentence_encodings, examples=examples + ) + bp_context(d_scores) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def get_loss(self, examples: Iterable[Example], sentence_encodings: Floats2d): + validate_examples(examples, "EntityLinker_v1.get_loss") + entity_encodings = [] + for eg in examples: + kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True) + for ent in eg.reference.ents: + kb_id = kb_ids[ent.start] + if kb_id: + entity_encoding = self.kb.get_vector(kb_id) + entity_encodings.append(entity_encoding) + entity_encodings = self.model.ops.asarray2f(entity_encodings) + if sentence_encodings.shape != entity_encodings.shape: + err = Errors.E147.format( + method="get_loss", msg="gold entities do not match up" + ) + raise RuntimeError(err) + gradients = self.distance.get_grad(sentence_encodings, entity_encodings) + loss = self.distance.get_loss(sentence_encodings, entity_encodings) + loss = loss / len(entity_encodings) + return float(loss), gradients + + def predict(self, docs: Iterable[Doc]) -> List[str]: + """Apply the pipeline's model to a batch of docs, without modifying them. + Returns the KB IDs for each entity in each doc, including NIL if there is + no prediction. + + docs (Iterable[Doc]): The documents to predict. + RETURNS (List[str]): The models prediction for each document. + + DOCS: https://spacy.io/api/entitylinker#predict + """ + self.validate_kb() + entity_count = 0 + final_kb_ids: List[str] = [] + if not docs: + return final_kb_ids + if isinstance(docs, Doc): + docs = [docs] + for i, doc in enumerate(docs): + sentences = [s for s in doc.sents] + if len(doc) > 0: + # Looping through each entity (TODO: rewrite) + for ent in doc.ents: + sent = ent.sent + sent_index = sentences.index(sent) + assert sent_index >= 0 + # get n_neighbour sentences, clipped to the length of the document + start_sentence = max(0, sent_index - self.n_sents) + end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + sent_doc = doc[start_token:end_token].as_doc() + # currently, the context is the same for each entity in a sentence (should be refined) + xp = self.model.ops.xp + if self.incl_context: + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(self.get_candidates(self.kb, ent)) + if not candidates: + # no prediction possible for this entity - setting to NIL + final_kb_ids.append(self.NIL) + elif len(candidates) == 1: + # shortcut for efficiency reasons: take the 1 candidate + final_kb_ids.append(candidates[0].entity_) + else: + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", + ) + ) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + best_index = scores.argmax().item() + best_candidate = candidates[best_index] + final_kb_ids.append(best_candidate.entity_) + if not (len(final_kb_ids) == entity_count): + err = Errors.E147.format( + method="predict", msg="result variables not of equal length" + ) + raise RuntimeError(err) + return final_kb_ids + + def set_annotations(self, docs: Iterable[Doc], kb_ids: List[str]) -> None: + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + kb_ids (List[str]): The IDs to set, produced by EntityLinker.predict. + + DOCS: https://spacy.io/api/entitylinker#set_annotations + """ + count_ents = len([ent for doc in docs for ent in doc.ents]) + if count_ents != len(kb_ids): + raise ValueError(Errors.E148.format(ents=count_ents, ids=len(kb_ids))) + i = 0 + overwrite = self.cfg["overwrite"] + for doc in docs: + for ent in doc.ents: + kb_id = kb_ids[i] + i += 1 + for token in ent: + if token.ent_kb_id == 0 or overwrite: + token.ent_kb_id_ = kb_id + + def to_bytes(self, *, exclude=tuple()): + """Serialize the pipe to a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/entitylinker#to_bytes + """ + self._validate_serialization_attrs() + serialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serialize["kb"] = self.kb.to_bytes + serialize["model"] = self.model.to_bytes + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load the pipe from a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (TrainablePipe): The loaded object. + + DOCS: https://spacy.io/api/entitylinker#from_bytes + """ + self._validate_serialization_attrs() + + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b)) + deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) + deserialize["kb"] = lambda b: self.kb.from_bytes(b) + deserialize["model"] = load_model + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Serialize the pipe to disk. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/entitylinker#to_disk + """ + serialize = {} + serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) + serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) + serialize["kb"] = lambda p: self.kb.to_disk(p) + serialize["model"] = lambda p: self.model.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "EntityLinker_v1": + """Load the pipe from disk. Modifies the object in place and returns it. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (EntityLinker): The modified EntityLinker object. + + DOCS: https://spacy.io/api/entitylinker#from_disk + """ + + def load_model(p): + try: + with p.open("rb") as infile: + self.model.from_bytes(infile.read()) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize: Dict[str, Callable[[Any], Any]] = {} + deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p)) + deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude) + deserialize["kb"] = lambda p: self.kb.from_disk(p) + deserialize["model"] = load_model + util.from_disk(path, deserialize, exclude) + return self + + def rehearse(self, examples, *, sgd=None, losses=None, **config): + raise NotImplementedError + + def add_label(self, label): + raise NotImplementedError diff --git a/spacy/pipeline/lemmatizer.py b/spacy/pipeline/lemmatizer.py new file mode 100644 index 0000000..f518e10 --- /dev/null +++ b/spacy/pipeline/lemmatizer.py @@ -0,0 +1,323 @@ +import importlib +import warnings +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + +from thinc.api import Model + +from .. import util +from ..errors import Errors, Warnings +from ..language import Language +from ..lookups import Lookups, load_lookups +from ..scorer import Scorer +from ..tokens import Doc, Token +from ..training import Example +from ..util import SimpleFrozenList, logger +from ..vocab import Vocab +from .pipe import Pipe + + +def lemmatizer_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + return Scorer.score_token_attr(examples, "lemma", **kwargs) + + +def make_lemmatizer_scorer(): + return lemmatizer_score + + +class Lemmatizer(Pipe): + """ + The Lemmatizer supports simple part-of-speech-sensitive suffix rules and + lookup tables. + + DOCS: https://spacy.io/api/lemmatizer + """ + + @classmethod + def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]: + """Returns the lookups configuration settings for a given mode for use + in Lemmatizer.load_lookups. + + mode (str): The lemmatizer mode. + RETURNS (Tuple[List[str], List[str]]): The required and optional + lookup tables for this mode. + """ + if mode == "lookup": + return (["lemma_lookup"], []) + elif mode == "rule": + return (["lemma_rules"], ["lemma_exc", "lemma_index"]) + return ([], []) + + def __init__( + self, + vocab: Vocab, + model: Optional[Model], + name: str = "lemmatizer", + *, + mode: str = "lookup", + overwrite: bool = False, + scorer: Optional[Callable] = lemmatizer_score, + ) -> None: + """Initialize a Lemmatizer. + + vocab (Vocab): The vocab. + model (Model): A model (not yet implemented). + name (str): The component name. Defaults to "lemmatizer". + mode (str): The lemmatizer mode: "lookup", "rule". Defaults to "lookup". + overwrite (bool): Whether to overwrite existing lemmas. Defaults to + `False`. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_token_attr for the attribute "lemma". + + DOCS: https://spacy.io/api/lemmatizer#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._mode = mode + self.lookups = Lookups() + self.overwrite = overwrite + self._validated = False + if self.mode == "lookup": + self.lemmatize = self.lookup_lemmatize + elif self.mode == "rule": + self.lemmatize = self.rule_lemmatize + else: + mode_attr = f"{self.mode}_lemmatize" + if not hasattr(self, mode_attr): + raise ValueError(Errors.E1003.format(mode=mode)) + self.lemmatize = getattr(self, mode_attr) + self.cache = {} # type: ignore[var-annotated] + self.scorer = scorer + + @property + def mode(self): + return self._mode + + def __call__(self, doc: Doc) -> Doc: + """Apply the lemmatizer to one document. + + doc (Doc): The Doc to process. + RETURNS (Doc): The processed Doc. + + DOCS: https://spacy.io/api/lemmatizer#call + """ + if not self._validated: + self._validate_tables(Errors.E1004) + error_handler = self.get_error_handler() + try: + for token in doc: + if self.overwrite or token.lemma == 0: + token.lemma_ = self.lemmatize(token)[0] + return doc + except Exception as e: + error_handler(self.name, self, [doc], e) + + def initialize( + self, + get_examples: Optional[Callable[[], Iterable[Example]]] = None, + *, + nlp: Optional[Language] = None, + lookups: Optional[Lookups] = None, + ): + """Initialize the lemmatizer and load in data. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + lookups (Lookups): The lookups object containing the (optional) tables + such as "lemma_rules", "lemma_index", "lemma_exc" and + "lemma_lookup". Defaults to None. + """ + required_tables, optional_tables = self.get_lookups_config(self.mode) + if lookups is None: + logger.debug("Lemmatizer: loading tables from spacy-lookups-data") + lookups = load_lookups(lang=self.vocab.lang, tables=required_tables) + optional_lookups = load_lookups( + lang=self.vocab.lang, tables=optional_tables, strict=False + ) + for table in optional_lookups.tables: + lookups.set_table(table, optional_lookups.get_table(table)) + self.lookups = lookups + self._validate_tables(Errors.E1004) + + def _validate_tables(self, error_message: str = Errors.E912) -> None: + """Check that the lookups are correct for the current mode.""" + required_tables, optional_tables = self.get_lookups_config(self.mode) + for table in required_tables: + if table not in self.lookups: + raise ValueError( + error_message.format( + mode=self.mode, + tables=required_tables, + found=self.lookups.tables, + ) + ) + self._validated = True + + def lookup_lemmatize(self, token: Token) -> List[str]: + """Lemmatize using a lookup-based approach. + + token (Token): The token to lemmatize. + RETURNS (list): The available lemmas for the string. + + DOCS: https://spacy.io/api/lemmatizer#lookup_lemmatize + """ + lookup_table = self.lookups.get_table("lemma_lookup", {}) + result = lookup_table.get(token.text, token.text) + if isinstance(result, str): + result = [result] + return result + + def rule_lemmatize(self, token: Token) -> List[str]: + """Lemmatize using a rule-based approach. + + token (Token): The token to lemmatize. + RETURNS (list): The available lemmas for the string. + + DOCS: https://spacy.io/api/lemmatizer#rule_lemmatize + """ + cache_key = (token.orth, token.pos, token.morph.key) # type: ignore[attr-defined] + if cache_key in self.cache: + return self.cache[cache_key] + string = token.text + univ_pos = token.pos_.lower() + if univ_pos in ("", "eol", "space"): + if univ_pos == "": + warnings.warn(Warnings.W108) + return [string.lower()] + # See Issue #435 for example of where this logic is requied. + if self.is_base_form(token): + return [string.lower()] + index_table = self.lookups.get_table("lemma_index", {}) + exc_table = self.lookups.get_table("lemma_exc", {}) + rules_table = self.lookups.get_table("lemma_rules", {}) + if not any( + ( + index_table.get(univ_pos), + exc_table.get(univ_pos), + rules_table.get(univ_pos), + ) + ): + if univ_pos == "propn": + return [string] + else: + return [string.lower()] + + index = index_table.get(univ_pos, {}) + exceptions = exc_table.get(univ_pos, {}) + rules = rules_table.get(univ_pos, {}) + orig = string + string = string.lower() + forms = [] # type: ignore + oov_forms = [] + for old, new in rules: + if string.endswith(old): + form = string[: len(string) - len(old)] + new + if not form: + pass + elif form in index or not form.isalpha(): + if form in index: + forms.insert(0, form) + else: + forms.append(form) + else: + oov_forms.append(form) + # Remove duplicates but preserve the ordering of applied "rules" + forms = list(dict.fromkeys(forms)) + # Put exceptions at the front of the list, so they get priority. + # This is a dodgy heuristic -- but it's the best we can do until we get + # frequencies on this. We can at least prune out problematic exceptions, + # if they shadow more frequent analyses. + for form in exceptions.get(string, []): + if form not in forms: + forms.insert(0, form) + if not forms: + forms.extend(oov_forms) + if not forms: + forms.append(orig) + self.cache[cache_key] = forms + return forms + + def is_base_form(self, token: Token) -> bool: + """Check whether the token is a base form that does not need further + analysis for lemmatization. + + token (Token): The token. + RETURNS (bool): Whether the token is a base form. + + DOCS: https://spacy.io/api/lemmatizer#is_base_form + """ + return False + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ): + """Serialize the pipe to disk. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/lemmatizer#to_disk + """ + serialize = {} + serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) + serialize["lookups"] = lambda p: self.lookups.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "Lemmatizer": + """Load the pipe from disk. Modifies the object in place and returns it. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Lemmatizer): The modified Lemmatizer object. + + DOCS: https://spacy.io/api/lemmatizer#from_disk + """ + deserialize: Dict[str, Callable[[Any], Any]] = {} + deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude) + deserialize["lookups"] = lambda p: self.lookups.from_disk(p) + util.from_disk(path, deserialize, exclude) + self._validate_tables() + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the pipe to a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/lemmatizer#to_bytes + """ + serialize = {} + serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serialize["lookups"] = self.lookups.to_bytes + return util.to_bytes(serialize, exclude) + + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "Lemmatizer": + """Load the pipe from a bytestring. + + bytes_data (bytes): The serialized pipe. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Lemmatizer): The loaded Lemmatizer. + + DOCS: https://spacy.io/api/lemmatizer#from_bytes + """ + deserialize: Dict[str, Callable[[Any], Any]] = {} + deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) + deserialize["lookups"] = lambda b: self.lookups.from_bytes(b) + util.from_bytes(bytes_data, deserialize, exclude) + self._validate_tables() + return self + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_lemmatizer": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_lemmatizer + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/morphologizer.pyx b/spacy/pipeline/morphologizer.pyx new file mode 100644 index 0000000..333f64d --- /dev/null +++ b/spacy/pipeline/morphologizer.pyx @@ -0,0 +1,301 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from itertools import islice +from typing import Callable, Dict, Optional, Union + +from thinc.api import Config, Model, SequenceCategoricalCrossentropy + +from ..morphology cimport Morphology +from ..tokens.doc cimport Doc +from ..vocab cimport Vocab + +from .. import util +from ..errors import Errors +from ..language import Language +from ..parts_of_speech import IDS as POS_IDS +from ..scorer import Scorer +from ..training import validate_examples, validate_get_examples +from ..util import registry +from .tagger import Tagger + +# See #9050 +BACKWARD_OVERWRITE = True +BACKWARD_EXTEND = False + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v2" + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[model.tok2vec.embed] +@architectures = "spacy.CharacterEmbed.v2" +width = 128 +rows = 7000 +nM = 64 +nC = 8 +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = 128 +depth = 4 +window_size = 1 +maxout_pieces = 3 +""" + +DEFAULT_MORPH_MODEL = Config().from_str(default_model_config)["model"] + + +def morphologizer_score(examples, **kwargs): + def morph_key_getter(token, attr): + return getattr(token, attr).key + + results = {} + results.update(Scorer.score_token_attr(examples, "pos", **kwargs)) + results.update(Scorer.score_token_attr(examples, "morph", getter=morph_key_getter, **kwargs)) + results.update( + Scorer.score_token_attr_per_feat( + examples, "morph", getter=morph_key_getter, **kwargs + ) + ) + return results + + +def make_morphologizer_scorer(): + return morphologizer_score + + +class Morphologizer(Tagger): + POS_FEAT = "POS" + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "morphologizer", + *, + overwrite: bool = BACKWARD_OVERWRITE, + extend: bool = BACKWARD_EXTEND, + label_smoothing: float = 0.0, + scorer: Optional[Callable] = morphologizer_score, + ): + """Initialize a morphologizer. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_token_attr for the attributes "pos" and "morph" and + Scorer.score_token_attr_per_feat for the attribute "morph". + + DOCS: https://spacy.io/api/morphologizer#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + # to be able to set annotations without string operations on labels, + # store mappings from morph+POS labels to token-level annotations: + # 1) labels_morph stores a mapping from morph+POS->morph + # 2) labels_pos stores a mapping from morph+POS->POS + cfg = { + "labels_morph": {}, + "labels_pos": {}, + "overwrite": overwrite, + "extend": extend, + "label_smoothing": label_smoothing, + } + self.cfg = dict(sorted(cfg.items())) + self.scorer = scorer + + @property + def labels(self): + """RETURNS (Tuple[str]): The labels currently added to the component.""" + return tuple(self.cfg["labels_morph"].keys()) + + @property + def label_data(self) -> Dict[str, Dict[str, Union[str, float, int, None]]]: + """A dictionary with all labels data.""" + return {"morph": self.cfg["labels_morph"], "pos": self.cfg["labels_pos"]} + + def add_label(self, label): + """Add a new label to the pipe. + + label (str): The label to add. + RETURNS (int): 0 if label is already present, otherwise 1. + + DOCS: https://spacy.io/api/morphologizer#add_label + """ + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + self._allow_extra_label() + # normalize label + norm_label = self.vocab.morphology.normalize_features(label) + # extract separate POS and morph tags + label_dict = Morphology.feats_to_dict(label) + pos = label_dict.get(self.POS_FEAT, "") + if self.POS_FEAT in label_dict: + label_dict.pop(self.POS_FEAT) + # normalize morph string and add to morphology table + norm_morph = self.vocab.strings[self.vocab.morphology.add(label_dict)] + # add label mappings + if norm_label not in self.cfg["labels_morph"]: + self.cfg["labels_morph"][norm_label] = norm_morph + self.cfg["labels_pos"][norm_label] = POS_IDS[pos] + return 1 + + def initialize(self, get_examples, *, nlp=None, labels=None): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + + DOCS: https://spacy.io/api/morphologizer#initialize + """ + validate_get_examples(get_examples, "Morphologizer.initialize") + util.check_lexeme_norms(self.vocab, "morphologizer") + if labels is not None: + self.cfg["labels_morph"] = labels["morph"] + self.cfg["labels_pos"] = labels["pos"] + else: + # First, fetch all labels from the data + for example in get_examples(): + for i, token in enumerate(example.reference): + pos = token.pos_ + # if both are unset, annotation is missing, so do not add + # an empty label + if pos == "" and not token.has_morph(): + continue + morph = str(token.morph) + # create and add the combined morph+POS label + morph_dict = Morphology.feats_to_dict(morph) + if pos: + morph_dict[self.POS_FEAT] = pos + norm_label = self.vocab.strings[self.vocab.morphology.add(morph_dict)] + # add label->morph and label->POS mappings + if norm_label not in self.cfg["labels_morph"]: + self.cfg["labels_morph"][norm_label] = morph + self.cfg["labels_pos"][norm_label] = POS_IDS[pos] + if len(self.labels) < 1: + raise ValueError(Errors.E143.format(name=self.name)) + doc_sample = [] + label_sample = [] + for example in islice(get_examples(), 10): + gold_array = [] + for i, token in enumerate(example.reference): + pos = token.pos_ + morph = str(token.morph) + morph_dict = Morphology.feats_to_dict(morph) + if pos: + morph_dict[self.POS_FEAT] = pos + norm_label = self.vocab.strings[self.vocab.morphology.add(morph_dict)] + gold_array.append([1.0 if label == norm_label else 0.0 for label in self.labels]) + doc_sample.append(example.x) + label_sample.append(self.model.ops.asarray(gold_array, dtype="float32")) + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample, Y=label_sample) + + def set_annotations(self, docs, batch_tag_ids): + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + batch_tag_ids: The IDs to set, produced by Morphologizer.predict. + + DOCS: https://spacy.io/api/morphologizer#set_annotations + """ + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef bint overwrite = self.cfg["overwrite"] + cdef bint extend = self.cfg["extend"] + labels = self.labels + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + if hasattr(doc_tag_ids, "get"): + doc_tag_ids = doc_tag_ids.get() + for j, tag_id in enumerate(doc_tag_ids): + morph = labels[tag_id] + # set morph + if doc.c[j].morph == 0 or overwrite or extend: + if overwrite and extend: + # morphologizer morph overwrites any existing features + # while extending + extended_morph = Morphology.feats_to_dict(self.vocab.strings[doc.c[j].morph]) + extended_morph.update(Morphology.feats_to_dict(self.cfg["labels_morph"].get(morph, 0))) + doc.c[j].morph = self.vocab.morphology.add(extended_morph) + elif extend: + # existing features are preserved and any new features + # are added + extended_morph = Morphology.feats_to_dict(self.cfg["labels_morph"].get(morph, 0)) + extended_morph.update(Morphology.feats_to_dict(self.vocab.strings[doc.c[j].morph])) + doc.c[j].morph = self.vocab.morphology.add(extended_morph) + else: + # clobber + doc.c[j].morph = self.vocab.morphology.add(self.cfg["labels_morph"].get(morph, 0)) + # set POS + if doc.c[j].pos == 0 or overwrite: + doc.c[j].pos = self.cfg["labels_pos"].get(morph, 0) + + def get_loss(self, examples, scores): + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/morphologizer#get_loss + """ + validate_examples(examples, "Morphologizer.get_loss") + loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, + label_smoothing=self.cfg["label_smoothing"]) + truths = [] + for eg in examples: + eg_truths = [] + pos_tags = eg.get_aligned("POS", as_string=True) + morphs = eg.get_aligned("MORPH", as_string=True) + for i in range(len(morphs)): + pos = pos_tags[i] + morph = morphs[i] + # POS may align (same value for multiple tokens) when morph + # doesn't, so if either is misaligned (None), treat the + # annotation as missing so that truths doesn't end up with an + # unknown morph+POS combination + if pos is None or morph is None: + label = None + # If both are unset, the annotation is missing (empty morph + # converted from int is "_" rather than "") + elif pos == "" and morph == "": + label = None + # Otherwise, generate the combined label + else: + label_dict = Morphology.feats_to_dict(morph) + if pos: + label_dict[self.POS_FEAT] = pos + label = self.vocab.strings[self.vocab.morphology.add(label_dict)] + # As a fail-safe, skip any unrecognized labels + if label not in self.labels: + label = None + eg_truths.append(label) + truths.append(eg_truths) + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError(Errors.E910.format(name=self.name)) + return float(loss), d_scores + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_morphologizer": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_morphologizer + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/multitask.pyx b/spacy/pipeline/multitask.pyx new file mode 100644 index 0000000..1ba84b2 --- /dev/null +++ b/spacy/pipeline/multitask.pyx @@ -0,0 +1,217 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from typing import Optional + +import numpy +from thinc.api import Config, CosineDistance, Model, set_dropout_rate, to_categorical + +from ..attrs import ID +from ..errors import Errors +from ..language import Language +from ..training import validate_examples +from .tagger import Tagger +from .trainable_pipe import TrainablePipe + +default_model_config = """ +[model] +@architectures = "spacy.MultiTask.v1" +maxout_pieces = 3 +token_vector_width = 96 + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 2 +subword_features = true +""" +DEFAULT_MT_MODEL = Config().from_str(default_model_config)["model"] + + +class MultitaskObjective(Tagger): + """Experimental: Assist training of a parser or tagger, by training a + side-objective. + """ + + def __init__(self, vocab, model, name="nn_labeller", *, target): + self.vocab = vocab + self.model = model + self.name = name + if target == "dep": + self.make_label = self.make_dep + elif target == "tag": + self.make_label = self.make_tag + elif target == "ent": + self.make_label = self.make_ent + elif target == "dep_tag_offset": + self.make_label = self.make_dep_tag_offset + elif target == "ent_tag": + self.make_label = self.make_ent_tag + elif target == "sent_start": + self.make_label = self.make_sent_start + elif hasattr(target, "__call__"): + self.make_label = target + else: + raise ValueError(Errors.E016) + cfg = {"labels": {}, "target": target} + self.cfg = dict(cfg) + + @property + def labels(self): + return self.cfg.setdefault("labels", {}) + + @labels.setter + def labels(self, value): + self.cfg["labels"] = value + + def set_annotations(self, docs, dep_ids): + pass + + def initialize(self, get_examples, nlp=None, labels=None): + if not hasattr(get_examples, "__call__"): + err = Errors.E930.format(name="MultitaskObjective", obj=type(get_examples)) + raise ValueError(err) + if labels is not None: + self.labels = labels + else: + for example in get_examples(): + for token in example.y: + label = self.make_label(token) + if label is not None and label not in self.labels: + self.labels[label] = len(self.labels) + self.model.initialize() # TODO: fix initialization by defining X and Y + + def predict(self, docs): + tokvecs = self.model.get_ref("tok2vec")(docs) + scores = self.model.get_ref("softmax")(tokvecs) + return tokvecs, scores + + def get_loss(self, examples, scores): + cdef int idx = 0 + correct = numpy.zeros((scores.shape[0],), dtype="i") + guesses = scores.argmax(axis=1) + for i, eg in enumerate(examples): + # Handles alignment for tokenization differences + _doc_annots = eg.get_aligned() # TODO + for j in range(len(eg.predicted)): + tok_annots = {key: values[j] for key, values in tok_annots.items()} + label = self.make_label(j, tok_annots) + if label is None or label not in self.labels: + correct[idx] = guesses[idx] + else: + correct[idx] = self.labels[label] + idx += 1 + correct = self.model.ops.xp.array(correct, dtype="i") + d_scores = scores - to_categorical(correct, n_classes=scores.shape[1]) + loss = (d_scores**2).sum() + return float(loss), d_scores + + @staticmethod + def make_dep(token): + return token.dep_ + + @staticmethod + def make_tag(token): + return token.tag_ + + @staticmethod + def make_ent(token): + if token.ent_iob_ == "O": + return "O" + else: + return token.ent_iob_ + "-" + token.ent_type_ + + @staticmethod + def make_dep_tag_offset(token): + dep = token.dep_ + tag = token.tag_ + offset = token.head.i - token.i + offset = min(offset, 2) + offset = max(offset, -2) + return f"{dep}-{tag}:{offset}" + + @staticmethod + def make_ent_tag(token): + if token.ent_iob_ == "O": + ent = "O" + else: + ent = token.ent_iob_ + "-" + token.ent_type_ + tag = token.tag_ + return f"{tag}-{ent}" + + @staticmethod + def make_sent_start(token): + """A multi-task objective for representing sentence boundaries, + using BILU scheme. (O is impossible) + """ + if token.is_sent_start and token.is_sent_end: + return "U-SENT" + elif token.is_sent_start: + return "B-SENT" + else: + return "I-SENT" + + +class ClozeMultitask(TrainablePipe): + def __init__(self, vocab, model, **cfg): + self.vocab = vocab + self.model = model + self.cfg = cfg + self.distance = CosineDistance(ignore_zeros=True, normalize=False) # TODO: in config + + def set_annotations(self, docs, dep_ids): + pass + + def initialize(self, get_examples, nlp=None): + self.model.initialize() # TODO: fix initialization by defining X and Y + X = self.model.ops.alloc((5, self.model.get_ref("tok2vec").get_dim("nO"))) + self.model.output_layer.initialize(X) + + def predict(self, docs): + tokvecs = self.model.get_ref("tok2vec")(docs) + vectors = self.model.get_ref("output_layer")(tokvecs) + return tokvecs, vectors + + def get_loss(self, examples, vectors, prediction): + validate_examples(examples, "ClozeMultitask.get_loss") + # The simplest way to implement this would be to vstack the + # token.vector values, but that's a bit inefficient, especially on GPU. + # Instead we fetch the index into the vectors table for each of our tokens, + # and look them up all at once. This prevents data copying. + ids = self.model.ops.flatten([eg.predicted.to_array(ID).ravel() for eg in examples]) + target = vectors[ids] + gradient = self.distance.get_grad(prediction, target) + loss = self.distance.get_loss(prediction, target) + return float(loss), gradient + + def update(self, examples, *, drop=0., sgd=None, losses=None): + pass + + def rehearse(self, examples, drop=0., sgd=None, losses=None): + if losses is not None and self.name not in losses: + losses[self.name] = 0. + set_dropout_rate(self.model, drop) + validate_examples(examples, "ClozeMultitask.rehearse") + predictions, bp_predictions = self.model.begin_update() + loss, d_predictions = self.get_loss(examples, self.vocab.vectors.data, predictions) + bp_predictions(d_predictions) + if sgd is not None: + self.finish_update(sgd) + if losses is not None: + losses[self.name] += loss + return losses + + def add_label(self, label): + raise NotImplementedError + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_nn_labeller": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_nn_labeller + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/ner.pyx b/spacy/pipeline/ner.pyx new file mode 100644 index 0000000..1257a64 --- /dev/null +++ b/spacy/pipeline/ner.pyx @@ -0,0 +1,132 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from collections import defaultdict +from typing import Callable, Optional + +from thinc.api import Config, Model + +from ._parser_internals.transition_system import TransitionSystem + +from ._parser_internals.ner cimport BiluoPushDown +from .transition_parser cimport Parser + +from ..language import Language +from ..scorer import get_ner_prf +from ..training import remove_bilu_prefix +from ..util import registry + +default_model_config = """ +[model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +hidden_width = 64 +maxout_pieces = 2 +use_upper = true + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_NER_MODEL = Config().from_str(default_model_config)["model"] + + +def ner_score(examples, **kwargs): + return get_ner_prf(examples, **kwargs) + + +def make_ner_scorer(): + return ner_score + + +cdef class EntityRecognizer(Parser): + """Pipeline component for named entity recognition. + + DOCS: https://spacy.io/api/entityrecognizer + """ + TransitionSystem = BiluoPushDown + + def __init__( + self, + vocab, + model, + name="ner", + moves=None, + *, + update_with_oracle_cut_size=100, + beam_width=1, + beam_density=0.0, + beam_update_prob=0.0, + multitasks=tuple(), + incorrect_spans_key=None, + scorer=ner_score, + ): + """Create an EntityRecognizer. + """ + super().__init__( + vocab, + model, + name, + moves, + update_with_oracle_cut_size=update_with_oracle_cut_size, + min_action_freq=1, # not relevant for NER + learn_tokens=False, # not relevant for NER + beam_width=beam_width, + beam_density=beam_density, + beam_update_prob=beam_update_prob, + multitasks=multitasks, + incorrect_spans_key=incorrect_spans_key, + scorer=scorer, + ) + + def add_multitask_objective(self, mt_component): + """Register another component as a multi-task objective. Experimental.""" + self._multitasks.append(mt_component) + + def init_multitask_objectives(self, get_examples, nlp=None, **cfg): + """Setup multi-task objective components. Experimental and internal.""" + # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ? + for labeller in self._multitasks: + labeller.model.set_dim("nO", len(self.labels)) + if labeller.model.has_ref("output_layer"): + labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels)) + labeller.initialize(get_examples, nlp=nlp) + + @property + def labels(self): + # Get the labels from the model by looking at the available moves, e.g. + # B-PERSON, I-PERSON, L-PERSON, U-PERSON + labels = set(remove_bilu_prefix(move) for move in self.move_names + if move[0] in ("B", "I", "L", "U")) + return tuple(sorted(labels)) + + def scored_ents(self, beams): + """Return a dictionary of (start, end, label) tuples with corresponding scores + for each beam/doc that was processed. + """ + entity_scores = [] + for beam in beams: + score_dict = defaultdict(float) + for score, ents in self.moves.get_beam_parses(beam): + for start, end, label in ents: + score_dict[(start, end, label)] += score + entity_scores.append(score_dict) + return entity_scores + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_ner": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_ner + elif name == "make_beam_ner": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_beam_ner + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/pipe.pxd b/spacy/pipeline/pipe.pxd new file mode 100644 index 0000000..bb97f79 --- /dev/null +++ b/spacy/pipeline/pipe.pxd @@ -0,0 +1,2 @@ +cdef class Pipe: + cdef public str name diff --git a/spacy/pipeline/pipe.pyi b/spacy/pipeline/pipe.pyi new file mode 100644 index 0000000..55cfd1f --- /dev/null +++ b/spacy/pipeline/pipe.pyi @@ -0,0 +1,48 @@ +from pathlib import Path +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + NoReturn, + Tuple, + Union, +) + +from ..language import Language +from ..tokens.doc import Doc +from ..training import Example + +class Pipe: + def __call__(self, doc: Doc) -> Doc: ... + def pipe( + self, stream: Iterable[Doc], *, batch_size: int = ... + ) -> Iterator[Doc]: ... + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Language = ..., + ) -> None: ... + def score( + self, examples: Iterable[Example], **kwargs: Any + ) -> Dict[str, Union[float, Dict[str, float]]]: ... + @property + def is_trainable(self) -> bool: ... + @property + def labels(self) -> Tuple[str, ...]: ... + @property + def hide_labels(self) -> bool: ... + @property + def label_data(self) -> Any: ... + def _require_labels(self) -> None: ... + def set_error_handler( + self, error_handler: Callable[[str, "Pipe", List[Doc], Exception], NoReturn] + ) -> None: ... + def get_error_handler( + self, + ) -> Callable[[str, "Pipe", List[Doc], Exception], NoReturn]: ... + +def deserialize_config(path: Path) -> Any: ... diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx new file mode 100644 index 0000000..ea5fc52 --- /dev/null +++ b/spacy/pipeline/pipe.pyx @@ -0,0 +1,144 @@ +# cython: infer_types=True, binding=True +import warnings +from typing import Callable, Dict, Iterable, Iterator, Tuple, Union + +import srsly + +from ..tokens.doc cimport Doc + +from ..errors import Errors, Warnings +from ..language import Language +from ..training import Example +from ..util import raise_error + + +cdef class Pipe: + """This class is a base class and not instantiated directly. It provides + an interface for pipeline components to implement. + Trainable pipeline components like the EntityRecognizer or TextCategorizer + should inherit from the subclass 'TrainablePipe'. + + DOCS: https://spacy.io/api/pipe + """ + + def __call__(self, Doc doc) -> Doc: + """Apply the pipe to one document. The document is modified in place, + and returned. This usually happens under the hood when the nlp object + is called on a text and all components are applied to the Doc. + + doc (Doc): The Doc to process. + RETURNS (Doc): The processed Doc. + + DOCS: https://spacy.io/api/pipe#call + """ + raise NotImplementedError(Errors.E931.format(parent="Pipe", method="__call__", name=self.name)) + + def pipe(self, stream: Iterable[Doc], *, batch_size: int = 128) -> Iterator[Doc]: + """Apply the pipe to a stream of documents. This usually happens under + the hood when the nlp object is called on a text and all components are + applied to the Doc. + + stream (Iterable[Doc]): A stream of documents. + batch_size (int): The number of documents to buffer. + YIELDS (Doc): Processed documents in order. + + DOCS: https://spacy.io/api/pipe#pipe + """ + error_handler = self.get_error_handler() + for doc in stream: + try: + doc = self(doc) + yield doc + except Exception as e: + error_handler(self.name, self, [doc], e) + + def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language = None): + """Initialize the pipe. For non-trainable components, this method + is optional. For trainable components, which should inherit + from the subclass TrainablePipe, the provided data examples + should be used to ensure that the internal model is initialized + properly and all input/output dimensions throughout the network are + inferred. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + + DOCS: https://spacy.io/api/pipe#initialize + """ + pass + + def score(self, examples: Iterable[Example], **kwargs) -> Dict[str, Union[float, Dict[str, float]]]: + """Score a batch of examples. + + examples (Iterable[Example]): The examples to score. + RETURNS (Dict[str, Any]): The scores. + + DOCS: https://spacy.io/api/pipe#score + """ + if hasattr(self, "scorer") and self.scorer is not None: + scorer_kwargs = {} + # use default settings from cfg (e.g., threshold) + if hasattr(self, "cfg") and isinstance(self.cfg, dict): + scorer_kwargs.update(self.cfg) + # override self.cfg["labels"] with self.labels + if hasattr(self, "labels"): + scorer_kwargs["labels"] = self.labels + # override with kwargs settings + scorer_kwargs.update(kwargs) + return self.scorer(examples, **scorer_kwargs) + return {} + + @property + def is_trainable(self) -> bool: + return False + + @property + def labels(self) -> Tuple[str, ...]: + return tuple() + + @property + def hide_labels(self) -> bool: + return False + + @property + def label_data(self): + """Optional JSON-serializable data that would be sufficient to recreate + the label set if provided to the `pipe.initialize()` method. + """ + return None + + def _require_labels(self) -> None: + """Raise an error if this component has no labels defined.""" + if not self.labels or list(self.labels) == [""]: + raise ValueError(Errors.E143.format(name=self.name)) + + def set_error_handler(self, error_handler: Callable) -> None: + """Set an error handler function. + + error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], None]): + Function that deals with a failing batch of documents. This callable function should take in + the component's name, the component itself, the offending batch of documents, and the exception + that was thrown. + + DOCS: https://spacy.io/api/pipe#set_error_handler + """ + self.error_handler = error_handler + + def get_error_handler(self) -> Callable: + """Retrieve the error handler function. + + RETURNS (Callable): The error handler, or if it's not set a default function that just reraises. + + DOCS: https://spacy.io/api/pipe#get_error_handler + """ + if hasattr(self, "error_handler"): + return self.error_handler + return raise_error + + +def deserialize_config(path): + if path.exists(): + return srsly.read_json(path) + else: + return {} diff --git a/spacy/pipeline/sentencizer.pyx b/spacy/pipeline/sentencizer.pyx new file mode 100644 index 0000000..d2b0a8d --- /dev/null +++ b/spacy/pipeline/sentencizer.pyx @@ -0,0 +1,177 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from typing import Callable, List, Optional + +import srsly + +from ..tokens.doc cimport Doc + +from .. import util +from ..language import Language +from .pipe import Pipe +from .senter import senter_score + +# see #9050 +BACKWARD_OVERWRITE = False + + +class Sentencizer(Pipe): + """Segment the Doc into sentences using a rule-based strategy. + + DOCS: https://spacy.io/api/sentencizer + """ + + default_punct_chars = [ + '!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', + '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', + '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', + '‼', '‽', '⁇', '⁈', '⁉', '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶', + '꡷', '꣎', '꣏', '꤯', '꧈', '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒', + '﹖', '﹗', '！', '．', '？', '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀', + '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼', + '𑊩', '𑑋', '𑑌', '𑗂', '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐', + '𑗑', '𑗒', '𑗓', '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂', + '𑩃', '𑪛', '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', + '｡', '。' + ] + + def __init__( + self, + name="sentencizer", + *, + punct_chars=None, + overwrite=BACKWARD_OVERWRITE, + scorer=senter_score, + ): + """Initialize the sentencizer. + + punct_chars (list): Punctuation characters to split on. Will be + serialized with the nlp object. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the attribute "sents". + + DOCS: https://spacy.io/api/sentencizer#init + """ + self.name = name + if punct_chars: + self.punct_chars = set(punct_chars) + else: + self.punct_chars = set(self.default_punct_chars) + self.overwrite = overwrite + self.scorer = scorer + + def __call__(self, doc): + """Apply the sentencizer to a Doc and set Token.is_sent_start. + + doc (Doc): The document to process. + RETURNS (Doc): The processed Doc. + + DOCS: https://spacy.io/api/sentencizer#call + """ + error_handler = self.get_error_handler() + try: + tags = self.predict([doc]) + self.set_annotations([doc], tags) + return doc + except Exception as e: + error_handler(self.name, self, [doc], e) + + def predict(self, docs): + """Apply the pipe to a batch of docs, without modifying them. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: The predictions for each document. + """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + guesses = [[] for doc in docs] + return guesses + guesses = [] + for doc in docs: + doc_guesses = [False] * len(doc) + if len(doc) > 0: + start = 0 + seen_period = False + doc_guesses[0] = True + for i, token in enumerate(doc): + is_in_punct_chars = token.text in self.punct_chars + if seen_period and not token.is_punct and not is_in_punct_chars: + doc_guesses[start] = True + start = token.i + seen_period = False + elif is_in_punct_chars: + seen_period = True + if start < len(doc): + doc_guesses[start] = True + guesses.append(doc_guesses) + return guesses + + def set_annotations(self, docs, batch_tag_ids): + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + scores: The tag IDs produced by Sentencizer.predict. + """ + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + for j, tag_id in enumerate(doc_tag_ids): + if doc.c[j].sent_start == 0 or self.overwrite: + if tag_id: + doc.c[j].sent_start = 1 + else: + doc.c[j].sent_start = -1 + + def to_bytes(self, *, exclude=tuple()): + """Serialize the sentencizer to a bytestring. + + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/sentencizer#to_bytes + """ + return srsly.msgpack_dumps({"punct_chars": list(self.punct_chars), "overwrite": self.overwrite}) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load the sentencizer from a bytestring. + + bytes_data (bytes): The data to load. + returns (Sentencizer): The loaded object. + + DOCS: https://spacy.io/api/sentencizer#from_bytes + """ + cfg = srsly.msgpack_loads(bytes_data) + self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) + self.overwrite = cfg.get("overwrite", self.overwrite) + return self + + def to_disk(self, path, *, exclude=tuple()): + """Serialize the sentencizer to disk. + + DOCS: https://spacy.io/api/sentencizer#to_disk + """ + path = util.ensure_path(path) + path = path.with_suffix(".json") + srsly.write_json(path, {"punct_chars": list(self.punct_chars), "overwrite": self.overwrite}) + + def from_disk(self, path, *, exclude=tuple()): + """Load the sentencizer from disk. + + DOCS: https://spacy.io/api/sentencizer#from_disk + """ + path = util.ensure_path(path) + path = path.with_suffix(".json") + cfg = srsly.read_json(path) + self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars)) + self.overwrite = cfg.get("overwrite", self.overwrite) + return self + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_sentencizer": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_sentencizer + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/senter.pyx b/spacy/pipeline/senter.pyx new file mode 100644 index 0000000..a5d85f4 --- /dev/null +++ b/spacy/pipeline/senter.pyx @@ -0,0 +1,186 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from itertools import islice +from typing import Callable, Optional + +from thinc.api import Config, Model, SequenceCategoricalCrossentropy + +from ..tokens.doc cimport Doc + +from .. import util +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..training import validate_examples, validate_get_examples +from ..util import registry +from .tagger import Tagger + +# See #9050 +BACKWARD_OVERWRITE = False + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v2" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 12 +depth = 1 +embed_size = 2000 +window_size = 1 +maxout_pieces = 2 +subword_features = true +""" +DEFAULT_SENTER_MODEL = Config().from_str(default_model_config)["model"] + + +def senter_score(examples, **kwargs): + def has_sents(doc): + return doc.has_annotation("SENT_START") + + results = Scorer.score_spans(examples, "sents", has_annotation=has_sents, **kwargs) + del results["sents_per_type"] + return results + + +def make_senter_scorer(): + return senter_score + + +class SentenceRecognizer(Tagger): + """Pipeline component for sentence segmentation. + + DOCS: https://spacy.io/api/sentencerecognizer + """ + def __init__( + self, + vocab, + model, + name="senter", + *, + overwrite=BACKWARD_OVERWRITE, + scorer=senter_score, + ): + """Initialize a sentence recognizer. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the attribute "sents". + + DOCS: https://spacy.io/api/sentencerecognizer#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + self.cfg = {"overwrite": overwrite} + self.scorer = scorer + + @property + def labels(self): + """RETURNS (Tuple[str]): The labels.""" + # labels are numbered by index internally, so this matches GoldParse + # and Example where the sentence-initial tag is 1 and other positions + # are 0 + return tuple(["I", "S"]) + + @property + def hide_labels(self): + return True + + @property + def label_data(self): + return None + + def set_annotations(self, docs, batch_tag_ids): + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + batch_tag_ids: The IDs to set, produced by SentenceRecognizer.predict. + + DOCS: https://spacy.io/api/sentencerecognizer#set_annotations + """ + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef bint overwrite = self.cfg["overwrite"] + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + if hasattr(doc_tag_ids, "get"): + doc_tag_ids = doc_tag_ids.get() + for j, tag_id in enumerate(doc_tag_ids): + if doc.c[j].sent_start == 0 or overwrite: + if tag_id == 1: + doc.c[j].sent_start = 1 + else: + doc.c[j].sent_start = -1 + + def get_loss(self, examples, scores): + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/sentencerecognizer#get_loss + """ + validate_examples(examples, "SentenceRecognizer.get_loss") + labels = self.labels + loss_func = SequenceCategoricalCrossentropy(names=labels, normalize=False) + truths = [] + for eg in examples: + eg_truth = [] + for x in eg.get_aligned("SENT_START"): + if x is None: + eg_truth.append(None) + elif x == 1: + eg_truth.append(labels[1]) + else: + # anything other than 1: 0, -1, -1 as uint64 + eg_truth.append(labels[0]) + truths.append(eg_truth) + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError(Errors.E910.format(name=self.name)) + return float(loss), d_scores + + def initialize(self, get_examples, *, nlp=None): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + + DOCS: https://spacy.io/api/sentencerecognizer#initialize + """ + validate_get_examples(get_examples, "SentenceRecognizer.initialize") + util.check_lexeme_norms(self.vocab, "senter") + doc_sample = [] + label_sample = [] + assert self.labels, Errors.E924.format(name=self.name) + for example in islice(get_examples(), 10): + doc_sample.append(example.x) + gold_tags = example.get_aligned("SENT_START") + gold_array = [[1.0 if tag == gold_tag else 0.0 for tag in self.labels] for gold_tag in gold_tags] + label_sample.append(self.model.ops.asarray(gold_array, dtype="float32")) + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample, Y=label_sample) + + def add_label(self, label, values=None): + raise NotImplementedError + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_senter": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_senter + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/span_finder.py b/spacy/pipeline/span_finder.py new file mode 100644 index 0000000..7ee19de --- /dev/null +++ b/spacy/pipeline/span_finder.py @@ -0,0 +1,286 @@ +import importlib +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +from thinc.api import Config, Model, Optimizer, set_dropout_rate +from thinc.types import Floats2d + +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..tokens import Doc, Span +from ..training import Example +from .spancat import DEFAULT_SPANS_KEY +from .trainable_pipe import TrainablePipe + +span_finder_default_config = """ +[model] +@architectures = "spacy.SpanFinder.v1" + +[model.scorer] +@layers = "spacy.LinearLogistic.v1" +nO = 2 + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 96 +rows = [5000, 1000, 2500, 1000] +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 4 +""" + +DEFAULT_SPAN_FINDER_MODEL = Config().from_str(span_finder_default_config)["model"] + + +def make_span_finder_scorer(): + return span_finder_score + + +def span_finder_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + kwargs = dict(kwargs) + attr_prefix = "spans_" + key = kwargs["spans_key"] + kwargs.setdefault("attr", f"{attr_prefix}{key}") + kwargs.setdefault( + "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], []) + ) + kwargs.setdefault("has_annotation", lambda doc: key in doc.spans) + kwargs.setdefault("allow_overlap", True) + kwargs.setdefault("labeled", False) + scores = Scorer.score_spans(examples, **kwargs) + scores.pop(f"{kwargs['attr']}_per_type", None) + return scores + + +def _char_indices(span: Span) -> Tuple[int, int]: + start = span[0].idx + end = span[-1].idx + len(span[-1]) + return start, end + + +class SpanFinder(TrainablePipe): + """Pipeline that learns span boundaries. + + DOCS: https://spacy.io/api/spanfinder + """ + + def __init__( + self, + nlp: Language, + model: Model[Iterable[Doc], Floats2d], + name: str = "span_finder", + *, + spans_key: str = DEFAULT_SPANS_KEY, + threshold: float = 0.5, + max_length: Optional[int] = None, + min_length: Optional[int] = None, + scorer: Optional[Callable] = span_finder_score, + ) -> None: + """Initialize the span finder. + model (thinc.api.Model): The Thinc Model powering the pipeline + component. + name (str): The component instance name, used to add entries to the + losses during training. + threshold (float): Minimum probability to consider a prediction + positive. + scorer (Optional[Callable]): The scoring method. + spans_key (str): Key of the doc.spans dict to save the spans under. + During initialization and training, the component will look for + spans on the reference document under the same key. + max_length (Optional[int]): Maximum length of the produced spans, + defaults to None meaning unlimited length. + min_length (Optional[int]): Minimum length of the produced spans, + defaults to None meaning shortest span length is 1. + + DOCS: https://spacy.io/api/spanfinder#init + """ + self.vocab = nlp.vocab + if (max_length is not None and max_length < 1) or ( + min_length is not None and min_length < 1 + ): + raise ValueError( + Errors.E1053.format(min_length=min_length, max_length=max_length) + ) + self.model = model + self.name = name + self.scorer = scorer + self.cfg: Dict[str, Any] = { + "min_length": min_length, + "max_length": max_length, + "threshold": threshold, + "spans_key": spans_key, + } + + def predict(self, docs: Iterable[Doc]): + """Apply the pipeline's model to a batch of docs, without modifying + them. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: The models prediction for each document. + + DOCS: https://spacy.io/api/spanfinder#predict + """ + scores = self.model.predict(docs) + return scores + + def set_annotations(self, docs: Iterable[Doc], scores: Floats2d) -> None: + """Modify a batch of Doc objects, using pre-computed scores. + docs (Iterable[Doc]): The documents to modify. + scores: The scores to set, produced by SpanFinder predict method. + + DOCS: https://spacy.io/api/spanfinder#set_annotations + """ + offset = 0 + for i, doc in enumerate(docs): + doc.spans[self.cfg["spans_key"]] = [] + starts = [] + ends = [] + doc_scores = scores[offset : offset + len(doc)] + + for token, token_score in zip(doc, doc_scores): + if token_score[0] >= self.cfg["threshold"]: + starts.append(token.i) + if token_score[1] >= self.cfg["threshold"]: + ends.append(token.i) + + for start in starts: + for end in ends: + span_length = end + 1 - start + if span_length < 1: + continue + if ( + self.cfg["min_length"] is None + or self.cfg["min_length"] <= span_length + ) and ( + self.cfg["max_length"] is None + or span_length <= self.cfg["max_length"] + ): + doc.spans[self.cfg["spans_key"]].append(doc[start : end + 1]) + offset += len(doc) + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (Optional[thinc.api.Optimizer]): The optimizer. + losses (Optional[Dict[str, float]]): Optional record of the loss during + training. Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/spanfinder#update + """ + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + predicted = [eg.predicted for eg in examples] + set_dropout_rate(self.model, drop) + scores, backprop_scores = self.model.begin_update(predicted) + loss, d_scores = self.get_loss(examples, scores) + backprop_scores(d_scores) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def get_loss(self, examples, scores) -> Tuple[float, Floats2d]: + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, Floats2d]): The loss and the gradient. + + DOCS: https://spacy.io/api/spanfinder#get_loss + """ + truths, masks = self._get_aligned_truth_scores(examples, self.model.ops) + d_scores = scores - self.model.ops.asarray2f(truths) + d_scores *= masks + loss = float((d_scores**2).sum()) + return loss, d_scores + + def _get_aligned_truth_scores(self, examples, ops) -> Tuple[Floats2d, Floats2d]: + """Align scores of the predictions to the references for calculating + the loss. + """ + truths = [] + masks = [] + for eg in examples: + if eg.x.text != eg.y.text: + raise ValueError(Errors.E1054.format(component="span_finder")) + n_tokens = len(eg.predicted) + truth = ops.xp.zeros((n_tokens, 2), dtype="float32") + mask = ops.xp.ones((n_tokens, 2), dtype="float32") + if self.cfg["spans_key"] in eg.reference.spans: + for span in eg.reference.spans[self.cfg["spans_key"]]: + ref_start_char, ref_end_char = _char_indices(span) + pred_span = eg.predicted.char_span( + ref_start_char, ref_end_char, alignment_mode="expand" + ) + pred_start_char, pred_end_char = _char_indices(pred_span) + start_match = pred_start_char == ref_start_char + end_match = pred_end_char == ref_end_char + if start_match: + truth[pred_span[0].i, 0] = 1 + else: + mask[pred_span[0].i, 0] = 0 + if end_match: + truth[pred_span[-1].i, 1] = 1 + else: + mask[pred_span[-1].i, 1] = 0 + truths.append(truth) + masks.append(mask) + truths = ops.xp.concatenate(truths, axis=0) + masks = ops.xp.concatenate(masks, axis=0) + return truths, masks + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + ) -> None: + """Initialize the pipe for training, using a representative set + of data examples. + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Optional[Language]): The current nlp object the component is part + of. + + DOCS: https://spacy.io/api/spanfinder#initialize + """ + subbatch: List[Example] = [] + + for eg in get_examples(): + if len(subbatch) < 10: + subbatch.append(eg) + + if subbatch: + docs = [eg.reference for eg in subbatch] + Y, _ = self._get_aligned_truth_scores(subbatch, self.model.ops) + self.model.initialize(X=docs, Y=Y) + else: + self.model.initialize() + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_span_finder": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_span_finder + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py new file mode 100644 index 0000000..703eda6 --- /dev/null +++ b/spacy/pipeline/span_ruler.py @@ -0,0 +1,507 @@ +import importlib +import warnings +from functools import partial +from pathlib import Path +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, +) + +import srsly + +from .. import util +from ..errors import Errors, Warnings +from ..language import Language +from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare +from ..scorer import Scorer +from ..tokens import Doc, Span +from ..training import Example +from ..util import SimpleFrozenList, ensure_path +from .pipe import Pipe + +PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] +DEFAULT_SPANS_KEY = "ruler" + + +def prioritize_new_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by allowing + spans to overwrite any entities that they overlap with. Intended to + replicate the overwrite_ents=True behavior from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + entities = [e for e in entities if not (e.start < end and e.end > start)] + seen_tokens.update(range(start, end)) + return entities + new_entities + + +def make_prioritize_new_ents_filter(): + return prioritize_new_ents_filter + + +def prioritize_existing_ents_filter( + entities: Iterable[Span], spans: Iterable[Span] +) -> List[Span]: + """Merge entities and spans into one list without overlaps by prioritizing + existing entities. Intended to replicate the overwrite_ents=False behavior + from the EntityRuler. + + entities (Iterable[Span]): The entities, already filtered for overlaps. + spans (Iterable[Span]): The spans to merge, may contain overlaps. + RETURNS (List[Span]): Filtered list of non-overlapping spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + spans = sorted(spans, key=get_sort_key, reverse=True) + entities = list(entities) + new_entities = [] + seen_tokens: Set[int] = set() + seen_tokens.update(*(range(ent.start, ent.end) for ent in entities)) + for span in spans: + start = span.start + end = span.end + if all(token.i not in seen_tokens for token in span): + new_entities.append(span) + seen_tokens.update(range(start, end)) + return entities + new_entities + + +def make_preserve_existing_ents_filter(): + return prioritize_existing_ents_filter + + +def overlapping_labeled_spans_score( + examples: Iterable[Example], *, spans_key=DEFAULT_SPANS_KEY, **kwargs +) -> Dict[str, Any]: + kwargs = dict(kwargs) + attr_prefix = f"spans_" + kwargs.setdefault("attr", f"{attr_prefix}{spans_key}") + kwargs.setdefault("allow_overlap", True) + kwargs.setdefault("labeled", True) + kwargs.setdefault( + "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], []) + ) + kwargs.setdefault("has_annotation", lambda doc: spans_key in doc.spans) + return Scorer.score_spans(examples, **kwargs) + + +def make_overlapping_labeled_spans_scorer(spans_key: str = DEFAULT_SPANS_KEY): + return partial(overlapping_labeled_spans_score, spans_key=spans_key) + + +class SpanRuler(Pipe): + """The SpanRuler lets you add spans to the `Doc.spans` using token-based + rules or exact phrase matches. + + DOCS: https://spacy.io/api/spanruler + USAGE: https://spacy.io/usage/rule-based-matching#spanruler + """ + + def __init__( + self, + nlp: Language, + name: str = "span_ruler", + *, + spans_key: Optional[str] = DEFAULT_SPANS_KEY, + spans_filter: Optional[ + Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]] + ] = None, + annotate_ents: bool = False, + ents_filter: Callable[ + [Iterable[Span], Iterable[Span]], Iterable[Span] + ] = util.filter_chain_spans, + phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, + validate: bool = False, + overwrite: bool = False, + scorer: Optional[Callable] = partial( + overlapping_labeled_spans_score, spans_key=DEFAULT_SPANS_KEY + ), + ) -> None: + """Initialize the span ruler. If patterns are supplied here, they + need to be a list of dictionaries with a `"label"` and `"pattern"` + key. A pattern can either be a token pattern (list) or a phrase pattern + (string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`. + + nlp (Language): The shared nlp object to pass the vocab to the matchers + and process phrase patterns. + name (str): Instance name of the current pipeline component. Typically + passed in automatically from the factory when the component is + added. Used to disable the current span ruler while creating + phrase patterns with the nlp object. + spans_key (Optional[str]): The spans key to save the spans under. If + `None`, no spans are saved. Defaults to "ruler". + spans_filter (Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The optional method to filter spans before they are assigned to + doc.spans. Defaults to `None`. + annotate_ents (bool): Whether to save spans to doc.ents. Defaults to + `False`. + ents_filter (Callable[[Iterable[Span], Iterable[Span]], List[Span]]): + The method to filter spans before they are assigned to doc.ents. + Defaults to `util.filter_chain_spans`. + phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to + match on, passed to the internal PhraseMatcher as `attr`. Defaults + to `None`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. + validate (bool): Whether patterns should be validated, passed to + Matcher and PhraseMatcher as `validate`. + overwrite (bool): Whether to remove any existing spans under this spans + key if `spans_key` is set, and/or to remove any ents under `doc.ents` if + `annotate_ents` is set. Defaults to `True`. + scorer (Optional[Callable]): The scoring method. Defaults to + spacy.pipeline.span_ruler.overlapping_labeled_spans_score. + + DOCS: https://spacy.io/api/spanruler#init + """ + self.nlp = nlp + self.name = name + self.spans_key = spans_key + self.annotate_ents = annotate_ents + self.phrase_matcher_attr = phrase_matcher_attr + self.validate = validate + self.overwrite = overwrite + self.spans_filter = spans_filter + self.ents_filter = ents_filter + self.scorer = scorer + self.matcher_fuzzy_compare = matcher_fuzzy_compare + self._match_label_id_map: Dict[int, Dict[str, str]] = {} + self.clear() + + def __len__(self) -> int: + """The number of all labels added to the span ruler.""" + return len(self._patterns) + + def __contains__(self, label: str) -> bool: + """Whether a label is present in the patterns.""" + for label_id in self._match_label_id_map.values(): + if label_id["label"] == label: + return True + return False + + @property + def key(self) -> Optional[str]: + """Key of the doc.spans dict to save the spans under.""" + return self.spans_key + + def __call__(self, doc: Doc) -> Doc: + """Find matches in document and add them as entities. + + doc (Doc): The Doc object in the pipeline. + RETURNS (Doc): The Doc with added entities, if available. + + DOCS: https://spacy.io/api/spanruler#call + """ + error_handler = self.get_error_handler() + try: + matches = self.match(doc) + self.set_annotations(doc, matches) + return doc + except Exception as e: + return error_handler(self.name, self, [doc], e) + + def match(self, doc: Doc): + self._require_patterns() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="\\[W036") + matches = cast( + List[Tuple[int, int, int]], + list(self.matcher(doc)) + list(self.phrase_matcher(doc)), + ) + deduplicated_matches = set( + Span( + doc, + start, + end, + label=self._match_label_id_map[m_id]["label"], + span_id=self._match_label_id_map[m_id]["id"], + ) + for m_id, start, end in matches + if start != end + ) + return sorted(list(deduplicated_matches)) + + def set_annotations(self, doc, matches): + """Modify the document in place""" + # set doc.spans if spans_key is set + if self.key: + spans = [] + if self.key in doc.spans and not self.overwrite: + spans = doc.spans[self.key] + spans.extend( + self.spans_filter(spans, matches) if self.spans_filter else matches + ) + doc.spans[self.key] = spans + # set doc.ents if annotate_ents is set + if self.annotate_ents: + spans = [] + if not self.overwrite: + spans = list(doc.ents) + spans = self.ents_filter(spans, matches) + try: + doc.ents = sorted(spans) + except ValueError: + raise ValueError(Errors.E854) + + @property + def labels(self) -> Tuple[str, ...]: + """All labels present in the match patterns. + + RETURNS (set): The string labels. + + DOCS: https://spacy.io/api/spanruler#labels + """ + return tuple(sorted(set([cast(str, p["label"]) for p in self._patterns]))) + + @property + def ids(self) -> Tuple[str, ...]: + """All IDs present in the match patterns. + + RETURNS (set): The string IDs. + + DOCS: https://spacy.io/api/spanruler#ids + """ + return tuple( + sorted(set([cast(str, p.get("id")) for p in self._patterns]) - set([None])) + ) + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + patterns: Optional[Sequence[PatternType]] = None, + ): + """Initialize the pipe for training. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + patterns (Optional[Iterable[PatternType]]): The list of patterns. + + DOCS: https://spacy.io/api/spanruler#initialize + """ + self.clear() + if patterns: + self.add_patterns(patterns) # type: ignore[arg-type] + + @property + def patterns(self) -> List[PatternType]: + """Get all patterns that were added to the span ruler. + + RETURNS (list): The original patterns, one dictionary per pattern. + + DOCS: https://spacy.io/api/spanruler#patterns + """ + return self._patterns + + def add_patterns(self, patterns: List[PatternType]) -> None: + """Add patterns to the span ruler. A pattern can either be a token + pattern (list of dicts) or a phrase pattern (string). For example: + {'label': 'ORG', 'pattern': 'Apple'} + {'label': 'ORG', 'pattern': 'Apple', 'id': 'apple'} + {'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]} + + patterns (list): The patterns to add. + + DOCS: https://spacy.io/api/spanruler#add_patterns + """ + + # disable the nlp components after this one in case they haven't been + # initialized / deserialized yet + try: + current_index = -1 + for i, (name, pipe) in enumerate(self.nlp.pipeline): + if self == pipe: + current_index = i + break + subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]] + except ValueError: + subsequent_pipes = [] + with self.nlp.select_pipes(disable=subsequent_pipes): + phrase_pattern_labels = [] + phrase_pattern_texts = [] + for entry in patterns: + p_label = cast(str, entry["label"]) + p_id = cast(str, entry.get("id", "")) + label = repr((p_label, p_id)) + self._match_label_id_map[self.nlp.vocab.strings.as_int(label)] = { + "label": p_label, + "id": p_id, + } + if isinstance(entry["pattern"], str): + phrase_pattern_labels.append(label) + phrase_pattern_texts.append(entry["pattern"]) + elif isinstance(entry["pattern"], list): + self.matcher.add(label, [entry["pattern"]]) + else: + raise ValueError(Errors.E097.format(pattern=entry["pattern"])) + self._patterns.append(entry) + for label, pattern in zip( + phrase_pattern_labels, + self.nlp.pipe(phrase_pattern_texts), + ): + self.phrase_matcher.add(label, [pattern]) + + def clear(self) -> None: + """Reset all patterns. + + RETURNS: None + DOCS: https://spacy.io/api/spanruler#clear + """ + self._patterns: List[PatternType] = [] + self.matcher: Matcher = Matcher( + self.nlp.vocab, + validate=self.validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) + self.phrase_matcher: PhraseMatcher = PhraseMatcher( + self.nlp.vocab, + attr=self.phrase_matcher_attr, + validate=self.validate, + ) + + def remove(self, label: str) -> None: + """Remove a pattern by its label. + + label (str): Label of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove + """ + if label not in self: + raise ValueError( + Errors.E1024.format(attr_type="label", label=label, component=self.name) + ) + self._patterns = [p for p in self._patterns if p["label"] != label] + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["label"] == label: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def remove_by_id(self, pattern_id: str) -> None: + """Remove a pattern by its pattern ID. + + pattern_id (str): ID of the pattern to be removed. + RETURNS: None + DOCS: https://spacy.io/api/spanruler#remove_by_id + """ + orig_len = len(self) + self._patterns = [p for p in self._patterns if p.get("id") != pattern_id] + if orig_len == len(self): + raise ValueError( + Errors.E1024.format( + attr_type="ID", label=pattern_id, component=self.name + ) + ) + for m_label in self._match_label_id_map: + if self._match_label_id_map[m_label]["id"] == pattern_id: + m_label_str = self.nlp.vocab.strings.as_string(m_label) + if m_label_str in self.phrase_matcher: + self.phrase_matcher.remove(m_label_str) + if m_label_str in self.matcher: + self.matcher.remove(m_label_str) + + def _require_patterns(self) -> None: + """Raise a warning if this component has no patterns defined.""" + if len(self) == 0: + warnings.warn(Warnings.W036.format(name=self.name)) + + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a bytestring. + + bytes_data (bytes): The bytestring to load. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_bytes + """ + self.clear() + deserializers = { + "patterns": lambda b: self.add_patterns(srsly.json_loads(b)), + } + util.from_bytes(bytes_data, deserializers, exclude) + return self + + def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes: + """Serialize the span ruler to a bytestring. + + RETURNS (bytes): The serialized patterns. + + DOCS: https://spacy.io/api/spanruler#to_bytes + """ + serializers = { + "patterns": lambda: srsly.json_dumps(self.patterns), + } + return util.to_bytes(serializers, exclude) + + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> "SpanRuler": + """Load the span ruler from a directory. + + path (Union[str, Path]): A path to a directory. + RETURNS (SpanRuler): The loaded span ruler. + + DOCS: https://spacy.io/api/spanruler#from_disk + """ + self.clear() + path = ensure_path(path) + deserializers = { + "patterns": lambda p: self.add_patterns(srsly.read_jsonl(p)), + } + util.from_disk(path, deserializers, {}) + return self + + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList() + ) -> None: + """Save the span ruler patterns to a directory. + + path (Union[str, Path]): A path to a directory. + + DOCS: https://spacy.io/api/spanruler#to_disk + """ + path = ensure_path(path) + serializers = { + "patterns": lambda p: srsly.write_jsonl(p, self.patterns), + } + util.to_disk(path, serializers, {}) + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_span_ruler": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_span_ruler + elif name == "make_entity_ruler": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_future_entity_ruler + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py new file mode 100644 index 0000000..9b945df --- /dev/null +++ b/spacy/pipeline/spancat.py @@ -0,0 +1,663 @@ +import importlib +from dataclasses import dataclass +from functools import partial +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union, cast + +import numpy +from thinc.api import Config, Model, Ops, Optimizer, get_current_ops, set_dropout_rate +from thinc.types import Floats2d, Ints1d, Ints2d, Ragged + +from ..compat import Protocol, runtime_checkable +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..tokens import Doc, Span, SpanGroup +from ..training import Example, validate_examples +from ..vocab import Vocab +from .trainable_pipe import TrainablePipe + +spancat_default_config = """ +[model] +@architectures = "spacy.SpanCategorizer.v1" +scorer = {"@layers": "spacy.LinearLogistic.v1"} + +[model.reducer] +@layers = spacy.mean_max_reducer.v1 +hidden_size = 128 + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 96 +rows = [5000, 1000, 2500, 1000] +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 4 +""" + +spancat_singlelabel_default_config = """ +[model] +@architectures = "spacy.SpanCategorizer.v1" +scorer = {"@layers": "Softmax.v2"} + +[model.reducer] +@layers = spacy.mean_max_reducer.v1 +hidden_size = 128 + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = 96 +rows = [5000, 1000, 2500, 1000] +attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 4 +""" + +DEFAULT_SPANS_KEY = "sc" +DEFAULT_SPANCAT_MODEL = Config().from_str(spancat_default_config)["model"] +DEFAULT_SPANCAT_SINGLELABEL_MODEL = Config().from_str( + spancat_singlelabel_default_config +)["model"] + + +@runtime_checkable +class Suggester(Protocol): + def __call__(self, docs: Iterable[Doc], *, ops: Optional[Ops] = None) -> Ragged: ... + + +def ngram_suggester( + docs: Iterable[Doc], sizes: List[int], *, ops: Optional[Ops] = None +) -> Ragged: + if ops is None: + ops = get_current_ops() + spans = [] + lengths = [] + for doc in docs: + starts = ops.xp.arange(len(doc), dtype="i") + starts = starts.reshape((-1, 1)) + length = 0 + for size in sizes: + if size <= len(doc): + starts_size = starts[: len(doc) - (size - 1)] + spans.append(ops.xp.hstack((starts_size, starts_size + size))) + length += spans[-1].shape[0] + if spans: + assert spans[-1].ndim == 2, spans[-1].shape + lengths.append(length) + lengths_array = ops.asarray1i(lengths) + if len(spans) > 0: + output = Ragged(ops.xp.vstack(spans), lengths_array) + else: + output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) + + assert output.dataXd.ndim == 2 + return output + + +def preset_spans_suggester( + docs: Iterable[Doc], spans_key: str, *, ops: Optional[Ops] = None +) -> Ragged: + if ops is None: + ops = get_current_ops() + spans = [] + lengths = [] + for doc in docs: + length = 0 + if doc.spans[spans_key]: + for span in doc.spans[spans_key]: + spans.append([span.start, span.end]) + length += 1 + + lengths.append(length) + lengths_array = cast(Ints1d, ops.asarray(lengths, dtype="i")) + if len(spans) > 0: + output = Ragged(ops.asarray(spans, dtype="i"), lengths_array) + else: + output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) + return output + + +def build_ngram_suggester(sizes: List[int]) -> Suggester: + """Suggest all spans of the given lengths. Spans are returned as a ragged + array of integers. The array has two columns, indicating the start and end + position.""" + + return partial(ngram_suggester, sizes=sizes) + + +def build_ngram_range_suggester(min_size: int, max_size: int) -> Suggester: + """Suggest all spans of the given lengths between a given min and max value - both inclusive. + Spans are returned as a ragged array of integers. The array has two columns, + indicating the start and end position.""" + sizes = list(range(min_size, max_size + 1)) + return build_ngram_suggester(sizes) + + +def build_preset_spans_suggester(spans_key: str) -> Suggester: + """Suggest all spans that are already stored in doc.spans[spans_key]. + This is useful when an upstream component is used to set the spans + on the Doc such as a SpanRuler or SpanFinder.""" + return partial(preset_spans_suggester, spans_key=spans_key) + + +def spancat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + kwargs = dict(kwargs) + attr_prefix = "spans_" + key = kwargs["spans_key"] + kwargs.setdefault("attr", f"{attr_prefix}{key}") + kwargs.setdefault("allow_overlap", True) + kwargs.setdefault( + "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], []) + ) + kwargs.setdefault("has_annotation", lambda doc: key in doc.spans) + return Scorer.score_spans(examples, **kwargs) + + +def make_spancat_scorer(): + return spancat_score + + +@dataclass +class _Intervals: + """ + Helper class to avoid storing overlapping spans. + """ + + def __init__(self): + self.ranges = set() + + def add(self, i, j): + for e in range(i, j): + self.ranges.add(e) + + def __contains__(self, rang): + i, j = rang + for e in range(i, j): + if e in self.ranges: + return True + return False + + +class SpanCategorizer(TrainablePipe): + """Pipeline component to label spans of text. + + DOCS: https://spacy.io/api/spancategorizer + """ + + def __init__( + self, + vocab: Vocab, + model: Model[Tuple[List[Doc], Ragged], Floats2d], + suggester: Suggester, + name: str = "spancat", + *, + add_negative_label: bool = False, + spans_key: str = "spans", + negative_weight: Optional[float] = 1.0, + allow_overlap: Optional[bool] = True, + max_positive: Optional[int] = None, + threshold: Optional[float] = 0.5, + scorer: Optional[Callable] = spancat_score, + ) -> None: + """Initialize the multi-label or multi-class span categorizer. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + For multi-class classification (single label per span) we recommend + using a Softmax classifier as a the final layer, while for multi-label + classification (multiple possible labels per span) we recommend Logistic. + suggester (Callable[[Iterable[Doc], Optional[Ops]], Ragged]): A function that suggests spans. + Spans are returned as a ragged array with two integer columns, for the + start and end positions. + name (str): The component instance name, used to add entries to the + losses during training. + spans_key (str): Key of the Doc.spans dict to save the spans under. + During initialization and training, the component will look for + spans on the reference document under the same key. Defaults to + `"spans"`. + add_negative_label (bool): Learn to predict a special 'negative_label' + when a Span is not annotated. + threshold (Optional[float]): Minimum probability to consider a prediction + positive. Defaults to 0.5. Spans with a positive prediction will be saved + on the Doc. + max_positive (Optional[int]): Maximum number of labels to consider + positive per span. Defaults to None, indicating no limit. + negative_weight (float): Multiplier for the loss terms. + Can be used to downweight the negative samples if there are too many + when add_negative_label is True. Otherwise its unused. + allow_overlap (bool): If True the data is assumed to contain overlapping spans. + Otherwise it produces non-overlapping spans greedily prioritizing + higher assigned label scores. Only used when max_positive is 1. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the Doc.spans[spans_key] with overlapping + spans allowed. + + DOCS: https://spacy.io/api/spancategorizer#init + """ + self.cfg = { + "labels": [], + "spans_key": spans_key, + "threshold": threshold, + "max_positive": max_positive, + "negative_weight": negative_weight, + "allow_overlap": allow_overlap, + } + self.vocab = vocab + self.suggester = suggester + self.model = model + self.name = name + self.scorer = scorer + self.add_negative_label = add_negative_label + if not allow_overlap and max_positive is not None and max_positive > 1: + raise ValueError(Errors.E1051.format(max_positive=max_positive)) + + @property + def key(self) -> str: + """Key of the doc.spans dict to save the spans under. During + initialization and training, the component will look for spans on the + reference document under the same key. + """ + return str(self.cfg["spans_key"]) + + def _allow_extra_label(self) -> None: + """Raise an error if the component can not add any more labels.""" + nO = None + if self.model.has_dim("nO"): + nO = self.model.get_dim("nO") + elif self.model.has_ref("output_layer") and self.model.get_ref( + "output_layer" + ).has_dim("nO"): + nO = self.model.get_ref("output_layer").get_dim("nO") + if nO is not None and nO == self._n_labels: + if not self.is_resizable: + raise ValueError( + Errors.E922.format(name=self.name, nO=self.model.get_dim("nO")) + ) + + def add_label(self, label: str) -> int: + """Add a new label to the pipe. + + label (str): The label to add. + RETURNS (int): 0 if label is already present, otherwise 1. + + DOCS: https://spacy.io/api/spancategorizer#add_label + """ + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + self._allow_extra_label() + self.cfg["labels"].append(label) # type: ignore + self.vocab.strings.add(label) + return 1 + + @property + def labels(self) -> Tuple[str]: + """RETURNS (Tuple[str]): The labels currently added to the component. + + DOCS: https://spacy.io/api/spancategorizer#labels + """ + return tuple(self.cfg["labels"]) # type: ignore + + @property + def label_data(self) -> List[str]: + """RETURNS (List[str]): Information about the component's labels. + + DOCS: https://spacy.io/api/spancategorizer#label_data + """ + return list(self.labels) + + @property + def _label_map(self) -> Dict[str, int]: + """RETURNS (Dict[str, int]): The label map.""" + return {label: i for i, label in enumerate(self.labels)} + + @property + def _n_labels(self) -> int: + """RETURNS (int): Number of labels.""" + if self.add_negative_label: + return len(self.labels) + 1 + else: + return len(self.labels) + + @property + def _negative_label_i(self) -> Union[int, None]: + """RETURNS (Union[int, None]): Index of the negative label.""" + if self.add_negative_label: + return len(self.label_data) + else: + return None + + def predict(self, docs: Iterable[Doc]): + """Apply the pipeline's model to a batch of docs, without modifying them. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: The models prediction for each document. + + DOCS: https://spacy.io/api/spancategorizer#predict + """ + indices = self.suggester(docs, ops=self.model.ops) + if indices.lengths.sum() == 0: + scores = self.model.ops.alloc2f(0, 0) + else: + scores = self.model.predict((docs, indices)) # type: ignore + return indices, scores + + def set_candidates( + self, docs: Iterable[Doc], *, candidates_key: str = "candidates" + ) -> None: + """Use the spancat suggester to add a list of span candidates to a list of docs. + This method is intended to be used for debugging purposes. + + docs (Iterable[Doc]): The documents to modify. + candidates_key (str): Key of the Doc.spans dict to save the candidate spans under. + + DOCS: https://spacy.io/api/spancategorizer#set_candidates + """ + suggester_output = self.suggester(docs, ops=self.model.ops) + + for candidates, doc in zip(suggester_output, docs): # type: ignore + doc.spans[candidates_key] = [] + for index in candidates.dataXd: + doc.spans[candidates_key].append(doc[index[0] : index[1]]) + + def set_annotations(self, docs: Iterable[Doc], indices_scores) -> None: + """Modify a batch of Doc objects, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + scores: The scores to set, produced by SpanCategorizer.predict. + + DOCS: https://spacy.io/api/spancategorizer#set_annotations + """ + indices, scores = indices_scores + offset = 0 + for i, doc in enumerate(docs): + indices_i = indices[i].dataXd + allow_overlap = cast(bool, self.cfg["allow_overlap"]) + if self.cfg["max_positive"] == 1: + doc.spans[self.key] = self._make_span_group_singlelabel( + doc, + indices_i, + scores[offset : offset + indices.lengths[i]], + allow_overlap, + ) + else: + doc.spans[self.key] = self._make_span_group_multilabel( + doc, + indices_i, + scores[offset : offset + indices.lengths[i]], + ) + offset += indices.lengths[i] + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/spancategorizer#update + """ + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + validate_examples(examples, "SpanCategorizer.update") + self._validate_categories(examples) + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return losses + docs = [eg.predicted for eg in examples] + spans = self.suggester(docs, ops=self.model.ops) + if spans.lengths.sum() == 0: + return losses + set_dropout_rate(self.model, drop) + scores, backprop_scores = self.model.begin_update((docs, spans)) + loss, d_scores = self.get_loss(examples, (spans, scores)) + backprop_scores(d_scores) # type: ignore + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def get_loss( + self, examples: Iterable[Example], spans_scores: Tuple[Ragged, Floats2d] + ) -> Tuple[float, float]: + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + spans_scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/spancategorizer#get_loss + """ + spans, scores = spans_scores + spans = Ragged( + self.model.ops.to_numpy(spans.data), self.model.ops.to_numpy(spans.lengths) + ) + target = numpy.zeros(scores.shape, dtype=scores.dtype) + if self.add_negative_label: + negative_spans = numpy.ones((scores.shape[0])) + offset = 0 + label_map = self._label_map + for i, eg in enumerate(examples): + # Map (start, end) offset of spans to the row in the d_scores array, + # so that we can adjust the gradient for predictions that were + # in the gold standard. + spans_index = {} + spans_i = spans[i].dataXd + for j in range(spans.lengths[i]): + start = int(spans_i[j, 0]) # type: ignore + end = int(spans_i[j, 1]) # type: ignore + spans_index[(start, end)] = offset + j + for gold_span in self._get_aligned_spans(eg): + key = (gold_span.start, gold_span.end) + if key in spans_index: + row = spans_index[key] + k = label_map[gold_span.label_] + target[row, k] = 1.0 + if self.add_negative_label: + # delete negative label target. + negative_spans[row] = 0.0 + # The target is a flat array for all docs. Track the position + # we're at within the flat array. + offset += spans.lengths[i] + target = self.model.ops.asarray(target, dtype="f") # type: ignore + if self.add_negative_label: + negative_samples = numpy.nonzero(negative_spans)[0] + target[negative_samples, self._negative_label_i] = 1.0 # type: ignore + # The target will have the values 0 (for untrue predictions) or 1 + # (for true predictions). + # The scores should be in the range [0, 1]. + # If the prediction is 0.9 and it's true, the gradient + # will be -0.1 (0.9 - 1.0). + # If the prediction is 0.9 and it's false, the gradient will be + # 0.9 (0.9 - 0.0) + d_scores = scores - target + if self.add_negative_label: + neg_weight = cast(float, self.cfg["negative_weight"]) + if neg_weight != 1.0: + d_scores[negative_samples] *= neg_weight + loss = float((d_scores**2).sum()) + return loss, d_scores + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + labels: Optional[List[str]] = None, + ) -> None: + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Optional[Language]): The current nlp object the component is part of. + labels (Optional[List[str]]): The labels to add to the component, typically generated by the + `init labels` command. If no labels are provided, the get_examples + callback is used to extract the labels from the data. + + DOCS: https://spacy.io/api/spancategorizer#initialize + """ + subbatch: List[Example] = [] + if labels is not None: + for label in labels: + self.add_label(label) + for eg in get_examples(): + if labels is None: + for span in eg.reference.spans.get(self.key, []): + self.add_label(span.label_) + if len(subbatch) < 10: + subbatch.append(eg) + self._require_labels() + if subbatch: + docs = [eg.x for eg in subbatch] + spans = build_ngram_suggester(sizes=[1])(docs) + Y = self.model.ops.alloc2f(spans.dataXd.shape[0], self._n_labels) + self.model.initialize(X=(docs, spans), Y=Y) + else: + self.model.initialize() + + def _validate_categories(self, examples: Iterable[Example]): + # TODO + pass + + def _get_aligned_spans(self, eg: Example): + return eg.get_aligned_spans_y2x( + eg.reference.spans.get(self.key, []), allow_overlap=True + ) + + def _make_span_group_multilabel( + self, + doc: Doc, + indices: Ints2d, + scores: Floats2d, + ) -> SpanGroup: + """Find the top-k labels for each span (k=max_positive).""" + spans = SpanGroup(doc, name=self.key) + if scores.size == 0: + return spans + scores = self.model.ops.to_numpy(scores) + indices = self.model.ops.to_numpy(indices) + threshold = self.cfg["threshold"] + max_positive = self.cfg["max_positive"] + + keeps = scores >= threshold + if max_positive is not None: + assert isinstance(max_positive, int) + if self.add_negative_label: + negative_scores = numpy.copy(scores[:, self._negative_label_i]) + scores[:, self._negative_label_i] = -numpy.inf + ranked = (scores * -1).argsort() # type: ignore + scores[:, self._negative_label_i] = negative_scores + else: + ranked = (scores * -1).argsort() # type: ignore + span_filter = ranked[:, max_positive:] + for i, row in enumerate(span_filter): + keeps[i, row] = False + + attrs_scores = [] + for i in range(indices.shape[0]): + start = indices[i, 0] + end = indices[i, 1] + for j, keep in enumerate(keeps[i]): + if keep: + if j != self._negative_label_i: + spans.append(Span(doc, start, end, label=self.labels[j])) + attrs_scores.append(scores[i, j]) + spans.attrs["scores"] = numpy.array(attrs_scores) + return spans + + def _make_span_group_singlelabel( + self, + doc: Doc, + indices: Ints2d, + scores: Floats2d, + allow_overlap: bool = True, + ) -> SpanGroup: + """Find the argmax label for each span.""" + # Handle cases when there are zero suggestions + if scores.size == 0: + return SpanGroup(doc, name=self.key) + scores = self.model.ops.to_numpy(scores) + indices = self.model.ops.to_numpy(indices) + predicted = scores.argmax(axis=1) + argmax_scores = numpy.take_along_axis( + scores, numpy.expand_dims(predicted, 1), axis=1 + ) + keeps = numpy.ones(predicted.shape, dtype=bool) + # Remove samples where the negative label is the argmax. + if self.add_negative_label: + keeps = numpy.logical_and(keeps, predicted != self._negative_label_i) + # Filter samples according to threshold. + threshold = self.cfg["threshold"] + if threshold is not None: + keeps = numpy.logical_and(keeps, (argmax_scores >= threshold).squeeze()) + # Sort spans according to argmax probability + if not allow_overlap: + # Get the probabilities + sort_idx = (argmax_scores.squeeze() * -1).argsort() + argmax_scores = argmax_scores[sort_idx] + predicted = predicted[sort_idx] + indices = indices[sort_idx] + keeps = keeps[sort_idx] + seen = _Intervals() + spans = SpanGroup(doc, name=self.key) + attrs_scores = [] + for i in range(indices.shape[0]): + if not keeps[i]: + continue + + label = predicted[i] + start = indices[i, 0] + end = indices[i, 1] + + if not allow_overlap: + if (start, end) in seen: + continue + else: + seen.add(start, end) + attrs_scores.append(argmax_scores[i]) + spans.append(Span(doc, start, end, label=self.labels[label])) + + spans.attrs["scores"] = numpy.array(attrs_scores) + return spans + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_spancat": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_spancat + elif name == "make_spancat_singlelabel": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_spancat_singlelabel + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/tagger.pyx b/spacy/pipeline/tagger.pyx new file mode 100644 index 0000000..f7a16e0 --- /dev/null +++ b/spacy/pipeline/tagger.pyx @@ -0,0 +1,303 @@ +# cython: infer_types=True, binding=True +import importlib +import sys +from itertools import islice +from typing import Callable, Optional + +import numpy +from thinc.api import Config, Model, SequenceCategoricalCrossentropy, set_dropout_rate + +from ..tokens.doc cimport Doc + +from .. import util +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..training import validate_examples, validate_get_examples +from ..util import registry +from .trainable_pipe import TrainablePipe + +# See #9050 +BACKWARD_OVERWRITE = False + +default_model_config = """ +[model] +@architectures = "spacy.Tagger.v2" + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_TAGGER_MODEL = Config().from_str(default_model_config)["model"] + + +def tagger_score(examples, **kwargs): + return Scorer.score_token_attr(examples, "tag", **kwargs) + + +def make_tagger_scorer(): + return tagger_score + + +class Tagger(TrainablePipe): + """Pipeline component for part-of-speech tagging. + + DOCS: https://spacy.io/api/tagger + """ + def __init__( + self, + vocab, + model, + name="tagger", + *, + overwrite=BACKWARD_OVERWRITE, + scorer=tagger_score, + neg_prefix="!", + label_smoothing=0.0, + ): + """Initialize a part-of-speech tagger. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_token_attr for the attribute "tag". + + DOCS: https://spacy.io/api/tagger#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + cfg = {"labels": [], "overwrite": overwrite, "neg_prefix": neg_prefix, "label_smoothing": label_smoothing} + self.cfg = dict(sorted(cfg.items())) + self.scorer = scorer + + @property + def labels(self): + """The labels currently added to the component. Note that even for a + blank component, this will always include the built-in coarse-grained + part-of-speech tags by default. + + RETURNS (Tuple[str]): The labels. + + DOCS: https://spacy.io/api/tagger#labels + """ + return tuple(self.cfg["labels"]) + + @property + def label_data(self): + """Data about the labels currently added to the component.""" + return tuple(self.cfg["labels"]) + + def predict(self, docs): + """Apply the pipeline's model to a batch of docs, without modifying them. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: The models prediction for each document. + + DOCS: https://spacy.io/api/tagger#predict + """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + n_labels = len(self.labels) + guesses = [self.model.ops.alloc((0, n_labels)) for doc in docs] + assert len(guesses) == len(docs) + return guesses + scores = self.model.predict(docs) + assert len(scores) == len(docs), (len(scores), len(docs)) + guesses = self._scores2guesses(scores) + assert len(guesses) == len(docs) + return guesses + + def _scores2guesses(self, scores): + guesses = [] + for doc_scores in scores: + doc_guesses = doc_scores.argmax(axis=1) + if not isinstance(doc_guesses, numpy.ndarray): + doc_guesses = doc_guesses.get() + guesses.append(doc_guesses) + return guesses + + def set_annotations(self, docs, batch_tag_ids): + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + batch_tag_ids: The IDs to set, produced by Tagger.predict. + + DOCS: https://spacy.io/api/tagger#set_annotations + """ + if isinstance(docs, Doc): + docs = [docs] + cdef Doc doc + cdef bint overwrite = self.cfg["overwrite"] + labels = self.labels + for i, doc in enumerate(docs): + doc_tag_ids = batch_tag_ids[i] + if hasattr(doc_tag_ids, "get"): + doc_tag_ids = doc_tag_ids.get() + for j, tag_id in enumerate(doc_tag_ids): + if doc.c[j].tag == 0 or overwrite: + doc.c[j].tag = self.vocab.strings[labels[tag_id]] + + def update(self, examples, *, drop=0., sgd=None, losses=None): + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/tagger#update + """ + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + validate_examples(examples, "Tagger.update") + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return losses + set_dropout_rate(self.model, drop) + tag_scores, bp_tag_scores = self.model.begin_update([eg.predicted for eg in examples]) + for sc in tag_scores: + if self.model.ops.xp.isnan(sc.sum()): + raise ValueError(Errors.E940) + loss, d_tag_scores = self.get_loss(examples, tag_scores) + bp_tag_scores(d_tag_scores) + if sgd not in (None, False): + self.finish_update(sgd) + + losses[self.name] += loss + return losses + + def rehearse(self, examples, *, drop=0., sgd=None, losses=None): + """Perform a "rehearsal" update from a batch of data. Rehearsal updates + teach the current model to make predictions similar to an initial model, + to try to address the "catastrophic forgetting" problem. This feature is + experimental. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/tagger#rehearse + """ + loss_func = SequenceCategoricalCrossentropy() + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + validate_examples(examples, "Tagger.rehearse") + docs = [eg.predicted for eg in examples] + if self._rehearsal_model is None: + return losses + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + return losses + set_dropout_rate(self.model, drop) + tag_scores, bp_tag_scores = self.model.begin_update(docs) + tutor_tag_scores, _ = self._rehearsal_model.begin_update(docs) + grads, loss = loss_func(tag_scores, tutor_tag_scores) + bp_tag_scores(grads) + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def get_loss(self, examples, scores): + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/tagger#get_loss + """ + validate_examples(examples, "Tagger.get_loss") + loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, neg_prefix=self.cfg["neg_prefix"], label_smoothing=self.cfg["label_smoothing"]) + # Convert empty tag "" to missing value None so that both misaligned + # tokens and tokens with missing annotation have the default missing + # value None. + truths = [] + for eg in examples: + eg_truths = [tag if tag is not "" else None for tag in eg.get_aligned("TAG", as_string=True)] + truths.append(eg_truths) + d_scores, loss = loss_func(scores, truths) + if self.model.ops.xp.isnan(loss): + raise ValueError(Errors.E910.format(name=self.name)) + return float(loss), d_scores + + def initialize(self, get_examples, *, nlp=None, labels=None): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects.. + nlp (Language): The current nlp object the component is part of. + labels: The labels to add to the component, typically generated by the + `init labels` command. If no labels are provided, the get_examples + callback is used to extract the labels from the data. + + DOCS: https://spacy.io/api/tagger#initialize + """ + validate_get_examples(get_examples, "Tagger.initialize") + util.check_lexeme_norms(self.vocab, "tagger") + if labels is not None: + for tag in labels: + self.add_label(tag) + else: + tags = set() + for example in get_examples(): + for token in example.y: + if token.tag_: + tags.add(token.tag_) + for tag in sorted(tags): + self.add_label(tag) + doc_sample = [] + label_sample = [] + for example in islice(get_examples(), 10): + doc_sample.append(example.x) + gold_tags = example.get_aligned("TAG", as_string=True) + gold_array = [[1.0 if tag == gold_tag else 0.0 for tag in self.labels] for gold_tag in gold_tags] + label_sample.append(self.model.ops.asarray(gold_array, dtype="float32")) + self._require_labels() + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample, Y=label_sample) + + def add_label(self, label): + """Add a new label to the pipe. + + label (str): The label to add. + RETURNS (int): 0 if label is already present, otherwise 1. + + DOCS: https://spacy.io/api/tagger#add_label + """ + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + self._allow_extra_label() + self.cfg["labels"].append(label) + self.vocab.strings.add(label) + return 1 + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_tagger": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_tagger + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py new file mode 100644 index 0000000..7b03c7e --- /dev/null +++ b/spacy/pipeline/textcat.py @@ -0,0 +1,380 @@ +import importlib +from itertools import islice +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +import numpy +from thinc.api import Config, Model, Optimizer, set_dropout_rate + +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..tokens import Doc +from ..training import Example, validate_examples, validate_get_examples +from ..vocab import Vocab +from .trainable_pipe import TrainablePipe + +single_label_default_config = """ +[model] +@architectures = "spacy.TextCatEnsemble.v2" + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 64 +rows = [2000, 2000, 500, 1000, 500] +attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 2 + +[model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false +""" +DEFAULT_SINGLE_TEXTCAT_MODEL = Config().from_str(single_label_default_config)["model"] + +single_label_bow_config = """ +[model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false +""" + +single_label_cnn_config = """ +[model] +@architectures = "spacy.TextCatReduce.v1" +exclusive_classes = true +use_reduce_first = false +use_reduce_last = false +use_reduce_max = false +use_reduce_mean = true + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" + + +def textcat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + return Scorer.score_cats( + examples, + "cats", + multi_label=False, + **kwargs, + ) + + +def make_textcat_scorer(): + return textcat_score + + +class TextCategorizer(TrainablePipe): + """Pipeline component for single-label text classification. + + DOCS: https://spacy.io/api/textcategorizer + """ + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "textcat", + *, + threshold: float, + scorer: Optional[Callable] = textcat_score, + ) -> None: + """Initialize a text categorizer for single-label classification. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + threshold (float): Unused, not needed for single-label (exclusive + classes) classification. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_cats for the attribute "cats". + + DOCS: https://spacy.io/api/textcategorizer#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + cfg: Dict[str, Any] = { + "labels": [], + "threshold": threshold, + "positive_label": None, + } + self.cfg = dict(cfg) + self.scorer = scorer + + @property + def support_missing_values(self): + # There are no missing values as the textcat should always + # predict exactly one label. All other labels are 0.0 + # Subclasses may override this property to change internal behaviour. + return False + + @property + def labels(self) -> Tuple[str]: + """RETURNS (Tuple[str]): The labels currently added to the component. + + DOCS: https://spacy.io/api/textcategorizer#labels + """ + return tuple(self.cfg["labels"]) # type: ignore[arg-type, return-value] + + @property + def label_data(self) -> List[str]: + """RETURNS (List[str]): Information about the component's labels. + + DOCS: https://spacy.io/api/textcategorizer#label_data + """ + return self.labels # type: ignore[return-value] + + def predict(self, docs: Iterable[Doc]): + """Apply the pipeline's model to a batch of docs, without modifying them. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: The models prediction for each document. + + DOCS: https://spacy.io/api/textcategorizer#predict + """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + tensors = [doc.tensor for doc in docs] + xp = self.model.ops.xp + scores = xp.zeros((len(list(docs)), len(self.labels))) + return scores + scores = self.model.predict(docs) + scores = self.model.ops.asarray(scores) + return scores + + def set_annotations(self, docs: Iterable[Doc], scores) -> None: + """Modify a batch of Doc objects, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + scores: The scores to set, produced by TextCategorizer.predict. + + DOCS: https://spacy.io/api/textcategorizer#set_annotations + """ + for i, doc in enumerate(docs): + for j, label in enumerate(self.labels): + doc.cats[label] = float(scores[i, j]) + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/textcategorizer#update + """ + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + validate_examples(examples, "TextCategorizer.update") + self._validate_categories(examples) + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return losses + set_dropout_rate(self.model, drop) + scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples]) + loss, d_scores = self.get_loss(examples, scores) + bp_scores(d_scores) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def rehearse( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: + """Perform a "rehearsal" update from a batch of data. Rehearsal updates + teach the current model to make predictions similar to an initial model, + to try to address the "catastrophic forgetting" problem. This feature is + experimental. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/textcategorizer#rehearse + """ + if losses is None: + losses = {} + losses.setdefault(self.name, 0.0) + if self._rehearsal_model is None: + return losses + validate_examples(examples, "TextCategorizer.rehearse") + self._validate_categories(examples) + docs = [eg.predicted for eg in examples] + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + return losses + set_dropout_rate(self.model, drop) + scores, bp_scores = self.model.begin_update(docs) + target, _ = self._rehearsal_model.begin_update(docs) + gradient = scores - target + bp_scores(gradient) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += (gradient**2).sum() + return losses + + def _examples_to_truth( + self, examples: Iterable[Example] + ) -> Tuple[numpy.ndarray, numpy.ndarray]: + nr_examples = len(list(examples)) + truths = numpy.zeros((nr_examples, len(self.labels)), dtype="f") + not_missing = numpy.ones((nr_examples, len(self.labels)), dtype="f") + for i, eg in enumerate(examples): + for j, label in enumerate(self.labels): + if label in eg.reference.cats: + truths[i, j] = eg.reference.cats[label] + elif self.support_missing_values: + not_missing[i, j] = 0.0 + truths = self.model.ops.asarray(truths) # type: ignore + return truths, not_missing # type: ignore + + def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]: + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/textcategorizer#get_loss + """ + validate_examples(examples, "TextCategorizer.get_loss") + self._validate_categories(examples) + truths, not_missing = self._examples_to_truth(examples) + not_missing = self.model.ops.asarray(not_missing) # type: ignore + d_scores = scores - truths + d_scores *= not_missing + mean_square_error = (d_scores**2).mean() + return float(mean_square_error), d_scores + + def add_label(self, label: str) -> int: + """Add a new label to the pipe. + + label (str): The label to add. + RETURNS (int): 0 if label is already present, otherwise 1. + + DOCS: https://spacy.io/api/textcategorizer#add_label + """ + if not isinstance(label, str): + raise ValueError(Errors.E187) + if label in self.labels: + return 0 + self._allow_extra_label() + self.cfg["labels"].append(label) # type: ignore[attr-defined] + if self.model and "resize_output" in self.model.attrs: + self.model = self.model.attrs["resize_output"](self.model, len(self.labels)) + self.vocab.strings.add(label) + return 1 + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + labels: Optional[Iterable[str]] = None, + positive_label: Optional[str] = None, + ) -> None: + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + labels (Optional[Iterable[str]]): The labels to add to the component, typically generated by the + `init labels` command. If no labels are provided, the get_examples + callback is used to extract the labels from the data. + positive_label (Optional[str]): The positive label for a binary task with exclusive classes, + `None` otherwise and by default. + + DOCS: https://spacy.io/api/textcategorizer#initialize + """ + validate_get_examples(get_examples, "TextCategorizer.initialize") + self._validate_categories(get_examples()) + if labels is None: + for example in get_examples(): + for cat in example.y.cats: + self.add_label(cat) + else: + for label in labels: + self.add_label(label) + if len(self.labels) < 2: + raise ValueError(Errors.E867) + if positive_label is not None: + if positive_label not in self.labels: + err = Errors.E920.format(pos_label=positive_label, labels=self.labels) + raise ValueError(err) + if len(self.labels) != 2: + err = Errors.E919.format(pos_label=positive_label, labels=self.labels) + raise ValueError(err) + self.cfg["positive_label"] = positive_label + subbatch = list(islice(get_examples(), 10)) + doc_sample = [eg.reference for eg in subbatch] + label_sample, _ = self._examples_to_truth(subbatch) + self._require_labels() + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample, Y=label_sample) + + def _validate_categories(self, examples: Iterable[Example]): + """Check whether the provided examples all have single-label cats annotations.""" + for ex in examples: + vals = list(ex.reference.cats.values()) + if vals.count(1.0) > 1: + raise ValueError(Errors.E895.format(value=ex.reference.cats)) + for val in vals: + if not (val == 1.0 or val == 0.0): + raise ValueError(Errors.E851.format(val=val)) + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_textcat": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_textcat + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py new file mode 100644 index 0000000..cc094bf --- /dev/null +++ b/spacy/pipeline/textcat_multilabel.py @@ -0,0 +1,176 @@ +import importlib +from itertools import islice +from typing import Any, Callable, Dict, Iterable, Optional + +from thinc.api import Config, Model + +from ..errors import Errors +from ..language import Language +from ..scorer import Scorer +from ..training import Example, validate_get_examples +from ..vocab import Vocab +from .textcat import TextCategorizer + +multi_label_default_config = """ +[model] +@architectures = "spacy.TextCatEnsemble.v2" + +[model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 64 +rows = [2000, 2000, 500, 1000, 500] +attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"] +include_static_vectors = false + +[model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 2 + +[model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = false +length = 262144 +ngram_size = 1 +no_output_layer = false +""" +DEFAULT_MULTI_TEXTCAT_MODEL = Config().from_str(multi_label_default_config)["model"] + +multi_label_bow_config = """ +[model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = false +ngram_size = 1 +no_output_layer = false +""" + +multi_label_cnn_config = """ +[model] +@architectures = "spacy.TextCatReduce.v1" +exclusive_classes = false +use_reduce_first = false +use_reduce_last = false +use_reduce_max = false +use_reduce_mean = true + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" + + +def textcat_multilabel_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + return Scorer.score_cats( + examples, + "cats", + multi_label=True, + **kwargs, + ) + + +def make_textcat_multilabel_scorer(): + return textcat_multilabel_score + + +class MultiLabel_TextCategorizer(TextCategorizer): + """Pipeline component for multi-label text classification. + + DOCS: https://spacy.io/api/textcategorizer + """ + + def __init__( + self, + vocab: Vocab, + model: Model, + name: str = "textcat_multilabel", + *, + threshold: float, + scorer: Optional[Callable] = textcat_multilabel_score, + ) -> None: + """Initialize a text categorizer for multi-label classification. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name, used to add entries to the + losses during training. + threshold (float): Cutoff to consider a prediction "positive". + scorer (Optional[Callable]): The scoring method. + + DOCS: https://spacy.io/api/textcategorizer#init + """ + self.vocab = vocab + self.model = model + self.name = name + self._rehearsal_model = None + cfg = {"labels": [], "threshold": threshold} + self.cfg = dict(cfg) + self.scorer = scorer + + @property + def support_missing_values(self): + return True + + def initialize( # type: ignore[override] + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + labels: Optional[Iterable[str]] = None, + ): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + labels: The labels to add to the component, typically generated by the + `init labels` command. If no labels are provided, the get_examples + callback is used to extract the labels from the data. + + DOCS: https://spacy.io/api/textcategorizer#initialize + """ + validate_get_examples(get_examples, "MultiLabel_TextCategorizer.initialize") + if labels is None: + for example in get_examples(): + for cat in example.y.cats: + self.add_label(cat) + else: + for label in labels: + self.add_label(label) + subbatch = list(islice(get_examples(), 10)) + self._validate_categories(subbatch) + + doc_sample = [eg.reference for eg in subbatch] + label_sample, _ = self._examples_to_truth(subbatch) + self._require_labels() + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + assert len(label_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample, Y=label_sample) + + def _validate_categories(self, examples: Iterable[Example]): + """This component allows any type of single- or multi-label annotations. + This method overwrites the more strict one from 'textcat'.""" + # check that annotation values are valid + for ex in examples: + for val in ex.reference.cats.values(): + if not (val == 1.0 or val == 0.0): + raise ValueError(Errors.E851.format(val=val)) + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_multilabel_textcat": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_multilabel_textcat + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/tok2vec.py b/spacy/pipeline/tok2vec.py new file mode 100644 index 0000000..4e2e5af --- /dev/null +++ b/spacy/pipeline/tok2vec.py @@ -0,0 +1,324 @@ +import importlib +from itertools import islice +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence + +from thinc.api import Config, Model, Optimizer, set_dropout_rate + +from ..errors import Errors +from ..language import Language +from ..tokens import Doc +from ..training import Example, validate_examples, validate_get_examples +from ..vocab import Vocab +from .trainable_pipe import TrainablePipe + +default_model_config = """ +[model] +@architectures = "spacy.HashEmbedCNN.v2" +pretrained_vectors = null +width = 96 +depth = 4 +embed_size = 2000 +window_size = 1 +maxout_pieces = 3 +subword_features = true +""" +DEFAULT_TOK2VEC_MODEL = Config().from_str(default_model_config)["model"] + + +class Tok2Vec(TrainablePipe): + """Apply a "token-to-vector" model and set its outputs in the doc.tensor + attribute. This is mostly useful to share a single subnetwork between multiple + components, e.g. to have one embedding and CNN network shared between a + parser, tagger and NER. + + In order to use the `Tok2Vec` predictions, subsequent components should use + the `Tok2VecListener` layer as the tok2vec subnetwork of their model. This + layer will read data from the `doc.tensor` attribute during prediction. + During training, the `Tok2Vec` component will save its prediction and backprop + callback for each batch, so that the subsequent components can backpropagate + to the shared weights. This implementation is used because it allows us to + avoid relying on object identity within the models to achieve the parameter + sharing. + """ + + def __init__(self, vocab: Vocab, model: Model, name: str = "tok2vec") -> None: + """Initialize a tok2vec component. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model[List[Doc], List[Floats2d]]): + The Thinc Model powering the pipeline component. It should take + a list of Doc objects as input, and output a list of 2d float arrays. + name (str): The component instance name. + + DOCS: https://spacy.io/api/tok2vec#init + """ + self.vocab = vocab + self.model = model + self.name = name + self.listener_map: Dict[str, List["Tok2VecListener"]] = {} + self.cfg: Dict[str, Any] = {} + + @property + def listeners(self) -> List["Tok2VecListener"]: + """RETURNS (List[Tok2VecListener]): The listener models listening to this + component. Usually internals. + """ + return [m for c in self.listening_components for m in self.listener_map[c]] + + @property + def listening_components(self) -> List[str]: + """RETURNS (List[str]): The downstream components listening to this + component. Usually internals. + """ + return list(self.listener_map.keys()) + + def add_listener(self, listener: "Tok2VecListener", component_name: str) -> None: + """Add a listener for a downstream component. Usually internals.""" + self.listener_map.setdefault(component_name, []) + if listener not in self.listener_map[component_name]: + self.listener_map[component_name].append(listener) + + def remove_listener(self, listener: "Tok2VecListener", component_name: str) -> bool: + """Remove a listener for a downstream component. Usually internals.""" + if component_name in self.listener_map: + if listener in self.listener_map[component_name]: + self.listener_map[component_name].remove(listener) + # If no listeners are left, remove entry + if not self.listener_map[component_name]: + del self.listener_map[component_name] + return True + return False + + def find_listeners(self, component) -> None: + """Walk over a model of a processing component, looking for layers that + are Tok2vecListener subclasses that have an upstream_name that matches + this component. Listeners can also set their upstream_name attribute to + the wildcard string '*' to match any `Tok2Vec`. + + You're unlikely to ever need multiple `Tok2Vec` components, so it's + fine to leave your listeners upstream_name on '*'. + """ + names = ("*", self.name) + if isinstance(getattr(component, "model", None), Model): + for node in component.model.walk(): + if isinstance(node, Tok2VecListener) and node.upstream_name in names: + self.add_listener(node, component.name) + + def predict(self, docs: Iterable[Doc]): + """Apply the pipeline's model to a batch of docs, without modifying them. + Returns a single tensor for a batch of documents. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: Vector representations for each token in the documents. + + DOCS: https://spacy.io/api/tok2vec#predict + """ + if not any(len(doc) for doc in docs): + # Handle cases where there are no tokens in any docs. + width = self.model.get_dim("nO") + return [self.model.ops.alloc((0, width)) for doc in docs] + tokvecs = self.model.predict(docs) + return tokvecs + + def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None: + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + tokvecses: The tensors to set, produced by Tok2Vec.predict. + + DOCS: https://spacy.io/api/tok2vec#set_annotations + """ + for doc, tokvecs in zip(docs, tokvecses): + assert tokvecs.shape[0] == len(doc) + doc.tensor = tokvecs + + def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ): + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/tok2vec#update + """ + if losses is None: + losses = {} + validate_examples(examples, "Tok2Vec.update") + docs = [eg.predicted for eg in examples] + set_dropout_rate(self.model, drop) + tokvecs, bp_tokvecs = self.model.begin_update(docs) + d_tokvecs = [self.model.ops.alloc2f(*t2v.shape) for t2v in tokvecs] + losses.setdefault(self.name, 0.0) + + def accumulate_gradient(one_d_tokvecs): + """Accumulate tok2vec loss and gradient. This is passed as a callback + to all but the last listener. Only the last one does the backprop. + """ + nonlocal d_tokvecs + for i in range(len(one_d_tokvecs)): + d_tokvecs[i] += one_d_tokvecs[i] + losses[self.name] += float((one_d_tokvecs[i] ** 2).sum()) + return [self.model.ops.alloc2f(*t2v.shape) for t2v in tokvecs] + + def backprop(one_d_tokvecs): + """Callback to actually do the backprop. Passed to last listener.""" + accumulate_gradient(one_d_tokvecs) + d_docs = bp_tokvecs(d_tokvecs) + if sgd is not None: + self.finish_update(sgd) + return d_docs + + batch_id = Tok2VecListener.get_batch_id(docs) + for listener in self.listeners[:-1]: + listener.receive(batch_id, tokvecs, accumulate_gradient) + if self.listeners: + self.listeners[-1].receive(batch_id, tokvecs, backprop) + return losses + + def get_loss(self, examples, scores) -> None: + pass + + def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Optional[Language] = None, + ): + """Initialize the pipe for training, using a representative set + of data examples. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + + DOCS: https://spacy.io/api/tok2vec#initialize + """ + validate_get_examples(get_examples, "Tok2Vec.initialize") + doc_sample = [] + for example in islice(get_examples(), 10): + doc_sample.append(example.x) + assert doc_sample, Errors.E923.format(name=self.name) + self.model.initialize(X=doc_sample) + + def add_label(self, label): + raise NotImplementedError + + +class Tok2VecListener(Model): + """A layer that gets fed its answers from an upstream connection, + for instance from a component earlier in the pipeline. + + The Tok2VecListener layer is used as a sublayer within a component such + as a parser, NER or text categorizer. Usually you'll have multiple listeners + connecting to a single upstream Tok2Vec component, that's earlier in the + pipeline. The Tok2VecListener layers act as proxies, passing the predictions + from the Tok2Vec component into downstream components, and communicating + gradients back upstream. + """ + + name = "tok2vec-listener" + + def __init__(self, upstream_name: str, width: int) -> None: + """ + upstream_name (str): A string to identify the 'upstream' Tok2Vec component + to communicate with. The upstream name should either be the wildcard + string '*', or the name of the `Tok2Vec` component. You'll almost + never have multiple upstream Tok2Vec components, so the wildcard + string will almost always be fine. + width (int): + The width of the vectors produced by the upstream tok2vec component. + """ + Model.__init__(self, name=self.name, forward=forward, dims={"nO": width}) + self.upstream_name = upstream_name + self._batch_id: Optional[int] = None + self._outputs = None + self._backprop = None + + @classmethod + def get_batch_id(cls, inputs: Iterable[Doc]) -> int: + """Calculate a content-sensitive hash of the batch of documents, to check + whether the next batch of documents is unexpected. + """ + return sum(sum(token.orth for token in doc) for doc in inputs) + + def receive(self, batch_id: int, outputs, backprop) -> None: + """Store a batch of training predictions and a backprop callback. The + predictions and callback are produced by the upstream Tok2Vec component, + and later will be used when the listener's component's model is called. + """ + self._batch_id = batch_id + self._outputs = outputs + self._backprop = backprop + + def verify_inputs(self, inputs) -> bool: + """Check that the batch of Doc objects matches the ones we have a + prediction for. + """ + if self._batch_id is None and self._outputs is None: + raise ValueError(Errors.E954) + else: + batch_id = self.get_batch_id(inputs) + if batch_id != self._batch_id: + raise ValueError(Errors.E953.format(id1=batch_id, id2=self._batch_id)) + else: + return True + + +def forward(model: Tok2VecListener, inputs, is_train: bool): + """Supply the outputs from the upstream Tok2Vec component.""" + if is_train: + # This might occur during training when the tok2vec layer is frozen / hasn't been updated. + # In that case, it should be set to "annotating" so we can retrieve the embeddings from the doc. + if model._batch_id is None: + outputs = [] + for doc in inputs: + if doc.tensor.size == 0: + raise ValueError(Errors.E203.format(name="tok2vec")) + else: + outputs.append(doc.tensor) + return outputs, _empty_backprop + else: + model.verify_inputs(inputs) + return model._outputs, model._backprop + else: + # This is pretty grim, but it's hard to do better :(. + # It's hard to avoid relying on the doc.tensor attribute, because the + # pipeline components can batch the data differently during prediction. + # That doesn't happen in update, where the nlp object works on batches + # of data. + # When the components batch differently, we don't receive a matching + # prediction from the upstream, so we can't predict. + outputs = [] + width = model.get_dim("nO") + for doc in inputs: + if doc.tensor.size == 0: + # But we do need to do *something* if the tensor hasn't been set. + # The compromise is to at least return data of the right shape, + # so the output is valid. + outputs.append(model.ops.alloc2f(len(doc), width)) + else: + outputs.append(doc.tensor) + return outputs, _empty_backprop + + +def _empty_backprop(dX): # for pickling + return [] + + +# Setup backwards compatibility hook for factories +def __getattr__(name): + if name == "make_tok2vec": + module = importlib.import_module("spacy.pipeline.factories") + return module.make_tok2vec + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/spacy/pipeline/trainable_pipe.pxd b/spacy/pipeline/trainable_pipe.pxd new file mode 100644 index 0000000..b1d2550 --- /dev/null +++ b/spacy/pipeline/trainable_pipe.pxd @@ -0,0 +1,9 @@ +from ..vocab cimport Vocab +from .pipe cimport Pipe + + +cdef class TrainablePipe(Pipe): + cdef public Vocab vocab + cdef public object model + cdef public object cfg + cdef public object scorer diff --git a/spacy/pipeline/trainable_pipe.pyx b/spacy/pipeline/trainable_pipe.pyx new file mode 100644 index 0000000..8f219b3 --- /dev/null +++ b/spacy/pipeline/trainable_pipe.pyx @@ -0,0 +1,344 @@ +# cython: infer_types=True, binding=True +from typing import Callable, Dict, Iterable, Iterator, Optional, Tuple + +import srsly +from thinc.api import Model, Optimizer, set_dropout_rate + +from ..tokens.doc cimport Doc + +from .. import util +from ..errors import Errors +from ..language import Language +from ..training import Example, validate_examples +from ..vocab import Vocab +from .pipe import Pipe, deserialize_config + + +cdef class TrainablePipe(Pipe): + """This class is a base class and not instantiated directly. Trainable + pipeline components like the EntityRecognizer or TextCategorizer inherit + from it and it defines the interface that components should follow to + function as trainable components in a spaCy pipeline. + + DOCS: https://spacy.io/api/pipe + """ + def __init__(self, vocab: Vocab, model: Model, name: str, **cfg): + """Initialize a pipeline component. + + vocab (Vocab): The shared vocabulary. + model (thinc.api.Model): The Thinc Model powering the pipeline component. + name (str): The component instance name. + **cfg: Additional settings and config parameters. + + DOCS: https://spacy.io/api/pipe#init + """ + self.vocab = vocab + self.model = model + self.name = name + self.cfg = dict(cfg) + + def __call__(self, Doc doc) -> Doc: + """Apply the pipe to one document. The document is modified in place, + and returned. This usually happens under the hood when the nlp object + is called on a text and all components are applied to the Doc. + + docs (Doc): The Doc to process. + RETURNS (Doc): The processed Doc. + + DOCS: https://spacy.io/api/pipe#call + """ + error_handler = self.get_error_handler() + try: + scores = self.predict([doc]) + self.set_annotations([doc], scores) + return doc + except Exception as e: + error_handler(self.name, self, [doc], e) + + def pipe(self, stream: Iterable[Doc], *, batch_size: int = 128) -> Iterator[Doc]: + """Apply the pipe to a stream of documents. This usually happens under + the hood when the nlp object is called on a text and all components are + applied to the Doc. + + stream (Iterable[Doc]): A stream of documents. + batch_size (int): The number of documents to buffer. + error_handler (Callable[[str, List[Doc], Exception], Any]): Function that + deals with a failing batch of documents. The default function just reraises + the exception. + YIELDS (Doc): Processed documents in order. + + DOCS: https://spacy.io/api/pipe#pipe + """ + error_handler = self.get_error_handler() + for docs in util.minibatch(stream, size=batch_size): + try: + scores = self.predict(docs) + self.set_annotations(docs, scores) + yield from docs + except Exception as e: + error_handler(self.name, self, docs, e) + + def predict(self, docs: Iterable[Doc]): + """Apply the pipeline's model to a batch of docs, without modifying them. + Returns a single tensor for a batch of documents. + + docs (Iterable[Doc]): The documents to predict. + RETURNS: Vector representations of the predictions. + + DOCS: https://spacy.io/api/pipe#predict + """ + raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="predict", name=self.name)) + + def set_annotations(self, docs: Iterable[Doc], scores): + """Modify a batch of documents, using pre-computed scores. + + docs (Iterable[Doc]): The documents to modify. + scores: The scores to assign. + + DOCS: https://spacy.io/api/pipe#set_annotations + """ + raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="set_annotations", name=self.name)) + + def update(self, + examples: Iterable["Example"], + *, + drop: float = 0.0, + sgd: Optimizer = None, + losses: Optional[Dict[str, float]] = None) -> Dict[str, float]: + """Learn from a batch of documents and gold-standard information, + updating the pipe's model. Delegates to predict and get_loss. + + examples (Iterable[Example]): A batch of Example objects. + drop (float): The dropout rate. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/pipe#update + """ + if losses is None: + losses = {} + if not hasattr(self, "model") or self.model in (None, True, False): + return losses + losses.setdefault(self.name, 0.0) + validate_examples(examples, "TrainablePipe.update") + if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples): + # Handle cases where there are no tokens in any docs. + return losses + set_dropout_rate(self.model, drop) + scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples]) + loss, d_scores = self.get_loss(examples, scores) + bp_scores(d_scores) + if sgd not in (None, False): + self.finish_update(sgd) + losses[self.name] += loss + return losses + + def rehearse(self, + examples: Iterable[Example], + *, + sgd: Optimizer = None, + losses: Dict[str, float] = None, + **config) -> Dict[str, float]: + """Perform a "rehearsal" update from a batch of data. Rehearsal updates + teach the current model to make predictions similar to an initial model, + to try to address the "catastrophic forgetting" problem. This feature is + experimental. + + examples (Iterable[Example]): A batch of Example objects. + sgd (thinc.api.Optimizer): The optimizer. + losses (Dict[str, float]): Optional record of the loss during training. + Updated using the component name as the key. + RETURNS (Dict[str, float]): The updated losses dictionary. + + DOCS: https://spacy.io/api/pipe#rehearse + """ + pass + + def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]: + """Find the loss and gradient of loss for the batch of documents and + their predicted scores. + + examples (Iterable[Examples]): The batch of examples. + scores: Scores representing the model's predictions. + RETURNS (Tuple[float, float]): The loss and the gradient. + + DOCS: https://spacy.io/api/pipe#get_loss + """ + raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="get_loss", name=self.name)) + + def create_optimizer(self) -> Optimizer: + """Create an optimizer for the pipeline component. + + RETURNS (thinc.api.Optimizer): The optimizer. + + DOCS: https://spacy.io/api/pipe#create_optimizer + """ + return util.create_default_optimizer() + + def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language = None): + """Initialize the pipe for training, using data examples if available. + This method needs to be implemented by each TrainablePipe component, + ensuring the internal model (if available) is initialized properly + using the provided sample of Example objects. + + get_examples (Callable[[], Iterable[Example]]): Function that + returns a representative sample of gold-standard Example objects. + nlp (Language): The current nlp object the component is part of. + + DOCS: https://spacy.io/api/pipe#initialize + """ + raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="initialize", name=self.name)) + + def add_label(self, label: str) -> int: + """Add an output label. + For TrainablePipe components, it is possible to + extend pretrained models with new labels, but care should be taken to + avoid the "catastrophic forgetting" problem. + + label (str): The label to add. + RETURNS (int): 0 if label is already present, otherwise 1. + + DOCS: https://spacy.io/api/pipe#add_label + """ + raise NotImplementedError(Errors.E931.format(parent="Pipe", method="add_label", name=self.name)) + + @property + def is_trainable(self) -> bool: + return True + + @property + def is_resizable(self) -> bool: + return getattr(self, "model", None) and "resize_output" in self.model.attrs + + def _allow_extra_label(self) -> None: + """Raise an error if the component can not add any more labels.""" + nO = None + if self.model.has_dim("nO"): + nO = self.model.get_dim("nO") + elif self.model.has_ref("output_layer") and self.model.get_ref("output_layer").has_dim("nO"): + nO = self.model.get_ref("output_layer").get_dim("nO") + if nO is not None and nO == len(self.labels): + if not self.is_resizable: + raise ValueError(Errors.E922.format(name=self.name, nO=self.model.get_dim("nO"))) + + def set_output(self, nO: int) -> None: + if self.is_resizable: + self.model.attrs["resize_output"](self.model, nO) + else: + raise NotImplementedError(Errors.E921) + + def use_params(self, params: dict): + """Modify the pipe's model, to use the given parameter values. At the + end of the context, the original parameters are restored. + + params (dict): The parameter values to use in the model. + + DOCS: https://spacy.io/api/pipe#use_params + """ + with self.model.use_params(params): + yield + + def finish_update(self, sgd: Optimizer) -> None: + """Update parameters using the current parameter gradients. + The Optimizer instance contains the functionality to perform + the stochastic gradient descent. + + sgd (thinc.api.Optimizer): The optimizer. + + DOCS: https://spacy.io/api/pipe#finish_update + """ + self.model.finish_update(sgd) + + def _validate_serialization_attrs(self): + """Check that the pipe implements the required attributes. If a subclass + implements a custom __init__ method but doesn't set these attributes, + they currently default to None, so we need to perform additonal checks. + """ + if not hasattr(self, "vocab") or self.vocab is None: + raise ValueError(Errors.E899.format(name=util.get_object_name(self))) + if not hasattr(self, "model") or self.model is None: + raise ValueError(Errors.E898.format(name=util.get_object_name(self))) + + def to_bytes(self, *, exclude=tuple()): + """Serialize the pipe to a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized object. + + DOCS: https://spacy.io/api/pipe#to_bytes + """ + self._validate_serialization_attrs() + serialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + serialize["cfg"] = lambda: srsly.json_dumps(self.cfg) + serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude) + serialize["model"] = self.model.to_bytes + return util.to_bytes(serialize, exclude) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load the pipe from a bytestring. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (TrainablePipe): The loaded object. + + DOCS: https://spacy.io/api/pipe#from_bytes + """ + self._validate_serialization_attrs() + + def load_model(b): + try: + self.model.from_bytes(b) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b)) + deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) + deserialize["model"] = load_model + util.from_bytes(bytes_data, deserialize, exclude) + return self + + def to_disk(self, path, *, exclude=tuple()): + """Serialize the pipe to disk. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/pipe#to_disk + """ + self._validate_serialization_attrs() + serialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg) + serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude) + serialize["model"] = lambda p: self.model.to_disk(p) + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, *, exclude=tuple()): + """Load the pipe from disk. + + path (str / Path): Path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (TrainablePipe): The loaded object. + + DOCS: https://spacy.io/api/pipe#from_disk + """ + self._validate_serialization_attrs() + + def load_model(p): + try: + with open(p, "rb") as mfile: + self.model.from_bytes(mfile.read()) + except AttributeError: + raise ValueError(Errors.E149) from None + + deserialize = {} + if hasattr(self, "cfg") and self.cfg is not None: + deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p)) + deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude) + deserialize["model"] = load_model + util.from_disk(path, deserialize, exclude) + return self diff --git a/spacy/pipeline/transition_parser.pxd b/spacy/pipeline/transition_parser.pxd new file mode 100644 index 0000000..62c2bfb --- /dev/null +++ b/spacy/pipeline/transition_parser.pxd @@ -0,0 +1,30 @@ +from cymem.cymem cimport Pool +from thinc.backends.cblas cimport CBlas + +from ..ml.parser_model cimport ActivationsC, SizesC, WeightsC +from ..vocab cimport Vocab +from ._parser_internals._state cimport StateC +from ._parser_internals.transition_system cimport Transition, TransitionSystem +from .trainable_pipe cimport TrainablePipe + + +cdef class Parser(TrainablePipe): + cdef public object _rehearsal_model + cdef readonly TransitionSystem moves + cdef public object _multitasks + + cdef void _parseC( + self, + CBlas cblas, + StateC** states, + WeightsC weights, + SizesC sizes + ) noexcept nogil + + cdef void c_transition_batch( + self, + StateC** states, + const float* scores, + int nr_class, + int batch_size + ) noexcept nogil diff --git a/spacy/pipeline/transition_parser.pyx b/spacy/pipeline/transition_parser.pyx new file mode 100644 index 0000000..24a5bc1 --- /dev/null +++ b/spacy/pipeline/transition_parser.pyx @@ -0,0 +1,695 @@ +# cython: infer_types=True, cdivision=True, boundscheck=False, binding=True +# cython: profile=False +from __future__ import print_function + +cimport numpy as np +from cymem.cymem cimport Pool + +from itertools import islice + +from libc.stdlib cimport calloc, free +from libc.string cimport memset +from libcpp.vector cimport vector + +import random + +import numpy +import numpy.random +import srsly +from thinc.api import CupyOps, NumpyOps, set_dropout_rate + +from ..ml.parser_model cimport ( + ActivationsC, + SizesC, + WeightsC, + alloc_activations, + arg_max_if_valid, + cpu_log_loss, + free_activations, + get_c_sizes, + get_c_weights, + predict_states, +) +from ..tokens.doc cimport Doc +from ._parser_internals.stateclass cimport StateClass + +from .trainable_pipe import TrainablePipe + +from ._parser_internals cimport _beam_utils + +from .. import util +from ..errors import Errors +from ..training import validate_examples, validate_get_examples +from ._parser_internals import _beam_utils + +NUMPY_OPS = NumpyOps() + + +cdef class Parser(TrainablePipe): + """ + Base class of the DependencyParser and EntityRecognizer. + """ + + def __init__( + self, + Vocab vocab, + model, + name="base_parser", + moves=None, + *, + update_with_oracle_cut_size, + min_action_freq, + learn_tokens, + beam_width=1, + beam_density=0.0, + beam_update_prob=0.0, + multitasks=tuple(), + incorrect_spans_key=None, + scorer=None, + ): + """Create a Parser. + + vocab (Vocab): The vocabulary object. Must be shared with documents + to be processed. The value is set to the `.vocab` attribute. + model (Model): The model for the transition-based parser. The model needs + to have a specific substructure of named components --- see the + spacy.ml.tb_framework.TransitionModel for details. + name (str): The name of the pipeline component + moves (Optional[TransitionSystem]): This defines how the parse-state is created, + updated and evaluated. If 'moves' is None, a new instance is + created with `self.TransitionSystem()`. Defaults to `None`. + update_with_oracle_cut_size (int): During training, cut long sequences into + shorter segments by creating intermediate states based on the gold-standard + history. The model is not very sensitive to this parameter, so you usually + won't need to change it. 100 is a good default. + min_action_freq (int): The minimum frequency of labelled actions to retain. + Rarer labelled actions have their label backed-off to "dep". While this + primarily affects the label accuracy, it can also affect the attachment + structure, as the labels are used to represent the pseudo-projectivity + transformation. + learn_tokens (bool): Whether to learn to merge subtokens that are split + relative to the gold standard. Experimental. + beam_width (int): The number of candidate analyses to maintain. + beam_density (float): The minimum ratio between the scores of the first and + last candidates in the beam. This allows the parser to avoid exploring + candidates that are too far behind. This is mostly intended to improve + efficiency, but it can also improve accuracy as deeper search is not + always better. + beam_update_prob (float): The chance of making a beam update, instead of a + greedy update. Greedy updates are an approximation for the beam updates, + and are faster to compute. + multitasks: additional multi-tasking components. Experimental. + incorrect_spans_key (Optional[str]): Identifies spans that are known + to be incorrect entity annotations. The incorrect entity annotations + can be stored in the span group, under this key. + scorer (Optional[Callable]): The scoring method. Defaults to None. + """ + self.vocab = vocab + self.name = name + cfg = { + "moves": moves, + "update_with_oracle_cut_size": update_with_oracle_cut_size, + "multitasks": list(multitasks), + "min_action_freq": min_action_freq, + "learn_tokens": learn_tokens, + "beam_width": beam_width, + "beam_density": beam_density, + "beam_update_prob": beam_update_prob, + "incorrect_spans_key": incorrect_spans_key + } + if moves is None: + # EntityRecognizer -> BiluoPushDown + # DependencyParser -> ArcEager + moves = self.TransitionSystem( + self.vocab.strings, + incorrect_spans_key=incorrect_spans_key + ) + self.moves = moves + self.model = model + if self.moves.n_moves != 0: + self.set_output(self.moves.n_moves) + self.cfg = cfg + self._multitasks = [] + for multitask in cfg["multitasks"]: + self.add_multitask_objective(multitask) + + self._rehearsal_model = None + self.scorer = scorer + + def __getnewargs_ex__(self): + """This allows pickling the Parser and its keyword-only init arguments""" + args = (self.vocab, self.model, self.name, self.moves) + return args, self.cfg + + @property + def move_names(self): + names = [] + for i in range(self.moves.n_moves): + name = self.moves.move_name(self.moves.c[i].move, self.moves.c[i].label) + # Explicitly removing the internal "U-" token used for blocking entities + if name != "U-": + names.append(name) + return names + + @property + def labels(self): + class_names = [self.moves.get_class_name(i) for i in range(self.moves.n_moves)] + return class_names + + @property + def label_data(self): + return self.moves.labels + + @property + def tok2vec(self): + """Return the embedding and convolutional layer of the model.""" + return self.model.get_ref("tok2vec") + + @property + def postprocesses(self): + # Available for subclasses, e.g. to deprojectivize + return [] + + @property + def incorrect_spans_key(self): + return self.cfg["incorrect_spans_key"] + + def add_label(self, label): + resized = False + for action in self.moves.action_types: + added = self.moves.add_action(action, label) + if added: + resized = True + if resized: + self._resize() + self.vocab.strings.add(label) + return 1 + return 0 + + def _ensure_labels_are_added(self, docs): + """Ensure that all labels for a batch of docs are added.""" + resized = False + labels = set() + for doc in docs: + labels.update(self.moves.get_doc_labels(doc)) + for label in labels: + for action in self.moves.action_types: + added = self.moves.add_action(action, label) + if added: + self.vocab.strings.add(label) + resized = True + if resized: + self._resize() + return 1 + return 0 + + def _resize(self): + self.model.attrs["resize_output"](self.model, self.moves.n_moves) + if self._rehearsal_model not in (True, False, None): + self._rehearsal_model.attrs["resize_output"]( + self._rehearsal_model, self.moves.n_moves + ) + + def add_multitask_objective(self, target): + # Defined in subclasses, to avoid circular import + raise NotImplementedError + + def init_multitask_objectives(self, get_examples, pipeline, **cfg): + """Setup models for secondary objectives, to benefit from multi-task + learning. This method is intended to be overridden by subclasses. + + For instance, the dependency parser can benefit from sharing + an input representation with a label prediction model. These auxiliary + models are discarded after training. + """ + pass + + def use_params(self, params): + # Can't decorate cdef class :(. Workaround. + with self.model.use_params(params): + yield + + def pipe(self, docs, *, int batch_size=256): + """Process a stream of documents. + + stream: The sequence of documents to process. + batch_size (int): Number of documents to accumulate into a working set. + error_handler (Callable[[str, List[Doc], Exception], Any]): Function that + deals with a failing batch of documents. The default function just reraises + the exception. + + YIELDS (Doc): Documents, in order. + """ + cdef Doc doc + error_handler = self.get_error_handler() + for batch in util.minibatch(docs, size=batch_size): + batch_in_order = list(batch) + try: + by_length = sorted(batch, key=lambda doc: len(doc)) + for subbatch in util.minibatch(by_length, size=max(batch_size//4, 2)): + subbatch = list(subbatch) + parse_states = self.predict(subbatch) + self.set_annotations(subbatch, parse_states) + yield from batch_in_order + except Exception as e: + error_handler(self.name, self, batch_in_order, e) + + def predict(self, docs): + if isinstance(docs, Doc): + docs = [docs] + if not any(len(doc) for doc in docs): + result = self.moves.init_batch(docs) + return result + if self.cfg["beam_width"] == 1: + return self.greedy_parse(docs, drop=0.0) + else: + return self.beam_parse( + docs, + drop=0.0, + beam_width=self.cfg["beam_width"], + beam_density=self.cfg["beam_density"] + ) + + def greedy_parse(self, docs, drop=0.): + cdef vector[StateC*] states + cdef StateClass state + ops = self.model.ops + cdef CBlas cblas + if isinstance(ops, CupyOps): + cblas = NUMPY_OPS.cblas() + else: + cblas = ops.cblas() + self._ensure_labels_are_added(docs) + set_dropout_rate(self.model, drop) + batch = self.moves.init_batch(docs) + model = self.model.predict(docs) + weights = get_c_weights(model) + for state in batch: + if not state.is_final(): + states.push_back(state.c) + sizes = get_c_sizes(model, states.size()) + with nogil: + self._parseC(cblas, &states[0], weights, sizes) + model.clear_memory() + del model + return batch + + def beam_parse(self, docs, int beam_width, float drop=0., beam_density=0.): + self._ensure_labels_are_added(docs) + batch = _beam_utils.BeamBatch( + self.moves, + self.moves.init_batch(docs), + None, + beam_width, + density=beam_density + ) + model = self.model.predict(docs) + while not batch.is_done: + states = batch.get_unfinished_states() + if not states: + break + scores = model.predict(states) + batch.advance(scores) + model.clear_memory() + del model + return list(batch) + + cdef void _parseC( + self, CBlas cblas, StateC** states, WeightsC weights, SizesC sizes + ) noexcept nogil: + cdef int i + cdef vector[StateC*] unfinished + cdef ActivationsC activations = alloc_activations(sizes) + while sizes.states >= 1: + predict_states(cblas, &activations, states, &weights, sizes) + # Validate actions, argmax, take action. + self.c_transition_batch( + states, activations.scores, sizes.classes, sizes.states + ) + for i in range(sizes.states): + if not states[i].is_final(): + unfinished.push_back(states[i]) + for i in range(unfinished.size()): + states[i] = unfinished[i] + sizes.states = unfinished.size() + unfinished.clear() + free_activations(&activations) + + def set_annotations(self, docs, states_or_beams): + cdef StateClass state + cdef Doc doc + states = _beam_utils.collect_states(states_or_beams, docs) + for i, (state, doc) in enumerate(zip(states, docs)): + self.moves.set_annotations(state, doc) + for hook in self.postprocesses: + hook(doc) + + def transition_states(self, states, float[:, ::1] scores): + cdef StateClass state + cdef float* c_scores = &scores[0, 0] + cdef vector[StateC*] c_states + for state in states: + c_states.push_back(state.c) + self.c_transition_batch(&c_states[0], c_scores, scores.shape[1], scores.shape[0]) + return [state for state in states if not state.c.is_final()] + + cdef void c_transition_batch( + self, + StateC** states, + const float* scores, + int nr_class, + int batch_size + ) noexcept nogil: + # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc + with gil: + assert self.moves.n_moves > 0, Errors.E924.format(name=self.name) + is_valid = calloc(self.moves.n_moves, sizeof(int)) + cdef int i, guess + cdef Transition action + for i in range(batch_size): + self.moves.set_valid(is_valid, states[i]) + guess = arg_max_if_valid(&scores[i*nr_class], is_valid, nr_class) + if guess == -1: + # This shouldn't happen, but it's hard to raise an error here, + # and we don't want to infinite loop. So, force to end state. + states[i].force_final() + else: + action = self.moves.c[guess] + action.do(states[i], action.label) + free(is_valid) + + def update(self, examples, *, drop=0., sgd=None, losses=None): + if losses is None: + losses = {} + losses.setdefault(self.name, 0.) + validate_examples(examples, "Parser.update") + self._ensure_labels_are_added( + [eg.x for eg in examples] + [eg.y for eg in examples] + ) + for multitask in self._multitasks: + multitask.update(examples, drop=drop, sgd=sgd) + n_examples = len([eg for eg in examples if self.moves.has_gold(eg)]) + if n_examples == 0: + return losses + set_dropout_rate(self.model, drop) + # The probability we use beam update, instead of falling back to + # a greedy update + beam_update_prob = self.cfg["beam_update_prob"] + if self.cfg['beam_width'] >= 2 and numpy.random.random() < beam_update_prob: + return self.update_beam( + examples, + beam_width=self.cfg["beam_width"], + sgd=sgd, + losses=losses, + beam_density=self.cfg["beam_density"] + ) + max_moves = self.cfg["update_with_oracle_cut_size"] + if max_moves >= 1: + # Chop sequences into lengths of this many words, to make the + # batch uniform length. + max_moves = int(random.uniform(max_moves // 2, max_moves * 2)) + states, golds, _ = self._init_gold_batch( + examples, + max_length=max_moves + ) + else: + states, golds, _ = self.moves.init_gold_batch(examples) + if not states: + return losses + model, backprop_tok2vec = self.model.begin_update([eg.x for eg in examples]) + + states_golds = list(zip(states, golds)) + n_moves = 0 + while states_golds: + states, golds = zip(*states_golds) + scores, backprop = model.begin_update(states) + d_scores = self.get_batch_loss(states, golds, scores, losses) + # Note that the gradient isn't normalized by the batch size + # here, because our "samples" are really the states...But we + # can't normalize by the number of states either, as then we'd + # be getting smaller gradients for states in long sequences. + backprop(d_scores) + # Follow the predicted action + self.transition_states(states, scores) + states_golds = [(s, g) for (s, g) in zip(states, golds) if not s.is_final()] + if max_moves >= 1 and n_moves >= max_moves: + break + n_moves += 1 + + backprop_tok2vec(golds) + if sgd not in (None, False): + self.finish_update(sgd) + # Ugh, this is annoying. If we're working on GPU, we want to free the + # memory ASAP. It seems that Python doesn't necessarily get around to + # removing these in time if we don't explicitly delete? It's confusing. + del backprop + del backprop_tok2vec + model.clear_memory() + del model + return losses + + def rehearse(self, examples, sgd=None, losses=None, **cfg): + """Perform a "rehearsal" update, to prevent catastrophic forgetting.""" + if losses is None: + losses = {} + for multitask in self._multitasks: + if hasattr(multitask, 'rehearse'): + multitask.rehearse(examples, losses=losses, sgd=sgd) + if self._rehearsal_model is None: + return None + losses.setdefault(self.name, 0.) + validate_examples(examples, "Parser.rehearse") + docs = [eg.predicted for eg in examples] + states = self.moves.init_batch(docs) + # This is pretty dirty, but the NER can resize itself in init_batch, + # if labels are missing. We therefore have to check whether we need to + # expand our model output. + self._resize() + # Prepare the stepwise model, and get the callback for finishing the batch + set_dropout_rate(self._rehearsal_model, 0.0) + set_dropout_rate(self.model, 0.0) + tutor, _ = self._rehearsal_model.begin_update(docs) + model, backprop_tok2vec = self.model.begin_update(docs) + n_scores = 0. + loss = 0. + while states: + targets, _ = tutor.begin_update(states) + guesses, backprop = model.begin_update(states) + d_scores = (guesses - targets) / targets.shape[0] + # If all weights for an output are 0 in the original model, don't + # supervise that output. This allows us to add classes. + loss += (d_scores**2).sum() + backprop(d_scores) + # Follow the predicted action + self.transition_states(states, guesses) + states = [state for state in states if not state.is_final()] + n_scores += d_scores.size + # Do the backprop + backprop_tok2vec(docs) + if sgd is not None: + self.finish_update(sgd) + losses[self.name] += loss / n_scores + del backprop + del backprop_tok2vec + model.clear_memory() + tutor.clear_memory() + del model + del tutor + return losses + + def update_beam( + self, + examples, + *, + beam_width, + drop=0., + sgd=None, + losses=None, + beam_density=0.0 + ): + states, golds, _ = self.moves.init_gold_batch(examples) + if not states: + return losses + # Prepare the stepwise model, and get the callback for finishing the batch + model, backprop_tok2vec = self.model.begin_update( + [eg.predicted for eg in examples]) + loss = _beam_utils.update_beam( + self.moves, + states, + golds, + model, + beam_width, + beam_density=beam_density, + ) + losses[self.name] += loss + backprop_tok2vec(golds) + if sgd is not None: + self.finish_update(sgd) + + def get_batch_loss(self, states, golds, float[:, ::1] scores, losses): + cdef StateClass state + cdef Pool mem = Pool() + cdef int i + + # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc + assert self.moves.n_moves > 0, Errors.E924.format(name=self.name) + + is_valid = mem.alloc(self.moves.n_moves, sizeof(int)) + costs = mem.alloc(self.moves.n_moves, sizeof(float)) + cdef np.ndarray d_scores = numpy.zeros( + (len(states), self.moves.n_moves), dtype='f', order='C' + ) + c_d_scores = d_scores.data + unseen_classes = self.model.attrs["unseen_classes"] + for i, (state, gold) in enumerate(zip(states, golds)): + memset(is_valid, 0, self.moves.n_moves * sizeof(int)) + memset(costs, 0, self.moves.n_moves * sizeof(float)) + self.moves.set_costs(is_valid, costs, state.c, gold) + for j in range(self.moves.n_moves): + if costs[j] <= 0.0 and j in unseen_classes: + unseen_classes.remove(j) + cpu_log_loss( + c_d_scores, costs, is_valid, &scores[i, 0], d_scores.shape[1] + ) + c_d_scores += d_scores.shape[1] + # Note that we don't normalize this. See comment in update() for why. + if losses is not None: + losses.setdefault(self.name, 0.) + losses[self.name] += (d_scores**2).sum() + return d_scores + + def set_output(self, nO): + self.model.attrs["resize_output"](self.model, nO) + + def initialize(self, get_examples, nlp=None, labels=None): + validate_get_examples(get_examples, "Parser.initialize") + util.check_lexeme_norms(self.vocab, "parser or NER") + if labels is not None: + actions = dict(labels) + else: + actions = self.moves.get_actions( + examples=get_examples(), + min_freq=self.cfg['min_action_freq'], + learn_tokens=self.cfg["learn_tokens"] + ) + for action, labels in self.moves.labels.items(): + actions.setdefault(action, {}) + for label, freq in labels.items(): + if label not in actions[action]: + actions[action][label] = freq + self.moves.initialize_actions(actions) + # make sure we resize so we have an appropriate upper layer + self._resize() + doc_sample = [] + if nlp is not None: + for name, component in nlp.pipeline: + if component is self: + break + # non-trainable components may have a pipe() implementation that refers to dummy + # predict and set_annotations methods + if hasattr(component, "pipe"): + doc_sample = list(component.pipe(doc_sample, batch_size=8)) + else: + doc_sample = [component(doc) for doc in doc_sample] + if not doc_sample: + for example in islice(get_examples(), 10): + doc_sample.append(example.predicted) + assert len(doc_sample) > 0, Errors.E923.format(name=self.name) + self.model.initialize(doc_sample) + if nlp is not None: + self.init_multitask_objectives(get_examples, nlp.pipeline) + + def to_disk(self, path, exclude=tuple()): + serializers = { + "model": lambda p: (self.model.to_disk(p) if self.model is not True else True), + "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude), + "moves": lambda p: self.moves.to_disk(p, exclude=["strings"]), + "cfg": lambda p: srsly.write_json(p, self.cfg) + } + util.to_disk(path, serializers, exclude) + + def from_disk(self, path, exclude=tuple()): + deserializers = { + "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude), + "moves": lambda p: self.moves.from_disk(p, exclude=["strings"]), + "cfg": lambda p: self.cfg.update(srsly.read_json(p)), + "model": lambda p: None, + } + util.from_disk(path, deserializers, exclude) + if "model" not in exclude: + path = util.ensure_path(path) + with (path / "model").open("rb") as file_: + bytes_data = file_.read() + try: + self._resize() + self.model.from_bytes(bytes_data) + except AttributeError: + raise ValueError(Errors.E149) + return self + + def to_bytes(self, exclude=tuple()): + serializers = { + "model": lambda: (self.model.to_bytes()), + "vocab": lambda: self.vocab.to_bytes(exclude=exclude), + "moves": lambda: self.moves.to_bytes(exclude=["strings"]), + "cfg": lambda: srsly.json_dumps(self.cfg, indent=2, sort_keys=True) + } + return util.to_bytes(serializers, exclude) + + def from_bytes(self, bytes_data, exclude=tuple()): + deserializers = { + "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude), + "moves": lambda b: self.moves.from_bytes(b, exclude=["strings"]), + "cfg": lambda b: self.cfg.update(srsly.json_loads(b)), + "model": lambda b: None, + } + msg = util.from_bytes(bytes_data, deserializers, exclude) + if 'model' not in exclude: + if 'model' in msg: + try: + self.model.from_bytes(msg['model']) + except AttributeError: + raise ValueError(Errors.E149) from None + return self + + def _init_gold_batch(self, examples, max_length): + """Make a square batch, of length equal to the shortest transition + sequence or a cap. A long + doc will get multiple states. Let's say we have a doc of length 2*N, + where N is the shortest doc. We'll make two states, one representing + long_doc[:N], and another representing long_doc[N:].""" + cdef: + StateClass start_state + StateClass state + Transition action + all_states = self.moves.init_batch([eg.predicted for eg in examples]) + states = [] + golds = [] + to_cut = [] + for state, eg in zip(all_states, examples): + if self.moves.has_gold(eg) and not state.is_final(): + gold = self.moves.init_gold(state, eg) + if len(eg.x) < max_length: + states.append(state) + golds.append(gold) + else: + oracle_actions = self.moves.get_oracle_sequence_from_state( + state.copy(), gold) + to_cut.append((eg, state, gold, oracle_actions)) + if not to_cut: + return states, golds, 0 + cdef int clas + for eg, state, gold, oracle_actions in to_cut: + for i in range(0, len(oracle_actions), max_length): + start_state = state.copy() + for clas in oracle_actions[i:i+max_length]: + action = self.moves.c[clas] + action.do(state.c, action.label) + if state.is_final(): + break + if self.moves.has_gold(eg, start_state.B(0), state.B(0)): + states.append(start_state) + golds.append(gold) + if state.is_final(): + break + return states, golds, max_length diff --git a/spacy/py.typed b/spacy/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/spacy/registrations.py b/spacy/registrations.py new file mode 100644 index 0000000..7e29486 --- /dev/null +++ b/spacy/registrations.py @@ -0,0 +1,246 @@ +"""Centralized registry population for spaCy config + +This module centralizes registry decorations to prevent circular import issues +with Cython annotation changes from __future__ import annotations. Functions +remain in their original locations, but decoration is moved here. + +Component definitions and registrations are in spacy/pipeline/factories.py +""" + +# Global flag to track if registry has been populated +REGISTRY_POPULATED = False + + +def populate_registry() -> None: + """Populate the registry with all necessary components. + + This function should be called before accessing the registry, to ensure + it's populated. The function uses a global flag to prevent repopulation. + """ + global REGISTRY_POPULATED + if REGISTRY_POPULATED: + return + + # Import all necessary modules + from .lang.ja import create_tokenizer as create_japanese_tokenizer + from .lang.ko import create_tokenizer as create_korean_tokenizer + from .lang.th import create_thai_tokenizer + from .lang.vi import create_vietnamese_tokenizer + from .lang.zh import create_chinese_tokenizer + from .language import load_lookups_data + from .matcher.levenshtein import make_levenshtein_compare + from .ml.models.entity_linker import ( + create_candidates, + create_candidates_batch, + empty_kb, + empty_kb_for_config, + load_kb, + ) + from .pipeline.attributeruler import make_attribute_ruler_scorer + from .pipeline.dep_parser import make_parser_scorer + + # Import the functions we refactored by removing direct registry decorators + from .pipeline.entity_linker import make_entity_linker_scorer + from .pipeline.entityruler import ( + make_entity_ruler_scorer as make_entityruler_scorer, + ) + from .pipeline.lemmatizer import make_lemmatizer_scorer + from .pipeline.morphologizer import make_morphologizer_scorer + from .pipeline.ner import make_ner_scorer + from .pipeline.senter import make_senter_scorer + from .pipeline.span_finder import make_span_finder_scorer + from .pipeline.span_ruler import ( + make_overlapping_labeled_spans_scorer, + make_preserve_existing_ents_filter, + make_prioritize_new_ents_filter, + ) + from .pipeline.spancat import ( + build_ngram_range_suggester, + build_ngram_suggester, + build_preset_spans_suggester, + make_spancat_scorer, + ) + + # Import all pipeline components that were using registry decorators + from .pipeline.tagger import make_tagger_scorer + from .pipeline.textcat import make_textcat_scorer + from .pipeline.textcat_multilabel import make_textcat_multilabel_scorer + from .util import make_first_longest_spans_filter, registry + + # Register miscellaneous components + registry.misc("spacy.first_longest_spans_filter.v1")( + make_first_longest_spans_filter + ) + registry.misc("spacy.ngram_suggester.v1")(build_ngram_suggester) + registry.misc("spacy.ngram_range_suggester.v1")(build_ngram_range_suggester) + registry.misc("spacy.preset_spans_suggester.v1")(build_preset_spans_suggester) + registry.misc("spacy.prioritize_new_ents_filter.v1")( + make_prioritize_new_ents_filter + ) + registry.misc("spacy.prioritize_existing_ents_filter.v1")( + make_preserve_existing_ents_filter + ) + registry.misc("spacy.levenshtein_compare.v1")(make_levenshtein_compare) + # KB-related registrations + registry.misc("spacy.KBFromFile.v1")(load_kb) + registry.misc("spacy.EmptyKB.v2")(empty_kb_for_config) + registry.misc("spacy.EmptyKB.v1")(empty_kb) + registry.misc("spacy.CandidateGenerator.v1")(create_candidates) + registry.misc("spacy.CandidateBatchGenerator.v1")(create_candidates_batch) + registry.misc("spacy.LookupsDataLoader.v1")(load_lookups_data) + + # Need to get references to the existing functions in registry by importing the function that is there + # For the registry that was previously decorated + + # Import ML components that use registry + from .language import create_tokenizer + from .ml._precomputable_affine import PrecomputableAffine + from .ml.callbacks import ( + create_models_and_pipes_with_nvtx_range, + create_models_with_nvtx_range, + ) + from .ml.extract_ngrams import extract_ngrams + from .ml.extract_spans import extract_spans + + # Import decorator-removed ML components + from .ml.featureextractor import FeatureExtractor + from .ml.models.entity_linker import build_nel_encoder + from .ml.models.multi_task import ( + create_pretrain_characters, + create_pretrain_vectors, + ) + from .ml.models.parser import build_tb_parser_model + from .ml.models.span_finder import build_finder_model + from .ml.models.spancat import ( + build_linear_logistic, + build_mean_max_reducer, + build_spancat_model, + ) + from .ml.models.tagger import build_tagger_model + from .ml.models.textcat import ( + build_bow_text_classifier, + build_bow_text_classifier_v3, + build_reduce_text_classifier, + build_simple_cnn_text_classifier, + build_text_classifier_lowdata, + build_text_classifier_v2, + build_textcat_parametric_attention_v1, + ) + from .ml.models.tok2vec import ( + BiLSTMEncoder, + CharacterEmbed, + MaxoutWindowEncoder, + MishWindowEncoder, + MultiHashEmbed, + build_hash_embed_cnn_tok2vec, + build_Tok2Vec_model, + tok2vec_listener_v1, + ) + from .ml.staticvectors import StaticVectors + from .ml.tb_framework import TransitionModel + from .training.augment import ( + create_combined_augmenter, + create_lower_casing_augmenter, + create_orth_variants_augmenter, + ) + from .training.batchers import ( + configure_minibatch, + configure_minibatch_by_padded_size, + configure_minibatch_by_words, + ) + from .training.callbacks import create_copy_from_base_model + from .training.loggers import console_logger, console_logger_v3 + + # Register scorers + registry.scorers("spacy.tagger_scorer.v1")(make_tagger_scorer) + registry.scorers("spacy.ner_scorer.v1")(make_ner_scorer) + # span_ruler_scorer removed as it's not in span_ruler.py + registry.scorers("spacy.entity_ruler_scorer.v1")(make_entityruler_scorer) + registry.scorers("spacy.senter_scorer.v1")(make_senter_scorer) + registry.scorers("spacy.textcat_scorer.v1")(make_textcat_scorer) + registry.scorers("spacy.textcat_scorer.v2")(make_textcat_scorer) + registry.scorers("spacy.textcat_multilabel_scorer.v1")( + make_textcat_multilabel_scorer + ) + registry.scorers("spacy.textcat_multilabel_scorer.v2")( + make_textcat_multilabel_scorer + ) + registry.scorers("spacy.lemmatizer_scorer.v1")(make_lemmatizer_scorer) + registry.scorers("spacy.span_finder_scorer.v1")(make_span_finder_scorer) + registry.scorers("spacy.spancat_scorer.v1")(make_spancat_scorer) + registry.scorers("spacy.entity_linker_scorer.v1")(make_entity_linker_scorer) + registry.scorers("spacy.overlapping_labeled_spans_scorer.v1")( + make_overlapping_labeled_spans_scorer + ) + registry.scorers("spacy.attribute_ruler_scorer.v1")(make_attribute_ruler_scorer) + registry.scorers("spacy.parser_scorer.v1")(make_parser_scorer) + registry.scorers("spacy.morphologizer_scorer.v1")(make_morphologizer_scorer) + + # Register tokenizers + registry.tokenizers("spacy.Tokenizer.v1")(create_tokenizer) + registry.tokenizers("spacy.ja.JapaneseTokenizer")(create_japanese_tokenizer) + registry.tokenizers("spacy.zh.ChineseTokenizer")(create_chinese_tokenizer) + registry.tokenizers("spacy.ko.KoreanTokenizer")(create_korean_tokenizer) + registry.tokenizers("spacy.vi.VietnameseTokenizer")(create_vietnamese_tokenizer) + registry.tokenizers("spacy.th.ThaiTokenizer")(create_thai_tokenizer) + + # Register tok2vec architectures we've modified + registry.architectures("spacy.Tok2VecListener.v1")(tok2vec_listener_v1) + registry.architectures("spacy.HashEmbedCNN.v2")(build_hash_embed_cnn_tok2vec) + registry.architectures("spacy.Tok2Vec.v2")(build_Tok2Vec_model) + registry.architectures("spacy.MultiHashEmbed.v2")(MultiHashEmbed) + registry.architectures("spacy.CharacterEmbed.v2")(CharacterEmbed) + registry.architectures("spacy.MaxoutWindowEncoder.v2")(MaxoutWindowEncoder) + registry.architectures("spacy.MishWindowEncoder.v2")(MishWindowEncoder) + registry.architectures("spacy.TorchBiLSTMEncoder.v1")(BiLSTMEncoder) + registry.architectures("spacy.EntityLinker.v2")(build_nel_encoder) + registry.architectures("spacy.TextCatCNN.v2")(build_simple_cnn_text_classifier) + registry.architectures("spacy.TextCatBOW.v2")(build_bow_text_classifier) + registry.architectures("spacy.TextCatBOW.v3")(build_bow_text_classifier_v3) + registry.architectures("spacy.TextCatEnsemble.v2")(build_text_classifier_v2) + registry.architectures("spacy.TextCatLowData.v1")(build_text_classifier_lowdata) + registry.architectures("spacy.TextCatParametricAttention.v1")( + build_textcat_parametric_attention_v1 + ) + registry.architectures("spacy.TextCatReduce.v1")(build_reduce_text_classifier) + registry.architectures("spacy.SpanCategorizer.v1")(build_spancat_model) + registry.architectures("spacy.SpanFinder.v1")(build_finder_model) + registry.architectures("spacy.TransitionBasedParser.v2")(build_tb_parser_model) + registry.architectures("spacy.PretrainVectors.v1")(create_pretrain_vectors) + registry.architectures("spacy.PretrainCharacters.v1")(create_pretrain_characters) + registry.architectures("spacy.Tagger.v2")(build_tagger_model) + + # Register layers + registry.layers("spacy.FeatureExtractor.v1")(FeatureExtractor) + registry.layers("spacy.extract_spans.v1")(extract_spans) + registry.layers("spacy.extract_ngrams.v1")(extract_ngrams) + registry.layers("spacy.LinearLogistic.v1")(build_linear_logistic) + registry.layers("spacy.mean_max_reducer.v1")(build_mean_max_reducer) + registry.layers("spacy.StaticVectors.v2")(StaticVectors) + registry.layers("spacy.PrecomputableAffine.v1")(PrecomputableAffine) + registry.layers("spacy.CharEmbed.v1")(CharacterEmbed) + registry.layers("spacy.TransitionModel.v1")(TransitionModel) + + # Register callbacks + registry.callbacks("spacy.copy_from_base_model.v1")(create_copy_from_base_model) + registry.callbacks("spacy.models_with_nvtx_range.v1")(create_models_with_nvtx_range) + registry.callbacks("spacy.models_and_pipes_with_nvtx_range.v1")( + create_models_and_pipes_with_nvtx_range + ) + + # Register loggers + registry.loggers("spacy.ConsoleLogger.v2")(console_logger) + registry.loggers("spacy.ConsoleLogger.v3")(console_logger_v3) + + # Register batchers + registry.batchers("spacy.batch_by_padded.v1")(configure_minibatch_by_padded_size) + registry.batchers("spacy.batch_by_words.v1")(configure_minibatch_by_words) + registry.batchers("spacy.batch_by_sequence.v1")(configure_minibatch) + + # Register augmenters + registry.augmenters("spacy.combined_augmenter.v1")(create_combined_augmenter) + registry.augmenters("spacy.lower_case.v1")(create_lower_casing_augmenter) + registry.augmenters("spacy.orth_variants.v1")(create_orth_variants_augmenter) + + # Set the flag to indicate that the registry has been populated + REGISTRY_POPULATED = True diff --git a/spacy/schemas.py b/spacy/schemas.py new file mode 100644 index 0000000..359c3fd --- /dev/null +++ b/spacy/schemas.py @@ -0,0 +1,500 @@ +import inspect +from collections import defaultdict +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, +) + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + constr, + create_model, + field_validator, +) +from thinc.api import ConfigValidationError, Model, Optimizer +from thinc.config import Promise + +from .attrs import NAMES +from .compat import Literal +from .lookups import Lookups +from .util import is_cython_func + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language # noqa: F401 + from .training import Example # noqa: F401 + from .vocab import Vocab # noqa: F401 + + +# fmt: off +ItemT = TypeVar("ItemT") +Batcher = Union[Callable[[Iterable[ItemT]], Iterable[List[ItemT]]], Promise] +Reader = Union[Callable[["Language", str], Iterable["Example"]], Promise] +Logger = Union[Callable[["Language"], Tuple[Callable[[Dict[str, Any]], None], Callable]], Promise] +# fmt: on + + +def validate(schema: Type[BaseModel], obj: Dict[str, Any]) -> List[str]: + """Validate data against a given pydantic schema. + + obj (Dict[str, Any]): JSON-serializable data to validate. + schema (pydantic.BaseModel): The schema to validate against. + RETURNS (List[str]): A list of error messages, if available. + """ + try: + schema(**obj) + return [] + except ValidationError as e: + errors = e.errors() + data = defaultdict(list) + for error in errors: + err_loc = " -> ".join([str(p) for p in error.get("loc", [])]) + data[err_loc].append(error.get("msg")) + return [f"[{loc}] {', '.join(msg)}" for loc, msg in data.items()] # type: ignore[arg-type] + + +# Initialization + + +ArgSchemaConfig = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + +ArgSchemaConfigExtra = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + +def get_arg_model( + func: Callable, + *, + exclude: Iterable[str] = tuple(), + name: str = "ArgModel", + strict: bool = True, +) -> type[BaseModel]: + """Generate a pydantic model for function arguments. + + func (Callable): The function to generate the schema for. + exclude (Iterable[str]): Parameter names to ignore. + name (str): Name of created model class. + strict (bool): Don't allow extra arguments if no variable keyword arguments + are allowed on the function. + RETURNS (type[BaseModel]): A pydantic model. + """ + sig_args = {} + try: + sig = inspect.signature(func) + except ValueError: + # Typically happens if the method is part of a Cython module without + # binding=True. Here we just use an empty model that allows everything. + return create_model(name, __config__=ArgSchemaConfigExtra) # type: ignore[arg-type, return-value] + has_variable = False + for param in sig.parameters.values(): + if param.name in exclude: + continue + if param.kind == param.VAR_KEYWORD: + # The function allows variable keyword arguments so we shouldn't + # include **kwargs etc. in the schema and switch to non-strict + # mode and pass through all other values + has_variable = True + continue + # If no annotation is specified assume it's anything + annotation = param.annotation if param.annotation != param.empty else Any + # If no default value is specified assume that it's required. Cython + # functions/methods will have param.empty for default value None so we + # need to treat them differently + default_empty = None if is_cython_func(func) else ... + default = param.default if param.default != param.empty else default_empty + sig_args[param.name] = (annotation, default) + is_strict = strict and not has_variable + sig_args["__config__"] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment] + return create_model(name, **sig_args) # type: ignore[call-overload, arg-type, return-value] + + +def validate_init_settings( + func: Callable, + settings: Dict[str, Any], + *, + section: Optional[str] = None, + name: str = "", + exclude: Iterable[str] = ("get_examples", "nlp"), +) -> Dict[str, Any]: + """Validate initialization settings against the expected arguments in + the method signature. Will parse values if possible (e.g. int to string) + and return the updated settings dict. Will raise a ConfigValidationError + if types don't match or required values are missing. + + func (Callable): The initialize method of a given component etc. + settings (Dict[str, Any]): The settings from the respective [initialize] block. + section (str): Initialize section, for error message. + name (str): Name of the block in the section. + exclude (Iterable[str]): Parameter names to exclude from schema. + RETURNS (Dict[str, Any]): The validated settings. + """ + schema = get_arg_model(func, exclude=exclude, name="InitArgModel") + try: + return schema.model_validate(settings).model_dump() + except ValidationError as e: + block = "initialize" if not section else f"initialize.{section}" + title = f"Error validating initialization settings in [{block}]" + raise ConfigValidationError( + title=title, errors=e.errors(), config=settings, parent=name + ) from None + + +# Matcher token patterns + + +def validate_token_pattern(obj: list) -> List[str]: + # Try to convert non-string keys (e.g. {ORTH: "foo"} -> {"ORTH": "foo"}) + get_key = lambda k: NAMES[k] if isinstance(k, int) and k < len(NAMES) else k + if isinstance(obj, list): + converted = [] + for pattern in obj: + if isinstance(pattern, dict): + pattern = {get_key(k): v for k, v in pattern.items()} + converted.append(pattern) + obj = converted + return validate(TokenPatternSchema, {"pattern": obj}) + + +class TokenPatternString(BaseModel): + REGEX: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="regex") + IN: Optional[List[StrictStr]] = Field(None, alias="in") + NOT_IN: Optional[List[StrictStr]] = Field(None, alias="not_in") + IS_SUBSET: Optional[List[StrictStr]] = Field(None, alias="is_subset") + IS_SUPERSET: Optional[List[StrictStr]] = Field(None, alias="is_superset") + INTERSECTS: Optional[List[StrictStr]] = Field(None, alias="intersects") + FUZZY: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy") + FUZZY1: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy1" + ) + FUZZY2: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy2" + ) + FUZZY3: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy3" + ) + FUZZY4: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy4" + ) + FUZZY5: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy5" + ) + FUZZY6: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy6" + ) + FUZZY7: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy7" + ) + FUZZY8: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy8" + ) + FUZZY9: Optional[Union[StrictStr, "TokenPatternString"]] = Field( + None, alias="fuzzy9" + ) + + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + @field_validator("*", mode="before") + @classmethod + def raise_for_none(cls, v): + if v is None: + raise ValueError("None / null is not allowed") + return v + + +class TokenPatternNumber(BaseModel): + REGEX: Optional[StrictStr] = Field(None, alias="regex") + IN: Optional[List[StrictInt]] = Field(None, alias="in") + NOT_IN: Optional[List[StrictInt]] = Field(None, alias="not_in") + IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset") + IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset") + INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects") + EQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="==") + NEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="!=") + GEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">=") + LEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<=") + GT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">") + LT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<") + + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + @field_validator("*", mode="before") + @classmethod + def raise_for_none(cls, v): + if v is None: + raise ValueError("None / null is not allowed") + return v + + +class TokenPatternOperatorSimple(str, Enum): + plus: StrictStr = StrictStr("+") + star: StrictStr = StrictStr("*") + question: StrictStr = StrictStr("?") + exclamation: StrictStr = StrictStr("!") + + +TokenPatternOperatorMinMax = constr(pattern=r"^(\{\d+\}|\{\d+,\d*\}|\{\d*,\d+\})$") + + +TokenPatternOperator = Union[TokenPatternOperatorSimple, TokenPatternOperatorMinMax] # type: ignore[valid-type] +StringValue = Union[TokenPatternString, StrictStr] +NumberValue = Union[TokenPatternNumber, StrictInt, StrictFloat] +UnderscoreValue = Union[ + TokenPatternString, TokenPatternNumber, str, int, float, list, bool +] +IobValue = Literal["", "I", "O", "B", 0, 1, 2, 3] + + +class TokenPattern(BaseModel): + orth: Optional[StringValue] = None + text: Optional[StringValue] = None + lower: Optional[StringValue] = None + pos: Optional[StringValue] = None + tag: Optional[StringValue] = None + morph: Optional[StringValue] = None + dep: Optional[StringValue] = None + lemma: Optional[StringValue] = None + shape: Optional[StringValue] = None + ent_type: Optional[StringValue] = None + ent_iob: Optional[IobValue] = None + ent_id: Optional[StringValue] = None + ent_kb_id: Optional[StringValue] = None + norm: Optional[StringValue] = None + length: Optional[NumberValue] = None + spacy: Optional[StrictBool] = None + is_alpha: Optional[StrictBool] = None + is_ascii: Optional[StrictBool] = None + is_digit: Optional[StrictBool] = None + is_lower: Optional[StrictBool] = None + is_upper: Optional[StrictBool] = None + is_title: Optional[StrictBool] = None + is_punct: Optional[StrictBool] = None + is_space: Optional[StrictBool] = None + is_bracket: Optional[StrictBool] = None + is_quote: Optional[StrictBool] = None + is_left_punct: Optional[StrictBool] = None + is_right_punct: Optional[StrictBool] = None + is_currency: Optional[StrictBool] = None + is_stop: Optional[StrictBool] = None + is_sent_start: Optional[StrictBool] = None + sent_start: Optional[StrictBool] = None + like_num: Optional[StrictBool] = None + like_url: Optional[StrictBool] = None + like_email: Optional[StrictBool] = None + op: Optional[TokenPatternOperator] = None + underscore: Optional[Dict[StrictStr, UnderscoreValue]] = Field(None, alias="_") + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + alias_generator=lambda value: value.upper(), + ) + + @field_validator("*", mode="before") + @classmethod + def raise_for_none(cls, v): + if v is None: + raise ValueError("None / null is not allowed") + return v + + +class TokenPatternSchema(BaseModel): + pattern: List[TokenPattern] = Field(..., min_length=1) + + model_config = ConfigDict(extra="forbid") + + +# Model meta + + +class ModelMetaSchema(BaseModel): + # fmt: off + lang: StrictStr = Field(..., title="Two-letter language code, e.g. 'en'") + name: StrictStr = Field(..., title="Model name") + version: StrictStr = Field(..., title="Model version") + spacy_version: StrictStr = Field("", title="Compatible spaCy version identifier") + parent_package: StrictStr = Field("spacy", title="Name of parent spaCy package, e.g. spacy or spacy-nightly") + requirements: List[StrictStr] = Field([], title="Additional Python package dependencies, used for the Python package setup") + pipeline: List[StrictStr] = Field([], title="Names of pipeline components") + description: StrictStr = Field("", title="Model description") + license: StrictStr = Field("", title="Model license") + author: StrictStr = Field("", title="Model author name") + email: StrictStr = Field("", title="Model author email") + url: StrictStr = Field("", title="Model author URL") + sources: Optional[Union[List[StrictStr], List[Dict[str, str]]]] = Field(None, title="Training data sources") + vectors: Dict[str, Any] = Field({}, title="Included word vectors") + labels: Dict[str, List[str]] = Field({}, title="Component labels, keyed by component name") + performance: Dict[str, Any] = Field({}, title="Accuracy and speed numbers") + spacy_git_version: StrictStr = Field("", title="Commit of spaCy version used") + # fmt: on + + +# Config schema +# We're not setting any defaults here (which is too messy) and are making all +# fields required, so we can raise validation errors for missing values. To +# provide a default, we include a separate .cfg file with all values and +# check that against this schema in the test suite to make sure it's always +# up to date. + + +class ConfigSchemaTraining(BaseModel): + # fmt: off + dev_corpus: StrictStr = Field(..., title="Path in the config to the dev data") + train_corpus: StrictStr = Field(..., title="Path in the config to the training data") + batcher: Batcher = Field(..., title="Batcher for the training data") + dropout: StrictFloat = Field(..., title="Dropout rate") + patience: StrictInt = Field(..., title="How many steps to continue without improvement in evaluation score") + max_epochs: StrictInt = Field(..., title="Maximum number of epochs to train for") + max_steps: StrictInt = Field(..., title="Maximum number of update steps to train for") + eval_frequency: StrictInt = Field(..., title="How often to evaluate during training (steps)") + seed: Optional[StrictInt] = Field(..., title="Random seed") + gpu_allocator: Optional[StrictStr] = Field(..., title="Memory allocator when running on GPU") + accumulate_gradient: StrictInt = Field(..., title="Whether to divide the batch up into substeps") + score_weights: Dict[StrictStr, Optional[Union[StrictFloat, StrictInt]]] = Field(..., title="Scores to report and their weights for selecting final model") + optimizer: Optimizer = Field(..., title="The optimizer to use") + logger: Logger = Field(..., title="The logger to track training progress") + frozen_components: List[str] = Field(..., title="Pipeline components that shouldn't be updated during training") + annotating_components: List[str] = Field(..., title="Pipeline components that should set annotations during training") + before_to_disk: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object after training, before it's saved to disk") + before_update: Optional[Callable[["Language", Dict[str, Any]], None]] = Field(..., title="Optional callback that is invoked at the start of each training step") + # fmt: on + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + +class ConfigSchemaNlp(BaseModel): + # fmt: off + lang: StrictStr = Field(..., title="The base language to use") + pipeline: List[StrictStr] = Field(..., title="The pipeline component names in order") + disabled: List[StrictStr] = Field(..., title="Pipeline components to disable by default") + tokenizer: Callable = Field(..., title="The tokenizer to use") + before_creation: Optional[Callable[[Type["Language"]], Type["Language"]]] = Field(..., title="Optional callback to modify Language class before initialization") + after_creation: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object after creation and before the pipeline is constructed") + after_pipeline_creation: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object after the pipeline is constructed") + batch_size: Optional[int] = Field(..., title="Default batch size") + vectors: Callable = Field(..., title="Vectors implementation") + # fmt: on + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + +class ConfigSchemaPretrainEmpty(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class ConfigSchemaPretrain(BaseModel): + # fmt: off + max_epochs: StrictInt = Field(..., title="Maximum number of epochs to train for") + dropout: StrictFloat = Field(..., title="Dropout rate") + n_save_every: Optional[StrictInt] = Field(..., title="Saving additional temporary model after n batches within an epoch") + n_save_epoch: Optional[StrictInt] = Field(..., title="Saving model after every n epoch") + optimizer: Optimizer = Field(..., title="The optimizer to use") + corpus: StrictStr = Field(..., title="Path in the config to the training data") + batcher: Batcher = Field(..., title="Batcher for the training data") + component: str = Field(..., title="Component to find the layer to pretrain") + layer: str = Field(..., title="Layer to pretrain. Whole model if empty.") + objective: Callable[["Vocab", Model], Model] = Field(..., title="A function that creates the pretraining objective.") + # fmt: on + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + +class ConfigSchemaInit(BaseModel): + # fmt: off + vocab_data: Optional[StrictStr] = Field(..., title="Path to JSON-formatted vocabulary file") + lookups: Optional[Lookups] = Field(..., title="Vocabulary lookups, e.g. lexeme normalization") + vectors: Optional[StrictStr] = Field(..., title="Path to vectors") + init_tok2vec: Optional[StrictStr] = Field(..., title="Path to pretrained tok2vec weights") + tokenizer: Dict[StrictStr, Any] = Field(..., title="Arguments to be passed into Tokenizer.initialize") + components: Dict[StrictStr, Dict[StrictStr, Any]] = Field(..., title="Arguments for TrainablePipe.initialize methods of pipeline components, keyed by component") + before_init: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object before initialization") + after_init: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object after initialization") + # fmt: on + + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + + +class ConfigSchema(BaseModel): + training: ConfigSchemaTraining + nlp: ConfigSchemaNlp + pretraining: Union[ConfigSchemaPretrain, ConfigSchemaPretrainEmpty] = {} # type: ignore[assignment] + components: Dict[str, Dict[str, Any]] + corpora: Dict[str, Reader] + initialize: ConfigSchemaInit + + model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) + + +CONFIG_SCHEMAS = { + "nlp": ConfigSchemaNlp, + "training": ConfigSchemaTraining, + "pretraining": ConfigSchemaPretrain, + "initialize": ConfigSchemaInit, +} + +# Recommendations for init config workflows + + +class RecommendationTrfItem(BaseModel): + name: str + size_factor: int + + +class RecommendationTrf(BaseModel): + efficiency: RecommendationTrfItem + accuracy: RecommendationTrfItem + + +class RecommendationSchema(BaseModel): + word_vectors: Optional[str] = None + transformer: Optional[RecommendationTrf] = None + has_letters: bool = True + + +class DocJSONSchema(BaseModel): + """ + JSON/dict format for JSON representation of Doc objects. + """ + + cats: Optional[Dict[StrictStr, StrictFloat]] = Field( + None, title="Categories with corresponding probabilities" + ) + ents: Optional[List[Dict[StrictStr, Union[StrictInt, StrictStr]]]] = Field( + None, title="Information on entities" + ) + sents: Optional[List[Dict[StrictStr, StrictInt]]] = Field( + None, title="Indices of sentences' start and end indices" + ) + text: StrictStr = Field(..., title="Document text") + spans: Optional[ + Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] + ] = Field(None, title="Span information - end/start indices, label, KB ID") + tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( + ..., title="Token information - ID, start, annotations" + ) + underscore_doc: Optional[Dict[StrictStr, Any]] = Field( + None, + title="Any custom data stored in the document's _ attribute", + alias="_", + ) + underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( + None, title="Any custom data stored in the token's _ attribute" + ) + underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( + None, title="Any custom data stored in the span's _ attribute" + ) diff --git a/spacy/scorer.py b/spacy/scorer.py new file mode 100644 index 0000000..9ab116d --- /dev/null +++ b/spacy/scorer.py @@ -0,0 +1,1167 @@ +from collections import defaultdict +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, +) + +import numpy as np + +from .errors import Errors +from .morphology import Morphology +from .tokens import Doc, Span, Token +from .training import Example +from .util import SimpleFrozenList, get_lang_class + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language # noqa: F401 + + +DEFAULT_PIPELINE = ("senter", "tagger", "morphologizer", "parser", "ner", "textcat") +MISSING_VALUES = frozenset([None, 0, ""]) + + +class PRFScore: + """A precision / recall / F score.""" + + def __init__( + self, + *, + tp: int = 0, + fp: int = 0, + fn: int = 0, + ) -> None: + self.tp = tp + self.fp = fp + self.fn = fn + + def __len__(self) -> int: + return self.tp + self.fp + self.fn + + def __iadd__(self, other): + self.tp += other.tp + self.fp += other.fp + self.fn += other.fn + return self + + def __add__(self, other): + return PRFScore( + tp=self.tp + other.tp, fp=self.fp + other.fp, fn=self.fn + other.fn + ) + + def score_set(self, cand: set, gold: set) -> None: + self.tp += len(cand.intersection(gold)) + self.fp += len(cand - gold) + self.fn += len(gold - cand) + + @property + def precision(self) -> float: + return self.tp / (self.tp + self.fp + 1e-100) + + @property + def recall(self) -> float: + return self.tp / (self.tp + self.fn + 1e-100) + + @property + def fscore(self) -> float: + p = self.precision + r = self.recall + return 2 * ((p * r) / (p + r + 1e-100)) + + def to_dict(self) -> Dict[str, float]: + return {"p": self.precision, "r": self.recall, "f": self.fscore} + + +class ROCAUCScore: + """An AUC ROC score. This is only defined for binary classification. + Use the method is_binary before calculating the score, otherwise it + may throw an error.""" + + def __init__(self) -> None: + self.golds: List[Any] = [] + self.cands: List[Any] = [] + self.saved_score = 0.0 + self.saved_score_at_len = 0 + + def score_set(self, cand, gold) -> None: + self.cands.append(cand) + self.golds.append(gold) + + def is_binary(self): + return len(np.unique(self.golds)) == 2 + + @property + def score(self): + if not self.is_binary(): + raise ValueError(Errors.E165.format(label=set(self.golds))) + if len(self.golds) == self.saved_score_at_len: + return self.saved_score + self.saved_score = _roc_auc_score(self.golds, self.cands) + self.saved_score_at_len = len(self.golds) + return self.saved_score + + +class Scorer: + """Compute evaluation scores.""" + + def __init__( + self, + nlp: Optional["Language"] = None, + default_lang: str = "xx", + default_pipeline: Iterable[str] = DEFAULT_PIPELINE, + **cfg, + ) -> None: + """Initialize the Scorer. + + DOCS: https://spacy.io/api/scorer#init + """ + self.cfg = cfg + if nlp: + self.nlp = nlp + else: + nlp = get_lang_class(default_lang)() + for pipe in default_pipeline: + nlp.add_pipe(pipe) + self.nlp = nlp + + def score( + self, examples: Iterable[Example], *, per_component: bool = False + ) -> Dict[str, Any]: + """Evaluate a list of Examples. + + examples (Iterable[Example]): The predicted annotations + correct annotations. + per_component (bool): Whether to return the scores keyed by component + name. Defaults to False. + RETURNS (Dict): A dictionary of scores. + + DOCS: https://spacy.io/api/scorer#score + """ + scores = {} + if hasattr(self.nlp.tokenizer, "score"): + if per_component: + scores["tokenizer"] = self.nlp.tokenizer.score(examples, **self.cfg) + else: + scores.update(self.nlp.tokenizer.score(examples, **self.cfg)) # type: ignore + for name, component in self.nlp.pipeline: + if hasattr(component, "score"): + if per_component: + scores[name] = component.score(examples, **self.cfg) + else: + scores.update(component.score(examples, **self.cfg)) + return scores + + @staticmethod + def score_tokenization(examples: Iterable[Example], **cfg) -> Dict[str, Any]: + """Returns accuracy and PRF scores for tokenization. + * token_acc: # correct tokens / # gold tokens + * token_p/r/f: PRF for token character spans + + examples (Iterable[Example]): Examples to score + RETURNS (Dict[str, Any]): A dictionary containing the scores + token_acc/p/r/f. + + DOCS: https://spacy.io/api/scorer#score_tokenization + """ + acc_score = PRFScore() + prf_score = PRFScore() + for example in examples: + gold_doc = example.reference + pred_doc = example.predicted + if gold_doc.has_unknown_spaces: + continue + align = example.alignment + gold_spans = set() + pred_spans = set() + for token in gold_doc: + if token.orth_.isspace(): + continue + gold_spans.add((token.idx, token.idx + len(token))) + for token in pred_doc: + if token.orth_.isspace(): + continue + pred_spans.add((token.idx, token.idx + len(token))) + if align.x2y.lengths[token.i] != 1: + acc_score.fp += 1 + else: + acc_score.tp += 1 + prf_score.score_set(pred_spans, gold_spans) + if len(acc_score) > 0: + return { + "token_acc": acc_score.precision, + "token_p": prf_score.precision, + "token_r": prf_score.recall, + "token_f": prf_score.fscore, + } + else: + return { + "token_acc": None, + "token_p": None, + "token_r": None, + "token_f": None, + } + + @staticmethod + def score_token_attr( + examples: Iterable[Example], + attr: str, + *, + getter: Callable[[Token, str], Any] = getattr, + missing_values: Set[Any] = MISSING_VALUES, # type: ignore[assignment] + **cfg, + ) -> Dict[str, Any]: + """Returns an accuracy score for a token-level attribute. + + examples (Iterable[Example]): Examples to score + attr (str): The attribute to score. + getter (Callable[[Token, str], Any]): Defaults to getattr. If provided, + getter(token, attr) should return the value of the attribute for an + individual token. + missing_values (Set[Any]): Attribute values to treat as missing annotation + in the reference annotation. + RETURNS (Dict[str, Any]): A dictionary containing the accuracy score + under the key attr_acc. + + DOCS: https://spacy.io/api/scorer#score_token_attr + """ + tag_score = PRFScore() + for example in examples: + gold_doc = example.reference + pred_doc = example.predicted + align = example.alignment + gold_tags = set() + missing_indices = set() + for gold_i, token in enumerate(gold_doc): + value = getter(token, attr) + if value not in missing_values: + gold_tags.add((gold_i, getter(token, attr))) + else: + missing_indices.add(gold_i) + pred_tags = set() + for token in pred_doc: + if token.orth_.isspace(): + continue + if align.x2y.lengths[token.i] == 1: + gold_i = align.x2y[token.i][0] + if gold_i not in missing_indices: + pred_tags.add((gold_i, getter(token, attr))) + tag_score.score_set(pred_tags, gold_tags) + score_key = f"{attr}_acc" + if len(tag_score) == 0: + return {score_key: None} + else: + return {score_key: tag_score.fscore} + + @staticmethod + def score_token_attr_per_feat( + examples: Iterable[Example], + attr: str, + *, + getter: Callable[[Token, str], Any] = getattr, + missing_values: Set[Any] = MISSING_VALUES, # type: ignore[assignment] + **cfg, + ) -> Dict[str, Any]: + """Return micro PRF and PRF scores per feat for a token attribute in + UFEATS format. + + examples (Iterable[Example]): Examples to score + attr (str): The attribute to score. + getter (Callable[[Token, str], Any]): Defaults to getattr. If provided, + getter(token, attr) should return the value of the attribute for an + individual token. + missing_values (Set[Any]): Attribute values to treat as missing + annotation in the reference annotation. + RETURNS (dict): A dictionary containing the micro PRF scores under the + key attr_micro_p/r/f and the per-feat PRF scores under + attr_per_feat. + """ + micro_score = PRFScore() + per_feat = {} + for example in examples: + pred_doc = example.predicted + gold_doc = example.reference + align = example.alignment + gold_per_feat: Dict[str, Set] = {} + missing_indices = set() + for gold_i, token in enumerate(gold_doc): + value = getter(token, attr) + morph = gold_doc.vocab.strings[value] + if value not in missing_values and morph != Morphology.EMPTY_MORPH: + for feat in morph.split(Morphology.FEATURE_SEP): + field, values = feat.split(Morphology.FIELD_SEP) + if field not in per_feat: + per_feat[field] = PRFScore() + if field not in gold_per_feat: + gold_per_feat[field] = set() + gold_per_feat[field].add((gold_i, feat)) + else: + missing_indices.add(gold_i) + pred_per_feat: Dict[str, Set] = {} + for token in pred_doc: + if token.orth_.isspace(): + continue + if align.x2y.lengths[token.i] == 1: + gold_i = align.x2y[token.i][0] + if gold_i not in missing_indices: + value = getter(token, attr) + morph = gold_doc.vocab.strings[value] + if ( + value not in missing_values + and morph != Morphology.EMPTY_MORPH + ): + for feat in morph.split(Morphology.FEATURE_SEP): + field, values = feat.split(Morphology.FIELD_SEP) + if field not in per_feat: + per_feat[field] = PRFScore() + if field not in pred_per_feat: + pred_per_feat[field] = set() + pred_per_feat[field].add((gold_i, feat)) + for field in per_feat: + micro_score.score_set( + pred_per_feat.get(field, set()), gold_per_feat.get(field, set()) + ) + per_feat[field].score_set( + pred_per_feat.get(field, set()), gold_per_feat.get(field, set()) + ) + result: Dict[str, Any] = {} + if len(micro_score) > 0: + result[f"{attr}_micro_p"] = micro_score.precision + result[f"{attr}_micro_r"] = micro_score.recall + result[f"{attr}_micro_f"] = micro_score.fscore + result[f"{attr}_per_feat"] = {k: v.to_dict() for k, v in per_feat.items()} + else: + result[f"{attr}_micro_p"] = None + result[f"{attr}_micro_r"] = None + result[f"{attr}_micro_f"] = None + result[f"{attr}_per_feat"] = None + return result + + @staticmethod + def score_spans( + examples: Iterable[Example], + attr: str, + *, + getter: Callable[[Doc, str], Iterable[Span]] = getattr, + has_annotation: Optional[Callable[[Doc], bool]] = None, + labeled: bool = True, + allow_overlap: bool = False, + **cfg, + ) -> Dict[str, Any]: + """Returns PRF scores for labeled spans. + + examples (Iterable[Example]): Examples to score + attr (str): The attribute to score. + getter (Callable[[Doc, str], Iterable[Span]]): Defaults to getattr. If + provided, getter(doc, attr) should return the spans for the + individual doc. + has_annotation (Optional[Callable[[Doc], bool]]) should return whether a `Doc` + has annotation for this `attr`. Docs without annotation are skipped for + scoring purposes. + labeled (bool): Whether or not to include label information in + the evaluation. If set to 'False', two spans will be considered + equal if their start and end match, irrespective of their label. + allow_overlap (bool): Whether or not to allow overlapping spans. + If set to 'False', the alignment will automatically resolve conflicts. + RETURNS (Dict[str, Any]): A dictionary containing the PRF scores under + the keys attr_p/r/f and the per-type PRF scores under attr_per_type. + + DOCS: https://spacy.io/api/scorer#score_spans + """ + score = PRFScore() + score_per_type = dict() + for example in examples: + pred_doc = example.predicted + gold_doc = example.reference + # Option to handle docs without annotation for this attribute + if has_annotation is not None and not has_annotation(gold_doc): + continue + # Find all labels in gold + labels = set([k.label_ for k in getter(gold_doc, attr)]) + # If labeled, find all labels in pred + if has_annotation is None or ( + has_annotation is not None and has_annotation(pred_doc) + ): + labels |= set([k.label_ for k in getter(pred_doc, attr)]) + # Set up all labels for per type scoring and prepare gold per type + gold_per_type: Dict[str, Set] = {label: set() for label in labels} + for label in labels: + if label not in score_per_type: + score_per_type[label] = PRFScore() + # Find all predidate labels, for all and per type + gold_spans = set() + pred_spans = set() + for span in getter(gold_doc, attr): + gold_span: Tuple + if labeled: + gold_span = (span.label_, span.start, span.end - 1) + else: + gold_span = (span.start, span.end - 1) + gold_spans.add(gold_span) + gold_per_type[span.label_].add(gold_span) + pred_per_type: Dict[str, Set] = {label: set() for label in labels} + if has_annotation is None or ( + has_annotation is not None and has_annotation(pred_doc) + ): + for span in example.get_aligned_spans_x2y( + getter(pred_doc, attr), allow_overlap + ): + pred_span: Tuple + if labeled: + pred_span = (span.label_, span.start, span.end - 1) + else: + pred_span = (span.start, span.end - 1) + pred_spans.add(pred_span) + pred_per_type[span.label_].add(pred_span) + # Scores per label + if labeled: + for k, v in score_per_type.items(): + if k in pred_per_type: + v.score_set(pred_per_type[k], gold_per_type[k]) + # Score for all labels + score.score_set(pred_spans, gold_spans) + # Assemble final result + final_scores: Dict[str, Any] = { + f"{attr}_p": None, + f"{attr}_r": None, + f"{attr}_f": None, + } + if labeled: + final_scores[f"{attr}_per_type"] = None + if len(score) > 0: + final_scores[f"{attr}_p"] = score.precision + final_scores[f"{attr}_r"] = score.recall + final_scores[f"{attr}_f"] = score.fscore + if labeled: + final_scores[f"{attr}_per_type"] = { + k: v.to_dict() for k, v in score_per_type.items() + } + return final_scores + + @staticmethod + def score_cats( + examples: Iterable[Example], + attr: str, + *, + getter: Callable[[Doc, str], Any] = getattr, + labels: Iterable[str] = SimpleFrozenList(), + multi_label: bool = True, + positive_label: Optional[str] = None, + threshold: Optional[float] = None, + **cfg, + ) -> Dict[str, Any]: + """Returns PRF and ROC AUC scores for a doc-level attribute with a + dict with scores for each label like Doc.cats. The reported overall + score depends on the scorer settings. + + examples (Iterable[Example]): Examples to score + attr (str): The attribute to score. + getter (Callable[[Doc, str], Any]): Defaults to getattr. If provided, + getter(doc, attr) should return the values for the individual doc. + labels (Iterable[str]): The set of possible labels. Defaults to []. + multi_label (bool): Whether the attribute allows multiple labels. + Defaults to True. When set to False (exclusive labels), missing + gold labels are interpreted as 0.0 and the threshold is set to 0.0. + positive_label (str): The positive label for a binary task with + exclusive classes. Defaults to None. + threshold (float): Cutoff to consider a prediction "positive". Defaults + to 0.5 for multi-label, and 0.0 (i.e. whatever's highest scoring) + otherwise. + RETURNS (Dict[str, Any]): A dictionary containing the scores, with + inapplicable scores as None: + for all: + attr_score (one of attr_micro_f / attr_macro_f / attr_macro_auc), + attr_score_desc (text description of the overall score), + attr_micro_p, + attr_micro_r, + attr_micro_f, + attr_macro_p, + attr_macro_r, + attr_macro_f, + attr_macro_auc, + attr_f_per_type, + attr_auc_per_type + + DOCS: https://spacy.io/api/scorer#score_cats + """ + if threshold is None: + threshold = 0.5 if multi_label else 0.0 + if not multi_label: + threshold = 0.0 + f_per_type = {label: PRFScore() for label in labels} + auc_per_type = {label: ROCAUCScore() for label in labels} + labels = set(labels) + for example in examples: + # Through this loop, None in the gold_cats indicates missing label. + pred_cats = getter(example.predicted, attr) + pred_cats = {k: v for k, v in pred_cats.items() if k in labels} + gold_cats = getter(example.reference, attr) + gold_cats = {k: v for k, v in gold_cats.items() if k in labels} + + for label in labels: + pred_score = pred_cats.get(label, 0.0) + gold_score = gold_cats.get(label) + if not gold_score and not multi_label: + gold_score = 0.0 + if gold_score is not None: + auc_per_type[label].score_set(pred_score, gold_score) + if multi_label: + for label in labels: + pred_score = pred_cats.get(label, 0.0) + gold_score = gold_cats.get(label) + if gold_score is not None: + if pred_score >= threshold and gold_score > 0: + f_per_type[label].tp += 1 + elif pred_score >= threshold and gold_score == 0: + f_per_type[label].fp += 1 + elif pred_score < threshold and gold_score > 0: + f_per_type[label].fn += 1 + elif pred_cats and gold_cats: + # Get the highest-scoring for each. + pred_label, pred_score = max(pred_cats.items(), key=lambda it: it[1]) + gold_label, gold_score = max(gold_cats.items(), key=lambda it: it[1]) + if pred_label == gold_label: + f_per_type[pred_label].tp += 1 + else: + f_per_type[gold_label].fn += 1 + f_per_type[pred_label].fp += 1 + elif gold_cats: + gold_label, gold_score = max(gold_cats, key=lambda it: it[1]) + if gold_score > 0: + f_per_type[gold_label].fn += 1 + elif pred_cats: + pred_label, pred_score = max(pred_cats.items(), key=lambda it: it[1]) + f_per_type[pred_label].fp += 1 + micro_prf = PRFScore() + for label_prf in f_per_type.values(): + micro_prf.tp += label_prf.tp + micro_prf.fn += label_prf.fn + micro_prf.fp += label_prf.fp + n_cats = len(f_per_type) + 1e-100 + macro_p = sum(prf.precision for prf in f_per_type.values()) / n_cats + macro_r = sum(prf.recall for prf in f_per_type.values()) / n_cats + macro_f = sum(prf.fscore for prf in f_per_type.values()) / n_cats + # Limit macro_auc to those labels with gold annotations, + # but still divide by all cats to avoid artificial boosting of datasets with missing labels + macro_auc = ( + sum(auc.score if auc.is_binary() else 0.0 for auc in auc_per_type.values()) + / n_cats + ) + results: Dict[str, Any] = { + f"{attr}_score": None, + f"{attr}_score_desc": None, + f"{attr}_micro_p": micro_prf.precision, + f"{attr}_micro_r": micro_prf.recall, + f"{attr}_micro_f": micro_prf.fscore, + f"{attr}_macro_p": macro_p, + f"{attr}_macro_r": macro_r, + f"{attr}_macro_f": macro_f, + f"{attr}_macro_auc": macro_auc, + f"{attr}_f_per_type": {k: v.to_dict() for k, v in f_per_type.items()}, + f"{attr}_auc_per_type": { + k: v.score if v.is_binary() else None for k, v in auc_per_type.items() + }, + } + if len(labels) == 2 and not multi_label and positive_label: + positive_label_f = results[f"{attr}_f_per_type"][positive_label]["f"] + results[f"{attr}_score"] = positive_label_f + results[f"{attr}_score_desc"] = f"F ({positive_label})" + elif not multi_label: + results[f"{attr}_score"] = results[f"{attr}_macro_f"] + results[f"{attr}_score_desc"] = "macro F" + else: + results[f"{attr}_score"] = results[f"{attr}_macro_auc"] + results[f"{attr}_score_desc"] = "macro AUC" + return results + + @staticmethod + def score_links( + examples: Iterable[Example], *, negative_labels: Iterable[str], **cfg + ) -> Dict[str, Any]: + """Returns PRF for predicted links on the entity level. + To disentangle the performance of the NEL from the NER, + this method only evaluates NEL links for entities that overlap + between the gold reference and the predictions. + + examples (Iterable[Example]): Examples to score + negative_labels (Iterable[str]): The string values that refer to no annotation (e.g. "NIL") + RETURNS (Dict[str, Any]): A dictionary containing the scores. + + DOCS: https://spacy.io/api/scorer#score_links + """ + f_per_type = {} + for example in examples: + gold_ent_by_offset = {} + for gold_ent in example.reference.ents: + gold_ent_by_offset[(gold_ent.start_char, gold_ent.end_char)] = gold_ent + + for pred_ent in example.predicted.ents: + gold_span = gold_ent_by_offset.get( + (pred_ent.start_char, pred_ent.end_char), None + ) + if gold_span is not None: + label = gold_span.label_ + if label not in f_per_type: + f_per_type[label] = PRFScore() + gold = gold_span.kb_id_ + # only evaluating entities that overlap between gold and pred, + # to disentangle the performance of the NEL from the NER + if gold is not None: + pred = pred_ent.kb_id_ + if gold in negative_labels and pred in negative_labels: + # ignore true negatives + pass + elif gold == pred: + f_per_type[label].tp += 1 + elif gold in negative_labels: + f_per_type[label].fp += 1 + elif pred in negative_labels: + f_per_type[label].fn += 1 + else: + # a wrong prediction (e.g. Q42 != Q3) counts as both a FP as well as a FN + f_per_type[label].fp += 1 + f_per_type[label].fn += 1 + micro_prf = PRFScore() + for label_prf in f_per_type.values(): + micro_prf.tp += label_prf.tp + micro_prf.fn += label_prf.fn + micro_prf.fp += label_prf.fp + n_labels = len(f_per_type) + 1e-100 + macro_p = sum(prf.precision for prf in f_per_type.values()) / n_labels + macro_r = sum(prf.recall for prf in f_per_type.values()) / n_labels + macro_f = sum(prf.fscore for prf in f_per_type.values()) / n_labels + results = { + f"nel_score": micro_prf.fscore, + f"nel_score_desc": "micro F", + f"nel_micro_p": micro_prf.precision, + f"nel_micro_r": micro_prf.recall, + f"nel_micro_f": micro_prf.fscore, + f"nel_macro_p": macro_p, + f"nel_macro_r": macro_r, + f"nel_macro_f": macro_f, + f"nel_f_per_type": {k: v.to_dict() for k, v in f_per_type.items()}, + } + return results + + @staticmethod + def score_deps( + examples: Iterable[Example], + attr: str, + *, + getter: Callable[[Token, str], Any] = getattr, + head_attr: str = "head", + head_getter: Callable[[Token, str], Token] = getattr, + ignore_labels: Iterable[str] = SimpleFrozenList(), + missing_values: Set[Any] = MISSING_VALUES, # type: ignore[assignment] + **cfg, + ) -> Dict[str, Any]: + """Returns the UAS, LAS, and LAS per type scores for dependency + parses. + + examples (Iterable[Example]): Examples to score + attr (str): The attribute containing the dependency label. + getter (Callable[[Token, str], Any]): Defaults to getattr. If provided, + getter(token, attr) should return the value of the attribute for an + individual token. + head_attr (str): The attribute containing the head token. Defaults to + 'head'. + head_getter (Callable[[Token, str], Token]): Defaults to getattr. If provided, + head_getter(token, attr) should return the value of the head for an + individual token. + ignore_labels (Tuple): Labels to ignore while scoring (e.g., punct). + missing_values (Set[Any]): Attribute values to treat as missing annotation + in the reference annotation. + RETURNS (Dict[str, Any]): A dictionary containing the scores: + attr_uas, attr_las, and attr_las_per_type. + + DOCS: https://spacy.io/api/scorer#score_deps + """ + unlabelled = PRFScore() + labelled = PRFScore() + labelled_per_dep = dict() + missing_indices = set() + for example in examples: + gold_doc = example.reference + pred_doc = example.predicted + align = example.alignment + gold_deps = set() + gold_deps_per_dep: Dict[str, Set] = {} + for gold_i, token in enumerate(gold_doc): + dep = getter(token, attr) + head = head_getter(token, head_attr) + if dep not in missing_values: + if dep not in ignore_labels: + gold_deps.add((gold_i, head.i, dep)) + if dep not in labelled_per_dep: + labelled_per_dep[dep] = PRFScore() + if dep not in gold_deps_per_dep: + gold_deps_per_dep[dep] = set() + gold_deps_per_dep[dep].add((gold_i, head.i, dep)) + else: + missing_indices.add(gold_i) + pred_deps = set() + pred_deps_per_dep: Dict[str, Set] = {} + for token in pred_doc: + if token.orth_.isspace(): + continue + if align.x2y.lengths[token.i] != 1: + gold_i = None # type: ignore + else: + gold_i = align.x2y[token.i][0] + if gold_i not in missing_indices: + dep = getter(token, attr) + head = head_getter(token, head_attr) + if dep not in ignore_labels and token.orth_.strip(): + if align.x2y.lengths[head.i] == 1: + gold_head = align.x2y[head.i][0] + else: + gold_head = None + # None is indistinct, so we can't just add it to the set + # Multiple (None, None) deps are possible + if gold_i is None or gold_head is None: + unlabelled.fp += 1 + labelled.fp += 1 + else: + pred_deps.add((gold_i, gold_head, dep)) + if dep not in labelled_per_dep: + labelled_per_dep[dep] = PRFScore() + if dep not in pred_deps_per_dep: + pred_deps_per_dep[dep] = set() + pred_deps_per_dep[dep].add((gold_i, gold_head, dep)) + labelled.score_set(pred_deps, gold_deps) + for dep in labelled_per_dep: + labelled_per_dep[dep].score_set( + pred_deps_per_dep.get(dep, set()), gold_deps_per_dep.get(dep, set()) + ) + unlabelled.score_set( + set(item[:2] for item in pred_deps), set(item[:2] for item in gold_deps) + ) + if len(unlabelled) > 0: + return { + f"{attr}_uas": unlabelled.fscore, + f"{attr}_las": labelled.fscore, + f"{attr}_las_per_type": { + k: v.to_dict() for k, v in labelled_per_dep.items() + }, + } + else: + return { + f"{attr}_uas": None, + f"{attr}_las": None, + f"{attr}_las_per_type": None, + } + + +def get_ner_prf(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: + """Compute micro-PRF and per-entity PRF scores for a sequence of examples.""" + score_per_type = defaultdict(PRFScore) + for eg in examples: + if not eg.y.has_annotation("ENT_IOB"): + continue + golds = {(e.label_, e.start, e.end) for e in eg.y.ents} + align_x2y = eg.alignment.x2y + for pred_ent in eg.x.ents: + if pred_ent.label_ not in score_per_type: + score_per_type[pred_ent.label_] = PRFScore() + indices = align_x2y[pred_ent.start : pred_ent.end] + if len(indices): + g_span = eg.y[indices[0] : indices[-1] + 1] + # Check we aren't missing annotation on this span. If so, + # our prediction is neither right nor wrong, we just + # ignore it. + if all(token.ent_iob != 0 for token in g_span): + key = (pred_ent.label_, indices[0], indices[-1] + 1) + if key in golds: + score_per_type[pred_ent.label_].tp += 1 + golds.remove(key) + else: + score_per_type[pred_ent.label_].fp += 1 + for label, start, end in golds: + score_per_type[label].fn += 1 + totals = PRFScore() + for prf in score_per_type.values(): + totals += prf + if len(totals) > 0: + return { + "ents_p": totals.precision, + "ents_r": totals.recall, + "ents_f": totals.fscore, + "ents_per_type": {k: v.to_dict() for k, v in score_per_type.items()}, + } + else: + return { + "ents_p": None, + "ents_r": None, + "ents_f": None, + "ents_per_type": None, + } + + +# The following implementation of trapezoid() is adapted from SciPy, +# which is distributed under the New BSD License. +# Copyright (c) 2001-2002 Enthought, Inc. 2003-2023, SciPy Developers. +# See licenses/3rd_party_licenses.txt +def trapezoid(y, x=None, dx=1.0, axis=-1): + r""" + Integrate along the given axis using the composite trapezoidal rule. + + If `x` is provided, the integration happens in sequence along its + elements - they are not sorted. + + Integrate `y` (`x`) along each 1d slice on the given axis, compute + :math:`\int y(x) dx`. + When `x` is specified, this integrates along the parametric curve, + computing :math:`\int_t y(t) dt = + \int_t y(t) \left.\frac{dx}{dt}\right|_{x=x(t)} dt`. + + Parameters + ---------- + y : array_like + Input array to integrate. + x : array_like, optional + The sample points corresponding to the `y` values. If `x` is None, + the sample points are assumed to be evenly spaced `dx` apart. The + default is None. + dx : scalar, optional + The spacing between sample points when `x` is None. The default is 1. + axis : int, optional + The axis along which to integrate. + + Returns + ------- + trapezoid : float or ndarray + Definite integral of `y` = n-dimensional array as approximated along + a single axis by the trapezoidal rule. If `y` is a 1-dimensional array, + then the result is a float. If `n` is greater than 1, then the result + is an `n`-1 dimensional array. + + See Also + -------- + cumulative_trapezoid, simpson, romb + + Notes + ----- + Image [2]_ illustrates trapezoidal rule -- y-axis locations of points + will be taken from `y` array, by default x-axis distances between + points will be 1.0, alternatively they can be provided with `x` array + or with `dx` scalar. Return value will be equal to combined area under + the red lines. + + References + ---------- + .. [1] Wikipedia page: https://en.wikipedia.org/wiki/Trapezoidal_rule + + .. [2] Illustration image: + https://en.wikipedia.org/wiki/File:Composite_trapezoidal_rule_illustration.png + + Examples + -------- + Use the trapezoidal rule on evenly spaced points: + + >>> import numpy as np + >>> from scipy import integrate + >>> integrate.trapezoid([1, 2, 3]) + 4.0 + + The spacing between sample points can be selected by either the + ``x`` or ``dx`` arguments: + + >>> integrate.trapezoid([1, 2, 3], x=[4, 6, 8]) + 8.0 + >>> integrate.trapezoid([1, 2, 3], dx=2) + 8.0 + + Using a decreasing ``x`` corresponds to integrating in reverse: + + >>> integrate.trapezoid([1, 2, 3], x=[8, 6, 4]) + -8.0 + + More generally ``x`` is used to integrate along a parametric curve. We can + estimate the integral :math:`\int_0^1 x^2 = 1/3` using: + + >>> x = np.linspace(0, 1, num=50) + >>> y = x**2 + >>> integrate.trapezoid(y, x) + 0.33340274885464394 + + Or estimate the area of a circle, noting we repeat the sample which closes + the curve: + + >>> theta = np.linspace(0, 2 * np.pi, num=1000, endpoint=True) + >>> integrate.trapezoid(np.cos(theta), x=np.sin(theta)) + 3.141571941375841 + + ``trapezoid`` can be applied along a specified axis to do multiple + computations in one call: + + >>> a = np.arange(6).reshape(2, 3) + >>> a + array([[0, 1, 2], + [3, 4, 5]]) + >>> integrate.trapezoid(a, axis=0) + array([1.5, 2.5, 3.5]) + >>> integrate.trapezoid(a, axis=1) + array([2., 8.]) + """ + y = np.asanyarray(y) + if x is None: + d = dx + else: + x = np.asanyarray(x) + if x.ndim == 1: + d = np.diff(x) + # reshape to correct shape + shape = [1] * y.ndim + shape[axis] = d.shape[0] + d = d.reshape(shape) + else: + d = np.diff(x, axis=axis) + nd = y.ndim + slice1 = [slice(None)] * nd + slice2 = [slice(None)] * nd + slice1[axis] = slice(1, None) + slice2[axis] = slice(None, -1) + try: + ret = (d * (y[tuple(slice1)] + y[tuple(slice2)]) / 2.0).sum(axis) + except ValueError: + # Operations didn't work, cast to ndarray + d = np.asarray(d) + y = np.asarray(y) + ret = np.add.reduce(d * (y[tuple(slice1)] + y[tuple(slice2)]) / 2.0, axis) + return ret + + +# The following implementation of roc_auc_score() is adapted from +# scikit-learn, which is distributed under the New BSD License. +# Copyright (c) 2007–2019 The scikit-learn developers. +# See licenses/3rd_party_licenses.txt +def _roc_auc_score(y_true, y_score): + """Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) + from prediction scores. + + Note: this implementation is restricted to the binary classification task + + Parameters + ---------- + y_true : array, shape = [n_samples] or [n_samples, n_classes] + True binary labels or binary label indicators. + The multiclass case expects shape = [n_samples] and labels + with values in ``range(n_classes)``. + + y_score : array, shape = [n_samples] or [n_samples, n_classes] + Target scores, can either be probability estimates of the positive + class, confidence values, or non-thresholded measure of decisions + (as returned by "decision_function" on some classifiers). For binary + y_true, y_score is supposed to be the score of the class with greater + label. The multiclass case expects shape = [n_samples, n_classes] + where the scores correspond to probability estimates. + + Returns + ------- + auc : float + + References + ---------- + .. [1] `Wikipedia entry for the Receiver operating characteristic + `_ + + .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition + Letters, 2006, 27(8):861-874. + + .. [3] `Analyzing a portion of the ROC curve. McClish, 1989 + `_ + """ + if len(np.unique(y_true)) != 2: + raise ValueError(Errors.E165.format(label=np.unique(y_true))) + fpr, tpr, _ = _roc_curve(y_true, y_score) + return _auc(fpr, tpr) + + +def _roc_curve(y_true, y_score): + """Compute Receiver operating characteristic (ROC) + + Note: this implementation is restricted to the binary classification task. + + Parameters + ---------- + + y_true : array, shape = [n_samples] + True binary labels. If labels are not either {-1, 1} or {0, 1}, then + pos_label should be explicitly given. + + y_score : array, shape = [n_samples] + Target scores, can either be probability estimates of the positive + class, confidence values, or non-thresholded measure of decisions + (as returned by "decision_function" on some classifiers). + + Returns + ------- + fpr : array, shape = [>2] + Increasing false positive rates such that element i is the false + positive rate of predictions with score >= thresholds[i]. + + tpr : array, shape = [>2] + Increasing true positive rates such that element i is the true + positive rate of predictions with score >= thresholds[i]. + + thresholds : array, shape = [n_thresholds] + Decreasing thresholds on the decision function used to compute + fpr and tpr. `thresholds[0]` represents no instances being predicted + and is arbitrarily set to `max(y_score) + 1`. + + Notes + ----- + Since the thresholds are sorted from low to high values, they + are reversed upon returning them to ensure they correspond to both ``fpr`` + and ``tpr``, which are sorted in reversed order during their calculation. + + References + ---------- + .. [1] `Wikipedia entry for the Receiver operating characteristic + `_ + + .. [2] Fawcett T. An introduction to ROC analysis[J]. Pattern Recognition + Letters, 2006, 27(8):861-874. + """ + fps, tps, thresholds = _binary_clf_curve(y_true, y_score) + + # Add an extra threshold position + # to make sure that the curve starts at (0, 0) + tps = np.r_[0, tps] + fps = np.r_[0, fps] + thresholds = np.r_[thresholds[0] + 1, thresholds] + + if fps[-1] <= 0: + fpr = np.repeat(np.nan, fps.shape) + else: + fpr = fps / fps[-1] + + if tps[-1] <= 0: + tpr = np.repeat(np.nan, tps.shape) + else: + tpr = tps / tps[-1] + + return fpr, tpr, thresholds + + +def _binary_clf_curve(y_true, y_score): + """Calculate true and false positives per binary classification threshold. + + Parameters + ---------- + y_true : array, shape = [n_samples] + True targets of binary classification + + y_score : array, shape = [n_samples] + Estimated probabilities or decision function + + Returns + ------- + fps : array, shape = [n_thresholds] + A count of false positives, at index i being the number of negative + samples assigned a score >= thresholds[i]. The total number of + negative samples is equal to fps[-1] (thus true negatives are given by + fps[-1] - fps). + + tps : array, shape = [n_thresholds <= len(np.unique(y_score))] + An increasing count of true positives, at index i being the number + of positive samples assigned a score >= thresholds[i]. The total + number of positive samples is equal to tps[-1] (thus false negatives + are given by tps[-1] - tps). + + thresholds : array, shape = [n_thresholds] + Decreasing score values. + """ + pos_label = 1.0 + + y_true = np.ravel(y_true) + y_score = np.ravel(y_score) + + # make y_true a boolean vector + y_true = y_true == pos_label + + # sort scores and corresponding truth values + desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1] + y_score = y_score[desc_score_indices] + y_true = y_true[desc_score_indices] + weight = 1.0 + + # y_score typically has many tied values. Here we extract + # the indices associated with the distinct values. We also + # concatenate a value for the end of the curve. + distinct_value_indices = np.where(np.diff(y_score))[0] + threshold_idxs = np.r_[distinct_value_indices, y_true.size - 1] + + # accumulate the true positives with decreasing threshold + tps = _stable_cumsum(y_true * weight)[threshold_idxs] + fps = 1 + threshold_idxs - tps + return fps, tps, y_score[threshold_idxs] + + +def _stable_cumsum(arr, axis=None, rtol=1e-05, atol=1e-08): + """Use high precision for cumsum and check that final value matches sum + + Parameters + ---------- + arr : array-like + To be cumulatively summed as flat + axis : int, optional + Axis along which the cumulative sum is computed. + The default (None) is to compute the cumsum over the flattened array. + rtol : float + Relative tolerance, see ``np.allclose`` + atol : float + Absolute tolerance, see ``np.allclose`` + """ + out = np.cumsum(arr, axis=axis, dtype=np.float64) + expected = np.sum(arr, axis=axis, dtype=np.float64) + if not np.all( + np.isclose( + out.take(-1, axis=axis), expected, rtol=rtol, atol=atol, equal_nan=True + ) + ): + raise ValueError(Errors.E163) + return out + + +def _auc(x, y): + """Compute Area Under the Curve (AUC) using the trapezoidal rule + + This is a general function, given points on a curve. For computing the + area under the ROC-curve, see :func:`roc_auc_score`. + + Parameters + ---------- + x : array, shape = [n] + x coordinates. These must be either monotonic increasing or monotonic + decreasing. + y : array, shape = [n] + y coordinates. + + Returns + ------- + auc : float + """ + x = np.ravel(x) + y = np.ravel(y) + + direction = 1 + dx = np.diff(x) + if np.any(dx < 0): + if np.all(dx <= 0): + direction = -1 + else: + raise ValueError(Errors.E164.format(x=x)) + + area = direction * trapezoid(y, x) + if isinstance(area, np.memmap): + # Reductions such as .sum used internally in trapezoid do not return a + # scalar by default for numpy.memmap instances contrary to + # regular numpy.ndarray instances. + area = area.dtype.type(area) + return area diff --git a/spacy/strings.pxd b/spacy/strings.pxd new file mode 100644 index 0000000..b015858 --- /dev/null +++ b/spacy/strings.pxd @@ -0,0 +1,31 @@ +from cymem.cymem cimport Pool +from libc.stdint cimport int64_t +from libcpp.set cimport set +from libcpp.vector cimport vector +from murmurhash.mrmr cimport hash64 +from preshed.maps cimport PreshMap + +from .typedefs cimport attr_t, hash_t + + +cpdef hash_t hash_string(str string) except 0 +cdef hash_t hash_utf8(char* utf8_string, int length) nogil + +cdef str decode_Utf8Str(const Utf8Str* string) + + +ctypedef union Utf8Str: + unsigned char[8] s + unsigned char* p + + +cdef class StringStore: + cdef Pool mem + + cdef vector[hash_t] keys + cdef public PreshMap _map + + cdef const Utf8Str* intern_unicode(self, str py_string, bint allow_transient) + cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash, bint allow_transient) + cdef vector[hash_t] _transient_keys + cdef Pool _non_temp_mem diff --git a/spacy/strings.pyi b/spacy/strings.pyi new file mode 100644 index 0000000..f8fe838 --- /dev/null +++ b/spacy/strings.pyi @@ -0,0 +1,25 @@ +from pathlib import Path +from typing import Any, Iterable, Iterator, Optional, Union, overload + +def get_string_id(key: Union[str, int]) -> int: ... + +class StringStore: + def __init__( + self, strings: Optional[Iterable[str]] = ..., freeze: bool = ... + ) -> None: ... + @overload + def __getitem__(self, string_or_id: Union[bytes, str]) -> int: ... + @overload + def __getitem__(self, string_or_id: int) -> str: ... + def as_int(self, key: Union[bytes, str, int]) -> int: ... + def as_string(self, key: Union[bytes, str, int]) -> str: ... + def add(self, string: str) -> int: ... + def __len__(self) -> int: ... + def __contains__(self, string: str) -> bool: ... + def __iter__(self) -> Iterator[str]: ... + def __reduce__(self) -> Any: ... + def to_disk(self, path: Union[str, Path]) -> None: ... + def from_disk(self, path: Union[str, Path]) -> StringStore: ... + def to_bytes(self, **kwargs: Any) -> bytes: ... + def from_bytes(self, bytes_data: bytes, **kwargs: Any) -> StringStore: ... + def _reset_and_load(self, strings: Iterable[str]) -> None: ... diff --git a/spacy/strings.pyx b/spacy/strings.pyx new file mode 100644 index 0000000..a65cdb6 --- /dev/null +++ b/spacy/strings.pyx @@ -0,0 +1,399 @@ +# cython: infer_types=True +# cython: profile=False +cimport cython + +from contextlib import contextmanager +from typing import List, Optional + +from libc.stdint cimport uint32_t +from libc.string cimport memcpy +from murmurhash.mrmr cimport hash32, hash64 +from preshed.maps cimport map_clear + +import srsly + +from .typedefs cimport hash_t + +from . import util +from .errors import Errors +from .symbols import IDS as SYMBOLS_BY_STR +from .symbols import NAMES as SYMBOLS_BY_INT + + +# Not particularly elegant, but this is faster than `isinstance(key, numbers.Integral)` +cdef inline bint _try_coerce_to_hash(object key, hash_t* out_hash): + try: + out_hash[0] = key + return True + except: # no-cython-lint + return False + + +def get_string_id(key): + """Get a string ID, handling the reserved symbols correctly. If the key is + already an ID, return it. + + This function optimises for convenience over performance, so shouldn't be + used in tight loops. + """ + cdef hash_t str_hash + if isinstance(key, str): + if len(key) == 0: + return 0 + + symbol = SYMBOLS_BY_STR.get(key, None) + if symbol is not None: + return symbol + else: + chars = key.encode("utf8") + return hash_utf8(chars, len(chars)) + elif _try_coerce_to_hash(key, &str_hash): + # Coerce the integral key to the expected primitive hash type. + # This ensures that custom/overloaded "primitive" data types + # such as those implemented by numpy are not inadvertently used + # downsteam (as these are internally implemented as custom PyObjects + # whose comparison operators can incur a significant overhead). + return str_hash + else: + # TODO: Raise an error instead + return key + + +cpdef hash_t hash_string(str string) except 0: + chars = string.encode("utf8") + return hash_utf8(chars, len(chars)) + + +cdef hash_t hash_utf8(char* utf8_string, int length) nogil: + return hash64(utf8_string, length, 1) + + +cdef uint32_t hash32_utf8(char* utf8_string, int length) nogil: + return hash32(utf8_string, length, 1) + + +cdef str decode_Utf8Str(const Utf8Str* string): + cdef int i, length + if string.s[0] < sizeof(string.s) and string.s[0] != 0: + return string.s[1:string.s[0]+1].decode("utf8") + elif string.p[0] < 255: + return string.p[1:string.p[0]+1].decode("utf8") + else: + i = 0 + length = 0 + while string.p[i] == 255: + i += 1 + length += 255 + length += string.p[i] + i += 1 + return string.p[i:length + i].decode("utf8") + + +cdef Utf8Str* _allocate(Pool mem, const unsigned char* chars, uint32_t length) except *: + cdef int n_length_bytes + cdef int i + cdef Utf8Str* string = mem.alloc(1, sizeof(Utf8Str)) + if length < sizeof(string.s): + string.s[0] = length + memcpy(&string.s[1], chars, length) + return string + elif length < 255: + string.p = mem.alloc(length + 1, sizeof(unsigned char)) + string.p[0] = length + memcpy(&string.p[1], chars, length) + return string + else: + i = 0 + n_length_bytes = (length // 255) + 1 + string.p = mem.alloc(length + n_length_bytes, sizeof(unsigned char)) + for i in range(n_length_bytes-1): + string.p[i] = 255 + string.p[n_length_bytes-1] = length % 255 + memcpy(&string.p[n_length_bytes], chars, length) + return string + + +cdef class StringStore: + """Look up strings by 64-bit hashes. + + DOCS: https://spacy.io/api/stringstore + """ + def __init__(self, strings=None, freeze=False): + """Create the StringStore. + + strings (iterable): A sequence of unicode strings to add to the store. + """ + self.mem = Pool() + self._non_temp_mem = self.mem + self._map = PreshMap() + if strings is not None: + for string in strings: + self.add(string, allow_transient=False) + + def __getitem__(self, object string_or_id): + """Retrieve a string from a given hash, or vice versa. + + string_or_id (bytes, str or uint64): The value to encode. + Returns (str / uint64): The value to be retrieved. + """ + cdef hash_t str_hash + cdef Utf8Str* utf8str = NULL + + if isinstance(string_or_id, str): + if len(string_or_id) == 0: + return 0 + + # Return early if the string is found in the symbols LUT. + symbol = SYMBOLS_BY_STR.get(string_or_id, None) + if symbol is not None: + return symbol + else: + return hash_string(string_or_id) + elif isinstance(string_or_id, bytes): + return hash_utf8(string_or_id, len(string_or_id)) + elif _try_coerce_to_hash(string_or_id, &str_hash): + if str_hash == 0: + return "" + elif str_hash < len(SYMBOLS_BY_INT): + return SYMBOLS_BY_INT[str_hash] + else: + utf8str = self._map.get(str_hash) + if utf8str is NULL: + raise KeyError(Errors.E018.format(hash_value=string_or_id)) + else: + return decode_Utf8Str(utf8str) + else: + # TODO: Raise an error instead + utf8str = self._map.get(string_or_id) + if utf8str is NULL: + raise KeyError(Errors.E018.format(hash_value=string_or_id)) + else: + return decode_Utf8Str(utf8str) + + def as_int(self, key): + """If key is an int, return it; otherwise, get the int value.""" + if not isinstance(key, str): + return key + else: + return self[key] + + def as_string(self, key): + """If key is a string, return it; otherwise, get the string value.""" + if isinstance(key, str): + return key + else: + return self[key] + + def __len__(self) -> int: + """The number of strings in the store. + + RETURNS (int): The number of strings in the store. + """ + return self.keys.size() + self._transient_keys.size() + + @contextmanager + def memory_zone(self, mem: Optional[Pool] = None) -> Pool: + """Begin a block where all resources allocated during the block will + be freed at the end of it. If a resources was created within the + memory zone block, accessing it outside the block is invalid. + Behaviour of this invalid access is undefined. Memory zones should + not be nested. + + The memory zone is helpful for services that need to process large + volumes of text with a defined memory budget. + """ + if mem is None: + mem = Pool() + self.mem = mem + try: + yield mem + finally: + for key in self._transient_keys: + map_clear(self._map.c_map, key) + self._transient_keys.clear() + self.mem = self._non_temp_mem + + def add(self, string: str, allow_transient: Optional[bool] = None) -> int: + """Add a string to the StringStore. + + string (str): The string to add. + allow_transient (bool): Allow the string to be stored in the 'transient' + map, which will be flushed at the end of the memory zone. Strings + encountered during arbitrary text processing should be added + with allow_transient=True, while labels and other strings used + internally should not. + RETURNS (uint64): The string's hash value. + """ + if not string: + return 0 + if allow_transient is None: + allow_transient = self.mem is not self._non_temp_mem + cdef hash_t str_hash + if isinstance(string, str): + if string in SYMBOLS_BY_STR: + return SYMBOLS_BY_STR[string] + + string = string.encode("utf8") + str_hash = hash_utf8(string, len(string)) + self._intern_utf8(string, len(string), &str_hash, allow_transient) + elif isinstance(string, bytes): + if string in SYMBOLS_BY_STR: + return SYMBOLS_BY_STR[string] + str_hash = hash_utf8(string, len(string)) + self._intern_utf8(string, len(string), &str_hash, allow_transient) + else: + raise TypeError(Errors.E017.format(value_type=type(string))) + return str_hash + + def __len__(self): + """The number of strings in the store. + if string in SYMBOLS_BY_STR: + return SYMBOLS_BY_STR[string] + else: + return self._intern_str(string, allow_transient) + + RETURNS (int): The number of strings in the store. + """ + return self.keys.size() + self._transient_keys.size() + + def __contains__(self, string_or_id not None): + """Check whether a string or ID is in the store. + + string_or_id (str or int): The string to check. + RETURNS (bool): Whether the store contains the string. + """ + cdef hash_t str_hash + if isinstance(string_or_id, str): + if len(string_or_id) == 0: + return True + elif string_or_id in SYMBOLS_BY_STR: + return True + str_hash = hash_string(string_or_id) + elif _try_coerce_to_hash(string_or_id, &str_hash): + pass + else: + # TODO: Raise an error instead + if self._map.get(string_or_id) is not NULL: + return True + else: + return False + if str_hash < len(SYMBOLS_BY_INT): + return True + else: + if self._map.get(str_hash) is not NULL: + return True + else: + return False + + def __iter__(self): + """Iterate over the strings in the store, in order. + + YIELDS (str): A string in the store. + """ + cdef int i + cdef hash_t key + for i in range(self.keys.size()): + key = self.keys[i] + utf8str = self._map.get(key) + yield decode_Utf8Str(utf8str) + for i in range(self._transient_keys.size()): + key = self._transient_keys[i] + utf8str = self._map.get(key) + yield decode_Utf8Str(utf8str) + + def __reduce__(self): + strings = list(self) + return (StringStore, (strings,), None, None, None) + + def values(self) -> List[int]: + """Iterate over the stored strings hashes in insertion order. + + RETURNS: A list of string hashs. + """ + cdef int i + hashes = [None] * self._keys.size() + for i in range(self._keys.size()): + hashes[i] = self._keys[i] + transient_hashes = [None] * self._transient_keys.size() + for i in range(self._transient_keys.size()): + transient_hashes[i] = self._transient_keys[i] + return hashes + transient_hashes + + def to_disk(self, path): + """Save the current state to a directory. + + path (str / Path): A path to a directory, which will be created if + it doesn't exist. Paths may be either strings or Path-like objects. + """ + path = util.ensure_path(path) + strings = sorted(self) + srsly.write_json(path, strings) + + def from_disk(self, path): + """Loads state from a directory. Modifies the object in place and + returns it. + + path (str / Path): A path to a directory. Paths may be either + strings or `Path`-like objects. + RETURNS (StringStore): The modified `StringStore` object. + """ + path = util.ensure_path(path) + strings = srsly.read_json(path) + prev = list(self) + self._reset_and_load(strings) + for word in prev: + self.add(word, allow_transient=False) + return self + + def to_bytes(self, **kwargs): + """Serialize the current state to a binary string. + + RETURNS (bytes): The serialized form of the `StringStore` object. + """ + return srsly.json_dumps(sorted(self)) + + def from_bytes(self, bytes_data, **kwargs): + """Load state from a binary string. + + bytes_data (bytes): The data to load from. + RETURNS (StringStore): The `StringStore` object. + """ + strings = srsly.json_loads(bytes_data) + prev = list(self) + self._reset_and_load(strings) + for word in prev: + self.add(word, allow_transient=False) + return self + + def _reset_and_load(self, strings): + self.mem = Pool() + self._non_temp_mem = self.mem + self._map = PreshMap() + self.keys.clear() + self._transient_keys.clear() + for string in strings: + self.add(string, allow_transient=False) + + cdef const Utf8Str* intern_unicode(self, str py_string, bint allow_transient): + # 0 means missing, but we don't bother offsetting the index. + cdef bytes byte_string = py_string.encode("utf8") + return self._intern_utf8(byte_string, len(byte_string), NULL, allow_transient) + + @cython.final + cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length, hash_t* precalculated_hash, bint allow_transient): + # TODO: This function's API/behaviour is an unholy mess... + # 0 means missing, but we don't bother offsetting the index. + cdef hash_t key = precalculated_hash[0] if precalculated_hash is not NULL else hash_utf8(utf8_string, length) + cdef Utf8Str* value = self._map.get(key) + if value is not NULL: + return value + if allow_transient: + value = _allocate(self.mem, utf8_string, length) + else: + value = _allocate(self._non_temp_mem, utf8_string, length) + self._map.set(key, value) + if allow_transient and self.mem is not self._non_temp_mem: + self._transient_keys.push_back(key) + else: + self.keys.push_back(key) + return value diff --git a/spacy/structs.pxd b/spacy/structs.pxd new file mode 100644 index 0000000..8cfcc29 --- /dev/null +++ b/spacy/structs.pxd @@ -0,0 +1,113 @@ +from libc.stdint cimport int32_t, int64_t, uint8_t, uint32_t, uint64_t +from libcpp.unordered_map cimport unordered_map +from libcpp.unordered_set cimport unordered_set +from libcpp.vector cimport vector + +from .parts_of_speech cimport univ_pos_t +from .typedefs cimport attr_t, flags_t, hash_t + + +cdef struct LexemeC: + flags_t flags + + attr_t lang + + attr_t id + attr_t length + + attr_t orth + attr_t lower + attr_t norm + attr_t shape + attr_t prefix + attr_t suffix + + +cdef struct SpanC: + hash_t id + int start + int end + int start_char + int end_char + attr_t label + attr_t kb_id + + +cdef struct TokenC: + const LexemeC* lex + uint64_t morph + univ_pos_t pos + bint spacy + attr_t tag + int idx + attr_t lemma + attr_t norm + int head + attr_t dep + + uint32_t l_kids + uint32_t r_kids + uint32_t l_edge + uint32_t r_edge + + int sent_start + int ent_iob + attr_t ent_type # TODO: Is there a better way to do this? Multiple sources of truth.. + attr_t ent_kb_id + hash_t ent_id + + +cdef struct MorphAnalysisC: + hash_t key + int length + + attr_t* fields + attr_t* features + + +# Internal struct, for storage and disambiguation of entities. +cdef struct KBEntryC: + + # The hash of this entry's unique ID/name in the kB + hash_t entity_hash + + # Allows retrieval of the entity vector, as an index into a vectors table of the KB. + # Can be expanded later to refer to multiple rows (compositional model to reduce storage footprint). + int32_t vector_index + + # Allows retrieval of a struct of non-vector features. + # This is currently not implemented and set to -1 for the common case where there are no features. + int32_t feats_row + + # log probability of entity, based on corpus frequency + float freq + + +# Each alias struct stores a list of Entry pointers with their prior probabilities +# for this specific mention/alias. +cdef struct AliasC: + + # All entry candidates for this alias + vector[int64_t] entry_indices + + # Prior probability P(entity|alias) - should sum up to (at most) 1. + vector[float] probs + + +cdef struct EdgeC: + hash_t label + int32_t head + int32_t tail + + +cdef struct GraphC: + vector[vector[int32_t]] nodes + vector[EdgeC] edges + vector[float] weights + vector[int] n_heads + vector[int] n_tails + vector[int] first_head + vector[int] first_tail + unordered_set[int]* roots + unordered_map[hash_t, int]* node_map + unordered_map[hash_t, int]* edge_map diff --git a/spacy/symbols.pxd b/spacy/symbols.pxd new file mode 100644 index 0000000..73be191 --- /dev/null +++ b/spacy/symbols.pxd @@ -0,0 +1,469 @@ +cdef enum symbol_t: + NIL + IS_ALPHA + IS_ASCII + IS_DIGIT + IS_LOWER + IS_PUNCT + IS_SPACE + IS_TITLE + IS_UPPER + LIKE_URL + LIKE_NUM + LIKE_EMAIL + IS_STOP + IS_OOV_DEPRECATED + IS_BRACKET + IS_QUOTE + IS_LEFT_PUNCT + IS_RIGHT_PUNCT + IS_CURRENCY + + FLAG19 = 19 + FLAG20 + FLAG21 + FLAG22 + FLAG23 + FLAG24 + FLAG25 + FLAG26 + FLAG27 + FLAG28 + FLAG29 + FLAG30 + FLAG31 + FLAG32 + FLAG33 + FLAG34 + FLAG35 + FLAG36 + FLAG37 + FLAG38 + FLAG39 + FLAG40 + FLAG41 + FLAG42 + FLAG43 + FLAG44 + FLAG45 + FLAG46 + FLAG47 + FLAG48 + FLAG49 + FLAG50 + FLAG51 + FLAG52 + FLAG53 + FLAG54 + FLAG55 + FLAG56 + FLAG57 + FLAG58 + FLAG59 + FLAG60 + FLAG61 + FLAG62 + FLAG63 + + ID + ORTH + LOWER + NORM + SHAPE + PREFIX + SUFFIX + + LENGTH + CLUSTER + LEMMA + POS + TAG + DEP + ENT_IOB + ENT_TYPE + HEAD + SENT_START + SPACY + PROB + LANG + + ADJ + ADP + ADV + AUX + CONJ + CCONJ # U20 + DET + INTJ + NOUN + NUM + PART + PRON + PROPN + PUNCT + SCONJ + SYM + VERB + X + EOL + SPACE + + DEPRECATED001 + DEPRECATED002 + DEPRECATED003 + DEPRECATED004 + DEPRECATED005 + DEPRECATED006 + DEPRECATED007 + DEPRECATED008 + DEPRECATED009 + DEPRECATED010 + DEPRECATED011 + DEPRECATED012 + DEPRECATED013 + DEPRECATED014 + DEPRECATED015 + DEPRECATED016 + DEPRECATED017 + DEPRECATED018 + DEPRECATED019 + DEPRECATED020 + DEPRECATED021 + DEPRECATED022 + DEPRECATED023 + DEPRECATED024 + DEPRECATED025 + DEPRECATED026 + DEPRECATED027 + DEPRECATED028 + DEPRECATED029 + DEPRECATED030 + DEPRECATED031 + DEPRECATED032 + DEPRECATED033 + DEPRECATED034 + DEPRECATED035 + DEPRECATED036 + DEPRECATED037 + DEPRECATED038 + DEPRECATED039 + DEPRECATED040 + DEPRECATED041 + DEPRECATED042 + DEPRECATED043 + DEPRECATED044 + DEPRECATED045 + DEPRECATED046 + DEPRECATED047 + DEPRECATED048 + DEPRECATED049 + DEPRECATED050 + DEPRECATED051 + DEPRECATED052 + DEPRECATED053 + DEPRECATED054 + DEPRECATED055 + DEPRECATED056 + DEPRECATED057 + DEPRECATED058 + DEPRECATED059 + DEPRECATED060 + DEPRECATED061 + DEPRECATED062 + DEPRECATED063 + DEPRECATED064 + DEPRECATED065 + DEPRECATED066 + DEPRECATED067 + DEPRECATED068 + DEPRECATED069 + DEPRECATED070 + DEPRECATED071 + DEPRECATED072 + DEPRECATED073 + DEPRECATED074 + DEPRECATED075 + DEPRECATED076 + DEPRECATED077 + DEPRECATED078 + DEPRECATED079 + DEPRECATED080 + DEPRECATED081 + DEPRECATED082 + DEPRECATED083 + DEPRECATED084 + DEPRECATED085 + DEPRECATED086 + DEPRECATED087 + DEPRECATED088 + DEPRECATED089 + DEPRECATED090 + DEPRECATED091 + DEPRECATED092 + DEPRECATED093 + DEPRECATED094 + DEPRECATED095 + DEPRECATED096 + DEPRECATED097 + DEPRECATED098 + DEPRECATED099 + DEPRECATED100 + DEPRECATED101 + DEPRECATED102 + DEPRECATED103 + DEPRECATED104 + DEPRECATED105 + DEPRECATED106 + DEPRECATED107 + DEPRECATED108 + DEPRECATED109 + DEPRECATED110 + DEPRECATED111 + DEPRECATED112 + DEPRECATED113 + DEPRECATED114 + DEPRECATED115 + DEPRECATED116 + DEPRECATED117 + DEPRECATED118 + DEPRECATED119 + DEPRECATED120 + DEPRECATED121 + DEPRECATED122 + DEPRECATED123 + DEPRECATED124 + DEPRECATED125 + DEPRECATED126 + DEPRECATED127 + DEPRECATED128 + DEPRECATED129 + DEPRECATED130 + DEPRECATED131 + DEPRECATED132 + DEPRECATED133 + DEPRECATED134 + DEPRECATED135 + DEPRECATED136 + DEPRECATED137 + DEPRECATED138 + DEPRECATED139 + DEPRECATED140 + DEPRECATED141 + DEPRECATED142 + DEPRECATED143 + DEPRECATED144 + DEPRECATED145 + DEPRECATED146 + DEPRECATED147 + DEPRECATED148 + DEPRECATED149 + DEPRECATED150 + DEPRECATED151 + DEPRECATED152 + DEPRECATED153 + DEPRECATED154 + DEPRECATED155 + DEPRECATED156 + DEPRECATED157 + DEPRECATED158 + DEPRECATED159 + DEPRECATED160 + DEPRECATED161 + DEPRECATED162 + DEPRECATED163 + DEPRECATED164 + DEPRECATED165 + DEPRECATED166 + DEPRECATED167 + DEPRECATED168 + DEPRECATED169 + DEPRECATED170 + DEPRECATED171 + DEPRECATED172 + DEPRECATED173 + DEPRECATED174 + DEPRECATED175 + DEPRECATED176 + DEPRECATED177 + DEPRECATED178 + DEPRECATED179 + DEPRECATED180 + DEPRECATED181 + DEPRECATED182 + DEPRECATED183 + DEPRECATED184 + DEPRECATED185 + DEPRECATED186 + DEPRECATED187 + DEPRECATED188 + DEPRECATED189 + DEPRECATED190 + DEPRECATED191 + DEPRECATED192 + DEPRECATED193 + DEPRECATED194 + DEPRECATED195 + DEPRECATED196 + DEPRECATED197 + DEPRECATED198 + DEPRECATED199 + DEPRECATED200 + DEPRECATED201 + DEPRECATED202 + DEPRECATED203 + DEPRECATED204 + DEPRECATED205 + DEPRECATED206 + DEPRECATED207 + DEPRECATED208 + DEPRECATED209 + DEPRECATED210 + DEPRECATED211 + DEPRECATED212 + DEPRECATED213 + DEPRECATED214 + DEPRECATED215 + DEPRECATED216 + DEPRECATED217 + DEPRECATED218 + DEPRECATED219 + DEPRECATED220 + DEPRECATED221 + DEPRECATED222 + DEPRECATED223 + DEPRECATED224 + DEPRECATED225 + DEPRECATED226 + DEPRECATED227 + DEPRECATED228 + DEPRECATED229 + DEPRECATED230 + DEPRECATED231 + DEPRECATED232 + DEPRECATED233 + DEPRECATED234 + DEPRECATED235 + DEPRECATED236 + DEPRECATED237 + DEPRECATED238 + DEPRECATED239 + DEPRECATED240 + DEPRECATED241 + DEPRECATED242 + DEPRECATED243 + DEPRECATED244 + DEPRECATED245 + DEPRECATED246 + DEPRECATED247 + DEPRECATED248 + DEPRECATED249 + DEPRECATED250 + DEPRECATED251 + DEPRECATED252 + DEPRECATED253 + DEPRECATED254 + DEPRECATED255 + DEPRECATED256 + DEPRECATED257 + DEPRECATED258 + DEPRECATED259 + DEPRECATED260 + DEPRECATED261 + DEPRECATED262 + DEPRECATED263 + DEPRECATED264 + DEPRECATED265 + DEPRECATED266 + DEPRECATED267 + DEPRECATED268 + DEPRECATED269 + DEPRECATED270 + DEPRECATED271 + DEPRECATED272 + DEPRECATED273 + DEPRECATED274 + DEPRECATED275 + DEPRECATED276 + + PERSON + NORP + FACILITY + ORG + GPE + LOC + PRODUCT + EVENT + WORK_OF_ART + LANGUAGE + LAW + + DATE + TIME + PERCENT + MONEY + QUANTITY + ORDINAL + CARDINAL + + acomp + advcl + advmod + agent + amod + appos + attr + aux + auxpass + cc + ccomp + complm + conj + cop # U20 + csubj + csubjpass + dep + det + dobj + expl + hmod + hyph + infmod + intj + iobj + mark + meta + neg + nmod + nn + npadvmod + nsubj + nsubjpass + num + number + oprd + obj # U20 + obl # U20 + parataxis + partmod + pcomp + pobj + poss + possessive + preconj + prep + prt + punct + quantmod + relcl + rcmod + root + xcomp + + acl + + ENT_KB_ID + MORPH + ENT_ID + + IDX + _ diff --git a/spacy/symbols.pyx b/spacy/symbols.pyx new file mode 100644 index 0000000..29c179d --- /dev/null +++ b/spacy/symbols.pyx @@ -0,0 +1,482 @@ +# cython: optimize.unpack_method_calls=False +# cython: profile=False +IDS = { + "": NIL, + "IS_ALPHA": IS_ALPHA, + "IS_ASCII": IS_ASCII, + "IS_DIGIT": IS_DIGIT, + "IS_LOWER": IS_LOWER, + "IS_PUNCT": IS_PUNCT, + "IS_SPACE": IS_SPACE, + "IS_TITLE": IS_TITLE, + "IS_UPPER": IS_UPPER, + "LIKE_URL": LIKE_URL, + "LIKE_NUM": LIKE_NUM, + "LIKE_EMAIL": LIKE_EMAIL, + "IS_STOP": IS_STOP, + "IS_OOV_DEPRECATED": IS_OOV_DEPRECATED, + "IS_BRACKET": IS_BRACKET, + "IS_QUOTE": IS_QUOTE, + "IS_LEFT_PUNCT": IS_LEFT_PUNCT, + "IS_RIGHT_PUNCT": IS_RIGHT_PUNCT, + "IS_CURRENCY": IS_CURRENCY, + + "FLAG19": FLAG19, + "FLAG20": FLAG20, + "FLAG21": FLAG21, + "FLAG22": FLAG22, + "FLAG23": FLAG23, + "FLAG24": FLAG24, + "FLAG25": FLAG25, + "FLAG26": FLAG26, + "FLAG27": FLAG27, + "FLAG28": FLAG28, + "FLAG29": FLAG29, + "FLAG30": FLAG30, + "FLAG31": FLAG31, + "FLAG32": FLAG32, + "FLAG33": FLAG33, + "FLAG34": FLAG34, + "FLAG35": FLAG35, + "FLAG36": FLAG36, + "FLAG37": FLAG37, + "FLAG38": FLAG38, + "FLAG39": FLAG39, + "FLAG40": FLAG40, + "FLAG41": FLAG41, + "FLAG42": FLAG42, + "FLAG43": FLAG43, + "FLAG44": FLAG44, + "FLAG45": FLAG45, + "FLAG46": FLAG46, + "FLAG47": FLAG47, + "FLAG48": FLAG48, + "FLAG49": FLAG49, + "FLAG50": FLAG50, + "FLAG51": FLAG51, + "FLAG52": FLAG52, + "FLAG53": FLAG53, + "FLAG54": FLAG54, + "FLAG55": FLAG55, + "FLAG56": FLAG56, + "FLAG57": FLAG57, + "FLAG58": FLAG58, + "FLAG59": FLAG59, + "FLAG60": FLAG60, + "FLAG61": FLAG61, + "FLAG62": FLAG62, + "FLAG63": FLAG63, + + "ID": ID, + "ORTH": ORTH, + "LOWER": LOWER, + "NORM": NORM, + "SHAPE": SHAPE, + "PREFIX": PREFIX, + "SUFFIX": SUFFIX, + + "LENGTH": LENGTH, + "CLUSTER": CLUSTER, + "LEMMA": LEMMA, + "POS": POS, + "TAG": TAG, + "DEP": DEP, + "ENT_IOB": ENT_IOB, + "ENT_TYPE": ENT_TYPE, + "ENT_ID": ENT_ID, + "ENT_KB_ID": ENT_KB_ID, + "HEAD": HEAD, + "SENT_START": SENT_START, + "SPACY": SPACY, + "PROB": PROB, + "LANG": LANG, + "IDX": IDX, + + "ADJ": ADJ, + "ADP": ADP, + "ADV": ADV, + "AUX": AUX, + "CONJ": CONJ, + "CCONJ": CCONJ, # U20 + "DET": DET, + "INTJ": INTJ, + "NOUN": NOUN, + "NUM": NUM, + "PART": PART, + "PRON": PRON, + "PROPN": PROPN, + "PUNCT": PUNCT, + "SCONJ": SCONJ, + "SYM": SYM, + "VERB": VERB, + "X": X, + "EOL": EOL, + "SPACE": SPACE, + + "DEPRECATED001": DEPRECATED001, + "DEPRECATED002": DEPRECATED002, + "DEPRECATED003": DEPRECATED003, + "DEPRECATED004": DEPRECATED004, + "DEPRECATED005": DEPRECATED005, + "DEPRECATED006": DEPRECATED006, + "DEPRECATED007": DEPRECATED007, + "DEPRECATED008": DEPRECATED008, + "DEPRECATED009": DEPRECATED009, + "DEPRECATED010": DEPRECATED010, + "DEPRECATED011": DEPRECATED011, + "DEPRECATED012": DEPRECATED012, + "DEPRECATED013": DEPRECATED013, + "DEPRECATED014": DEPRECATED014, + "DEPRECATED015": DEPRECATED015, + "DEPRECATED016": DEPRECATED016, + "DEPRECATED017": DEPRECATED017, + "DEPRECATED018": DEPRECATED018, + "DEPRECATED019": DEPRECATED019, + "DEPRECATED020": DEPRECATED020, + "DEPRECATED021": DEPRECATED021, + "DEPRECATED022": DEPRECATED022, + "DEPRECATED023": DEPRECATED023, + "DEPRECATED024": DEPRECATED024, + "DEPRECATED025": DEPRECATED025, + "DEPRECATED026": DEPRECATED026, + "DEPRECATED027": DEPRECATED027, + "DEPRECATED028": DEPRECATED028, + "DEPRECATED029": DEPRECATED029, + "DEPRECATED030": DEPRECATED030, + "DEPRECATED031": DEPRECATED031, + "DEPRECATED032": DEPRECATED032, + "DEPRECATED033": DEPRECATED033, + "DEPRECATED034": DEPRECATED034, + "DEPRECATED035": DEPRECATED035, + "DEPRECATED036": DEPRECATED036, + "DEPRECATED037": DEPRECATED037, + "DEPRECATED038": DEPRECATED038, + "DEPRECATED039": DEPRECATED039, + "DEPRECATED040": DEPRECATED040, + "DEPRECATED041": DEPRECATED041, + "DEPRECATED042": DEPRECATED042, + "DEPRECATED043": DEPRECATED043, + "DEPRECATED044": DEPRECATED044, + "DEPRECATED045": DEPRECATED045, + "DEPRECATED046": DEPRECATED046, + "DEPRECATED047": DEPRECATED047, + "DEPRECATED048": DEPRECATED048, + "DEPRECATED049": DEPRECATED049, + "DEPRECATED050": DEPRECATED050, + "DEPRECATED051": DEPRECATED051, + "DEPRECATED052": DEPRECATED052, + "DEPRECATED053": DEPRECATED053, + "DEPRECATED054": DEPRECATED054, + "DEPRECATED055": DEPRECATED055, + "DEPRECATED056": DEPRECATED056, + "DEPRECATED057": DEPRECATED057, + "DEPRECATED058": DEPRECATED058, + "DEPRECATED059": DEPRECATED059, + "DEPRECATED060": DEPRECATED060, + "DEPRECATED061": DEPRECATED061, + "DEPRECATED062": DEPRECATED062, + "DEPRECATED063": DEPRECATED063, + "DEPRECATED064": DEPRECATED064, + "DEPRECATED065": DEPRECATED065, + "DEPRECATED066": DEPRECATED066, + "DEPRECATED067": DEPRECATED067, + "DEPRECATED068": DEPRECATED068, + "DEPRECATED069": DEPRECATED069, + "DEPRECATED070": DEPRECATED070, + "DEPRECATED071": DEPRECATED071, + "DEPRECATED072": DEPRECATED072, + "DEPRECATED073": DEPRECATED073, + "DEPRECATED074": DEPRECATED074, + "DEPRECATED075": DEPRECATED075, + "DEPRECATED076": DEPRECATED076, + "DEPRECATED077": DEPRECATED077, + "DEPRECATED078": DEPRECATED078, + "DEPRECATED079": DEPRECATED079, + "DEPRECATED080": DEPRECATED080, + "DEPRECATED081": DEPRECATED081, + "DEPRECATED082": DEPRECATED082, + "DEPRECATED083": DEPRECATED083, + "DEPRECATED084": DEPRECATED084, + "DEPRECATED085": DEPRECATED085, + "DEPRECATED086": DEPRECATED086, + "DEPRECATED087": DEPRECATED087, + "DEPRECATED088": DEPRECATED088, + "DEPRECATED089": DEPRECATED089, + "DEPRECATED090": DEPRECATED090, + "DEPRECATED091": DEPRECATED091, + "DEPRECATED092": DEPRECATED092, + "DEPRECATED093": DEPRECATED093, + "DEPRECATED094": DEPRECATED094, + "DEPRECATED095": DEPRECATED095, + "DEPRECATED096": DEPRECATED096, + "DEPRECATED097": DEPRECATED097, + "DEPRECATED098": DEPRECATED098, + "DEPRECATED099": DEPRECATED099, + "DEPRECATED100": DEPRECATED100, + "DEPRECATED101": DEPRECATED101, + "DEPRECATED102": DEPRECATED102, + "DEPRECATED103": DEPRECATED103, + "DEPRECATED104": DEPRECATED104, + "DEPRECATED105": DEPRECATED105, + "DEPRECATED106": DEPRECATED106, + "DEPRECATED107": DEPRECATED107, + "DEPRECATED108": DEPRECATED108, + "DEPRECATED109": DEPRECATED109, + "DEPRECATED110": DEPRECATED110, + "DEPRECATED111": DEPRECATED111, + "DEPRECATED112": DEPRECATED112, + "DEPRECATED113": DEPRECATED113, + "DEPRECATED114": DEPRECATED114, + "DEPRECATED115": DEPRECATED115, + "DEPRECATED116": DEPRECATED116, + "DEPRECATED117": DEPRECATED117, + "DEPRECATED118": DEPRECATED118, + "DEPRECATED119": DEPRECATED119, + "DEPRECATED120": DEPRECATED120, + "DEPRECATED121": DEPRECATED121, + "DEPRECATED122": DEPRECATED122, + "DEPRECATED123": DEPRECATED123, + "DEPRECATED124": DEPRECATED124, + "DEPRECATED125": DEPRECATED125, + "DEPRECATED126": DEPRECATED126, + "DEPRECATED127": DEPRECATED127, + "DEPRECATED128": DEPRECATED128, + "DEPRECATED129": DEPRECATED129, + "DEPRECATED130": DEPRECATED130, + "DEPRECATED131": DEPRECATED131, + "DEPRECATED132": DEPRECATED132, + "DEPRECATED133": DEPRECATED133, + "DEPRECATED134": DEPRECATED134, + "DEPRECATED135": DEPRECATED135, + "DEPRECATED136": DEPRECATED136, + "DEPRECATED137": DEPRECATED137, + "DEPRECATED138": DEPRECATED138, + "DEPRECATED139": DEPRECATED139, + "DEPRECATED140": DEPRECATED140, + "DEPRECATED141": DEPRECATED141, + "DEPRECATED142": DEPRECATED142, + "DEPRECATED143": DEPRECATED143, + "DEPRECATED144": DEPRECATED144, + "DEPRECATED145": DEPRECATED145, + "DEPRECATED146": DEPRECATED146, + "DEPRECATED147": DEPRECATED147, + "DEPRECATED148": DEPRECATED148, + "DEPRECATED149": DEPRECATED149, + "DEPRECATED150": DEPRECATED150, + "DEPRECATED151": DEPRECATED151, + "DEPRECATED152": DEPRECATED152, + "DEPRECATED153": DEPRECATED153, + "DEPRECATED154": DEPRECATED154, + "DEPRECATED155": DEPRECATED155, + "DEPRECATED156": DEPRECATED156, + "DEPRECATED157": DEPRECATED157, + "DEPRECATED158": DEPRECATED158, + "DEPRECATED159": DEPRECATED159, + "DEPRECATED160": DEPRECATED160, + "DEPRECATED161": DEPRECATED161, + "DEPRECATED162": DEPRECATED162, + "DEPRECATED163": DEPRECATED163, + "DEPRECATED164": DEPRECATED164, + "DEPRECATED165": DEPRECATED165, + "DEPRECATED166": DEPRECATED166, + "DEPRECATED167": DEPRECATED167, + "DEPRECATED168": DEPRECATED168, + "DEPRECATED169": DEPRECATED169, + "DEPRECATED170": DEPRECATED170, + "DEPRECATED171": DEPRECATED171, + "DEPRECATED172": DEPRECATED172, + "DEPRECATED173": DEPRECATED173, + "DEPRECATED174": DEPRECATED174, + "DEPRECATED175": DEPRECATED175, + "DEPRECATED176": DEPRECATED176, + "DEPRECATED177": DEPRECATED177, + "DEPRECATED178": DEPRECATED178, + "DEPRECATED179": DEPRECATED179, + "DEPRECATED180": DEPRECATED180, + "DEPRECATED181": DEPRECATED181, + "DEPRECATED182": DEPRECATED182, + "DEPRECATED183": DEPRECATED183, + "DEPRECATED184": DEPRECATED184, + "DEPRECATED185": DEPRECATED185, + "DEPRECATED186": DEPRECATED186, + "DEPRECATED187": DEPRECATED187, + "DEPRECATED188": DEPRECATED188, + "DEPRECATED189": DEPRECATED189, + "DEPRECATED190": DEPRECATED190, + "DEPRECATED191": DEPRECATED191, + "DEPRECATED192": DEPRECATED192, + "DEPRECATED193": DEPRECATED193, + "DEPRECATED194": DEPRECATED194, + "DEPRECATED195": DEPRECATED195, + "DEPRECATED196": DEPRECATED196, + "DEPRECATED197": DEPRECATED197, + "DEPRECATED198": DEPRECATED198, + "DEPRECATED199": DEPRECATED199, + "DEPRECATED200": DEPRECATED200, + "DEPRECATED201": DEPRECATED201, + "DEPRECATED202": DEPRECATED202, + "DEPRECATED203": DEPRECATED203, + "DEPRECATED204": DEPRECATED204, + "DEPRECATED205": DEPRECATED205, + "DEPRECATED206": DEPRECATED206, + "DEPRECATED207": DEPRECATED207, + "DEPRECATED208": DEPRECATED208, + "DEPRECATED209": DEPRECATED209, + "DEPRECATED210": DEPRECATED210, + "DEPRECATED211": DEPRECATED211, + "DEPRECATED212": DEPRECATED212, + "DEPRECATED213": DEPRECATED213, + "DEPRECATED214": DEPRECATED214, + "DEPRECATED215": DEPRECATED215, + "DEPRECATED216": DEPRECATED216, + "DEPRECATED217": DEPRECATED217, + "DEPRECATED218": DEPRECATED218, + "DEPRECATED219": DEPRECATED219, + "DEPRECATED220": DEPRECATED220, + "DEPRECATED221": DEPRECATED221, + "DEPRECATED222": DEPRECATED222, + "DEPRECATED223": DEPRECATED223, + "DEPRECATED224": DEPRECATED224, + "DEPRECATED225": DEPRECATED225, + "DEPRECATED226": DEPRECATED226, + "DEPRECATED227": DEPRECATED227, + "DEPRECATED228": DEPRECATED228, + "DEPRECATED229": DEPRECATED229, + "DEPRECATED230": DEPRECATED230, + "DEPRECATED231": DEPRECATED231, + "DEPRECATED232": DEPRECATED232, + "DEPRECATED233": DEPRECATED233, + "DEPRECATED234": DEPRECATED234, + "DEPRECATED235": DEPRECATED235, + "DEPRECATED236": DEPRECATED236, + "DEPRECATED237": DEPRECATED237, + "DEPRECATED238": DEPRECATED238, + "DEPRECATED239": DEPRECATED239, + "DEPRECATED240": DEPRECATED240, + "DEPRECATED241": DEPRECATED241, + "DEPRECATED242": DEPRECATED242, + "DEPRECATED243": DEPRECATED243, + "DEPRECATED244": DEPRECATED244, + "DEPRECATED245": DEPRECATED245, + "DEPRECATED246": DEPRECATED246, + "DEPRECATED247": DEPRECATED247, + "DEPRECATED248": DEPRECATED248, + "DEPRECATED249": DEPRECATED249, + "DEPRECATED250": DEPRECATED250, + "DEPRECATED251": DEPRECATED251, + "DEPRECATED252": DEPRECATED252, + "DEPRECATED253": DEPRECATED253, + "DEPRECATED254": DEPRECATED254, + "DEPRECATED255": DEPRECATED255, + "DEPRECATED256": DEPRECATED256, + "DEPRECATED257": DEPRECATED257, + "DEPRECATED258": DEPRECATED258, + "DEPRECATED259": DEPRECATED259, + "DEPRECATED260": DEPRECATED260, + "DEPRECATED261": DEPRECATED261, + "DEPRECATED262": DEPRECATED262, + "DEPRECATED263": DEPRECATED263, + "DEPRECATED264": DEPRECATED264, + "DEPRECATED265": DEPRECATED265, + "DEPRECATED266": DEPRECATED266, + "DEPRECATED267": DEPRECATED267, + "DEPRECATED268": DEPRECATED268, + "DEPRECATED269": DEPRECATED269, + "DEPRECATED270": DEPRECATED270, + "DEPRECATED271": DEPRECATED271, + "DEPRECATED272": DEPRECATED272, + "DEPRECATED273": DEPRECATED273, + "DEPRECATED274": DEPRECATED274, + "DEPRECATED275": DEPRECATED275, + "DEPRECATED276": DEPRECATED276, + + "PERSON": PERSON, + "NORP": NORP, + "FACILITY": FACILITY, + "ORG": ORG, + "GPE": GPE, + "LOC": LOC, + "PRODUCT": PRODUCT, + "EVENT": EVENT, + "WORK_OF_ART": WORK_OF_ART, + "LANGUAGE": LANGUAGE, + + "DATE": DATE, + "TIME": TIME, + "PERCENT": PERCENT, + "MONEY": MONEY, + "QUANTITY": QUANTITY, + "ORDINAL": ORDINAL, + "CARDINAL": CARDINAL, + + "acomp": acomp, + "advcl": advcl, + "advmod": advmod, + "agent": agent, + "amod": amod, + "appos": appos, + "attr": attr, + "aux": aux, + "auxpass": auxpass, + "cc": cc, + "ccomp": ccomp, + "complm": complm, + "conj": conj, + "cop": cop, # U20 + "csubj": csubj, + "csubjpass": csubjpass, + "dep": dep, + "det": det, + "dobj": dobj, + "expl": expl, + "hmod": hmod, + "hyph": hyph, + "infmod": infmod, + "intj": intj, + "iobj": iobj, + "mark": mark, + "meta": meta, + "neg": neg, + "nmod": nmod, + "nn": nn, + "npadvmod": npadvmod, + "nsubj": nsubj, + "nsubjpass": nsubjpass, + "num": num, + "number": number, + "oprd": oprd, + "obj": obj, # U20 + "obl": obl, # U20 + "parataxis": parataxis, + "partmod": partmod, + "pcomp": pcomp, + "pobj": pobj, + "poss": poss, + "possessive": possessive, + "preconj": preconj, + "prep": prep, + "prt": prt, + "punct": punct, + "quantmod": quantmod, + "rcmod": rcmod, + "relcl": relcl, + "root": root, + "xcomp": xcomp, + + "acl": acl, + "LAW": LAW, + "MORPH": MORPH, + "_": _, +} + + +def sort_nums(x): + return x[1] + + +NAMES = [it[0] for it in sorted(IDS.items(), key=sort_nums)] +# Unfortunate hack here, to work around problem with long cpdef enum +# (which is generating an enormous amount of C++ in Cython 0.24+) +# We keep the enum cdef, and just make sure the names are available to Python +locals().update(IDS) + diff --git a/spacy/tests/README.md b/spacy/tests/README.md new file mode 100644 index 0000000..82fabcc --- /dev/null +++ b/spacy/tests/README.md @@ -0,0 +1,137 @@ + + +# spaCy tests + +spaCy uses the [pytest](http://doc.pytest.org/) framework for testing. For more info on this, see the [pytest documentation](http://docs.pytest.org/en/latest/contents.html). + +Tests for spaCy modules and classes live in their own directories of the same name. For example, tests for the `Tokenizer` can be found in [`/tests/tokenizer`](tokenizer). All test modules (i.e. directories) also need to be listed in spaCy's [`setup.py`](../setup.py). To be interpreted and run, all test files and test functions need to be prefixed with `test_`. + +> ⚠️ **Important note:** As part of our new model training infrastructure, we've moved all model tests to the [`spacy-models`](https://github.com/explosion/spacy-models) repository. This allows us to test the models separately from the core library functionality. + +## Table of contents + +1. [Running the tests](#running-the-tests) +2. [Dos and don'ts](#dos-and-donts) +3. [Parameters](#parameters) +4. [Fixtures](#fixtures) +5. [Helpers and utilities](#helpers-and-utilities) +6. [Contributing to the tests](#contributing-to-the-tests) + +## Running the tests + +To show print statements, run the tests with `py.test -s`. To abort after the +first failure, run them with `py.test -x`. + +```bash +py.test spacy # run basic tests +py.test spacy --slow # run basic and slow tests +``` + +You can also run tests in a specific file or directory, or even only one +specific test: + +```bash +py.test spacy/tests/tokenizer # run all tests in directory +py.test spacy/tests/tokenizer/test_exceptions.py # run all tests in file +py.test spacy/tests/tokenizer/test_exceptions.py::test_tokenizer_handles_emoji # run specific test +``` + +## Dos and don'ts + +To keep the behavior of the tests consistent and predictable, we try to follow a few basic conventions: + +- **Test names** should follow a pattern of `test_[module]_[tested behaviour]`. For example: `test_tokenizer_keeps_email` or `test_spans_override_sentiment`. +- If you're testing for a bug reported in a specific issue, always create a **regression test**. Regression tests should be named `test_issue[ISSUE NUMBER]` and live in the [`regression`](regression) directory. +- Only use `@pytest.mark.xfail` for tests that **should pass, but currently fail**. To test for desired negative behavior, use `assert not` in your test. +- Very **extensive tests** that take a long time to run should be marked with `@pytest.mark.slow`. If your slow test is testing important behavior, consider adding an additional simpler version. +- If tests require **loading the models**, they should be added to the [`spacy-models`](https://github.com/explosion/spacy-models) tests. +- Before requiring the models, always make sure there is no other way to test the particular behavior. In a lot of cases, it's sufficient to simply create a `Doc` object manually. See the section on [helpers and utility functions](#helpers-and-utilities) for more info on this. +- **Avoid unnecessary imports.** There should never be a need to explicitly import spaCy at the top of a file, and many components are available as [fixtures](#fixtures). You should also avoid wildcard imports (`from module import *`). +- If you're importing from spaCy, **always use absolute imports**. For example: `from spacy.language import Language`. +- Try to keep the tests **readable and concise**. Use clear and descriptive variable names (`doc`, `tokens` and `text` are great), keep it short and only test for one behavior at a time. + +## Parameters + +If the test cases can be extracted from the test, always `parametrize` them instead of hard-coding them into the test: + +```python +@pytest.mark.parametrize('text', ["google.com", "spacy.io"]) +def test_tokenizer_keep_urls(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 1 +``` + +This will run the test once for each `text` value. Even if you're only testing one example, it's usually best to specify it as a parameter. This will later make it easier for others to quickly add additional test cases without having to modify the test. + +You can also specify parameters as tuples to test with multiple values per test: + +```python +@pytest.mark.parametrize('text,length', [("U.S.", 1), ("us.", 2), ("(U.S.", 2)]) +``` + +To test for combinations of parameters, you can add several `parametrize` markers: + +```python +@pytest.mark.parametrize('text', ["A test sentence", "Another sentence"]) +@pytest.mark.parametrize('punct', ['.', '!', '?']) +``` + +This will run the test with all combinations of the two parameters `text` and `punct`. **Use this feature sparingly**, though, as it can easily cause unnecessary or undesired test bloat. + +## Fixtures + +Fixtures to create instances of spaCy objects and other components should only be defined once in the global [`conftest.py`](conftest.py). We avoid having per-directory conftest files, as this can easily lead to confusion. + +These are the main fixtures that are currently available: + +| Fixture | Description | +| ----------------------------------- | ---------------------------------------------------------------------------- | +| `tokenizer` | Basic, language-independent tokenizer. Identical to the `xx` language class. | +| `en_tokenizer`, `de_tokenizer`, ... | Creates an English, German etc. tokenizer. | +| `en_vocab` | Creates an instance of the English `Vocab`. | + +The fixtures can be used in all tests by simply setting them as an argument, like this: + +```python +def test_module_do_something(en_tokenizer): + tokens = en_tokenizer("Some text here") +``` + +If all tests in a file require a specific configuration, or use the same complex example, it can be helpful to create a separate fixture. This fixture should be added at the top of each file. Make sure to use descriptive names for these fixtures and don't override any of the global fixtures listed above. **From looking at a test, it should immediately be clear which fixtures are used, and where they are coming from.** + +## Helpers and utilities + +Our new test setup comes with a few handy utility functions that can be imported from [`util.py`](util.py). + +### Constructing a `Doc` object manually + +Loading the models is expensive and not necessary if you're not actually testing the model performance. If all you need is a `Doc` object with annotations like heads, POS tags or the dependency parse, you can construct it manually. + +```python +def test_doc_token_api_strings(en_vocab): + words = ["Give", "it", "back", "!", "He", "pleaded", "."] + pos = ['VERB', 'PRON', 'PART', 'PUNCT', 'PRON', 'VERB', 'PUNCT'] + heads = [0, 0, 0, 0, 5, 5, 5] + deps = ['ROOT', 'dobj', 'prt', 'punct', 'nsubj', 'ROOT', 'punct'] + + doc = Doc(en_vocab, words=words, pos=pos, heads=heads, deps=deps) + assert doc[0].text == 'Give' + assert doc[0].lower_ == 'give' + assert doc[0].pos_ == 'VERB' + assert doc[0].dep_ == 'ROOT' +``` + +### Other utilities + +| Name | Description | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `apply_transition_sequence(parser, doc, sequence)` | Perform a series of pre-specified transitions, to put the parser in a desired state. | +| `add_vecs_to_vocab(vocab, vectors)` | Add list of vector tuples (`[("text", [1, 2, 3])]`) to given vocab. All vectors need to have the same length. | +| `get_cosine(vec1, vec2)` | Get cosine for two given vectors. | +| `assert_docs_equal(doc1, doc2)` | Compare two `Doc` objects and `assert` that they're equal. Tests for tokens, tags, dependencies and entities. | + +## Contributing to the tests + +There's still a long way to go to finally reach **100% test coverage** – and we'd appreciate your help! 🙌 You can open an issue on our [issue tracker](https://github.com/explosion/spaCy/issues) and label it `tests`, or make a [pull request](https://github.com/explosion/spaCy/pulls) to this repository. + +📖 **For more information on contributing to spaCy in general, check out our [contribution guidelines](https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md).** diff --git a/spacy/tests/__init__.py b/spacy/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py new file mode 100644 index 0000000..ae5255c --- /dev/null +++ b/spacy/tests/conftest.py @@ -0,0 +1,534 @@ +import pytest +from hypothesis import settings + +from spacy.util import get_lang_class + +# Functionally disable deadline settings for tests +# to prevent spurious test failures in CI builds. +settings.register_profile("no_deadlines", deadline=2 * 60 * 1000) # in ms +settings.load_profile("no_deadlines") + + +def pytest_addoption(parser): + try: + parser.addoption("--slow", action="store_true", help="include slow tests") + parser.addoption("--issue", action="store", help="test specific issues") + # Options are already added, e.g. if conftest is copied in a build pipeline + # and runs twice + except ValueError: + pass + + +def pytest_runtest_setup(item): + def getopt(opt): + # When using 'pytest --pyargs spacy' to test an installed copy of + # spacy, pytest skips running our pytest_addoption() hook. Later, when + # we call getoption(), pytest raises an error, because it doesn't + # recognize the option we're asking about. To avoid this, we need to + # pass a default value. We default to False, i.e., we act like all the + # options weren't given. + return item.config.getoption(f"--{opt}", False) + + # Integration of boolean flags + for opt in ["slow"]: + if opt in item.keywords and not getopt(opt): + pytest.skip(f"need --{opt} option to run") + + # Special integration to mark tests with issue numbers + issues = getopt("issue") + if isinstance(issues, str): + if "issue" in item.keywords: + # Convert issues provided on the CLI to list of ints + issue_nos = [int(issue.strip()) for issue in issues.split(",")] + # Get all issues specified by decorators and check if they're provided + issue_refs = [mark.args[0] for mark in item.iter_markers(name="issue")] + if not any([ref in issue_nos for ref in issue_refs]): + pytest.skip(f"not referencing specified issues: {issue_nos}") + else: + pytest.skip("not referencing any issues") + + +# Fixtures for language tokenizers (languages sorted alphabetically) + + +@pytest.fixture(scope="module") +def tokenizer(): + return get_lang_class("xx")().tokenizer + + +@pytest.fixture(scope="session") +def af_tokenizer(): + return get_lang_class("af")().tokenizer + + +@pytest.fixture(scope="session") +def am_tokenizer(): + return get_lang_class("am")().tokenizer + + +@pytest.fixture(scope="session") +def ar_tokenizer(): + return get_lang_class("ar")().tokenizer + + +@pytest.fixture(scope="session") +def bg_tokenizer(): + return get_lang_class("bg")().tokenizer + + +@pytest.fixture(scope="session") +def bn_tokenizer(): + return get_lang_class("bn")().tokenizer + + +@pytest.fixture(scope="session") +def bo_tokenizer(): + return get_lang_class("bo")().tokenizer + + +@pytest.fixture(scope="session") +def ca_tokenizer(): + return get_lang_class("ca")().tokenizer + + +@pytest.fixture(scope="session") +def cs_tokenizer(): + return get_lang_class("cs")().tokenizer + + +@pytest.fixture(scope="session") +def da_tokenizer(): + return get_lang_class("da")().tokenizer + + +@pytest.fixture(scope="session") +def de_tokenizer(): + return get_lang_class("de")().tokenizer + + +@pytest.fixture(scope="session") +def de_vocab(): + return get_lang_class("de")().vocab + + +@pytest.fixture(scope="session") +def dsb_tokenizer(): + return get_lang_class("dsb")().tokenizer + + +@pytest.fixture(scope="session") +def el_tokenizer(): + return get_lang_class("el")().tokenizer + + +@pytest.fixture(scope="session") +def en_tokenizer(): + return get_lang_class("en")().tokenizer + + +@pytest.fixture(scope="session") +def en_vocab(): + return get_lang_class("en")().vocab + + +@pytest.fixture(scope="session") +def en_parser(en_vocab): + nlp = get_lang_class("en")(en_vocab) + return nlp.create_pipe("parser") + + +@pytest.fixture(scope="session") +def es_tokenizer(): + return get_lang_class("es")().tokenizer + + +@pytest.fixture(scope="session") +def es_vocab(): + return get_lang_class("es")().vocab + + +@pytest.fixture(scope="session") +def et_tokenizer(): + return get_lang_class("et")().tokenizer + + +@pytest.fixture(scope="session") +def eu_tokenizer(): + return get_lang_class("eu")().tokenizer + + +@pytest.fixture(scope="session") +def fa_tokenizer(): + return get_lang_class("fa")().tokenizer + + +@pytest.fixture(scope="session") +def fi_tokenizer(): + return get_lang_class("fi")().tokenizer + + +@pytest.fixture(scope="session") +def fo_tokenizer(): + return get_lang_class("fo")().tokenizer + + +@pytest.fixture(scope="session") +def fr_tokenizer(): + return get_lang_class("fr")().tokenizer + + +@pytest.fixture(scope="session") +def fr_vocab(): + return get_lang_class("fr")().vocab + + +@pytest.fixture(scope="session") +def ga_tokenizer(): + return get_lang_class("ga")().tokenizer + + +@pytest.fixture(scope="session") +def grc_tokenizer(): + return get_lang_class("grc")().tokenizer + + +@pytest.fixture(scope="session") +def gu_tokenizer(): + return get_lang_class("gu")().tokenizer + + +@pytest.fixture(scope="session") +def he_tokenizer(): + return get_lang_class("he")().tokenizer + + +@pytest.fixture(scope="session") +def hi_tokenizer(): + return get_lang_class("hi")().tokenizer + + +@pytest.fixture(scope="session") +def hr_tokenizer(): + return get_lang_class("hr")().tokenizer + + +@pytest.fixture(scope="session") +def ht_tokenizer(): + return get_lang_class("ht")().tokenizer + + +@pytest.fixture(scope="session") +def ht_vocab(): + return get_lang_class("ht")().vocab + + +@pytest.fixture +def hu_tokenizer(): + return get_lang_class("hu")().tokenizer + + +@pytest.fixture(scope="session") +def id_tokenizer(): + return get_lang_class("id")().tokenizer + + +@pytest.fixture(scope="session") +def is_tokenizer(): + return get_lang_class("is")().tokenizer + + +@pytest.fixture(scope="session") +def it_tokenizer(): + return get_lang_class("it")().tokenizer + + +@pytest.fixture(scope="session") +def it_vocab(): + return get_lang_class("it")().vocab + + +@pytest.fixture(scope="session") +def ja_tokenizer(): + pytest.importorskip("sudachipy") + return get_lang_class("ja")().tokenizer + + +@pytest.fixture(scope="session") +def hsb_tokenizer(): + return get_lang_class("hsb")().tokenizer + + +@pytest.fixture(scope="session") +def ko_tokenizer(): + pytest.importorskip("natto") + return get_lang_class("ko")().tokenizer + + +@pytest.fixture(scope="session") +def ko_tokenizer_tokenizer(): + config = { + "nlp": { + "tokenizer": { + "@tokenizers": "spacy.Tokenizer.v1", + } + } + } + nlp = get_lang_class("ko").from_config(config) + return nlp.tokenizer + + +@pytest.fixture(scope="module") +def la_tokenizer(): + return get_lang_class("la")().tokenizer + + +@pytest.fixture(scope="session") +def lb_tokenizer(): + return get_lang_class("lb")().tokenizer + + +@pytest.fixture(scope="session") +def lg_tokenizer(): + return get_lang_class("lg")().tokenizer + + +@pytest.fixture(scope="session") +def lt_tokenizer(): + return get_lang_class("lt")().tokenizer + + +@pytest.fixture(scope="session") +def lv_tokenizer(): + return get_lang_class("lv")().tokenizer + + +@pytest.fixture(scope="session") +def mk_tokenizer(): + return get_lang_class("mk")().tokenizer + + +@pytest.fixture(scope="session") +def ml_tokenizer(): + return get_lang_class("ml")().tokenizer + + +@pytest.fixture(scope="session") +def ms_tokenizer(): + return get_lang_class("ms")().tokenizer + + +@pytest.fixture(scope="session") +def nb_tokenizer(): + return get_lang_class("nb")().tokenizer + + +@pytest.fixture(scope="session") +def ne_tokenizer(): + return get_lang_class("ne")().tokenizer + + +@pytest.fixture(scope="session") +def nl_vocab(): + return get_lang_class("nl")().vocab + + +@pytest.fixture(scope="session") +def nl_tokenizer(): + return get_lang_class("nl")().tokenizer + + +@pytest.fixture(scope="session") +def nn_tokenizer(): + return get_lang_class("nn")().tokenizer + + +@pytest.fixture(scope="session") +def pl_tokenizer(): + return get_lang_class("pl")().tokenizer + + +@pytest.fixture(scope="session") +def pt_tokenizer(): + return get_lang_class("pt")().tokenizer + + +@pytest.fixture(scope="session") +def pt_vocab(): + return get_lang_class("pt")().vocab + + +@pytest.fixture(scope="session") +def ro_tokenizer(): + return get_lang_class("ro")().tokenizer + + +@pytest.fixture(scope="session") +def ru_tokenizer(): + pytest.importorskip("pymorphy3") + return get_lang_class("ru")().tokenizer + + +@pytest.fixture(scope="session") +def ru_lemmatizer(): + pytest.importorskip("pymorphy3") + return get_lang_class("ru")().add_pipe("lemmatizer") + + +@pytest.fixture(scope="session") +def ru_lookup_lemmatizer(): + pytest.importorskip("pymorphy3") + return get_lang_class("ru")().add_pipe( + "lemmatizer", config={"mode": "pymorphy3_lookup"} + ) + + +@pytest.fixture(scope="session") +def sa_tokenizer(): + return get_lang_class("sa")().tokenizer + + +@pytest.fixture(scope="session") +def sk_tokenizer(): + return get_lang_class("sk")().tokenizer + + +@pytest.fixture(scope="session") +def sl_tokenizer(): + return get_lang_class("sl")().tokenizer + + +@pytest.fixture(scope="session") +def sr_tokenizer(): + return get_lang_class("sr")().tokenizer + + +@pytest.fixture(scope="session") +def sq_tokenizer(): + return get_lang_class("sq")().tokenizer + + +@pytest.fixture(scope="session") +def sv_tokenizer(): + return get_lang_class("sv")().tokenizer + + +@pytest.fixture(scope="session") +def ta_tokenizer(): + return get_lang_class("ta")().tokenizer + + +@pytest.fixture(scope="session") +def th_tokenizer(): + pytest.importorskip("pythainlp") + return get_lang_class("th")().tokenizer + + +@pytest.fixture(scope="session") +def ti_tokenizer(): + return get_lang_class("ti")().tokenizer + + +@pytest.fixture(scope="session") +def tl_tokenizer(): + return get_lang_class("tl")().tokenizer + + +@pytest.fixture(scope="session") +def tr_tokenizer(): + return get_lang_class("tr")().tokenizer + + +@pytest.fixture(scope="session") +def tt_tokenizer(): + return get_lang_class("tt")().tokenizer + + +@pytest.fixture(scope="session") +def ky_tokenizer(): + return get_lang_class("ky")().tokenizer + + +@pytest.fixture(scope="session") +def uk_tokenizer(): + pytest.importorskip("pymorphy3") + return get_lang_class("uk")().tokenizer + + +@pytest.fixture(scope="session") +def uk_lemmatizer(): + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") + return get_lang_class("uk")().add_pipe("lemmatizer") + + +@pytest.fixture(scope="session") +def uk_lookup_lemmatizer(): + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") + return get_lang_class("uk")().add_pipe( + "lemmatizer", config={"mode": "pymorphy3_lookup"} + ) + + +@pytest.fixture(scope="session") +def ur_tokenizer(): + return get_lang_class("ur")().tokenizer + + +@pytest.fixture(scope="session") +def vi_tokenizer(): + pytest.importorskip("pyvi") + return get_lang_class("vi")().tokenizer + + +@pytest.fixture(scope="session") +def xx_tokenizer(): + return get_lang_class("xx")().tokenizer + + +@pytest.fixture(scope="session") +def yo_tokenizer(): + return get_lang_class("yo")().tokenizer + + +@pytest.fixture(scope="session") +def zh_tokenizer_char(): + nlp = get_lang_class("zh")() + return nlp.tokenizer + + +@pytest.fixture(scope="session") +def zh_tokenizer_jieba(): + pytest.importorskip("jieba") + config = { + "nlp": { + "tokenizer": { + "@tokenizers": "spacy.zh.ChineseTokenizer", + "segmenter": "jieba", + } + } + } + nlp = get_lang_class("zh").from_config(config) + return nlp.tokenizer + + +@pytest.fixture(scope="session") +def zh_tokenizer_pkuseg(): + pytest.importorskip("spacy_pkuseg") + config = { + "nlp": { + "tokenizer": { + "@tokenizers": "spacy.zh.ChineseTokenizer", + "segmenter": "pkuseg", + } + }, + "initialize": {"tokenizer": {"pkuseg_model": "web"}}, + } + nlp = get_lang_class("zh").from_config(config) + nlp.initialize() + return nlp.tokenizer + + +@pytest.fixture(scope="session") +def hy_tokenizer(): + return get_lang_class("hy")().tokenizer diff --git a/spacy/tests/doc/__init__.py b/spacy/tests/doc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/doc/test_add_entities.py b/spacy/tests/doc/test_add_entities.py new file mode 100644 index 0000000..259b21f --- /dev/null +++ b/spacy/tests/doc/test_add_entities.py @@ -0,0 +1,57 @@ +import pytest + +from spacy import registry +from spacy.pipeline import EntityRecognizer +from spacy.pipeline.ner import DEFAULT_NER_MODEL +from spacy.tokens import Doc, Span +from spacy.training import Example + + +def _ner_example(ner): + doc = Doc( + ner.vocab, + words=["Joe", "loves", "visiting", "London", "during", "the", "weekend"], + ) + gold = {"entities": [(0, 3, "PERSON"), (19, 25, "LOC")]} + return Example.from_dict(doc, gold) + + +def test_doc_add_entities_set_ents_iob(en_vocab): + text = ["This", "is", "a", "lion"] + doc = Doc(en_vocab, words=text) + cfg = {"model": DEFAULT_NER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + ner = EntityRecognizer(en_vocab, model) + ner.initialize(lambda: [_ner_example(ner)]) + ner(doc) + + doc.ents = [("ANIMAL", 3, 4)] + assert [w.ent_iob_ for w in doc] == ["O", "O", "O", "B"] + + doc.ents = [("WORD", 0, 2)] + assert [w.ent_iob_ for w in doc] == ["B", "I", "O", "O"] + + +def test_ents_reset(en_vocab): + """Ensure that resetting doc.ents does not change anything""" + text = ["This", "is", "a", "lion"] + doc = Doc(en_vocab, words=text) + cfg = {"model": DEFAULT_NER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + ner = EntityRecognizer(en_vocab, model) + ner.initialize(lambda: [_ner_example(ner)]) + ner(doc) + orig_iobs = [t.ent_iob_ for t in doc] + doc.ents = list(doc.ents) + assert [t.ent_iob_ for t in doc] == orig_iobs + + +def test_add_overlapping_entities(en_vocab): + text = ["Louisiana", "Office", "of", "Conservation"] + doc = Doc(en_vocab, words=text) + entity = Span(doc, 0, 4, label=391) + doc.ents = [entity] + + new_entity = Span(doc, 0, 1, label=392) + with pytest.raises(ValueError): + doc.ents = list(doc.ents) + [new_entity] diff --git a/spacy/tests/doc/test_array.py b/spacy/tests/doc/test_array.py new file mode 100644 index 0000000..757655f --- /dev/null +++ b/spacy/tests/doc/test_array.py @@ -0,0 +1,136 @@ +import numpy +import pytest + +from spacy.attrs import DEP, MORPH, ORTH, POS, SHAPE +from spacy.tokens import Doc + + +@pytest.mark.issue(2203) +def test_issue2203(en_vocab): + """Test that lemmas are set correctly in doc.from_array.""" + words = ["I", "'ll", "survive"] + tags = ["PRP", "MD", "VB"] + lemmas = ["-PRON-", "will", "survive"] + tag_ids = [en_vocab.strings.add(tag) for tag in tags] + lemma_ids = [en_vocab.strings.add(lemma) for lemma in lemmas] + doc = Doc(en_vocab, words=words) + # Work around lemma corruption problem and set lemmas after tags + doc.from_array("TAG", numpy.array(tag_ids, dtype="uint64")) + doc.from_array("LEMMA", numpy.array(lemma_ids, dtype="uint64")) + assert [t.tag_ for t in doc] == tags + assert [t.lemma_ for t in doc] == lemmas + # We need to serialize both tag and lemma, since this is what causes the bug + doc_array = doc.to_array(["TAG", "LEMMA"]) + new_doc = Doc(doc.vocab, words=words).from_array(["TAG", "LEMMA"], doc_array) + assert [t.tag_ for t in new_doc] == tags + assert [t.lemma_ for t in new_doc] == lemmas + + +def test_doc_array_attr_of_token(en_vocab): + doc = Doc(en_vocab, words=["An", "example", "sentence"]) + example = doc.vocab["example"] + assert example.orth != example.shape + feats_array = doc.to_array((ORTH, SHAPE)) + assert feats_array[0][0] != feats_array[0][1] + assert feats_array[0][0] != feats_array[0][1] + + +def test_doc_stringy_array_attr_of_token(en_vocab): + doc = Doc(en_vocab, words=["An", "example", "sentence"]) + example = doc.vocab["example"] + assert example.orth != example.shape + feats_array = doc.to_array((ORTH, SHAPE)) + feats_array_stringy = doc.to_array(("ORTH", "SHAPE")) + assert feats_array_stringy[0][0] == feats_array[0][0] + assert feats_array_stringy[0][1] == feats_array[0][1] + + +def test_doc_scalar_attr_of_token(en_vocab): + doc = Doc(en_vocab, words=["An", "example", "sentence"]) + example = doc.vocab["example"] + assert example.orth != example.shape + feats_array = doc.to_array(ORTH) + assert feats_array.shape == (3,) + + +def test_doc_array_tag(en_vocab): + words = ["A", "nice", "sentence", "."] + pos = ["DET", "ADJ", "NOUN", "PUNCT"] + doc = Doc(en_vocab, words=words, pos=pos) + assert doc[0].pos != doc[1].pos != doc[2].pos != doc[3].pos + feats_array = doc.to_array((ORTH, POS)) + assert feats_array[0][1] == doc[0].pos + assert feats_array[1][1] == doc[1].pos + assert feats_array[2][1] == doc[2].pos + assert feats_array[3][1] == doc[3].pos + + +def test_doc_array_morph(en_vocab): + words = ["Eat", "blue", "ham"] + morph = ["Feat=V", "Feat=J", "Feat=N"] + doc = Doc(en_vocab, words=words, morphs=morph) + assert morph[0] == str(doc[0].morph) + assert morph[1] == str(doc[1].morph) + assert morph[2] == str(doc[2].morph) + + feats_array = doc.to_array((ORTH, MORPH)) + assert feats_array[0][1] == doc[0].morph.key + assert feats_array[1][1] == doc[1].morph.key + assert feats_array[2][1] == doc[2].morph.key + + +def test_doc_array_dep(en_vocab): + words = ["A", "nice", "sentence", "."] + deps = ["det", "amod", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, deps=deps) + feats_array = doc.to_array((ORTH, DEP)) + assert feats_array[0][1] == doc[0].dep + assert feats_array[1][1] == doc[1].dep + assert feats_array[2][1] == doc[2].dep + assert feats_array[3][1] == doc[3].dep + + +@pytest.mark.parametrize("attrs", [["ORTH", "SHAPE"], "IS_ALPHA"]) +def test_doc_array_to_from_string_attrs(en_vocab, attrs): + """Test that both Doc.to_array and Doc.from_array accept string attrs, + as well as single attrs and sequences of attrs. + """ + words = ["An", "example", "sentence"] + doc = Doc(en_vocab, words=words) + Doc(en_vocab, words=words).from_array(attrs, doc.to_array(attrs)) + + +def test_doc_array_idx(en_vocab): + """Test that Doc.to_array can retrieve token start indices""" + words = ["An", "example", "sentence"] + offsets = Doc(en_vocab, words=words).to_array("IDX") + assert offsets[0] == 0 + assert offsets[1] == 3 + assert offsets[2] == 11 + + +def test_doc_from_array_heads_in_bounds(en_vocab): + """Test that Doc.from_array doesn't set heads that are out of bounds.""" + words = ["This", "is", "a", "sentence", "."] + doc = Doc(en_vocab, words=words) + for token in doc: + token.head = doc[0] + + # correct + arr = doc.to_array(["HEAD"]) + doc_from_array = Doc(en_vocab, words=words) + doc_from_array.from_array(["HEAD"], arr) + + # head before start + arr = doc.to_array(["HEAD"]) + arr[0] = numpy.int32(-1).astype(numpy.uint64) + doc_from_array = Doc(en_vocab, words=words) + with pytest.raises(ValueError): + doc_from_array.from_array(["HEAD"], arr) + + # head after end + arr = doc.to_array(["HEAD"]) + arr[0] = numpy.int32(5).astype(numpy.uint64) + doc_from_array = Doc(en_vocab, words=words) + with pytest.raises(ValueError): + doc_from_array.from_array(["HEAD"], arr) diff --git a/spacy/tests/doc/test_creation.py b/spacy/tests/doc/test_creation.py new file mode 100644 index 0000000..1fb5c73 --- /dev/null +++ b/spacy/tests/doc/test_creation.py @@ -0,0 +1,80 @@ +import pytest + +from spacy import util +from spacy.tokens import Doc +from spacy.vocab import Vocab + + +@pytest.fixture +def vocab(): + return Vocab() + + +def test_empty_doc(vocab): + doc = Doc(vocab) + assert len(doc) == 0 + + +def test_single_word(vocab): + doc = Doc(vocab, words=["a"]) + assert doc.text == "a " + doc = Doc(vocab, words=["a"], spaces=[False]) + assert doc.text == "a" + + +def test_create_from_words_and_text(vocab): + # no whitespace in words + words = ["'", "dogs", "'", "run"] + text = " 'dogs'\n\nrun " + words, spaces = util.get_words_and_spaces(words, text) + doc = Doc(vocab, words=words, spaces=spaces) + assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] + assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] + assert doc.text == text + assert [t.text for t in doc if not t.text.isspace()] == [ + word for word in words if not word.isspace() + ] + + # partial whitespace in words + words = [" ", "'", "dogs", "'", "\n\n", "run", " "] + text = " 'dogs'\n\nrun " + words, spaces = util.get_words_and_spaces(words, text) + doc = Doc(vocab, words=words, spaces=spaces) + assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] + assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] + assert doc.text == text + assert [t.text for t in doc if not t.text.isspace()] == [ + word for word in words if not word.isspace() + ] + + # non-standard whitespace tokens + words = [" ", " ", "'", "dogs", "'", "\n\n", "run"] + text = " 'dogs'\n\nrun " + words, spaces = util.get_words_and_spaces(words, text) + doc = Doc(vocab, words=words, spaces=spaces) + assert [t.text for t in doc] == [" ", "'", "dogs", "'", "\n\n", "run", " "] + assert [t.whitespace_ for t in doc] == ["", "", "", "", "", " ", ""] + assert doc.text == text + assert [t.text for t in doc if not t.text.isspace()] == [ + word for word in words if not word.isspace() + ] + + # mismatch between words and text + with pytest.raises(ValueError): + words = [" ", " ", "'", "dogs", "'", "\n\n", "run"] + text = " 'dogs'\n\nrun " + words, spaces = util.get_words_and_spaces(words + ["away"], text) + + +def test_create_with_heads_and_no_deps(vocab): + words = "I like ginger".split() + heads = list(range(len(words))) + with pytest.raises(ValueError): + Doc(vocab, words=words, heads=heads) + + +def test_create_invalid_pos(vocab): + words = "I like ginger".split() + pos = "QQ ZZ XX".split() + with pytest.raises(ValueError): + Doc(vocab, words=words, pos=pos) diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py new file mode 100644 index 0000000..ef098ec --- /dev/null +++ b/spacy/tests/doc/test_doc_api.py @@ -0,0 +1,1003 @@ +import warnings +import weakref + +import numpy +import pytest +from numpy.testing import assert_array_equal +from thinc.api import NumpyOps, get_current_ops + +from spacy.attrs import ( + DEP, + ENT_IOB, + ENT_TYPE, + HEAD, + IS_ALPHA, + MORPH, + POS, + SENT_START, + TAG, +) +from spacy.lang.en import English +from spacy.lang.xx import MultiLanguage +from spacy.language import Language +from spacy.lexeme import Lexeme +from spacy.tokens import Doc, Span, SpanGroup, Token +from spacy.vocab import Vocab + +from .test_underscore import clean_underscore # noqa: F401 + + +def test_doc_api_init(en_vocab): + words = ["a", "b", "c", "d"] + heads = [0, 0, 2, 2] + # set sent_start by sent_starts + doc = Doc(en_vocab, words=words, sent_starts=[True, False, True, False]) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + + # set sent_start by heads + doc = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * 4) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + # heads override sent_starts + doc = Doc( + en_vocab, words=words, sent_starts=[True] * 4, heads=heads, deps=["dep"] * 4 + ) + assert [t.is_sent_start for t in doc] == [True, False, True, False] + + +@pytest.mark.issue(1547) +def test_issue1547(): + """Test that entity labels still match after merging tokens.""" + words = ["\n", "worda", ".", "\n", "wordb", "-", "Biosphere", "2", "-", " \n"] + doc = Doc(Vocab(), words=words) + doc.ents = [Span(doc, 6, 8, label=doc.vocab.strings["PRODUCT"])] + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[5:7]) + assert [ent.text for ent in doc.ents] + + +@pytest.mark.issue(1757) +def test_issue1757(): + """Test comparison against None doesn't cause segfault.""" + doc = Doc(Vocab(), words=["a", "b", "c"]) + assert not doc[0] < None + assert doc[0] is not None + assert doc[0] >= None + assert not doc[:2] < None + assert doc[:2] is not None + assert doc[:2] >= None + assert doc.vocab["a"] is not None + assert not doc.vocab["a"] < None + + +@pytest.mark.issue(2396) +def test_issue2396(en_vocab): + words = ["She", "created", "a", "test", "for", "spacy"] + heads = [1, 1, 3, 1, 3, 4] + deps = ["dep"] * len(heads) + matrix = numpy.array( + [ + [0, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 2, 3, 3, 3], + [1, 1, 3, 3, 3, 3], + [1, 1, 3, 3, 4, 4], + [1, 1, 3, 3, 4, 5], + ], + dtype=numpy.int32, + ) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + span = doc[:] + assert (doc.get_lca_matrix() == matrix).all() + assert (span.get_lca_matrix() == matrix).all() + + +@pytest.mark.issue(11499) +def test_init_args_unmodified(en_vocab): + words = ["A", "sentence"] + ents = ["B-TYPE1", ""] + sent_starts = [True, False] + Doc( + vocab=en_vocab, + words=words, + ents=ents, + sent_starts=sent_starts, + ) + assert ents == ["B-TYPE1", ""] + assert sent_starts == [True, False] + + +@pytest.mark.parametrize("text", ["-0.23", "+123,456", "±1"]) +@pytest.mark.parametrize("lang_cls", [English, MultiLanguage]) +@pytest.mark.issue(2782) +def test_issue2782(text, lang_cls): + """Check that like_num handles + and - before number.""" + nlp = lang_cls() + doc = nlp(text) + assert len(doc) == 1 + assert doc[0].like_num + + +@pytest.mark.parametrize( + "sentence", + [ + "The story was to the effect that a young American student recently called on Professor Christlieb with a letter of introduction.", + "The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale's #1.", + "The next month Barry Siddall joined Stoke City on a free transfer, after Chris Pearce had established himself as the Vale's number one", + "Indeed, making the one who remains do all the work has installed him into a position of such insolent tyranny, it will take a month at least to reduce him to his proper proportions.", + "It was a missed assignment, but it shouldn't have resulted in a turnover ...", + ], +) +@pytest.mark.issue(3869) +def test_issue3869(sentence): + """Test that the Doc's count_by function works consistently""" + nlp = English() + doc = nlp(sentence) + count = 0 + for token in doc: + count += token.is_alpha + assert count == doc.count_by(IS_ALPHA).get(1, 0) + + +@pytest.mark.issue(3962) +def test_issue3962(en_vocab): + """Ensure that as_doc does not result in out-of-bound access of tokens. + This is achieved by setting the head to itself if it would lie out of the span otherwise. + """ + # fmt: off + words = ["He", "jests", "at", "scars", ",", "that", "never", "felt", "a", "wound", "."] + heads = [1, 7, 1, 2, 7, 7, 7, 7, 9, 7, 7] + deps = ["nsubj", "ccomp", "prep", "pobj", "punct", "nsubj", "neg", "ROOT", "det", "dobj", "punct"] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + span2 = doc[1:5] # "jests at scars ," + doc2 = span2.as_doc() + doc2_json = doc2.to_json() + assert doc2_json + # head set to itself, being the new artificial root + assert doc2[0].head.text == "jests" + assert doc2[0].dep_ == "dep" + assert doc2[1].head.text == "jests" + assert doc2[1].dep_ == "prep" + assert doc2[2].head.text == "at" + assert doc2[2].dep_ == "pobj" + assert doc2[3].head.text == "jests" # head set to the new artificial root + assert doc2[3].dep_ == "dep" + # We should still have 1 sentence + assert len(list(doc2.sents)) == 1 + span3 = doc[6:9] # "never felt a" + doc3 = span3.as_doc() + doc3_json = doc3.to_json() + assert doc3_json + assert doc3[0].head.text == "felt" + assert doc3[0].dep_ == "neg" + assert doc3[1].head.text == "felt" + assert doc3[1].dep_ == "ROOT" + assert doc3[2].head.text == "felt" # head set to ancestor + assert doc3[2].dep_ == "dep" + # We should still have 1 sentence as "a" can be attached to "felt" instead of "wound" + assert len(list(doc3.sents)) == 1 + + +@pytest.mark.issue(3962) +def test_issue3962_long(en_vocab): + """Ensure that as_doc does not result in out-of-bound access of tokens. + This is achieved by setting the head to itself if it would lie out of the span otherwise. + """ + # fmt: off + words = ["He", "jests", "at", "scars", ".", "They", "never", "felt", "a", "wound", "."] + heads = [1, 1, 1, 2, 1, 7, 7, 7, 9, 7, 7] + deps = ["nsubj", "ROOT", "prep", "pobj", "punct", "nsubj", "neg", "ROOT", "det", "dobj", "punct"] + # fmt: on + two_sent_doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + span2 = two_sent_doc[1:7] # "jests at scars. They never" + doc2 = span2.as_doc() + doc2_json = doc2.to_json() + assert doc2_json + # head set to itself, being the new artificial root (in sentence 1) + assert doc2[0].head.text == "jests" + assert doc2[0].dep_ == "ROOT" + assert doc2[1].head.text == "jests" + assert doc2[1].dep_ == "prep" + assert doc2[2].head.text == "at" + assert doc2[2].dep_ == "pobj" + assert doc2[3].head.text == "jests" + assert doc2[3].dep_ == "punct" + # head set to itself, being the new artificial root (in sentence 2) + assert doc2[4].head.text == "They" + assert doc2[4].dep_ == "dep" + # head set to the new artificial head (in sentence 2) + assert doc2[4].head.text == "They" + assert doc2[4].dep_ == "dep" + # We should still have 2 sentences + sents = list(doc2.sents) + assert len(sents) == 2 + assert sents[0].text == "jests at scars ." + assert sents[1].text == "They never" + + +@Language.factory("my_pipe") +class CustomPipe: + def __init__(self, nlp, name="my_pipe"): + self.name = name + Span.set_extension("my_ext", getter=self._get_my_ext) + Doc.set_extension("my_ext", default=None) + + def __call__(self, doc): + gathered_ext = [] + for sent in doc.sents: + sent_ext = self._get_my_ext(sent) + sent._.set("my_ext", sent_ext) + gathered_ext.append(sent_ext) + + doc._.set("my_ext", "\n".join(gathered_ext)) + return doc + + @staticmethod + def _get_my_ext(span): + return str(span.end) + + +@pytest.mark.issue(4903) +def test_issue4903(): + """Ensure that this runs correctly and doesn't hang or crash on Windows / + macOS.""" + nlp = English() + nlp.add_pipe("sentencizer") + nlp.add_pipe("my_pipe", after="sentencizer") + text = ["I like bananas.", "Do you like them?", "No, I prefer wasabi."] + if isinstance(get_current_ops(), NumpyOps): + docs = list(nlp.pipe(text, n_process=2)) + assert docs[0].text == "I like bananas." + assert docs[1].text == "Do you like them?" + assert docs[2].text == "No, I prefer wasabi." + + +@pytest.mark.issue(5048) +def test_issue5048(en_vocab): + words = ["This", "is", "a", "sentence"] + pos_s = ["DET", "VERB", "DET", "NOUN"] + spaces = [" ", " ", " ", ""] + deps_s = ["dep", "adj", "nn", "atm"] + tags_s = ["DT", "VBZ", "DT", "NN"] + strings = en_vocab.strings + for w in words: + strings.add(w) + deps = [strings.add(d) for d in deps_s] + pos = [strings.add(p) for p in pos_s] + tags = [strings.add(t) for t in tags_s] + attrs = [POS, DEP, TAG] + array = numpy.array(list(zip(pos, deps, tags)), dtype="uint64") + doc = Doc(en_vocab, words=words, spaces=spaces) + doc.from_array(attrs, array) + v1 = [(token.text, token.pos_, token.tag_) for token in doc] + doc2 = Doc(en_vocab, words=words, pos=pos_s, deps=deps_s, tags=tags_s) + v2 = [(token.text, token.pos_, token.tag_) for token in doc2] + assert v1 == v2 + + +@pytest.mark.parametrize("text", [["one", "two", "three"]]) +def test_doc_api_compare_by_string_position(en_vocab, text): + doc = Doc(en_vocab, words=text) + # Get the tokens in this order, so their ID ordering doesn't match the idx + token3 = doc[-1] + token2 = doc[-2] + token1 = doc[-1] + token1, token2, token3 = doc + assert token1 < token2 < token3 + assert not token1 > token2 + assert token2 > token1 + assert token2 <= token3 + assert token3 >= token1 + + +def test_doc_api_getitem(en_tokenizer): + text = "Give it back! He pleaded." + tokens = en_tokenizer(text) + assert tokens[0].text == "Give" + assert tokens[-1].text == "." + with pytest.raises(IndexError): + tokens[len(tokens)] + + def to_str(span): + return "/".join(token.text for token in span) + + span = tokens[1:1] + assert not to_str(span) + span = tokens[1:4] + assert to_str(span) == "it/back/!" + span = tokens[1:4:1] + assert to_str(span) == "it/back/!" + with pytest.raises(ValueError): + tokens[1:4:2] + with pytest.raises(ValueError): + tokens[1:4:-1] + + span = tokens[-3:6] + assert to_str(span) == "He/pleaded" + span = tokens[4:-1] + assert to_str(span) == "He/pleaded" + span = tokens[-5:-3] + assert to_str(span) == "back/!" + span = tokens[5:4] + assert span.start == span.end == 5 and not to_str(span) + span = tokens[4:-3] + assert span.start == span.end == 4 and not to_str(span) + + span = tokens[:] + assert to_str(span) == "Give/it/back/!/He/pleaded/." + span = tokens[4:] + assert to_str(span) == "He/pleaded/." + span = tokens[:4] + assert to_str(span) == "Give/it/back/!" + span = tokens[:-3] + assert to_str(span) == "Give/it/back/!" + span = tokens[-3:] + assert to_str(span) == "He/pleaded/." + + span = tokens[4:50] + assert to_str(span) == "He/pleaded/." + span = tokens[-50:4] + assert to_str(span) == "Give/it/back/!" + span = tokens[-50:-40] + assert span.start == span.end == 0 and not to_str(span) + span = tokens[40:50] + assert span.start == span.end == 7 and not to_str(span) + + span = tokens[1:4] + assert span[0].orth_ == "it" + subspan = span[:] + assert to_str(subspan) == "it/back/!" + subspan = span[:2] + assert to_str(subspan) == "it/back" + subspan = span[1:] + assert to_str(subspan) == "back/!" + subspan = span[:-1] + assert to_str(subspan) == "it/back" + subspan = span[-2:] + assert to_str(subspan) == "back/!" + subspan = span[1:2] + assert to_str(subspan) == "back" + subspan = span[-2:-1] + assert to_str(subspan) == "back" + subspan = span[-50:50] + assert to_str(subspan) == "it/back/!" + subspan = span[50:-50] + assert subspan.start == subspan.end == 4 and not to_str(subspan) + + +@pytest.mark.parametrize( + "text", ["Give it back! He pleaded.", " Give it back! He pleaded. "] +) +def test_doc_api_serialize(en_tokenizer, text): + tokens = en_tokenizer(text) + tokens[0].lemma_ = "lemma" + tokens[0].norm_ = "norm" + tokens.ents = [(tokens.vocab.strings["PRODUCT"], 0, 1)] + tokens[0].ent_kb_id_ = "ent_kb_id" + tokens[0].ent_id_ = "ent_id" + new_tokens = Doc(tokens.vocab).from_bytes(tokens.to_bytes()) + assert tokens.text == new_tokens.text + assert [t.text for t in tokens] == [t.text for t in new_tokens] + assert [t.orth for t in tokens] == [t.orth for t in new_tokens] + assert new_tokens[0].lemma_ == "lemma" + assert new_tokens[0].norm_ == "norm" + assert new_tokens[0].ent_kb_id_ == "ent_kb_id" + assert new_tokens[0].ent_id_ == "ent_id" + + new_tokens = Doc(tokens.vocab).from_bytes( + tokens.to_bytes(exclude=["tensor"]), exclude=["tensor"] + ) + assert tokens.text == new_tokens.text + assert [t.text for t in tokens] == [t.text for t in new_tokens] + assert [t.orth for t in tokens] == [t.orth for t in new_tokens] + + new_tokens = Doc(tokens.vocab).from_bytes( + tokens.to_bytes(exclude=["sentiment"]), exclude=["sentiment"] + ) + assert tokens.text == new_tokens.text + assert [t.text for t in tokens] == [t.text for t in new_tokens] + assert [t.orth for t in tokens] == [t.orth for t in new_tokens] + + def inner_func(d1, d2): + return "hello!" + + _ = tokens.to_bytes() # noqa: F841 + with pytest.warns(UserWarning): + tokens.user_hooks["similarity"] = inner_func + _ = tokens.to_bytes() # noqa: F841 + + +def test_doc_api_set_ents(en_tokenizer): + text = "I use goggle chrone to surf the web" + tokens = en_tokenizer(text) + assert len(tokens.ents) == 0 + tokens.ents = [(tokens.vocab.strings["PRODUCT"], 2, 4)] + assert len(list(tokens.ents)) == 1 + assert [t.ent_iob for t in tokens] == [2, 2, 3, 1, 2, 2, 2, 2] + assert tokens.ents[0].label_ == "PRODUCT" + assert tokens.ents[0].start == 2 + assert tokens.ents[0].end == 4 + + +def test_doc_api_sents_empty_string(en_tokenizer): + doc = en_tokenizer("") + sents = list(doc.sents) + assert len(sents) == 0 + + +def test_doc_api_runtime_error(en_tokenizer): + # Example that caused run-time error while parsing Reddit + # fmt: off + text = "67% of black households are single parent \n\n72% of all black babies born out of wedlock \n\n50% of all black kids don\u2019t finish high school" + deps = ["nummod", "nsubj", "prep", "amod", "pobj", "ROOT", "amod", "attr", "", "nummod", "appos", "prep", "det", + "amod", "pobj", "acl", "prep", "prep", "pobj", + "", "nummod", "nsubj", "prep", "det", "amod", "pobj", "aux", "neg", "ccomp", "amod", "dobj"] + # fmt: on + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], deps=deps) + nps = [] + for np in doc.noun_chunks: + while len(np) > 1 and np[0].dep_ not in ("advmod", "amod", "compound"): + np = np[1:] + if len(np) > 1: + nps.append(np) + with doc.retokenize() as retokenizer: + for np in nps: + attrs = { + "tag": np.root.tag_, + "lemma": np.text, + "ent_type": np.root.ent_type_, + } + retokenizer.merge(np, attrs=attrs) + + +def test_doc_api_right_edge(en_vocab): + """Test for bug occurring from Unshift action, causing incorrect right edge""" + # fmt: off + words = [ + "I", "have", "proposed", "to", "myself", ",", "for", "the", "sake", + "of", "such", "as", "live", "under", "the", "government", "of", "the", + "Romans", ",", "to", "translate", "those", "books", "into", "the", + "Greek", "tongue", "." + ] + heads = [2, 2, 2, 2, 3, 2, 21, 8, 6, 8, 11, 8, 11, 12, 15, 13, 15, 18, 16, 12, 21, 2, 23, 21, 21, 27, 27, 24, 2] + deps = ["dep"] * len(heads) + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert doc[6].text == "for" + subtree = [w.text for w in doc[6].subtree] + # fmt: off + assert subtree == ["for", "the", "sake", "of", "such", "as", "live", "under", "the", "government", "of", "the", "Romans", ","] + # fmt: on + assert doc[6].right_edge.text == "," + + +def test_doc_api_has_vector(): + vocab = Vocab() + vocab.reset_vectors(width=2) + vocab.set_vector("kitten", vector=numpy.asarray([0.0, 2.0], dtype="f")) + doc = Doc(vocab, words=["kitten"]) + assert doc.has_vector + + +def test_doc_api_similarity_match(): + doc = Doc(Vocab(), words=["a"]) + assert doc.similarity(doc[0]) == 1.0 + assert doc.similarity(doc.vocab["a"]) == 1.0 + doc2 = Doc(doc.vocab, words=["a", "b", "c"]) + with pytest.warns(UserWarning): + assert doc.similarity(doc2[:1]) == 1.0 + assert doc.similarity(doc2) == 0.0 + + +@pytest.mark.parametrize( + "words,heads,lca_matrix", + [ + ( + ["the", "lazy", "dog", "slept"], + [2, 2, 3, 3], + numpy.array([[0, 2, 2, 3], [2, 1, 2, 3], [2, 2, 2, 3], [3, 3, 3, 3]]), + ), + ( + ["The", "lazy", "dog", "slept", ".", "The", "quick", "fox", "jumped"], + [2, 2, 3, 3, 3, 7, 7, 8, 8], + numpy.array( + [ + [0, 2, 2, 3, 3, -1, -1, -1, -1], + [2, 1, 2, 3, 3, -1, -1, -1, -1], + [2, 2, 2, 3, 3, -1, -1, -1, -1], + [3, 3, 3, 3, 3, -1, -1, -1, -1], + [3, 3, 3, 3, 4, -1, -1, -1, -1], + [-1, -1, -1, -1, -1, 5, 7, 7, 8], + [-1, -1, -1, -1, -1, 7, 6, 7, 8], + [-1, -1, -1, -1, -1, 7, 7, 7, 8], + [-1, -1, -1, -1, -1, 8, 8, 8, 8], + ] + ), + ), + ], +) +def test_lowest_common_ancestor(en_vocab, words, heads, lca_matrix): + doc = Doc(en_vocab, words, heads=heads, deps=["dep"] * len(heads)) + lca = doc.get_lca_matrix() + assert (lca == lca_matrix).all() + assert lca[1, 1] == 1 + assert lca[0, 1] == 2 + assert lca[1, 2] == 2 + + +def test_doc_is_nered(en_vocab): + words = ["I", "live", "in", "New", "York"] + doc = Doc(en_vocab, words=words) + assert not doc.has_annotation("ENT_IOB") + doc.ents = [Span(doc, 3, 5, label="GPE")] + assert doc.has_annotation("ENT_IOB") + # Test creating doc from array with unknown values + arr = numpy.array([[0, 0], [0, 0], [0, 0], [384, 3], [384, 1]], dtype="uint64") + doc = Doc(en_vocab, words=words).from_array([ENT_TYPE, ENT_IOB], arr) + assert doc.has_annotation("ENT_IOB") + # Test serialization + new_doc = Doc(en_vocab).from_bytes(doc.to_bytes()) + assert new_doc.has_annotation("ENT_IOB") + + +def test_doc_from_array_sent_starts(en_vocab): + # fmt: off + words = ["I", "live", "in", "New", "York", ".", "I", "like", "cats", "."] + heads = [0, 0, 0, 0, 0, 0, 6, 6, 6, 6] + deps = ["ROOT", "dep", "dep", "dep", "dep", "dep", "ROOT", "dep", "dep", "dep"] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + # HEAD overrides SENT_START without warning + attrs = [SENT_START, HEAD] + arr = doc.to_array(attrs) + new_doc = Doc(en_vocab, words=words) + new_doc.from_array(attrs, arr) + # no warning using default attrs + attrs = doc._get_array_attrs() + arr = doc.to_array(attrs) + with warnings.catch_warnings(): + warnings.simplefilter("error") + new_doc.from_array(attrs, arr) + # only SENT_START uses SENT_START + attrs = [SENT_START] + arr = doc.to_array(attrs) + new_doc = Doc(en_vocab, words=words) + new_doc.from_array(attrs, arr) + assert [t.is_sent_start for t in doc] == [t.is_sent_start for t in new_doc] + assert not new_doc.has_annotation("DEP") + # only HEAD uses HEAD + attrs = [HEAD, DEP] + arr = doc.to_array(attrs) + new_doc = Doc(en_vocab, words=words) + new_doc.from_array(attrs, arr) + assert [t.is_sent_start for t in doc] == [t.is_sent_start for t in new_doc] + assert new_doc.has_annotation("DEP") + + +def test_doc_from_array_morph(en_vocab): + # fmt: off + words = ["I", "live", "in", "New", "York", "."] + morphs = ["Feat1=A", "Feat1=B", "Feat1=C", "Feat1=A|Feat2=D", "Feat2=E", "Feat3=F"] + # fmt: on + doc = Doc(en_vocab, words=words, morphs=morphs) + attrs = [MORPH] + arr = doc.to_array(attrs) + new_doc = Doc(en_vocab, words=words) + new_doc.from_array(attrs, arr) + assert [str(t.morph) for t in new_doc] == morphs + assert [str(t.morph) for t in doc] == [str(t.morph) for t in new_doc] + + +@pytest.mark.usefixtures("clean_underscore") +def test_doc_api_from_docs(en_tokenizer, de_tokenizer): + en_texts = [ + "Merging the docs is fun.", + "", + "They don't think alike. ", + "", + "Another doc.", + ] + en_texts_without_empty = [t for t in en_texts if len(t)] + de_text = "Wie war die Frage?" + en_docs = [en_tokenizer(text) for text in en_texts] + en_docs[0].spans["group"] = [en_docs[0][1:4]] + en_docs[2].spans["group"] = [en_docs[2][1:4]] + en_docs[4].spans["group"] = [en_docs[4][0:1]] + span_group_texts = sorted( + [en_docs[0][1:4].text, en_docs[2][1:4].text, en_docs[4][0:1].text] + ) + de_doc = de_tokenizer(de_text) + Token.set_extension("is_ambiguous", default=False) + en_docs[0][2]._.is_ambiguous = True # docs + en_docs[2][3]._.is_ambiguous = True # think + assert Doc.from_docs([]) is None + assert de_doc is not Doc.from_docs([de_doc]) + assert str(de_doc) == str(Doc.from_docs([de_doc])) + + with pytest.raises(ValueError): + Doc.from_docs(en_docs + [de_doc]) + + m_doc = Doc.from_docs(en_docs) + assert len(en_texts_without_empty) == len(list(m_doc.sents)) + assert len(m_doc.text) > len(en_texts[0]) + len(en_texts[1]) + assert m_doc.text == " ".join([t.strip() for t in en_texts_without_empty]) + p_token = m_doc[len(en_docs[0]) - 1] + assert p_token.text == "." and bool(p_token.whitespace_) + en_docs_tokens = [t for doc in en_docs for t in doc] + assert len(m_doc) == len(en_docs_tokens) + think_idx = len(en_texts[0]) + 1 + en_texts[2].index("think") + assert m_doc[2]._.is_ambiguous is True + assert m_doc[9].idx == think_idx + assert m_doc[9]._.is_ambiguous is True + assert not any([t._.is_ambiguous for t in m_doc[3:8]]) + assert "group" in m_doc.spans + assert span_group_texts == sorted([s.text for s in m_doc.spans["group"]]) + assert bool(m_doc[11].whitespace_) + + m_doc = Doc.from_docs(en_docs, ensure_whitespace=False) + assert len(en_texts_without_empty) == len(list(m_doc.sents)) + assert len(m_doc.text) == sum(len(t) for t in en_texts) + assert m_doc.text == "".join(en_texts_without_empty) + p_token = m_doc[len(en_docs[0]) - 1] + assert p_token.text == "." and not bool(p_token.whitespace_) + en_docs_tokens = [t for doc in en_docs for t in doc] + assert len(m_doc) == len(en_docs_tokens) + think_idx = len(en_texts[0]) + 0 + en_texts[2].index("think") + assert m_doc[9].idx == think_idx + assert "group" in m_doc.spans + assert span_group_texts == sorted([s.text for s in m_doc.spans["group"]]) + assert bool(m_doc[11].whitespace_) + + m_doc = Doc.from_docs(en_docs, attrs=["lemma", "length", "pos"]) + assert len(m_doc.text) > len(en_texts[0]) + len(en_texts[1]) + # space delimiter considered, although spacy attribute was missing + assert m_doc.text == " ".join([t.strip() for t in en_texts_without_empty]) + p_token = m_doc[len(en_docs[0]) - 1] + assert p_token.text == "." and bool(p_token.whitespace_) + en_docs_tokens = [t for doc in en_docs for t in doc] + assert len(m_doc) == len(en_docs_tokens) + think_idx = len(en_texts[0]) + 1 + en_texts[2].index("think") + assert m_doc[9].idx == think_idx + assert "group" in m_doc.spans + assert span_group_texts == sorted([s.text for s in m_doc.spans["group"]]) + + # can exclude spans + m_doc = Doc.from_docs(en_docs, exclude=["spans"]) + assert "group" not in m_doc.spans + + # can exclude user_data + m_doc = Doc.from_docs(en_docs, exclude=["user_data"]) + assert m_doc.user_data == {} + + # can merge empty docs + doc = Doc.from_docs([en_tokenizer("")] * 10) + + # empty but set spans keys are preserved + en_docs = [en_tokenizer(text) for text in en_texts] + m_doc = Doc.from_docs(en_docs) + assert "group" not in m_doc.spans + for doc in en_docs: + doc.spans["group"] = [] + m_doc = Doc.from_docs(en_docs) + assert "group" in m_doc.spans + assert len(m_doc.spans["group"]) == 0 + + # with tensor + ops = get_current_ops() + for doc in en_docs: + doc.tensor = ops.asarray([[len(t.text), 0.0] for t in doc]) + m_doc = Doc.from_docs(en_docs) + assert_array_equal( + ops.to_numpy(m_doc.tensor), + ops.to_numpy(ops.xp.vstack([doc.tensor for doc in en_docs if len(doc)])), + ) + + # can exclude tensor + m_doc = Doc.from_docs(en_docs, exclude=["tensor"]) + assert m_doc.tensor.shape == (0,) + + +def test_doc_api_from_docs_ents(en_tokenizer): + texts = ["Merging the docs is fun.", "They don't think alike."] + docs = [en_tokenizer(t) for t in texts] + docs[0].ents = () + docs[1].ents = (Span(docs[1], 0, 1, label="foo"),) + doc = Doc.from_docs(docs) + assert len(doc.ents) == 1 + + +def test_doc_lang(en_vocab): + doc = Doc(en_vocab, words=["Hello", "world"]) + assert doc.lang_ == "en" + assert doc.lang == en_vocab.strings["en"] + assert doc[0].lang_ == "en" + assert doc[0].lang == en_vocab.strings["en"] + nlp = English() + doc = nlp("Hello world") + assert doc.lang_ == "en" + assert doc.lang == en_vocab.strings["en"] + assert doc[0].lang_ == "en" + assert doc[0].lang == en_vocab.strings["en"] + + +def test_token_lexeme(en_vocab): + """Test that tokens expose their lexeme.""" + token = Doc(en_vocab, words=["Hello", "world"])[0] + assert isinstance(token.lex, Lexeme) + assert token.lex.text == token.text + assert en_vocab[token.orth] == token.lex + + +def test_has_annotation(en_vocab): + doc = Doc(en_vocab, words=["Hello", "world"]) + attrs = ("TAG", "POS", "MORPH", "LEMMA", "DEP", "HEAD", "ENT_IOB", "ENT_TYPE") + for attr in attrs: + assert not doc.has_annotation(attr) + assert not doc.has_annotation(attr, require_complete=True) + + doc[0].tag_ = "A" + doc[0].pos_ = "X" + doc[0].set_morph("Feat=Val") + doc[0].lemma_ = "a" + doc[0].dep_ = "dep" + doc[0].head = doc[1] + doc.set_ents([Span(doc, 0, 1, label="HELLO")], default="missing") + + for attr in attrs: + assert doc.has_annotation(attr) + assert not doc.has_annotation(attr, require_complete=True) + + doc[1].tag_ = "A" + doc[1].pos_ = "X" + doc[1].set_morph("") + doc[1].lemma_ = "a" + doc[1].dep_ = "dep" + doc.ents = [Span(doc, 0, 2, label="HELLO")] + + for attr in attrs: + assert doc.has_annotation(attr) + assert doc.has_annotation(attr, require_complete=True) + + +def test_has_annotation_sents(en_vocab): + doc = Doc(en_vocab, words=["Hello", "beautiful", "world"]) + attrs = ("SENT_START", "IS_SENT_START", "IS_SENT_END") + for attr in attrs: + assert not doc.has_annotation(attr) + assert not doc.has_annotation(attr, require_complete=True) + + # The first token (index 0) is always assumed to be a sentence start, + # and ignored by the check in doc.has_annotation + + doc[1].is_sent_start = False + for attr in attrs: + assert doc.has_annotation(attr) + assert not doc.has_annotation(attr, require_complete=True) + + doc[2].is_sent_start = False + for attr in attrs: + assert doc.has_annotation(attr) + assert doc.has_annotation(attr, require_complete=True) + + +def test_is_flags_deprecated(en_tokenizer): + doc = en_tokenizer("test") + with pytest.deprecated_call(): + doc.is_tagged + with pytest.deprecated_call(): + doc.is_parsed + with pytest.deprecated_call(): + doc.is_nered + with pytest.deprecated_call(): + doc.is_sentenced + + +def test_doc_set_ents(en_tokenizer): + # set ents + doc = en_tokenizer("a b c d e") + doc.set_ents([Span(doc, 0, 1, 10), Span(doc, 1, 3, 11)]) + assert [t.ent_iob for t in doc] == [3, 3, 1, 2, 2] + assert [t.ent_type for t in doc] == [10, 11, 11, 0, 0] + + # add ents, invalid IOB repaired + doc = en_tokenizer("a b c d e") + doc.set_ents([Span(doc, 0, 1, 10), Span(doc, 1, 3, 11)]) + doc.set_ents([Span(doc, 0, 2, 12)], default="unmodified") + assert [t.ent_iob for t in doc] == [3, 1, 3, 2, 2] + assert [t.ent_type for t in doc] == [12, 12, 11, 0, 0] + + # missing ents + doc = en_tokenizer("a b c d e") + doc.set_ents([Span(doc, 0, 1, 10), Span(doc, 1, 3, 11)], missing=[doc[4:5]]) + assert [t.ent_iob for t in doc] == [3, 3, 1, 2, 0] + assert [t.ent_type for t in doc] == [10, 11, 11, 0, 0] + + # outside ents + doc = en_tokenizer("a b c d e") + doc.set_ents( + [Span(doc, 0, 1, 10), Span(doc, 1, 3, 11)], + outside=[doc[4:5]], + default="missing", + ) + assert [t.ent_iob for t in doc] == [3, 3, 1, 0, 2] + assert [t.ent_type for t in doc] == [10, 11, 11, 0, 0] + + # blocked ents + doc = en_tokenizer("a b c d e") + doc.set_ents([], blocked=[doc[1:2], doc[3:5]], default="unmodified") + assert [t.ent_iob for t in doc] == [0, 3, 0, 3, 3] + assert [t.ent_type for t in doc] == [0, 0, 0, 0, 0] + assert doc.ents == tuple() + + # invalid IOB repaired after blocked + doc.ents = [Span(doc, 3, 5, "ENT")] + assert [t.ent_iob for t in doc] == [2, 2, 2, 3, 1] + doc.set_ents([], blocked=[doc[3:4]], default="unmodified") + assert [t.ent_iob for t in doc] == [2, 2, 2, 3, 3] + + # all types + doc = en_tokenizer("a b c d e") + doc.set_ents( + [Span(doc, 0, 1, 10)], + blocked=[doc[1:2]], + missing=[doc[2:3]], + outside=[doc[3:4]], + default="unmodified", + ) + assert [t.ent_iob for t in doc] == [3, 3, 0, 2, 0] + assert [t.ent_type for t in doc] == [10, 0, 0, 0, 0] + + doc = en_tokenizer("a b c d e") + # single span instead of a list + with pytest.raises(ValueError): + doc.set_ents([], missing=doc[1:2]) + # invalid default mode + with pytest.raises(ValueError): + doc.set_ents([], missing=[doc[1:2]], default="none") + # conflicting/overlapping specifications + with pytest.raises(ValueError): + doc.set_ents([], missing=[doc[1:2]], outside=[doc[1:2]]) + + +def test_doc_ents_setter(): + """Test that both strings and integers can be used to set entities in + tuple format via doc.ents.""" + words = ["a", "b", "c", "d", "e"] + doc = Doc(Vocab(), words=words) + doc.ents = [("HELLO", 0, 2), (doc.vocab.strings.add("WORLD"), 3, 5)] + assert [e.label_ for e in doc.ents] == ["HELLO", "WORLD"] + vocab = Vocab() + ents = [("HELLO", 0, 2), (vocab.strings.add("WORLD"), 3, 5)] + ents = ["B-HELLO", "I-HELLO", "O", "B-WORLD", "I-WORLD"] + doc = Doc(vocab, words=words, ents=ents) + assert [e.label_ for e in doc.ents] == ["HELLO", "WORLD"] + + +def test_doc_morph_setter(en_tokenizer, de_tokenizer): + doc1 = en_tokenizer("a b") + doc1b = en_tokenizer("c d") + doc2 = de_tokenizer("a b") + + # unset values can be copied + doc1[0].morph = doc1[1].morph + assert doc1[0].morph.key == 0 + assert doc1[1].morph.key == 0 + + # morph values from the same vocab can be copied + doc1[0].set_morph("Feat=Val") + doc1[1].morph = doc1[0].morph + assert doc1[0].morph == doc1[1].morph + + # ... also across docs + doc1b[0].morph = doc1[0].morph + assert doc1[0].morph == doc1b[0].morph + + doc2[0].set_morph("Feat2=Val2") + + # the morph value must come from the same vocab + with pytest.raises(ValueError): + doc1[0].morph = doc2[0].morph + + +def test_doc_init_iob(): + """Test ents validation/normalization in Doc.__init__""" + words = ["a", "b", "c", "d", "e"] + ents = ["O"] * len(words) + doc = Doc(Vocab(), words=words, ents=ents) + assert doc.ents == () + + ents = ["B-PERSON", "I-PERSON", "O", "I-PERSON", "I-PERSON"] + doc = Doc(Vocab(), words=words, ents=ents) + assert len(doc.ents) == 2 + + ents = ["B-PERSON", "I-PERSON", "O", "I-PERSON", "I-GPE"] + doc = Doc(Vocab(), words=words, ents=ents) + assert len(doc.ents) == 3 + + # None is missing + ents = ["B-PERSON", "I-PERSON", "O", None, "I-GPE"] + doc = Doc(Vocab(), words=words, ents=ents) + assert len(doc.ents) == 2 + + # empty tag is missing + ents = ["", "B-PERSON", "O", "B-PERSON", "I-PERSON"] + doc = Doc(Vocab(), words=words, ents=ents) + assert len(doc.ents) == 2 + + # invalid IOB + ents = ["Q-PERSON", "I-PERSON", "O", "I-PERSON", "I-GPE"] + with pytest.raises(ValueError): + doc = Doc(Vocab(), words=words, ents=ents) + + # no dash + ents = ["OPERSON", "I-PERSON", "O", "I-PERSON", "I-GPE"] + with pytest.raises(ValueError): + doc = Doc(Vocab(), words=words, ents=ents) + + # no ent type + ents = ["O", "B-", "O", "I-PERSON", "I-GPE"] + with pytest.raises(ValueError): + doc = Doc(Vocab(), words=words, ents=ents) + + # not strings or None + ents = [0, "B-", "O", "I-PERSON", "I-GPE"] + with pytest.raises(ValueError): + doc = Doc(Vocab(), words=words, ents=ents) + + +def test_doc_set_ents_invalid_spans(en_tokenizer): + doc = en_tokenizer("Some text about Colombia and the Czech Republic") + spans = [Span(doc, 3, 4, label="GPE"), Span(doc, 6, 8, label="GPE")] + with doc.retokenize() as retokenizer: + for span in spans: + retokenizer.merge(span) + with pytest.raises(IndexError): + doc.ents = spans + + +def test_doc_noun_chunks_not_implemented(): + """Test that a language without noun_chunk iterator, throws a NotImplementedError""" + text = "Může data vytvářet a spravovat, ale především je dokáže analyzovat, najít v nich nové vztahy a vše přehledně vizualizovat." + nlp = MultiLanguage() + doc = nlp(text) + with pytest.raises(NotImplementedError): + _ = list(doc.noun_chunks) # noqa: F841 + + +def test_span_groups(en_tokenizer): + doc = en_tokenizer("Some text about Colombia and the Czech Republic") + doc.spans["hi"] = [Span(doc, 3, 4, label="bye")] + assert "hi" in doc.spans + assert "bye" not in doc.spans + assert len(doc.spans["hi"]) == 1 + assert doc.spans["hi"][0].label_ == "bye" + doc.spans["hi"].append(doc[0:3]) + assert len(doc.spans["hi"]) == 2 + assert doc.spans["hi"][1].text == "Some text about" + assert [span.text for span in doc.spans["hi"]] == ["Colombia", "Some text about"] + assert not doc.spans["hi"].has_overlap + doc.ents = [Span(doc, 3, 4, label="GPE"), Span(doc, 6, 8, label="GPE")] + doc.spans["hi"].extend(doc.ents) + assert len(doc.spans["hi"]) == 4 + assert [span.label_ for span in doc.spans["hi"]] == ["bye", "", "GPE", "GPE"] + assert doc.spans["hi"].has_overlap + del doc.spans["hi"] + assert "hi" not in doc.spans + + +def test_doc_spans_copy(en_tokenizer): + doc1 = en_tokenizer("Some text about Colombia and the Czech Republic") + assert weakref.ref(doc1) == doc1.spans.doc_ref + doc2 = doc1.copy() + assert weakref.ref(doc2) == doc2.spans.doc_ref + + +def test_doc_spans_setdefault(en_tokenizer): + doc = en_tokenizer("Some text about Colombia and the Czech Republic") + doc.spans.setdefault("key1") + assert len(doc.spans["key1"]) == 0 + doc.spans.setdefault("key2", default=[doc[0:1]]) + assert len(doc.spans["key2"]) == 1 + doc.spans.setdefault("key3", default=SpanGroup(doc, spans=[doc[0:1], doc[1:2]])) + assert len(doc.spans["key3"]) == 2 diff --git a/spacy/tests/doc/test_graph.py b/spacy/tests/doc/test_graph.py new file mode 100644 index 0000000..d14a5b0 --- /dev/null +++ b/spacy/tests/doc/test_graph.py @@ -0,0 +1,48 @@ +from spacy.tokens.doc import Doc +from spacy.tokens.graph import Graph +from spacy.vocab import Vocab + + +def test_graph_init(): + doc = Doc(Vocab(), words=["a", "b", "c", "d"]) + graph = Graph(doc, name="hello") + assert graph.name == "hello" + assert graph.doc is doc + + +def test_graph_edges_and_nodes(): + doc = Doc(Vocab(), words=["a", "b", "c", "d"]) + graph = Graph(doc, name="hello") + node1 = graph.add_node((0,)) + assert graph.get_node((0,)) == node1 + node2 = graph.add_node((1, 3)) + assert list(node2) == [1, 3] + graph.add_edge(node1, node2, label="one", weight=-10.5) + assert graph.has_edge(node1, node2, label="one") + assert node1.heads() == [] + assert [tuple(h) for h in node2.heads()] == [(0,)] + assert [tuple(t) for t in node1.tails()] == [(1, 3)] + assert [tuple(t) for t in node2.tails()] == [] + + +def test_graph_walk(): + doc = Doc(Vocab(), words=["a", "b", "c", "d"]) + graph = Graph( + doc, + name="hello", + nodes=[(0,), (1,), (2,), (3,)], + edges=[(0, 1), (0, 2), (0, 3), (3, 0)], + labels=None, + weights=None, + ) + node0, node1, node2, node3 = list(graph.nodes) + assert [tuple(h) for h in node0.heads()] == [(3,)] + assert [tuple(h) for h in node1.heads()] == [(0,)] + assert [tuple(h) for h in node0.walk_heads()] == [(3,), (0,)] + assert [tuple(h) for h in node1.walk_heads()] == [(0,), (3,), (0,)] + assert [tuple(h) for h in node2.walk_heads()] == [(0,), (3,), (0,)] + assert [tuple(h) for h in node3.walk_heads()] == [(0,), (3,)] + assert [tuple(t) for t in node0.walk_tails()] == [(1,), (2,), (3,), (0,)] + assert [tuple(t) for t in node1.walk_tails()] == [] + assert [tuple(t) for t in node2.walk_tails()] == [] + assert [tuple(t) for t in node3.walk_tails()] == [(0,), (1,), (2,), (3,)] diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py new file mode 100644 index 0000000..a76472d --- /dev/null +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -0,0 +1,383 @@ +import pytest +import srsly + +import spacy +from spacy import schemas +from spacy.tokens import Doc, Span, Token + +from .test_underscore import clean_underscore # noqa: F401 + + +@pytest.fixture() +def doc(en_vocab): + words = ["c", "d", "e"] + spaces = [True, True, True] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + heads = [0, 0, 1] + deps = ["ROOT", "dobj", "dobj"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + spaces=spaces, + pos=pos, + tags=tags, + heads=heads, + deps=deps, + ents=ents, + morphs=morphs, + ) + + +@pytest.fixture() +def doc_without_deps(en_vocab): + words = ["c", "d", "e"] + pos = ["VERB", "NOUN", "NOUN"] + tags = ["VBP", "NN", "NN"] + ents = ["O", "B-ORG", "O"] + morphs = ["Feat1=A", "Feat1=B", "Feat1=A|Feat2=D"] + + return Doc( + en_vocab, + words=words, + pos=pos, + tags=tags, + ents=ents, + morphs=morphs, + sent_starts=[True, False, True], + ) + + +@pytest.fixture() +def doc_json(): + return { + "text": "c d e ", + "ents": [{"start": 2, "end": 3, "label": "ORG"}], + "sents": [{"start": 0, "end": 5}], + "tokens": [ + { + "id": 0, + "start": 0, + "end": 1, + "tag": "VBP", + "pos": "VERB", + "morph": "Feat1=A", + "dep": "ROOT", + "head": 0, + }, + { + "id": 1, + "start": 2, + "end": 3, + "tag": "NN", + "pos": "NOUN", + "morph": "Feat1=B", + "dep": "dobj", + "head": 0, + }, + { + "id": 2, + "start": 4, + "end": 5, + "tag": "NN", + "pos": "NOUN", + "morph": "Feat1=A|Feat2=D", + "dep": "dobj", + "head": 1, + }, + ], + } + + +def test_doc_to_json(doc): + json_doc = doc.to_json() + assert json_doc["text"] == "c d e " + assert len(json_doc["tokens"]) == 3 + assert json_doc["tokens"][0]["pos"] == "VERB" + assert json_doc["tokens"][0]["tag"] == "VBP" + assert json_doc["tokens"][0]["dep"] == "ROOT" + assert len(json_doc["ents"]) == 1 + assert json_doc["ents"][0]["start"] == 2 # character offset! + assert json_doc["ents"][0]["end"] == 3 # character offset! + assert json_doc["ents"][0]["label"] == "ORG" + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc + + +def test_doc_to_json_underscore(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + assert "_" in json_doc + assert json_doc["_"]["json_test1"] == "hello world" + assert json_doc["_"]["json_test2"] == [1, 2, 3] + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc + + +def test_doc_to_json_with_token_span_attributes(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + Token.set_extension("token_test", default=False) + Span.set_extension("span_test", default=False) + + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" + doc[0]._.token_test = 117 + doc[1]._.token_test = 118 + doc.spans["span_group"] = [doc[0:1]] + json_doc = doc.to_json( + underscore=["json_test1", "json_test2", "token_test", "span_test"] + ) + + assert "_" in json_doc + assert json_doc["_"]["json_test1"] == "hello world" + assert json_doc["_"]["json_test2"] == [1, 2, 3] + assert "underscore_token" in json_doc + assert "underscore_span" in json_doc + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_token"]["token_test"][1]["value"] == 118 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2" + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc + + +def test_doc_to_json_with_custom_user_data(doc): + Doc.set_extension("json_test", default=False) + Token.set_extension("token_test", default=False) + Span.set_extension("span_test", default=False) + + doc._.json_test = "hello world" + doc[0:1]._.span_test = "span_attribute" + doc[0]._.token_test = 117 + json_doc = doc.to_json(underscore=["json_test", "token_test", "span_test"]) + doc.user_data["user_data_test"] = 10 + doc.user_data[("user_data_test2", True)] = 10 + + assert "_" in json_doc + assert json_doc["_"]["json_test"] == "hello world" + assert "underscore_token" in json_doc + assert "underscore_span" in json_doc + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc + + +def test_doc_to_json_with_token_span_same_identifier(doc): + Doc.set_extension("my_ext", default=False) + Token.set_extension("my_ext", default=False) + Span.set_extension("my_ext", default=False) + + doc._.my_ext = "hello world" + doc[0:1]._.my_ext = "span_attribute" + doc[0]._.my_ext = 117 + json_doc = doc.to_json(underscore=["my_ext"]) + + assert "_" in json_doc + assert json_doc["_"]["my_ext"] == "hello world" + assert "underscore_token" in json_doc + assert "underscore_span" in json_doc + assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117 + assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute" + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc + + +def test_doc_to_json_with_token_attributes_missing(doc): + Token.set_extension("token_test", default=False) + Span.set_extension("span_test", default=False) + + doc[0:1]._.span_test = "span_attribute" + doc[0]._.token_test = 117 + json_doc = doc.to_json(underscore=["span_test"]) + + assert "underscore_span" in json_doc + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert "underscore_token" not in json_doc + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + + +def test_doc_to_json_underscore_error_attr(doc): + """Test that Doc.to_json() raises an error if a custom attribute doesn't + exist in the ._ space.""" + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test3"]) + + +def test_doc_to_json_underscore_error_serialize(doc): + """Test that Doc.to_json() raises an error if a custom attribute value + isn't JSON-serializable.""" + Doc.set_extension("json_test4", method=lambda doc: doc.text) + with pytest.raises(ValueError): + doc.to_json(underscore=["json_test4"]) + + +def test_doc_to_json_span(doc): + """Test that Doc.to_json() includes spans""" + doc.spans["test"] = [Span(doc, 0, 2, "test"), Span(doc, 0, 1, "test")] + json_doc = doc.to_json() + assert "spans" in json_doc + assert len(json_doc["spans"]) == 1 + assert len(json_doc["spans"]["test"]) == 2 + assert json_doc["spans"]["test"][0]["start"] == 0 + assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 + + +def test_json_to_doc(doc): + json_doc = doc.to_json() + json_doc = srsly.json_loads(srsly.json_dumps(json_doc)) + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert new_doc.text == doc.text == "c d e " + assert len(new_doc) == len(doc) == 3 + assert new_doc[0].pos == doc[0].pos + assert new_doc[0].tag == doc[0].tag + assert new_doc[0].dep == doc[0].dep + assert new_doc[0].head.idx == doc[0].head.idx + assert new_doc[0].lemma == doc[0].lemma + assert len(new_doc.ents) == 1 + assert new_doc.ents[0].start == 1 + assert new_doc.ents[0].end == 2 + assert new_doc.ents[0].label_ == "ORG" + assert doc.to_bytes() == new_doc.to_bytes() + + +def test_json_to_doc_compat(doc, doc_json): + new_doc = Doc(doc.vocab).from_json(doc_json, validate=True) + new_tokens = [token for token in new_doc] + assert new_doc.text == doc.text == "c d e " + assert len(new_tokens) == len([token for token in doc]) == 3 + assert new_tokens[0].pos == doc[0].pos + assert new_tokens[0].tag == doc[0].tag + assert new_tokens[0].dep == doc[0].dep + assert new_tokens[0].head.idx == doc[0].head.idx + assert new_tokens[0].lemma == doc[0].lemma + assert len(new_doc.ents) == 1 + assert new_doc.ents[0].start == 1 + assert new_doc.ents[0].end == 2 + assert new_doc.ents[0].label_ == "ORG" + + +def test_json_to_doc_underscore(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + json_doc = doc.to_json(underscore=["json_test1", "json_test2"]) + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)]) + assert new_doc._.json_test1 == "hello world" + assert new_doc._.json_test2 == [1, 2, 3] + assert doc.to_bytes() == new_doc.to_bytes() + + +def test_json_to_doc_with_token_span_attributes(doc): + Doc.set_extension("json_test1", default=False) + Doc.set_extension("json_test2", default=False) + Token.set_extension("token_test", default=False) + Span.set_extension("span_test", default=False) + doc._.json_test1 = "hello world" + doc._.json_test2 = [1, 2, 3] + doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" + doc[0]._.token_test = 117 + doc[1]._.token_test = 118 + + json_doc = doc.to_json( + underscore=["json_test1", "json_test2", "token_test", "span_test"] + ) + json_doc = srsly.json_loads(srsly.json_dumps(json_doc)) + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + + assert all([new_doc.has_extension(f"json_test{i}") for i in range(1, 3)]) + assert new_doc._.json_test1 == "hello world" + assert new_doc._.json_test2 == [1, 2, 3] + assert new_doc[0]._.token_test == 117 + assert new_doc[1]._.token_test == 118 + assert new_doc[0:1]._.span_test == "span_attribute" + assert new_doc[0:2]._.span_test == "span_attribute_2" + assert new_doc.user_data == doc.user_data + assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes( + exclude=["user_data"] + ) + + +def test_json_to_doc_spans(doc): + """Test that Doc.from_json() includes correct.spans.""" + doc.spans["test"] = [ + Span(doc, 0, 2, label="test"), + Span(doc, 0, 1, label="test", kb_id=7), + ] + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert len(new_doc.spans) == 1 + assert len(new_doc.spans["test"]) == 2 + for i in range(2): + assert new_doc.spans["test"][i].start == doc.spans["test"][i].start + assert new_doc.spans["test"][i].end == doc.spans["test"][i].end + assert new_doc.spans["test"][i].label == doc.spans["test"][i].label + assert new_doc.spans["test"][i].kb_id == doc.spans["test"][i].kb_id + + +def test_json_to_doc_sents(doc, doc_without_deps): + """Test that Doc.from_json() includes correct.sents.""" + for test_doc in (doc, doc_without_deps): + json_doc = test_doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert [sent.text for sent in test_doc.sents] == [ + sent.text for sent in new_doc.sents + ] + assert [token.is_sent_start for token in test_doc] == [ + token.is_sent_start for token in new_doc + ] + + +def test_json_to_doc_cats(doc): + """Test that Doc.from_json() includes correct .cats.""" + cats = {"A": 0.3, "B": 0.7} + doc.cats = cats + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert new_doc.cats == cats + + +def test_json_to_doc_spaces(): + """Test that Doc.from_json() preserves spaces correctly.""" + doc = spacy.blank("en")("This is just brilliant.") + json_doc = doc.to_json() + new_doc = Doc(doc.vocab).from_json(json_doc, validate=True) + assert doc.text == new_doc.text + + +def test_json_to_doc_attribute_consistency(doc): + """Test that Doc.from_json() raises an exception if tokens don't all have the same set of properties.""" + doc_json = doc.to_json() + doc_json["tokens"][1].pop("morph") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json) + + +def test_json_to_doc_validation_error(doc): + """Test that Doc.from_json() raises an exception when validating invalid input.""" + doc_json = doc.to_json() + doc_json.pop("tokens") + with pytest.raises(ValueError): + Doc(doc.vocab).from_json(doc_json, validate=True) + + +def test_to_json_underscore_doc_getters(doc): + def get_text_length(doc): + return len(doc.text) + + Doc.set_extension("text_length", getter=get_text_length) + doc_json = doc.to_json(underscore=["text_length"]) + assert doc_json["_"]["text_length"] == get_text_length(doc) diff --git a/spacy/tests/doc/test_morphanalysis.py b/spacy/tests/doc/test_morphanalysis.py new file mode 100644 index 0000000..49e32b9 --- /dev/null +++ b/spacy/tests/doc/test_morphanalysis.py @@ -0,0 +1,100 @@ +import pytest + + +@pytest.fixture +def i_has(en_tokenizer): + doc = en_tokenizer("I has") + doc[0].set_morph({"PronType": "prs"}) + doc[1].set_morph( + { + "VerbForm": "fin", + "Tense": "pres", + "Number": "sing", + "Person": "three", + } + ) + + return doc + + +def test_token_morph_eq(i_has): + assert i_has[0].morph is not i_has[0].morph + assert i_has[0].morph == i_has[0].morph + assert i_has[0].morph != i_has[1].morph + + +def test_token_morph_key(i_has): + assert i_has[0].morph.key != 0 + assert i_has[1].morph.key != 0 + assert i_has[0].morph.key == i_has[0].morph.key + assert i_has[0].morph.key != i_has[1].morph.key + + +def test_morph_props(i_has): + assert i_has[0].morph.get("PronType") == ["prs"] + assert i_has[1].morph.get("PronType") == [] + assert i_has[1].morph.get("AsdfType", ["asdf"]) == ["asdf"] + assert i_has[1].morph.get("AsdfType", default=["asdf", "qwer"]) == ["asdf", "qwer"] + + +def test_morph_iter(i_has): + assert set(i_has[0].morph) == set(["PronType=prs"]) + assert set(i_has[1].morph) == set( + ["Number=sing", "Person=three", "Tense=pres", "VerbForm=fin"] + ) + + +def test_morph_get(i_has): + assert i_has[0].morph.get("PronType") == ["prs"] + + +def test_morph_set(i_has): + assert i_has[0].morph.get("PronType") == ["prs"] + # set by string + i_has[0].set_morph("PronType=unk") + assert i_has[0].morph.get("PronType") == ["unk"] + # set by string, fields are alphabetized + i_has[0].set_morph("PronType=123|NounType=unk") + assert str(i_has[0].morph) == "NounType=unk|PronType=123" + # set by dict + i_has[0].set_morph({"AType": "123", "BType": "unk"}) + assert str(i_has[0].morph) == "AType=123|BType=unk" + # set by string with multiple values, fields and values are alphabetized + i_has[0].set_morph("BType=c|AType=b,a") + assert str(i_has[0].morph) == "AType=a,b|BType=c" + # set by dict with multiple values, fields and values are alphabetized + i_has[0].set_morph({"AType": "b,a", "BType": "c"}) + assert str(i_has[0].morph) == "AType=a,b|BType=c" + + +def test_morph_str(i_has): + assert str(i_has[0].morph) == "PronType=prs" + assert str(i_has[1].morph) == "Number=sing|Person=three|Tense=pres|VerbForm=fin" + + +def test_morph_property(tokenizer): + doc = tokenizer("a dog") + + # set through token.morph_ + doc[0].set_morph("PronType=prs") + assert str(doc[0].morph) == "PronType=prs" + assert doc.to_array(["MORPH"])[0] != 0 + + # unset with token.morph + doc[0].set_morph(None) + assert doc.to_array(["MORPH"])[0] == 0 + + # empty morph is equivalent to "_" + doc[0].set_morph("") + assert str(doc[0].morph) == "" + assert doc.to_array(["MORPH"])[0] == tokenizer.vocab.strings["_"] + + # "_" morph is also equivalent to empty morph + doc[0].set_morph("_") + assert str(doc[0].morph) == "" + assert doc.to_array(["MORPH"])[0] == tokenizer.vocab.strings["_"] + + # set through existing hash with token.morph + tokenizer.vocab.strings.add("Feat=Val") + doc[0].set_morph(tokenizer.vocab.strings.add("Feat=Val")) + assert str(doc[0].morph) == "Feat=Val" diff --git a/spacy/tests/doc/test_pickle_doc.py b/spacy/tests/doc/test_pickle_doc.py new file mode 100644 index 0000000..2e28162 --- /dev/null +++ b/spacy/tests/doc/test_pickle_doc.py @@ -0,0 +1,54 @@ +from spacy.compat import pickle +from spacy.language import Language + + +def test_pickle_single_doc(): + nlp = Language() + doc = nlp("pickle roundtrip") + data = pickle.dumps(doc, 1) + doc2 = pickle.loads(data) + assert doc2.text == "pickle roundtrip" + + +def test_list_of_docs_pickles_efficiently(): + nlp = Language() + for i in range(10000): + _ = nlp.vocab[str(i)] # noqa: F841 + one_pickled = pickle.dumps(nlp("0"), -1) + docs = list(nlp.pipe(str(i) for i in range(100))) + many_pickled = pickle.dumps(docs, -1) + assert len(many_pickled) < (len(one_pickled) * 2) + many_unpickled = pickle.loads(many_pickled) + assert many_unpickled[0].text == "0" + assert many_unpickled[-1].text == "99" + assert len(many_unpickled) == 100 + + +def test_user_data_from_disk(): + nlp = Language() + doc = nlp("Hello") + doc.user_data[(0, 1)] = False + b = doc.to_bytes() + doc2 = doc.__class__(doc.vocab).from_bytes(b) + assert doc2.user_data[(0, 1)] is False + + +def test_user_data_unpickles(): + nlp = Language() + doc = nlp("Hello") + doc.user_data[(0, 1)] = False + b = pickle.dumps(doc) + doc2 = pickle.loads(b) + assert doc2.user_data[(0, 1)] is False + + +def test_hooks_unpickle(): + def inner_func(d1, d2): + return "hello!" + + nlp = Language() + doc = nlp("Hello") + doc.user_hooks["similarity"] = inner_func + b = pickle.dumps(doc) + doc2 = pickle.loads(b) + assert doc2.similarity(None) == "hello!" diff --git a/spacy/tests/doc/test_retokenize_merge.py b/spacy/tests/doc/test_retokenize_merge.py new file mode 100644 index 0000000..45d5434 --- /dev/null +++ b/spacy/tests/doc/test_retokenize_merge.py @@ -0,0 +1,495 @@ +import pytest + +from spacy.attrs import LEMMA +from spacy.tokens import Doc, Token +from spacy.vocab import Vocab + + +def test_doc_retokenize_merge(en_tokenizer): + text = "WKRO played songs by the beach boys all night" + attrs = { + "tag": "NAMED", + "lemma": "LEMMA", + "ent_type": "TYPE", + "morph": "Number=Plur", + } + doc = en_tokenizer(text) + assert len(doc) == 9 + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[4:7], attrs=attrs) + retokenizer.merge(doc[7:9], attrs=attrs) + assert len(doc) == 6 + assert doc[4].text == "the beach boys" + assert doc[4].text_with_ws == "the beach boys " + assert doc[4].tag_ == "NAMED" + assert doc[4].lemma_ == "LEMMA" + assert str(doc[4].morph) == "Number=Plur" + assert doc[5].text == "all night" + assert doc[5].text_with_ws == "all night" + assert doc[5].tag_ == "NAMED" + assert str(doc[5].morph) == "Number=Plur" + assert doc[5].lemma_ == "LEMMA" + + +def test_doc_retokenize_merge_children(en_tokenizer): + """Test that attachments work correctly after merging.""" + text = "WKRO played songs by the beach boys all night" + attrs = {"tag": "NAMED", "lemma": "LEMMA", "ent_type": "TYPE"} + doc = en_tokenizer(text) + assert len(doc) == 9 + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[4:7], attrs=attrs) + for word in doc: + if word.i < word.head.i: + assert word in list(word.head.lefts) + elif word.i > word.head.i: + assert word in list(word.head.rights) + + +def test_doc_retokenize_merge_hang(en_tokenizer): + text = "through North and South Carolina" + doc = en_tokenizer(text) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[3:5], attrs={"lemma": "", "ent_type": "ORG"}) + retokenizer.merge(doc[1:2], attrs={"lemma": "", "ent_type": "ORG"}) + + +def test_doc_retokenize_retokenizer(en_tokenizer): + doc = en_tokenizer("WKRO played songs by the beach boys all night") + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[4:7]) + assert len(doc) == 7 + assert doc[4].text == "the beach boys" + + +def test_doc_retokenize_retokenizer_attrs(en_tokenizer): + doc = en_tokenizer("WKRO played songs by the beach boys all night") + # test both string and integer attributes and values + attrs = {LEMMA: "boys", "ENT_TYPE": doc.vocab.strings["ORG"]} + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[4:7], attrs=attrs) + assert len(doc) == 7 + assert doc[4].text == "the beach boys" + assert doc[4].lemma_ == "boys" + assert doc[4].ent_type_ == "ORG" + + +def test_doc_retokenize_lex_attrs(en_tokenizer): + """Test that lexical attributes can be changed (see #2390).""" + doc = en_tokenizer("WKRO played beach boys songs") + assert not any(token.is_stop for token in doc) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[2:4], attrs={"LEMMA": "boys", "IS_STOP": True}) + assert doc[2].text == "beach boys" + assert doc[2].lemma_ == "boys" + assert doc[2].is_stop + new_doc = Doc(doc.vocab, words=["beach boys"]) + assert new_doc[0].is_stop + + +def test_doc_retokenize_spans_merge_tokens(en_tokenizer): + text = "Los Angeles start." + heads = [1, 2, 2, 2] + deps = ["dep"] * len(heads) + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + assert len(doc) == 4 + assert doc[0].head.text == "Angeles" + assert doc[1].head.text == "start" + with doc.retokenize() as retokenizer: + attrs = {"tag": "NNP", "lemma": "Los Angeles", "ent_type": "GPE"} + retokenizer.merge(doc[0:2], attrs=attrs) + assert len(doc) == 3 + assert doc[0].text == "Los Angeles" + assert doc[0].head.text == "start" + assert doc[0].ent_type_ == "GPE" + + +def test_doc_retokenize_spans_merge_tokens_default_attrs(en_vocab): + words = ["The", "players", "start", "."] + lemmas = [t.lower() for t in words] + heads = [1, 2, 2, 2] + deps = ["dep"] * len(heads) + tags = ["DT", "NN", "VBZ", "."] + pos = ["DET", "NOUN", "VERB", "PUNCT"] + doc = Doc( + en_vocab, words=words, tags=tags, pos=pos, heads=heads, deps=deps, lemmas=lemmas + ) + assert len(doc) == 4 + assert doc[0].text == "The" + assert doc[0].tag_ == "DT" + assert doc[0].pos_ == "DET" + assert doc[0].lemma_ == "the" + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2]) + assert len(doc) == 3 + assert doc[0].text == "The players" + assert doc[0].tag_ == "NN" + assert doc[0].pos_ == "NOUN" + assert doc[0].lemma_ == "the players" + doc = Doc( + en_vocab, words=words, tags=tags, pos=pos, heads=heads, deps=deps, lemmas=lemmas + ) + assert len(doc) == 4 + assert doc[0].text == "The" + assert doc[0].tag_ == "DT" + assert doc[0].pos_ == "DET" + assert doc[0].lemma_ == "the" + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2]) + retokenizer.merge(doc[2:4]) + assert len(doc) == 2 + assert doc[0].text == "The players" + assert doc[0].tag_ == "NN" + assert doc[0].pos_ == "NOUN" + assert doc[0].lemma_ == "the players" + assert doc[1].text == "start ." + assert doc[1].tag_ == "VBZ" + assert doc[1].pos_ == "VERB" + assert doc[1].lemma_ == "start ." + + +def test_doc_retokenize_spans_merge_heads(en_vocab): + words = ["I", "found", "a", "pilates", "class", "near", "work", "."] + heads = [1, 1, 4, 6, 1, 4, 5, 1] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert len(doc) == 8 + with doc.retokenize() as retokenizer: + attrs = {"tag": doc[4].tag_, "lemma": "pilates class", "ent_type": "O"} + retokenizer.merge(doc[3:5], attrs=attrs) + assert len(doc) == 7 + assert doc[0].head.i == 1 + assert doc[1].head.i == 1 + assert doc[2].head.i == 3 + assert doc[3].head.i == 1 + assert doc[4].head.i in [1, 3] + assert doc[5].head.i == 4 + + +def test_doc_retokenize_spans_merge_non_disjoint(en_tokenizer): + text = "Los Angeles start." + doc = en_tokenizer(text) + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.merge( + doc[0:2], + attrs={"tag": "NNP", "lemma": "Los Angeles", "ent_type": "GPE"}, + ) + retokenizer.merge( + doc[0:1], + attrs={"tag": "NNP", "lemma": "Los Angeles", "ent_type": "GPE"}, + ) + + +def test_doc_retokenize_span_np_merges(en_tokenizer): + text = "displaCy is a parse tool built with Javascript" + heads = [1, 1, 4, 4, 1, 4, 5, 6] + deps = ["dep"] * len(heads) + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + assert doc[4].head.i == 1 + with doc.retokenize() as retokenizer: + attrs = {"tag": "NP", "lemma": "tool", "ent_type": "O"} + retokenizer.merge(doc[2:5], attrs=attrs) + assert doc[2].head.i == 1 + + text = "displaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript." + heads = [1, 1, 10, 7, 3, 3, 7, 10, 9, 10, 1, 10, 11, 12, 13, 13, 1] + deps = ["dep"] * len(heads) + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + with doc.retokenize() as retokenizer: + for ent in doc.ents: + attrs = {"tag": ent.label_, "lemma": ent.lemma_, "ent_type": ent.label_} + retokenizer.merge(ent, attrs=attrs) + + text = "One test with entities like New York City so the ents list is not void" + heads = [1, 1, 1, 2, 3, 6, 7, 4, 12, 11, 11, 12, 1, 12, 12] + deps = ["dep"] * len(heads) + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + with doc.retokenize() as retokenizer: + for ent in doc.ents: + retokenizer.merge(ent) + + +def test_doc_retokenize_spans_entity_merge(en_tokenizer): + # fmt: off + text = "Stewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.\n" + heads = [1, 2, 2, 4, 6, 4, 2, 8, 6, 8, 9, 8, 8, 14, 12, 2, 15] + deps = ["dep"] * len(heads) + tags = ["NNP", "NNP", "VBZ", "DT", "VB", "RP", "NN", "WP", "VBZ", "IN", "NNP", "CC", "VBZ", "NNP", "NNP", ".", "SP"] + ents = [("PERSON", 0, 2), ("GPE", 10, 11), ("PERSON", 13, 15)] + ents = ["O"] * len(heads) + ents[0] = "B-PERSON" + ents[1] = "I-PERSON" + ents[10] = "B-GPE" + ents[13] = "B-PERSON" + ents[14] = "I-PERSON" + # fmt: on + tokens = en_tokenizer(text) + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=heads, + deps=deps, + tags=tags, + ents=ents, + ) + assert len(doc) == 17 + with doc.retokenize() as retokenizer: + for ent in doc.ents: + ent_type = max(w.ent_type_ for w in ent) + attrs = {"lemma": ent.root.lemma_, "ent_type": ent_type} + retokenizer.merge(ent, attrs=attrs) + # check looping is ok + assert len(doc) == 15 + + +def test_doc_retokenize_spans_entity_merge_iob(en_vocab): + # Test entity IOB stays consistent after merging + words = ["a", "b", "c", "d", "e"] + doc = Doc(Vocab(), words=words) + doc.ents = [ + (doc.vocab.strings.add("ent-abc"), 0, 3), + (doc.vocab.strings.add("ent-d"), 3, 4), + ] + assert doc[0].ent_iob_ == "B" + assert doc[1].ent_iob_ == "I" + assert doc[2].ent_iob_ == "I" + assert doc[3].ent_iob_ == "B" + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2]) + assert len(doc) == len(words) - 1 + assert doc[0].ent_iob_ == "B" + assert doc[1].ent_iob_ == "I" + + # Test that IOB stays consistent with provided IOB + words = ["a", "b", "c", "d", "e"] + doc = Doc(Vocab(), words=words) + with doc.retokenize() as retokenizer: + attrs = {"ent_type": "ent-abc", "ent_iob": 1} + retokenizer.merge(doc[0:3], attrs=attrs) + retokenizer.merge(doc[3:5], attrs=attrs) + assert doc[0].ent_iob_ == "B" + assert doc[1].ent_iob_ == "I" + + # if no parse/heads, the first word in the span is the root and provides + # default values + words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"] + doc = Doc(Vocab(), words=words) + doc.ents = [ + (doc.vocab.strings.add("ent-de"), 3, 5), + (doc.vocab.strings.add("ent-fg"), 5, 7), + ] + assert doc[3].ent_iob_ == "B" + assert doc[4].ent_iob_ == "I" + assert doc[5].ent_iob_ == "B" + assert doc[6].ent_iob_ == "I" + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[2:4]) + retokenizer.merge(doc[4:6]) + retokenizer.merge(doc[7:9]) + assert len(doc) == 6 + assert doc[3].ent_iob_ == "B" + assert doc[3].ent_type_ == "ent-de" + assert doc[4].ent_iob_ == "B" + assert doc[4].ent_type_ == "ent-fg" + + # if there is a parse, span.root provides default values + words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"] + heads = [0, 0, 3, 0, 0, 0, 5, 0, 0] + ents = ["O"] * len(words) + ents[3] = "B-ent-de" + ents[4] = "I-ent-de" + ents[5] = "B-ent-fg" + ents[6] = "I-ent-fg" + deps = ["dep"] * len(words) + en_vocab.strings.add("ent-de") + en_vocab.strings.add("ent-fg") + en_vocab.strings.add("dep") + doc = Doc(en_vocab, words=words, heads=heads, deps=deps, ents=ents) + assert doc[2:4].root == doc[3] # root of 'c d' is d + assert doc[4:6].root == doc[4] # root is 'e f' is e + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[2:4]) + retokenizer.merge(doc[4:6]) + retokenizer.merge(doc[7:9]) + assert len(doc) == 6 + assert doc[2].ent_iob_ == "B" + assert doc[2].ent_type_ == "ent-de" + assert doc[3].ent_iob_ == "I" + assert doc[3].ent_type_ == "ent-de" + assert doc[4].ent_iob_ == "B" + assert doc[4].ent_type_ == "ent-fg" + + # check that B is preserved if span[start] is B + words = ["a", "b", "c", "d", "e", "f", "g", "h", "i"] + heads = [0, 0, 3, 4, 0, 0, 5, 0, 0] + ents = ["O"] * len(words) + ents[3] = "B-ent-de" + ents[4] = "I-ent-de" + ents[5] = "B-ent-de" + ents[6] = "I-ent-de" + deps = ["dep"] * len(words) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps, ents=ents) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[3:5]) + retokenizer.merge(doc[5:7]) + assert len(doc) == 7 + assert doc[3].ent_iob_ == "B" + assert doc[3].ent_type_ == "ent-de" + assert doc[4].ent_iob_ == "B" + assert doc[4].ent_type_ == "ent-de" + + +def test_doc_retokenize_spans_sentence_update_after_merge(en_tokenizer): + # fmt: off + text = "Stewart Lee is a stand up comedian. He lives in England and loves Joe Pasquale." + heads = [1, 2, 2, 4, 2, 4, 4, 2, 9, 9, 9, 10, 9, 9, 15, 13, 9] + deps = ['compound', 'nsubj', 'ROOT', 'det', 'amod', 'prt', 'attr', + 'punct', 'nsubj', 'ROOT', 'prep', 'pobj', 'cc', 'conj', + 'compound', 'dobj', 'punct'] + # fmt: on + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + sent1, sent2 = list(doc.sents) + init_len = len(sent1) + init_len2 = len(sent2) + with doc.retokenize() as retokenizer: + attrs = {"lemma": "none", "ent_type": "none"} + retokenizer.merge(doc[0:2], attrs=attrs) + retokenizer.merge(doc[-2:], attrs=attrs) + sent1, sent2 = list(doc.sents) + assert len(sent1) == init_len - 1 + assert len(sent2) == init_len2 - 1 + + +def test_doc_retokenize_spans_subtree_size_check(en_tokenizer): + # fmt: off + text = "Stewart Lee is a stand up comedian who lives in England and loves Joe Pasquale" + heads = [1, 2, 2, 4, 6, 4, 2, 8, 6, 8, 9, 8, 8, 14, 12] + deps = ["compound", "nsubj", "ROOT", "det", "amod", "prt", "attr", + "nsubj", "relcl", "prep", "pobj", "cc", "conj", "compound", + "dobj"] + # fmt: on + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + sent1 = list(doc.sents)[0] + init_len = len(list(sent1.root.subtree)) + with doc.retokenize() as retokenizer: + attrs = {"lemma": "none", "ent_type": "none"} + retokenizer.merge(doc[0:2], attrs=attrs) + assert len(list(sent1.root.subtree)) == init_len - 1 + + +def test_doc_retokenize_merge_extension_attrs(en_vocab): + Token.set_extension("a", default=False, force=True) + Token.set_extension("b", default="nothing", force=True) + doc = Doc(en_vocab, words=["hello", "world", "!"]) + # Test regular merging + with doc.retokenize() as retokenizer: + attrs = {"lemma": "hello world", "_": {"a": True, "b": "1"}} + retokenizer.merge(doc[0:2], attrs=attrs) + assert doc[0].lemma_ == "hello world" + assert doc[0]._.a is True + assert doc[0]._.b == "1" + # Test bulk merging + doc = Doc(en_vocab, words=["hello", "world", "!", "!"]) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2], attrs={"_": {"a": True, "b": "1"}}) + retokenizer.merge(doc[2:4], attrs={"_": {"a": None, "b": "2"}}) + assert doc[0]._.a is True + assert doc[0]._.b == "1" + assert doc[1]._.a is None + assert doc[1]._.b == "2" + + +@pytest.mark.parametrize("underscore_attrs", [{"a": "x"}, {"b": "x"}, {"c": "x"}, [1]]) +def test_doc_retokenize_merge_extension_attrs_invalid(en_vocab, underscore_attrs): + Token.set_extension("a", getter=lambda x: x, force=True) + Token.set_extension("b", method=lambda x: x, force=True) + doc = Doc(en_vocab, words=["hello", "world", "!"]) + attrs = {"_": underscore_attrs} + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2], attrs=attrs) + + +def test_doc_retokenizer_merge_lex_attrs(en_vocab): + """Test that retokenization also sets attributes on the lexeme if they're + lexical attributes. For example, if a user sets IS_STOP, it should mean that + "all tokens with that lexeme" are marked as a stop word, so the ambiguity + here is acceptable. Also see #2390. + """ + # Test regular merging + doc = Doc(en_vocab, words=["hello", "world", "!"]) + assert not any(t.is_stop for t in doc) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2], attrs={"lemma": "hello world", "is_stop": True}) + assert doc[0].lemma_ == "hello world" + assert doc[0].is_stop + # Test bulk merging + doc = Doc(en_vocab, words=["eins", "zwei", "!", "!"]) + assert not any(t.like_num for t in doc) + assert not any(t.is_stop for t in doc) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2], attrs={"like_num": True}) + retokenizer.merge(doc[2:4], attrs={"is_stop": True}) + assert doc[0].like_num + assert doc[1].is_stop + assert not doc[0].is_stop + assert not doc[1].like_num + # Test that norm is only set on tokens + doc = Doc(en_vocab, words=["eins", "zwei", "!", "!"]) + assert doc[0].norm_ == "eins" + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:1], attrs={"norm": "1"}) + assert doc[0].norm_ == "1" + assert en_vocab["eins"].norm_ == "eins" + + +def test_retokenize_skip_duplicates(en_vocab): + """Test that the retokenizer automatically skips duplicate spans instead + of complaining about overlaps. See #3687.""" + doc = Doc(en_vocab, words=["hello", "world", "!"]) + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2]) + retokenizer.merge(doc[0:2]) + assert len(doc) == 2 + assert doc[0].text == "hello world" + + +def test_retokenize_disallow_zero_length(en_vocab): + doc = Doc(en_vocab, words=["hello", "world", "!"]) + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[1:1]) + + +def test_doc_retokenize_merge_without_parse_keeps_sents(en_tokenizer): + text = "displaCy is a parse tool built with Javascript" + sent_starts = [1, 0, 0, 0, 1, 0, 0, 0] + tokens = en_tokenizer(text) + + # merging within a sentence keeps all sentence boundaries + doc = Doc(tokens.vocab, words=[t.text for t in tokens], sent_starts=sent_starts) + assert len(list(doc.sents)) == 2 + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[1:3]) + assert len(list(doc.sents)) == 2 + + # merging over a sentence boundary unsets it by default + doc = Doc(tokens.vocab, words=[t.text for t in tokens], sent_starts=sent_starts) + assert len(list(doc.sents)) == 2 + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[3:6]) + assert doc[3].is_sent_start is None + + # merging over a sentence boundary and setting sent_start + doc = Doc(tokens.vocab, words=[t.text for t in tokens], sent_starts=sent_starts) + assert len(list(doc.sents)) == 2 + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[3:6], attrs={"sent_start": True}) + assert len(list(doc.sents)) == 2 diff --git a/spacy/tests/doc/test_retokenize_split.py b/spacy/tests/doc/test_retokenize_split.py new file mode 100644 index 0000000..61ef599 --- /dev/null +++ b/spacy/tests/doc/test_retokenize_split.py @@ -0,0 +1,296 @@ +import numpy +import pytest + +from spacy.tokens import Doc, Token +from spacy.vocab import Vocab + + +@pytest.mark.issue(3540) +def test_issue3540(en_vocab): + words = ["I", "live", "in", "NewYork", "right", "now"] + tensor = numpy.asarray( + [[1.0, 1.1], [2.0, 2.1], [3.0, 3.1], [4.0, 4.1], [5.0, 5.1], [6.0, 6.1]], + dtype="f", + ) + doc = Doc(en_vocab, words=words) + doc.tensor = tensor + gold_text = ["I", "live", "in", "NewYork", "right", "now"] + assert [token.text for token in doc] == gold_text + gold_lemma = ["I", "live", "in", "NewYork", "right", "now"] + for i, lemma in enumerate(gold_lemma): + doc[i].lemma_ = lemma + assert [token.lemma_ for token in doc] == gold_lemma + vectors_1 = [token.vector for token in doc] + assert len(vectors_1) == len(doc) + + with doc.retokenize() as retokenizer: + heads = [(doc[3], 1), doc[2]] + attrs = { + "POS": ["PROPN", "PROPN"], + "LEMMA": ["New", "York"], + "DEP": ["pobj", "compound"], + } + retokenizer.split(doc[3], ["New", "York"], heads=heads, attrs=attrs) + + gold_text = ["I", "live", "in", "New", "York", "right", "now"] + assert [token.text for token in doc] == gold_text + gold_lemma = ["I", "live", "in", "New", "York", "right", "now"] + assert [token.lemma_ for token in doc] == gold_lemma + vectors_2 = [token.vector for token in doc] + assert len(vectors_2) == len(doc) + assert vectors_1[0].tolist() == vectors_2[0].tolist() + assert vectors_1[1].tolist() == vectors_2[1].tolist() + assert vectors_1[2].tolist() == vectors_2[2].tolist() + assert vectors_1[4].tolist() == vectors_2[5].tolist() + assert vectors_1[5].tolist() == vectors_2[6].tolist() + + +def test_doc_retokenize_split(en_vocab): + words = ["LosAngeles", "start", "."] + heads = [1, 2, 2] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert len(doc) == 3 + assert len(str(doc)) == 19 + assert doc[0].head.text == "start" + assert doc[1].head.text == "." + with doc.retokenize() as retokenizer: + retokenizer.split( + doc[0], + ["Los", "Angeles"], + [(doc[0], 1), doc[1]], + attrs={ + "tag": ["NNP"] * 2, + "lemma": ["Los", "Angeles"], + "ent_type": ["GPE"] * 2, + "morph": ["Number=Sing"] * 2, + }, + ) + assert len(doc) == 4 + assert doc[0].text == "Los" + assert doc[0].head.text == "Angeles" + assert doc[0].idx == 0 + assert str(doc[0].morph) == "Number=Sing" + assert doc[1].idx == 3 + assert doc[1].text == "Angeles" + assert doc[1].head.text == "start" + assert str(doc[1].morph) == "Number=Sing" + assert doc[2].text == "start" + assert doc[2].head.text == "." + assert doc[3].text == "." + assert doc[3].head.text == "." + assert len(str(doc)) == 19 + + +def test_doc_retokenize_split_lemmas(en_vocab): + # If lemmas are not set, leave unset + words = ["LosAngeles", "start", "."] + heads = [1, 2, 2] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + with doc.retokenize() as retokenizer: + retokenizer.split( + doc[0], + ["Los", "Angeles"], + [(doc[0], 1), doc[1]], + ) + assert doc[0].lemma_ == "" + assert doc[1].lemma_ == "" + + # If lemmas are set, use split orth as default lemma + words = ["LosAngeles", "start", "."] + heads = [1, 2, 2] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + for t in doc: + t.lemma_ = "a" + with doc.retokenize() as retokenizer: + retokenizer.split( + doc[0], + ["Los", "Angeles"], + [(doc[0], 1), doc[1]], + ) + assert doc[0].lemma_ == "Los" + assert doc[1].lemma_ == "Angeles" + + +def test_doc_retokenize_split_dependencies(en_vocab): + doc = Doc(en_vocab, words=["LosAngeles", "start", "."]) + dep1 = doc.vocab.strings.add("amod") + dep2 = doc.vocab.strings.add("subject") + with doc.retokenize() as retokenizer: + retokenizer.split( + doc[0], + ["Los", "Angeles"], + [(doc[0], 1), doc[1]], + attrs={"dep": [dep1, dep2]}, + ) + assert doc[0].dep == dep1 + assert doc[1].dep == dep2 + + +def test_doc_retokenize_split_heads_error(en_vocab): + doc = Doc(en_vocab, words=["LosAngeles", "start", "."]) + # Not enough heads + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.split(doc[0], ["Los", "Angeles"], [doc[1]]) + + # Too many heads + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.split(doc[0], ["Los", "Angeles"], [doc[1], doc[1], doc[1]]) + + +def test_doc_retokenize_spans_entity_split_iob(): + # Test entity IOB stays consistent after merging + words = ["abc", "d", "e"] + doc = Doc(Vocab(), words=words) + doc.ents = [(doc.vocab.strings.add("ent-abcd"), 0, 2)] + assert doc[0].ent_iob_ == "B" + assert doc[1].ent_iob_ == "I" + with doc.retokenize() as retokenizer: + retokenizer.split(doc[0], ["a", "b", "c"], [(doc[0], 1), (doc[0], 2), doc[1]]) + assert doc[0].ent_iob_ == "B" + assert doc[1].ent_iob_ == "I" + assert doc[2].ent_iob_ == "I" + assert doc[3].ent_iob_ == "I" + + +def test_doc_retokenize_spans_sentence_update_after_split(en_vocab): + # fmt: off + words = ["StewartLee", "is", "a", "stand", "up", "comedian", ".", "He", + "lives", "in", "England", "and", "loves", "JoePasquale", "."] + heads = [1, 1, 3, 5, 3, 1, 1, 8, 8, 8, 9, 8, 8, 14, 12] + deps = ["nsubj", "ROOT", "det", "amod", "prt", "attr", "punct", "nsubj", + "ROOT", "prep", "pobj", "cc", "conj", "compound", "punct"] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + sent1, sent2 = list(doc.sents) + init_len = len(sent1) + init_len2 = len(sent2) + with doc.retokenize() as retokenizer: + retokenizer.split( + doc[0], + ["Stewart", "Lee"], + [(doc[0], 1), doc[1]], + attrs={"dep": ["compound", "nsubj"]}, + ) + retokenizer.split( + doc[13], + ["Joe", "Pasquale"], + [(doc[13], 1), doc[12]], + attrs={"dep": ["compound", "dobj"]}, + ) + sent1, sent2 = list(doc.sents) + assert len(sent1) == init_len + 1 + assert len(sent2) == init_len2 + 1 + + +def test_doc_retokenize_split_orths_mismatch(en_vocab): + """Test that the regular retokenizer.split raises an error if the orths + don't match the original token text. There might still be a method that + allows this, but for the default use cases, merging and splitting should + always conform with spaCy's non-destructive tokenization policy. Otherwise, + it can lead to very confusing and unexpected results. + """ + doc = Doc(en_vocab, words=["LosAngeles", "start", "."]) + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + retokenizer.split(doc[0], ["L", "A"], [(doc[0], 0), (doc[0], 0)]) + + +def test_doc_retokenize_split_extension_attrs(en_vocab): + Token.set_extension("a", default=False, force=True) + Token.set_extension("b", default="nothing", force=True) + doc = Doc(en_vocab, words=["LosAngeles", "start"]) + with doc.retokenize() as retokenizer: + heads = [(doc[0], 1), doc[1]] + underscore = [{"a": True, "b": "1"}, {"b": "2"}] + attrs = {"lemma": ["los", "angeles"], "_": underscore} + retokenizer.split(doc[0], ["Los", "Angeles"], heads, attrs=attrs) + assert doc[0].lemma_ == "los" + assert doc[0]._.a is True + assert doc[0]._.b == "1" + assert doc[1].lemma_ == "angeles" + assert doc[1]._.a is False + assert doc[1]._.b == "2" + + +@pytest.mark.parametrize( + "underscore_attrs", + [ + [{"a": "x"}, {}], # Overwriting getter without setter + [{"b": "x"}, {}], # Overwriting method + [{"c": "x"}, {}], # Overwriting nonexistent attribute + [{"a": "x"}, {"x": "x"}], # Combination + [{"a": "x", "x": "x"}, {"x": "x"}], # Combination + {"x": "x"}, # Not a list of dicts + ], +) +def test_doc_retokenize_split_extension_attrs_invalid(en_vocab, underscore_attrs): + Token.set_extension("x", default=False, force=True) + Token.set_extension("a", getter=lambda x: x, force=True) + Token.set_extension("b", method=lambda x: x, force=True) + doc = Doc(en_vocab, words=["LosAngeles", "start"]) + attrs = {"_": underscore_attrs} + with pytest.raises(ValueError): + with doc.retokenize() as retokenizer: + heads = [(doc[0], 1), doc[1]] + retokenizer.split(doc[0], ["Los", "Angeles"], heads, attrs=attrs) + + +def test_doc_retokenizer_split_lex_attrs(en_vocab): + """Test that retokenization also sets attributes on the lexeme if they're + lexical attributes. For example, if a user sets IS_STOP, it should mean that + "all tokens with that lexeme" are marked as a stop word, so the ambiguity + here is acceptable. Also see #2390. + """ + assert not Doc(en_vocab, words=["Los"])[0].is_stop + assert not Doc(en_vocab, words=["Angeles"])[0].is_stop + doc = Doc(en_vocab, words=["LosAngeles", "start"]) + assert not doc[0].is_stop + with doc.retokenize() as retokenizer: + attrs = {"is_stop": [True, False]} + heads = [(doc[0], 1), doc[1]] + retokenizer.split(doc[0], ["Los", "Angeles"], heads, attrs=attrs) + assert doc[0].is_stop + assert not doc[1].is_stop + + +def test_doc_retokenizer_realloc(en_vocab): + """#4604: realloc correctly when new tokens outnumber original tokens""" + text = "Hyperglycemic adverse events following antipsychotic drug administration in the" + doc = Doc(en_vocab, words=text.split()[:-1]) + with doc.retokenize() as retokenizer: + token = doc[0] + heads = [(token, 0)] * len(token) + retokenizer.split(doc[token.i], list(token.text), heads=heads) + doc = Doc(en_vocab, words=text.split()) + with doc.retokenize() as retokenizer: + token = doc[0] + heads = [(token, 0)] * len(token) + retokenizer.split(doc[token.i], list(token.text), heads=heads) + + +def test_doc_retokenizer_split_norm(en_vocab): + """#6060: reset norm in split""" + text = "The quick brownfoxjumpsoverthe lazy dog w/ white spots" + doc = Doc(en_vocab, words=text.split()) + + # Set custom norm on the w/ token. + doc[5].norm_ = "with" + + # Retokenize to split out the words in the token at doc[2]. + token = doc[2] + with doc.retokenize() as retokenizer: + retokenizer.split( + token, + ["brown", "fox", "jumps", "over", "the"], + heads=[(token, idx) for idx in range(5)], + ) + + assert doc[9].text == "w/" + assert doc[9].norm_ == "with" + assert doc[5].text == "over" + assert doc[5].norm_ == "over" diff --git a/spacy/tests/doc/test_span.py b/spacy/tests/doc/test_span.py new file mode 100644 index 0000000..7167b68 --- /dev/null +++ b/spacy/tests/doc/test_span.py @@ -0,0 +1,757 @@ +import numpy +import pytest +from numpy.testing import assert_array_equal +from thinc.api import get_current_ops + +from spacy.attrs import LENGTH, ORTH +from spacy.lang.en import English +from spacy.tokens import Doc, Span, Token +from spacy.util import filter_spans +from spacy.vocab import Vocab + +from ..util import add_vecs_to_vocab +from .test_underscore import clean_underscore # noqa: F401 + + +@pytest.fixture +def doc(en_tokenizer): + # fmt: off + text = "This is a sentence. This is another sentence. And a third." + heads = [1, 1, 3, 1, 1, 6, 6, 8, 6, 6, 12, 12, 12, 12] + deps = ["nsubj", "ROOT", "det", "attr", "punct", "nsubj", "ROOT", "det", + "attr", "punct", "ROOT", "det", "npadvmod", "punct"] + ents = ["O", "O", "B-ENT", "I-ENT", "I-ENT", "I-ENT", "I-ENT", "O", "O", + "O", "O", "O", "O", "O"] + # fmt: on + tokens = en_tokenizer(text) + lemmas = [t.text for t in tokens] # this is not correct, just a placeholder + spaces = [bool(t.whitespace_) for t in tokens] + return Doc( + tokens.vocab, + words=[t.text for t in tokens], + spaces=spaces, + heads=heads, + deps=deps, + ents=ents, + lemmas=lemmas, + ) + + +@pytest.fixture +def doc_not_parsed(en_tokenizer): + text = "This is a sentence. This is another sentence. And a third." + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens]) + return doc + + +@pytest.mark.issue(1537) +def test_issue1537(): + """Test that Span.as_doc() doesn't segfault.""" + string = "The sky is blue . The man is pink . The dog is purple ." + doc = Doc(Vocab(), words=list(string.split())) + doc[0].sent_start = True + for word in doc[1:]: + if word.nbor(-1).text == ".": + word.sent_start = True + else: + word.sent_start = False + sents = list(doc.sents) + sent0 = sents[0].as_doc() + sent1 = sents[1].as_doc() + assert isinstance(sent0, Doc) + assert isinstance(sent1, Doc) + + +@pytest.mark.issue(1612) +def test_issue1612(en_tokenizer): + """Test that span.orth_ is identical to span.text""" + doc = en_tokenizer("The black cat purrs.") + span = doc[1:3] + assert span.orth_ == span.text + + +@pytest.mark.issue(3199) +def test_issue3199(): + """Test that Span.noun_chunks works correctly if no noun chunks iterator + is available. To make this test future-proof, we're constructing a Doc + with a new Vocab here and a parse tree to make sure the noun chunks run. + """ + words = ["This", "is", "a", "sentence"] + doc = Doc(Vocab(), words=words, heads=[0] * len(words), deps=["dep"] * len(words)) + with pytest.raises(NotImplementedError): + list(doc[0:3].noun_chunks) + + +@pytest.mark.issue(5152) +def test_issue5152(): + # Test that the comparison between a Span and a Token, goes well + # There was a bug when the number of tokens in the span equaled the number of characters in the token (!) + nlp = English() + text = nlp("Talk about being boring!") + text_var = nlp("Talk of being boring!") + y = nlp("Let") + span = text[0:3] # Talk about being + span_2 = text[0:3] # Talk about being + span_3 = text_var[0:3] # Talk of being + token = y[0] # Let + with pytest.warns(UserWarning): + assert span.similarity(token) == 0.0 + assert span.similarity(span_2) == 1.0 + with pytest.warns(UserWarning): + assert span_2.similarity(span_3) < 1.0 + + +@pytest.mark.issue(6755) +def test_issue6755(en_tokenizer): + doc = en_tokenizer("This is a magnificent sentence.") + span = doc[:0] + assert span.text_with_ws == "" + assert span.text == "" + + +@pytest.mark.parametrize( + "sentence, start_idx,end_idx,label", + [("Welcome to Mumbai, my friend", 11, 17, "GPE")], +) +@pytest.mark.issue(6815) +def test_issue6815_1(sentence, start_idx, end_idx, label): + nlp = English() + doc = nlp(sentence) + span = doc[:].char_span(start_idx, end_idx, label=label) + assert span.label_ == label + + +@pytest.mark.parametrize( + "sentence, start_idx,end_idx,kb_id", [("Welcome to Mumbai, my friend", 11, 17, 5)] +) +@pytest.mark.issue(6815) +def test_issue6815_2(sentence, start_idx, end_idx, kb_id): + nlp = English() + doc = nlp(sentence) + span = doc[:].char_span(start_idx, end_idx, kb_id=kb_id) + assert span.kb_id == kb_id + + +@pytest.mark.parametrize( + "sentence, start_idx,end_idx,vector", + [("Welcome to Mumbai, my friend", 11, 17, numpy.array([0.1, 0.2, 0.3]))], +) +@pytest.mark.issue(6815) +def test_issue6815_3(sentence, start_idx, end_idx, vector): + nlp = English() + doc = nlp(sentence) + span = doc[:].char_span(start_idx, end_idx, vector=vector) + assert (span.vector == vector).all() + + +@pytest.mark.parametrize( + "i_sent,i,j,text", + [ + (0, 0, len("This is a"), "This is a"), + (1, 0, len("This is another"), "This is another"), + (2, len("And "), len("And ") + len("a third"), "a third"), + (0, 1, 2, None), + ], +) +def test_char_span(doc, i_sent, i, j, text): + sents = list(doc.sents) + span = sents[i_sent].char_span(i, j) + if not text: + assert not span + else: + assert span.text == text + + +def test_char_span_attributes(doc): + label = "LABEL" + kb_id = "KB_ID" + span_id = "SPAN_ID" + span1 = doc.char_span(20, 45, label=label, kb_id=kb_id, span_id=span_id) + span2 = doc[1:].char_span(15, 40, label=label, kb_id=kb_id, span_id=span_id) + assert span1.text == span2.text + assert span1.label_ == span2.label_ == label + assert span1.kb_id_ == span2.kb_id_ == kb_id + assert span1.id_ == span2.id_ == span_id + + +def test_spans_sent_spans(doc): + sents = list(doc.sents) + assert sents[0].start == 0 + assert sents[0].end == 5 + assert len(sents) == 3 + assert sum(len(sent) for sent in sents) == len(doc) + + +def test_spans_root(doc): + span = doc[2:4] + assert len(span) == 2 + assert span.text == "a sentence" + assert span.root.text == "sentence" + assert span.root.head.text == "is" + + +def test_spans_string_fn(doc): + span = doc[0:4] + assert len(span) == 4 + assert span.text == "This is a sentence" + + +def test_spans_root2(en_tokenizer): + text = "through North and South Carolina" + heads = [0, 4, 1, 1, 0] + deps = ["dep"] * len(heads) + tokens = en_tokenizer(text) + doc = Doc(tokens.vocab, words=[t.text for t in tokens], heads=heads, deps=deps) + assert doc[-2:].root.text == "Carolina" + + +def test_spans_span_sent(doc, doc_not_parsed): + """Test span.sent property""" + assert len(list(doc.sents)) + assert doc[:2].sent.root.text == "is" + assert doc[:2].sent.text == "This is a sentence." + assert doc[6:7].sent.root.left_edge.text == "This" + assert doc[0 : len(doc)].sent == list(doc.sents)[0] + assert list(doc[0 : len(doc)].sents) == list(doc.sents) + + with pytest.raises(ValueError): + doc_not_parsed[:2].sent + + # test on manual sbd + doc_not_parsed[0].is_sent_start = True + doc_not_parsed[5].is_sent_start = True + assert doc_not_parsed[1:3].sent == doc_not_parsed[0:5] + assert doc_not_parsed[10:14].sent == doc_not_parsed[5:] + + +def test_issue13769(): + # Test issue 13769: Incorrect output of span.sents when final token is a sentence outside of the span. + doc = Doc( + Vocab(), + words=list("This is a sentence . This is another sentence . Third".split()), + ) + doc[0].is_sent_start = True + doc[5].is_sent_start = True + doc[10].is_sent_start = True + doc.ents = [("ENTITY", 7, 9)] # "another sentence" phrase in the second sentence + entity = doc.ents[0] + ent_sents = list(entity.sents) + assert len(ent_sents) == 1 + + +@pytest.mark.parametrize( + "start,end,expected_sentence", + [ + (0, 14, "This is"), # Entire doc + (1, 4, "This is"), # Overlapping with 2 sentences + (0, 2, "This is"), # Beginning of the Doc. Full sentence + (0, 1, "This is"), # Beginning of the Doc. Part of a sentence + (10, 14, "And a"), # End of the Doc. Overlapping with 2 senteces + (12, 14, "third."), # End of the Doc. Full sentence + (1, 1, "This is"), # Empty Span + ], +) +def test_spans_span_sent_user_hooks(doc, start, end, expected_sentence): + + # Doc-level sents hook + def user_hook(doc): + return [doc[ii : ii + 2] for ii in range(0, len(doc), 2)] + + doc.user_hooks["sents"] = user_hook + + # Make sure doc-level sents hook works + assert doc[start:end].sent.text == expected_sentence + + # Span-level sent hook + doc.user_span_hooks["sent"] = lambda x: x + # Now, span=level sent hook overrides the doc-level sents hook + assert doc[start:end].sent == doc[start:end] + + +def test_spans_lca_matrix(en_tokenizer): + """Test span's lca matrix generation""" + tokens = en_tokenizer("the lazy dog slept") + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=[2, 2, 3, 3], + deps=["dep"] * 4, + ) + lca = doc[:2].get_lca_matrix() + assert lca.shape == (2, 2) + assert lca[0, 0] == 0 # the & the -> the + assert lca[0, 1] == -1 # the & lazy -> dog (out of span) + assert lca[1, 0] == -1 # lazy & the -> dog (out of span) + assert lca[1, 1] == 1 # lazy & lazy -> lazy + + lca = doc[1:].get_lca_matrix() + assert lca.shape == (3, 3) + assert lca[0, 0] == 0 # lazy & lazy -> lazy + assert lca[0, 1] == 1 # lazy & dog -> dog + assert lca[0, 2] == 2 # lazy & slept -> slept + + lca = doc[2:].get_lca_matrix() + assert lca.shape == (2, 2) + assert lca[0, 0] == 0 # dog & dog -> dog + assert lca[0, 1] == 1 # dog & slept -> slept + assert lca[1, 0] == 1 # slept & dog -> slept + assert lca[1, 1] == 1 # slept & slept -> slept + + # example from Span API docs + tokens = en_tokenizer("I like New York in Autumn") + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=[1, 1, 3, 1, 3, 4], + deps=["dep"] * len(tokens), + ) + lca = doc[1:4].get_lca_matrix() + assert_array_equal(lca, numpy.asarray([[0, 0, 0], [0, 1, 2], [0, 2, 2]])) + + +def test_span_similarity_match(): + doc = Doc(Vocab(), words=["a", "b", "a", "b"]) + span1 = doc[:2] + span2 = doc[2:] + with pytest.warns(UserWarning): + assert span1.similarity(span2) == 1.0 + assert span1.similarity(doc) == 0.0 + assert span1[:1].similarity(doc.vocab["a"]) == 1.0 + + +def test_spans_default_sentiment(en_tokenizer): + """Test span.sentiment property's default averaging behaviour""" + text = "good stuff bad stuff" + tokens = en_tokenizer(text) + tokens.vocab[tokens[0].text].sentiment = 3.0 + tokens.vocab[tokens[2].text].sentiment = -2.0 + doc = Doc(tokens.vocab, words=[t.text for t in tokens]) + assert doc[:2].sentiment == 3.0 / 2 + assert doc[-2:].sentiment == -2.0 / 2 + assert doc[:-1].sentiment == (3.0 + -2) / 3.0 + + +def test_spans_override_sentiment(en_tokenizer): + """Test span.sentiment property's default averaging behaviour""" + text = "good stuff bad stuff" + tokens = en_tokenizer(text) + tokens.vocab[tokens[0].text].sentiment = 3.0 + tokens.vocab[tokens[2].text].sentiment = -2.0 + doc = Doc(tokens.vocab, words=[t.text for t in tokens]) + doc.user_span_hooks["sentiment"] = lambda span: 10.0 + assert doc[:2].sentiment == 10.0 + assert doc[-2:].sentiment == 10.0 + assert doc[:-1].sentiment == 10.0 + + +def test_spans_are_hashable(en_tokenizer): + """Test spans can be hashed.""" + text = "good stuff bad stuff" + tokens = en_tokenizer(text) + span1 = tokens[:2] + span2 = tokens[2:4] + assert hash(span1) != hash(span2) + span3 = tokens[0:2] + assert hash(span3) == hash(span1) + + +def test_spans_by_character(doc): + span1 = doc[1:-2] + + # default and specified alignment mode "strict" + span2 = doc.char_span(span1.start_char, span1.end_char, label="GPE") + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + + span2 = doc.char_span( + span1.start_char, span1.end_char, label="GPE", alignment_mode="strict" + ) + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + + # alignment mode "contract" + span2 = doc.char_span( + span1.start_char - 3, span1.end_char, label="GPE", alignment_mode="contract" + ) + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + + # alignment mode "expand" + span2 = doc.char_span( + span1.start_char + 1, span1.end_char, label="GPE", alignment_mode="expand" + ) + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + + # unsupported alignment mode + with pytest.raises(ValueError): + span2 = doc.char_span( + span1.start_char + 1, span1.end_char, label="GPE", alignment_mode="unk" + ) + + # Span.char_span + alignment mode "contract" + span2 = doc[0:2].char_span( + span1.start_char - 3, span1.end_char, label="GPE", alignment_mode="contract" + ) + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + assert span2.label_ == "GPE" + + +def test_span_to_array(doc): + span = doc[1:-2] + arr = span.to_array([ORTH, LENGTH]) + assert arr.shape == (len(span), 2) + assert arr[0, 0] == span[0].orth + assert arr[0, 1] == len(span[0]) + + +def test_span_as_doc(doc): + span = doc[4:10] + span_doc = span.as_doc() + assert span.text == span_doc.text.strip() + assert isinstance(span_doc, doc.__class__) + assert span_doc is not doc + assert span_doc[0].idx == 0 + + # partial initial entity is removed + assert len(span_doc.ents) == 0 + + # full entity is preserved + span_doc = doc[2:10].as_doc() + assert len(span_doc.ents) == 1 + + # partial final entity is removed + span_doc = doc[0:5].as_doc() + assert len(span_doc.ents) == 0 + + +@pytest.mark.usefixtures("clean_underscore") +def test_span_as_doc_user_data(doc): + """Test that the user_data can be preserved (but not by default).""" + my_key = "my_info" + my_value = 342 + doc.user_data[my_key] = my_value + Token.set_extension("is_x", default=False) + doc[7]._.is_x = True + + span = doc[4:10] + span_doc_with = span.as_doc(copy_user_data=True) + span_doc_without = span.as_doc() + + assert doc.user_data.get(my_key, None) is my_value + assert span_doc_with.user_data.get(my_key, None) is my_value + assert span_doc_without.user_data.get(my_key, None) is None + for i in range(len(span_doc_with)): + if i != 3: + assert span_doc_with[i]._.is_x is False + else: + assert span_doc_with[i]._.is_x is True + assert not any([t._.is_x for t in span_doc_without]) + + +def test_span_string_label_kb_id(doc): + span = Span(doc, 0, 1, label="hello", kb_id="Q342") + assert span.label_ == "hello" + assert span.label == doc.vocab.strings["hello"] + assert span.kb_id_ == "Q342" + assert span.kb_id == doc.vocab.strings["Q342"] + + +def test_span_string_label_id(doc): + span = Span(doc, 0, 1, label="hello", span_id="Q342") + assert span.label_ == "hello" + assert span.label == doc.vocab.strings["hello"] + assert span.id_ == "Q342" + assert span.id == doc.vocab.strings["Q342"] + + +def test_span_attrs_writable(doc): + span = Span(doc, 0, 1) + span.label_ = "label" + span.kb_id_ = "kb_id" + span.id_ = "id" + + +def test_span_ents_property(doc): + doc.ents = [ + (doc.vocab.strings["PRODUCT"], 0, 1), + (doc.vocab.strings["PRODUCT"], 7, 8), + (doc.vocab.strings["PRODUCT"], 11, 14), + ] + assert len(list(doc.ents)) == 3 + sentences = list(doc.sents) + assert len(sentences) == 3 + assert len(sentences[0].ents) == 1 + # First sentence, also tests start of sentence + assert sentences[0].ents[0].text == "This" + assert sentences[0].ents[0].label_ == "PRODUCT" + assert sentences[0].ents[0].start == 0 + assert sentences[0].ents[0].end == 1 + # Second sentence + assert len(sentences[1].ents) == 1 + assert sentences[1].ents[0].text == "another" + assert sentences[1].ents[0].label_ == "PRODUCT" + assert sentences[1].ents[0].start == 7 + assert sentences[1].ents[0].end == 8 + # Third sentence ents, Also tests end of sentence + assert sentences[2].ents[0].text == "a third." + assert sentences[2].ents[0].label_ == "PRODUCT" + assert sentences[2].ents[0].start == 11 + assert sentences[2].ents[0].end == 14 + + +def test_filter_spans(doc): + # Test filtering duplicates + spans = [doc[1:4], doc[6:8], doc[1:4], doc[10:14]] + filtered = filter_spans(spans) + assert len(filtered) == 3 + assert filtered[0].start == 1 and filtered[0].end == 4 + assert filtered[1].start == 6 and filtered[1].end == 8 + assert filtered[2].start == 10 and filtered[2].end == 14 + # Test filtering overlaps with longest preference + spans = [doc[1:4], doc[1:3], doc[5:10], doc[7:9], doc[1:4]] + filtered = filter_spans(spans) + assert len(filtered) == 2 + assert len(filtered[0]) == 3 + assert len(filtered[1]) == 5 + assert filtered[0].start == 1 and filtered[0].end == 4 + assert filtered[1].start == 5 and filtered[1].end == 10 + # Test filtering overlaps with earlier preference for identical length + spans = [doc[1:4], doc[2:5], doc[5:10], doc[7:9], doc[1:4]] + filtered = filter_spans(spans) + assert len(filtered) == 2 + assert len(filtered[0]) == 3 + assert len(filtered[1]) == 5 + assert filtered[0].start == 1 and filtered[0].end == 4 + assert filtered[1].start == 5 and filtered[1].end == 10 + + +def test_span_eq_hash(doc, doc_not_parsed): + assert doc[0:2] == doc[0:2] + assert doc[0:2] != doc[1:3] + assert doc[0:2] != doc_not_parsed[0:2] + assert hash(doc[0:2]) == hash(doc[0:2]) + assert hash(doc[0:2]) != hash(doc[1:3]) + assert hash(doc[0:2]) != hash(doc_not_parsed[0:2]) + + # check that an out-of-bounds is not equivalent to the span of the full doc + assert doc[0 : len(doc)] != doc[len(doc) : len(doc) + 1] + + +def test_span_boundaries(doc): + start = 1 + end = 5 + span = doc[start:end] + for i in range(start, end): + assert span[i - start] == doc[i] + with pytest.raises(IndexError): + span[-5] + with pytest.raises(IndexError): + span[5] + + empty_span_0 = doc[0:0] + assert empty_span_0.text == "" + assert empty_span_0.start == 0 + assert empty_span_0.end == 0 + assert empty_span_0.start_char == 0 + assert empty_span_0.end_char == 0 + + empty_span_1 = doc[1:1] + assert empty_span_1.text == "" + assert empty_span_1.start == 1 + assert empty_span_1.end == 1 + assert empty_span_1.start_char == empty_span_1.end_char + + oob_span_start = doc[-len(doc) - 1 : -len(doc) - 10] + assert oob_span_start.text == "" + assert oob_span_start.start == 0 + assert oob_span_start.end == 0 + assert oob_span_start.start_char == 0 + assert oob_span_start.end_char == 0 + + oob_span_end = doc[len(doc) + 1 : len(doc) + 10] + assert oob_span_end.text == "" + assert oob_span_end.start == len(doc) + assert oob_span_end.end == len(doc) + assert oob_span_end.start_char == len(doc.text) + assert oob_span_end.end_char == len(doc.text) + + +def test_span_lemma(doc): + # span lemmas should have the same number of spaces as the span + sp = doc[1:5] + assert len(sp.text.split(" ")) == len(sp.lemma_.split(" ")) + + +def test_sent(en_tokenizer): + doc = en_tokenizer("Check span.sent raises error if doc is not sentencized.") + span = doc[1:3] + assert not span.doc.has_annotation("SENT_START") + with pytest.raises(ValueError): + span.sent + + +def test_span_with_vectors(doc): + ops = get_current_ops() + prev_vectors = doc.vocab.vectors + vectors = [ + ("apple", ops.asarray([1, 2, 3])), + ("orange", ops.asarray([-1, -2, -3])), + ("And", ops.asarray([-1, -1, -1])), + ("juice", ops.asarray([5, 5, 10])), + ("pie", ops.asarray([7, 6.3, 8.9])), + ] + add_vecs_to_vocab(doc.vocab, vectors) + # 0-length span + assert_array_equal(ops.to_numpy(doc[0:0].vector), numpy.zeros((3,))) + # longer span with no vector + assert_array_equal(ops.to_numpy(doc[0:4].vector), numpy.zeros((3,))) + # single-token span with vector + assert_array_equal(ops.to_numpy(doc[10:11].vector), [-1, -1, -1]) + doc.vocab.vectors = prev_vectors + + +# fmt: off +def test_span_comparison(doc): + + # Identical start, end, only differ in label and kb_id + assert Span(doc, 0, 3) == Span(doc, 0, 3) + assert Span(doc, 0, 3, "LABEL") == Span(doc, 0, 3, "LABEL") + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") == Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + + assert Span(doc, 0, 3) != Span(doc, 0, 3, "LABEL") + assert Span(doc, 0, 3) != Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + assert Span(doc, 0, 3, "LABEL") != Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + + assert Span(doc, 0, 3) <= Span(doc, 0, 3) and Span(doc, 0, 3) >= Span(doc, 0, 3) + assert Span(doc, 0, 3, "LABEL") <= Span(doc, 0, 3, "LABEL") and Span(doc, 0, 3, "LABEL") >= Span(doc, 0, 3, "LABEL") + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") <= Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") >= Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + + assert (Span(doc, 0, 3) < Span(doc, 0, 3, "", kb_id="KB_ID") < Span(doc, 0, 3, "LABEL") < Span(doc, 0, 3, "LABEL", kb_id="KB_ID")) + assert (Span(doc, 0, 3) <= Span(doc, 0, 3, "", kb_id="KB_ID") <= Span(doc, 0, 3, "LABEL") <= Span(doc, 0, 3, "LABEL", kb_id="KB_ID")) + + assert (Span(doc, 0, 3, "LABEL", kb_id="KB_ID") > Span(doc, 0, 3, "LABEL") > Span(doc, 0, 3, "", kb_id="KB_ID") > Span(doc, 0, 3)) + assert (Span(doc, 0, 3, "LABEL", kb_id="KB_ID") >= Span(doc, 0, 3, "LABEL") >= Span(doc, 0, 3, "", kb_id="KB_ID") >= Span(doc, 0, 3)) + + # Different end + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") < Span(doc, 0, 4, "LABEL", kb_id="KB_ID") + + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") < Span(doc, 0, 4) + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") <= Span(doc, 0, 4) + assert Span(doc, 0, 4) > Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + assert Span(doc, 0, 4) >= Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + + # Different start + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") != Span(doc, 1, 3, "LABEL", kb_id="KB_ID") + + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") < Span(doc, 1, 3) + assert Span(doc, 0, 3, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3) + assert Span(doc, 1, 3) > Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + assert Span(doc, 1, 3) >= Span(doc, 0, 3, "LABEL", kb_id="KB_ID") + + # Different start & different end + assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") != Span(doc, 1, 3, "LABEL", kb_id="KB_ID") + + assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") < Span(doc, 1, 3) + assert Span(doc, 0, 4, "LABEL", kb_id="KB_ID") <= Span(doc, 1, 3) + assert Span(doc, 1, 3) > Span(doc, 0, 4, "LABEL", kb_id="KB_ID") + assert Span(doc, 1, 3) >= Span(doc, 0, 4, "LABEL", kb_id="KB_ID") + + # Different id + assert Span(doc, 1, 3, span_id="AAA") < Span(doc, 1, 3, span_id="BBB") +# fmt: on + + +@pytest.mark.parametrize( + "start,end,expected_sentences,expected_sentences_with_hook", + [ + (0, 14, 3, 7), # Entire doc + (3, 6, 2, 2), # Overlapping with 2 sentences + (0, 4, 1, 2), # Beginning of the Doc. Full sentence + (0, 3, 1, 2), # Beginning of the Doc. Part of a sentence + (9, 14, 2, 3), # End of the Doc. Overlapping with 2 senteces + (10, 14, 1, 2), # End of the Doc. Full sentence + (11, 14, 1, 2), # End of the Doc. Partial sentence + (0, 0, 1, 1), # Empty Span + ], +) +def test_span_sents(doc, start, end, expected_sentences, expected_sentences_with_hook): + + assert len(list(doc[start:end].sents)) == expected_sentences + + def user_hook(doc): + return [doc[ii : ii + 2] for ii in range(0, len(doc), 2)] + + doc.user_hooks["sents"] = user_hook + + assert len(list(doc[start:end].sents)) == expected_sentences_with_hook + + doc.user_span_hooks["sents"] = lambda x: [x] + + assert list(doc[start:end].sents)[0] == doc[start:end] + assert len(list(doc[start:end].sents)) == 1 + + +def test_span_sents_not_parsed(doc_not_parsed): + with pytest.raises(ValueError): + list(Span(doc_not_parsed, 0, 3).sents) + + +def test_span_group_copy(doc): + doc.spans["test"] = [doc[0:1], doc[2:4]] + assert len(doc.spans["test"]) == 2 + doc_copy = doc.copy() + # check that the spans were indeed copied + assert len(doc_copy.spans["test"]) == 2 + # add a new span to the original doc + doc.spans["test"].append(doc[3:4]) + assert len(doc.spans["test"]) == 3 + # check that the copy spans were not modified and this is an isolated doc + assert len(doc_copy.spans["test"]) == 2 + + +def test_for_partial_ent_sents(): + """Spans may be associated with multiple sentences. These .sents should always be complete, not partial, sentences, + which this tests for. + """ + doc = Doc( + English().vocab, + words=["Mahler's", "Symphony", "No.", "8", "was", "beautiful."], + sent_starts=[1, 0, 0, 1, 0, 0], + ) + doc.set_ents([Span(doc, 1, 4, "WORK")]) + # The specified entity is associated with both sentences in this doc, so we expect all sentences in the doc to be + # equal to the sentences referenced in ent.sents. + for doc_sent, ent_sent in zip(doc.sents, doc.ents[0].sents): + assert doc_sent == ent_sent + + +def test_for_no_ent_sents(): + """Span.sents() should set .sents correctly, even if Span in question is trailing and doesn't form a full + sentence. + """ + doc = Doc( + English().vocab, + words=["This", "is", "a", "test.", "ENTITY"], + sent_starts=[1, 0, 0, 0, 1], + ) + doc.set_ents([Span(doc, 4, 5, "WORK")]) + sents = list(doc.ents[0].sents) + assert len(sents) == 1 + assert str(sents[0]) == str(doc.ents[0].sent) == "ENTITY" + + +def test_span_api_richcmp_other(en_tokenizer): + doc1 = en_tokenizer("a b") + doc2 = en_tokenizer("b c") + assert not doc1[1:2] == doc1[1] + assert not doc1[1:2] == doc2[0] + assert not doc1[1:2] == doc2[0:1] + assert not doc1[0:1] == doc2 diff --git a/spacy/tests/doc/test_span_group.py b/spacy/tests/doc/test_span_group.py new file mode 100644 index 0000000..ef78172 --- /dev/null +++ b/spacy/tests/doc/test_span_group.py @@ -0,0 +1,280 @@ +from random import Random +from typing import List + +import pytest + +from spacy.matcher import Matcher +from spacy.tokens import Doc, Span, SpanGroup +from spacy.util import filter_spans + + +@pytest.fixture +def doc(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + matcher = Matcher(en_tokenizer.vocab, validate=True) + + # fmt: off + matcher.add("4", [[{}, {}, {}, {}]]) + matcher.add("2", [[{}, {}, ]]) + matcher.add("1", [[{}, ]]) + # fmt: on + matches = matcher(doc) + spans = [] + for match in matches: + spans.append( + Span(doc, match[1], match[2], en_tokenizer.vocab.strings[match[0]]) + ) + Random(42).shuffle(spans) + doc.spans["SPANS"] = SpanGroup( + doc, name="SPANS", attrs={"key": "value"}, spans=spans + ) + return doc + + +@pytest.fixture +def other_doc(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + matcher = Matcher(en_tokenizer.vocab, validate=True) + + # fmt: off + matcher.add("4", [[{}, {}, {}, {}]]) + matcher.add("2", [[{}, {}, ]]) + matcher.add("1", [[{}, ]]) + # fmt: on + + matches = matcher(doc) + spans = [] + for match in matches: + spans.append( + Span(doc, match[1], match[2], en_tokenizer.vocab.strings[match[0]]) + ) + Random(42).shuffle(spans) + doc.spans["SPANS"] = SpanGroup( + doc, name="SPANS", attrs={"key": "value"}, spans=spans + ) + return doc + + +@pytest.fixture +def span_group(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + matcher = Matcher(en_tokenizer.vocab, validate=True) + + # fmt: off + matcher.add("4", [[{}, {}, {}, {}]]) + matcher.add("2", [[{}, {}, ]]) + matcher.add("1", [[{}, ]]) + # fmt: on + + matches = matcher(doc) + spans = [] + for match in matches: + spans.append( + Span(doc, match[1], match[2], en_tokenizer.vocab.strings[match[0]]) + ) + Random(42).shuffle(spans) + doc.spans["SPANS"] = SpanGroup( + doc, name="SPANS", attrs={"key": "value"}, spans=spans + ) + + +def test_span_group_copy(doc): + span_group = doc.spans["SPANS"] + clone = span_group.copy() + assert clone != span_group + assert clone.name == span_group.name + assert clone.attrs == span_group.attrs + assert len(clone) == len(span_group) + assert list(span_group) == list(clone) + clone.name = "new_name" + clone.attrs["key"] = "new_value" + clone.append(Span(doc, 0, 6, "LABEL")) + assert clone.name != span_group.name + assert clone.attrs != span_group.attrs + assert span_group.attrs["key"] == "value" + assert list(span_group) != list(clone) + + # can't copy if the character offsets don't align to tokens + doc2 = Doc(doc.vocab, words=[t.text + "x" for t in doc]) + with pytest.raises(ValueError): + span_group.copy(doc=doc2) + + # can copy with valid character offsets despite different tokenization + doc3 = doc.copy() + with doc3.retokenize() as retokenizer: + retokenizer.merge(doc3[0:2]) + retokenizer.merge(doc3[3:6]) + span_group = SpanGroup(doc, spans=[doc[0:6], doc[3:6]]) + for span1, span2 in zip(span_group, span_group.copy(doc=doc3)): + assert span1.start_char == span2.start_char + assert span1.end_char == span2.end_char + + +def test_span_group_set_item(doc, other_doc): + span_group = doc.spans["SPANS"] + + index = 5 + span = span_group[index] + span.label_ = "NEW LABEL" + span.kb_id = doc.vocab.strings["KB_ID"] + + assert span_group[index].label != span.label + assert span_group[index].kb_id != span.kb_id + + span_group[index] = span + assert span_group[index].start == span.start + assert span_group[index].end == span.end + assert span_group[index].label == span.label + assert span_group[index].kb_id == span.kb_id + assert span_group[index] == span + + with pytest.raises(IndexError): + span_group[-100] = span + with pytest.raises(IndexError): + span_group[100] = span + + span = Span(other_doc, 0, 2) + with pytest.raises(ValueError): + span_group[index] = span + + +def test_span_group_has_overlap(doc): + span_group = doc.spans["SPANS"] + assert span_group.has_overlap + + +def test_span_group_concat(doc, other_doc): + span_group_1 = doc.spans["SPANS"] + spans = [doc[0:5], doc[0:6]] + span_group_2 = SpanGroup( + doc, + name="MORE_SPANS", + attrs={"key": "new_value", "new_key": "new_value"}, + spans=spans, + ) + span_group_3 = span_group_1._concat(span_group_2) + assert span_group_3.name == span_group_1.name + assert span_group_3.attrs == {"key": "value", "new_key": "new_value"} + span_list_expected = list(span_group_1) + list(span_group_2) + assert list(span_group_3) == list(span_list_expected) + + # Inplace + span_list_expected = list(span_group_1) + list(span_group_2) + span_group_3 = span_group_1._concat(span_group_2, inplace=True) + assert span_group_3 == span_group_1 + assert span_group_3.name == span_group_1.name + assert span_group_3.attrs == {"key": "value", "new_key": "new_value"} + assert list(span_group_3) == list(span_list_expected) + + span_group_2 = other_doc.spans["SPANS"] + with pytest.raises(ValueError): + span_group_1._concat(span_group_2) + + +def test_span_doc_delitem(doc): + span_group = doc.spans["SPANS"] + length = len(span_group) + index = 5 + span = span_group[index] + next_span = span_group[index + 1] + del span_group[index] + assert len(span_group) == length - 1 + assert span_group[index] != span + assert span_group[index] == next_span + + with pytest.raises(IndexError): + del span_group[-100] + with pytest.raises(IndexError): + del span_group[100] + + +def test_span_group_add(doc): + span_group_1 = doc.spans["SPANS"] + spans = [doc[0:5], doc[0:6]] + span_group_2 = SpanGroup( + doc, + name="MORE_SPANS", + attrs={"key": "new_value", "new_key": "new_value"}, + spans=spans, + ) + + span_group_3_expected = span_group_1._concat(span_group_2) + + span_group_3 = span_group_1 + span_group_2 + assert len(span_group_3) == len(span_group_3_expected) + assert span_group_3.attrs == {"key": "value", "new_key": "new_value"} + assert list(span_group_3) == list(span_group_3_expected) + + +def test_span_group_iadd(doc): + span_group_1 = doc.spans["SPANS"].copy() + spans = [doc[0:5], doc[0:6]] + span_group_2 = SpanGroup( + doc, + name="MORE_SPANS", + attrs={"key": "new_value", "new_key": "new_value"}, + spans=spans, + ) + + span_group_1_expected = span_group_1._concat(span_group_2) + + span_group_1 += span_group_2 + assert len(span_group_1) == len(span_group_1_expected) + assert span_group_1.attrs == {"key": "value", "new_key": "new_value"} + assert list(span_group_1) == list(span_group_1_expected) + + span_group_1 = doc.spans["SPANS"].copy() + span_group_1 += spans + assert len(span_group_1) == len(span_group_1_expected) + assert span_group_1.attrs == { + "key": "value", + } + assert list(span_group_1) == list(span_group_1_expected) + + +def test_span_group_extend(doc): + span_group_1 = doc.spans["SPANS"].copy() + spans = [doc[0:5], doc[0:6]] + span_group_2 = SpanGroup( + doc, + name="MORE_SPANS", + attrs={"key": "new_value", "new_key": "new_value"}, + spans=spans, + ) + + span_group_1_expected = span_group_1._concat(span_group_2) + + span_group_1.extend(span_group_2) + assert len(span_group_1) == len(span_group_1_expected) + assert span_group_1.attrs == {"key": "value", "new_key": "new_value"} + assert list(span_group_1) == list(span_group_1_expected) + + span_group_1 = doc.spans["SPANS"] + span_group_1.extend(spans) + assert len(span_group_1) == len(span_group_1_expected) + assert span_group_1.attrs == {"key": "value"} + assert list(span_group_1) == list(span_group_1_expected) + + +def test_span_group_dealloc(span_group): + with pytest.raises(AttributeError): + print(span_group.doc) + + +@pytest.mark.issue(11975) +def test_span_group_typing(doc: Doc): + """Tests whether typing of `SpanGroup` as `Iterable[Span]`-like object is accepted by mypy.""" + span_group: SpanGroup = doc.spans["SPANS"] + spans: List[Span] = list(span_group) + for i, span in enumerate(span_group): + assert span == span_group[i] == spans[i] + filter_spans(span_group) + + +def test_span_group_init_doc(en_tokenizer): + """Test that all spans must come from the specified doc.""" + doc1 = en_tokenizer("a b c") + doc2 = en_tokenizer("a b c") + span_group = SpanGroup(doc1, spans=[doc1[0:1], doc1[1:2]]) + with pytest.raises(ValueError): + span_group = SpanGroup(doc1, spans=[doc1[0:1], doc2[1:2]]) diff --git a/spacy/tests/doc/test_token_api.py b/spacy/tests/doc/test_token_api.py new file mode 100644 index 0000000..c10221e --- /dev/null +++ b/spacy/tests/doc/test_token_api.py @@ -0,0 +1,305 @@ +import numpy +import pytest + +from spacy.attrs import IS_ALPHA, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_STOP, IS_TITLE +from spacy.symbols import VERB +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +@pytest.fixture +def doc(en_vocab): + # fmt: off + words = ["This", "is", "a", "sentence", ".", "This", "is", "another", "sentence", ".", "And", "a", "third", "."] + heads = [1, 1, 3, 1, 1, 6, 6, 8, 6, 6, 10, 12, 10, 12] + deps = ["nsubj", "ROOT", "det", "attr", "punct", "nsubj", "ROOT", "det", + "attr", "punct", "ROOT", "det", "npadvmod", "punct"] + # fmt: on + return Doc(en_vocab, words=words, heads=heads, deps=deps) + + +def test_doc_token_api_strings(en_vocab): + words = ["Give", "it", "back", "!", "He", "pleaded", "."] + pos = ["VERB", "PRON", "PART", "PUNCT", "PRON", "VERB", "PUNCT"] + heads = [0, 0, 0, 0, 5, 5, 5] + deps = ["ROOT", "dobj", "prt", "punct", "nsubj", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, pos=pos, heads=heads, deps=deps) + assert doc[0].orth_ == "Give" + assert doc[0].text == "Give" + assert doc[0].text_with_ws == "Give " + assert doc[0].lower_ == "give" + assert doc[0].shape_ == "Xxxx" + assert doc[0].prefix_ == "G" + assert doc[0].suffix_ == "ive" + assert doc[0].pos_ == "VERB" + assert doc[0].dep_ == "ROOT" + + +def test_doc_token_api_flags(en_tokenizer): + text = "Give it back! He pleaded." + tokens = en_tokenizer(text) + assert tokens[0].check_flag(IS_ALPHA) + assert not tokens[0].check_flag(IS_DIGIT) + assert tokens[0].check_flag(IS_TITLE) + assert tokens[1].check_flag(IS_LOWER) + assert tokens[3].check_flag(IS_PUNCT) + assert tokens[2].check_flag(IS_STOP) + assert not tokens[5].check_flag(IS_STOP) + # TODO: Test more of these, esp. if a bug is found + + +@pytest.mark.parametrize("text", ["Give it back! He pleaded."]) +def test_doc_token_api_prob_inherited_from_vocab(en_tokenizer, text): + word = text.split()[0] + en_tokenizer.vocab[word].prob = -1 + tokens = en_tokenizer(text) + assert tokens[0].prob != 0 + + +@pytest.mark.parametrize("text", ["one two"]) +def test_doc_token_api_str_builtin(en_tokenizer, text): + tokens = en_tokenizer(text) + assert str(tokens[0]) == text.split(" ")[0] + assert str(tokens[1]) == text.split(" ")[1] + + +def test_doc_token_api_is_properties(en_vocab): + doc = Doc(en_vocab, words=["Hi", ",", "my", "email", "is", "test@me.com"]) + assert doc[0].is_title + assert doc[0].is_alpha + assert not doc[0].is_digit + assert doc[1].is_punct + assert doc[3].is_ascii + assert not doc[3].like_url + assert doc[4].is_lower + assert doc[5].like_email + + +def test_doc_token_api_vectors(): + vocab = Vocab() + vocab.reset_vectors(width=2) + vocab.set_vector("apples", vector=numpy.asarray([0.0, 2.0], dtype="f")) + vocab.set_vector("oranges", vector=numpy.asarray([0.0, 1.0], dtype="f")) + doc = Doc(vocab, words=["apples", "oranges", "oov"]) + assert doc.has_vector + assert doc[0].has_vector + assert doc[1].has_vector + assert not doc[2].has_vector + apples_norm = (0 * 0 + 2 * 2) ** 0.5 + oranges_norm = (0 * 0 + 1 * 1) ** 0.5 + cosine = ((0 * 0) + (2 * 1)) / (apples_norm * oranges_norm) + assert doc[0].similarity(doc[1]) == cosine + + +def test_doc_token_api_ancestors(en_vocab): + # the structure of this sentence depends on the English annotation scheme + words = ["Yesterday", "I", "saw", "a", "dog", "that", "barked", "loudly", "."] + heads = [2, 2, 2, 4, 2, 6, 4, 6, 2] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert [t.text for t in doc[6].ancestors] == ["dog", "saw"] + assert [t.text for t in doc[1].ancestors] == ["saw"] + assert [t.text for t in doc[2].ancestors] == [] + assert doc[2].is_ancestor(doc[7]) + assert not doc[6].is_ancestor(doc[2]) + + +def test_doc_token_api_head_setter(en_vocab): + words = ["Yesterday", "I", "saw", "a", "dog", "that", "barked", "loudly", "."] + heads = [2, 2, 2, 4, 2, 6, 4, 6, 2] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert doc[6].n_lefts == 1 + assert doc[6].n_rights == 1 + assert doc[6].left_edge.i == 5 + assert doc[6].right_edge.i == 7 + assert doc[4].n_lefts == 1 + assert doc[4].n_rights == 1 + assert doc[4].left_edge.i == 3 + assert doc[4].right_edge.i == 7 + assert doc[3].n_lefts == 0 + assert doc[3].n_rights == 0 + assert doc[3].left_edge.i == 3 + assert doc[3].right_edge.i == 3 + assert doc[2].left_edge.i == 0 + assert doc[2].right_edge.i == 8 + + doc[6].head = doc[3] + assert doc[6].n_lefts == 1 + assert doc[6].n_rights == 1 + assert doc[6].left_edge.i == 5 + assert doc[6].right_edge.i == 7 + assert doc[3].n_lefts == 0 + assert doc[3].n_rights == 1 + assert doc[3].left_edge.i == 3 + assert doc[3].right_edge.i == 7 + assert doc[4].n_lefts == 1 + assert doc[4].n_rights == 0 + assert doc[4].left_edge.i == 3 + assert doc[4].right_edge.i == 7 + assert doc[2].left_edge.i == 0 + assert doc[2].right_edge.i == 8 + + doc[0].head = doc[5] + assert doc[5].left_edge.i == 0 + assert doc[6].left_edge.i == 0 + assert doc[3].left_edge.i == 0 + assert doc[4].left_edge.i == 0 + assert doc[2].left_edge.i == 0 + # head token must be from the same document + doc2 = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads)) + with pytest.raises(ValueError): + doc[0].head = doc2[0] + # test sentence starts when two sentences are joined + # fmt: off + words = ["This", "is", "one", "sentence", ".", "This", "is", "another", "sentence", "."] + heads = [0, 0, 0, 0, 0, 5, 5, 5, 5, 5] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads)) + # initially two sentences + assert doc[0].is_sent_start + assert doc[5].is_sent_start + assert doc[0].left_edge == doc[0] + assert doc[0].right_edge == doc[4] + assert doc[5].left_edge == doc[5] + assert doc[5].right_edge == doc[9] + # modifying with a sentence doesn't change sent starts + doc[2].head = doc[3] + assert doc[0].is_sent_start + assert doc[5].is_sent_start + assert doc[0].left_edge == doc[0] + assert doc[0].right_edge == doc[4] + assert doc[5].left_edge == doc[5] + assert doc[5].right_edge == doc[9] + # attach the second sentence to the first, resulting in one sentence + doc[5].head = doc[0] + assert doc[0].is_sent_start + assert not doc[5].is_sent_start + assert doc[0].left_edge == doc[0] + assert doc[0].right_edge == doc[9] + + +def test_is_sent_start(en_tokenizer): + doc = en_tokenizer("This is a sentence. This is another.") + assert doc[5].is_sent_start is None + doc[5].is_sent_start = True + assert doc[5].is_sent_start is True + assert len(list(doc.sents)) == 2 + + +def test_is_sent_end(en_tokenizer): + doc = en_tokenizer("This is a sentence. This is another.") + assert doc[4].is_sent_end is None + doc[5].is_sent_start = True + assert doc[4].is_sent_end is True + assert len(list(doc.sents)) == 2 + + +def test_set_pos(): + doc = Doc(Vocab(), words=["hello", "world"]) + doc[0].pos_ = "NOUN" + assert doc[0].pos_ == "NOUN" + doc[1].pos = VERB + assert doc[1].pos_ == "VERB" + + +def test_set_invalid_pos(): + doc = Doc(Vocab(), words=["hello", "world"]) + with pytest.raises(ValueError): + doc[0].pos_ = "blah" + + +def test_tokens_sent(doc): + """Test token.sent property""" + assert len(list(doc.sents)) == 3 + assert doc[1].sent.text == "This is a sentence ." + assert doc[7].sent.text == "This is another sentence ." + assert doc[1].sent.root.left_edge.text == "This" + assert doc[7].sent.root.left_edge.text == "This" + + +def test_token0_has_sent_start_true(): + doc = Doc(Vocab(), words=["hello", "world"]) + assert doc[0].is_sent_start is True + assert doc[1].is_sent_start is None + assert not doc.has_annotation("SENT_START") + + +def test_tokenlast_has_sent_end_true(): + doc = Doc(Vocab(), words=["hello", "world"]) + assert doc[0].is_sent_end is None + assert doc[1].is_sent_end is True + assert not doc.has_annotation("SENT_START") + + +def test_token_api_conjuncts_chain(en_vocab): + words = ["The", "boy", "and", "the", "girl", "and", "the", "man", "went", "."] + heads = [1, 8, 1, 4, 1, 4, 7, 4, 8, 8] + deps = ["det", "nsubj", "cc", "det", "conj", "cc", "det", "conj", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert [w.text for w in doc[1].conjuncts] == ["girl", "man"] + assert [w.text for w in doc[4].conjuncts] == ["boy", "man"] + assert [w.text for w in doc[7].conjuncts] == ["boy", "girl"] + + +def test_token_api_conjuncts_simple(en_vocab): + words = ["They", "came", "and", "went", "."] + heads = [1, 1, 1, 1, 3] + deps = ["nsubj", "ROOT", "cc", "conj", "dep"] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert [w.text for w in doc[1].conjuncts] == ["went"] + assert [w.text for w in doc[3].conjuncts] == ["came"] + + +def test_token_api_non_conjuncts(en_vocab): + words = ["They", "came", "."] + heads = [1, 1, 1] + deps = ["nsubj", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert [w.text for w in doc[0].conjuncts] == [] + assert [w.text for w in doc[1].conjuncts] == [] + + +def test_missing_head_dep(en_vocab): + """Check that the Doc constructor and Example.from_dict parse missing information the same""" + heads = [1, 1, 1, 1, 2, None] # element 5 is missing + deps = ["", "ROOT", "dobj", "cc", "conj", None] # element 0 and 5 are missing + words = ["I", "like", "London", "and", "Berlin", "."] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + pred_has_heads = [t.has_head() for t in doc] + pred_has_deps = [t.has_dep() for t in doc] + pred_heads = [t.head.i for t in doc] + pred_deps = [t.dep_ for t in doc] + pred_sent_starts = [t.is_sent_start for t in doc] + assert pred_has_heads == [False, True, True, True, True, False] + assert pred_has_deps == [False, True, True, True, True, False] + assert pred_heads[1:5] == [1, 1, 1, 2] + assert pred_deps[1:5] == ["ROOT", "dobj", "cc", "conj"] + assert pred_sent_starts == [True, False, False, False, False, False] + example = Example.from_dict(doc, {"heads": heads, "deps": deps}) + ref_has_heads = [t.has_head() for t in example.reference] + ref_has_deps = [t.has_dep() for t in example.reference] + ref_heads = [t.head.i for t in example.reference] + ref_deps = [t.dep_ for t in example.reference] + ref_sent_starts = [t.is_sent_start for t in example.reference] + assert ref_has_heads == pred_has_heads + assert ref_has_deps == pred_has_heads + assert ref_heads == pred_heads + assert ref_deps == pred_deps + assert ref_sent_starts == pred_sent_starts + # check that the aligned parse preserves the missing information + aligned_heads, aligned_deps = example.get_aligned_parse(projectivize=True) + assert aligned_deps[0] == ref_deps[0] + assert aligned_heads[0] == ref_heads[0] + assert aligned_deps[5] == ref_deps[5] + assert aligned_heads[5] == ref_heads[5] + + +def test_token_api_richcmp_other(en_tokenizer): + doc1 = en_tokenizer("a b") + doc2 = en_tokenizer("b c") + assert not doc1[1] == doc1[0:1] + assert not doc1[1] == doc2[1:2] + assert not doc1[1] == doc2[0] + assert not doc1[0] == doc2 diff --git a/spacy/tests/doc/test_underscore.py b/spacy/tests/doc/test_underscore.py new file mode 100644 index 0000000..b79d2f0 --- /dev/null +++ b/spacy/tests/doc/test_underscore.py @@ -0,0 +1,174 @@ +import pytest +from mock import Mock + +from spacy.tokens import Doc, Span, Token +from spacy.tokens.underscore import Underscore + + +@pytest.fixture(scope="function", autouse=True) +def clean_underscore(): + # reset the Underscore object after the test, to avoid having state copied across tests + yield + Underscore.doc_extensions = {} + Underscore.span_extensions = {} + Underscore.token_extensions = {} + + +def test_create_doc_underscore(): + doc = Mock() + doc.doc = doc + uscore = Underscore(Underscore.doc_extensions, doc) + assert uscore._doc is doc + assert uscore._start is None + assert uscore._end is None + + +def test_doc_underscore_getattr_setattr(): + doc = Mock() + doc.doc = doc + doc.user_data = {} + Underscore.doc_extensions["hello"] = (False, None, None, None) + doc._ = Underscore(Underscore.doc_extensions, doc) + assert doc._.hello is False + doc._.hello = True + assert doc._.hello is True + + +def test_create_span_underscore(): + span = Mock(doc=Mock(), start=0, end=2) + uscore = Underscore( + Underscore.span_extensions, span, start=span.start, end=span.end + ) + assert uscore._doc is span.doc + assert uscore._start is span.start + assert uscore._end is span.end + + +def test_span_underscore_getter_setter(): + span = Mock(doc=Mock(), start=0, end=2) + Underscore.span_extensions["hello"] = ( + None, + None, + lambda s: (s.start, "hi"), + lambda s, value: setattr(s, "start", value), + ) + span._ = Underscore( + Underscore.span_extensions, span, start=span.start, end=span.end + ) + + assert span._.hello == (0, "hi") + span._.hello = 1 + assert span._.hello == (1, "hi") + + +def test_token_underscore_method(): + token = Mock(doc=Mock(), idx=7, say_cheese=lambda token: "cheese") + Underscore.token_extensions["hello"] = (None, token.say_cheese, None, None) + token._ = Underscore(Underscore.token_extensions, token, start=token.idx) + assert token._.hello() == "cheese" + + +@pytest.mark.parametrize("obj", [Doc, Span, Token]) +def test_doc_underscore_remove_extension(obj): + ext_name = "to_be_removed" + obj.set_extension(ext_name, default=False) + assert obj.has_extension(ext_name) + obj.remove_extension(ext_name) + assert not obj.has_extension(ext_name) + + +@pytest.mark.parametrize("obj", [Doc, Span, Token]) +def test_underscore_raises_for_dup(obj): + obj.set_extension("test", default=None) + with pytest.raises(ValueError): + obj.set_extension("test", default=None) + + +@pytest.mark.parametrize( + "invalid_kwargs", + [ + {"getter": None, "setter": lambda: None}, + {"default": None, "method": lambda: None, "getter": lambda: None}, + {"setter": lambda: None}, + {"default": None, "method": lambda: None}, + {"getter": True}, + ], +) +def test_underscore_raises_for_invalid(invalid_kwargs): + invalid_kwargs["force"] = True + with pytest.raises(ValueError): + Doc.set_extension("test", **invalid_kwargs) + + +@pytest.mark.parametrize( + "valid_kwargs", + [ + {"getter": lambda: None}, + {"getter": lambda: None, "setter": lambda: None}, + {"default": "hello"}, + {"default": None}, + {"method": lambda: None}, + ], +) +def test_underscore_accepts_valid(valid_kwargs): + valid_kwargs["force"] = True + Doc.set_extension("test", **valid_kwargs) + + +def test_underscore_mutable_defaults_list(en_vocab): + """Test that mutable default arguments are handled correctly (see #2581).""" + Doc.set_extension("mutable", default=[]) + doc1 = Doc(en_vocab, words=["one"]) + doc2 = Doc(en_vocab, words=["two"]) + doc1._.mutable.append("foo") + assert len(doc1._.mutable) == 1 + assert doc1._.mutable[0] == "foo" + assert len(doc2._.mutable) == 0 + doc1._.mutable = ["bar", "baz"] + doc1._.mutable.append("foo") + assert len(doc1._.mutable) == 3 + assert len(doc2._.mutable) == 0 + + +def test_underscore_mutable_defaults_dict(en_vocab): + """Test that mutable default arguments are handled correctly (see #2581).""" + Token.set_extension("mutable", default={}) + token1 = Doc(en_vocab, words=["one"])[0] + token2 = Doc(en_vocab, words=["two"])[0] + token1._.mutable["foo"] = "bar" + assert len(token1._.mutable) == 1 + assert token1._.mutable["foo"] == "bar" + assert len(token2._.mutable) == 0 + token1._.mutable["foo"] = "baz" + assert len(token1._.mutable) == 1 + assert token1._.mutable["foo"] == "baz" + token1._.mutable["x"] = [] + token1._.mutable["x"].append("y") + assert len(token1._.mutable) == 2 + assert token1._.mutable["x"] == ["y"] + assert len(token2._.mutable) == 0 + + +def test_underscore_dir(en_vocab): + """Test that dir() correctly returns extension attributes. This enables + things like tab-completion for the attributes in doc._.""" + Doc.set_extension("test_dir", default=None) + doc = Doc(en_vocab, words=["hello", "world"]) + assert "_" in dir(doc) + assert "test_dir" in dir(doc._) + assert "test_dir" not in dir(doc[0]._) + assert "test_dir" not in dir(doc[0:2]._) + + +def test_underscore_docstring(en_vocab): + """Test that docstrings are available for extension methods, even though + they're partials.""" + + def test_method(doc, arg1=1, arg2=2): + """I am a docstring""" + return (arg1, arg2) + + Doc.set_extension("test_docstrings", method=test_method) + doc = Doc(en_vocab, words=["hello", "world"]) + assert test_method.__doc__ == "I am a docstring" + assert doc._.test_docstrings.__doc__.rsplit(". ")[-1] == "I am a docstring" diff --git a/spacy/tests/enable_gpu.py b/spacy/tests/enable_gpu.py new file mode 100644 index 0000000..3d4fded --- /dev/null +++ b/spacy/tests/enable_gpu.py @@ -0,0 +1,3 @@ +from spacy import require_gpu + +require_gpu() diff --git a/spacy/tests/factory_registrations.json b/spacy/tests/factory_registrations.json new file mode 100644 index 0000000..475e480 --- /dev/null +++ b/spacy/tests/factory_registrations.json @@ -0,0 +1,132 @@ +{ + "attribute_ruler": { + "name": "attribute_ruler", + "module": "spacy.pipeline.attributeruler", + "function": "make_attribute_ruler" + }, + "beam_ner": { + "name": "beam_ner", + "module": "spacy.pipeline.ner", + "function": "make_beam_ner" + }, + "beam_parser": { + "name": "beam_parser", + "module": "spacy.pipeline.dep_parser", + "function": "make_beam_parser" + }, + "doc_cleaner": { + "name": "doc_cleaner", + "module": "spacy.pipeline.functions", + "function": "make_doc_cleaner" + }, + "entity_linker": { + "name": "entity_linker", + "module": "spacy.pipeline.entity_linker", + "function": "make_entity_linker" + }, + "entity_ruler": { + "name": "entity_ruler", + "module": "spacy.pipeline.entityruler", + "function": "make_entity_ruler" + }, + "future_entity_ruler": { + "name": "future_entity_ruler", + "module": "spacy.pipeline.span_ruler", + "function": "make_entity_ruler" + }, + "lemmatizer": { + "name": "lemmatizer", + "module": "spacy.pipeline.lemmatizer", + "function": "make_lemmatizer" + }, + "merge_entities": { + "name": "merge_entities", + "module": "spacy.language", + "function": "Language.component..add_component..factory_func" + }, + "merge_noun_chunks": { + "name": "merge_noun_chunks", + "module": "spacy.language", + "function": "Language.component..add_component..factory_func" + }, + "merge_subtokens": { + "name": "merge_subtokens", + "module": "spacy.language", + "function": "Language.component..add_component..factory_func" + }, + "morphologizer": { + "name": "morphologizer", + "module": "spacy.pipeline.morphologizer", + "function": "make_morphologizer" + }, + "ner": { + "name": "ner", + "module": "spacy.pipeline.ner", + "function": "make_ner" + }, + "parser": { + "name": "parser", + "module": "spacy.pipeline.dep_parser", + "function": "make_parser" + }, + "sentencizer": { + "name": "sentencizer", + "module": "spacy.pipeline.sentencizer", + "function": "make_sentencizer" + }, + "senter": { + "name": "senter", + "module": "spacy.pipeline.senter", + "function": "make_senter" + }, + "span_finder": { + "name": "span_finder", + "module": "spacy.pipeline.span_finder", + "function": "make_span_finder" + }, + "span_ruler": { + "name": "span_ruler", + "module": "spacy.pipeline.span_ruler", + "function": "make_span_ruler" + }, + "spancat": { + "name": "spancat", + "module": "spacy.pipeline.spancat", + "function": "make_spancat" + }, + "spancat_singlelabel": { + "name": "spancat_singlelabel", + "module": "spacy.pipeline.spancat", + "function": "make_spancat_singlelabel" + }, + "tagger": { + "name": "tagger", + "module": "spacy.pipeline.tagger", + "function": "make_tagger" + }, + "textcat": { + "name": "textcat", + "module": "spacy.pipeline.textcat", + "function": "make_textcat" + }, + "textcat_multilabel": { + "name": "textcat_multilabel", + "module": "spacy.pipeline.textcat_multilabel", + "function": "make_multilabel_textcat" + }, + "tok2vec": { + "name": "tok2vec", + "module": "spacy.pipeline.tok2vec", + "function": "make_tok2vec" + }, + "token_splitter": { + "name": "token_splitter", + "module": "spacy.pipeline.functions", + "function": "make_token_splitter" + }, + "trainable_lemmatizer": { + "name": "trainable_lemmatizer", + "module": "spacy.pipeline.edit_tree_lemmatizer", + "function": "make_edit_tree_lemmatizer" + } +} \ No newline at end of file diff --git a/spacy/tests/lang/__init__.py b/spacy/tests/lang/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/af/__init__.py b/spacy/tests/lang/af/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/af/test_text.py b/spacy/tests/lang/af/test_text.py new file mode 100644 index 0000000..99c2a9f --- /dev/null +++ b/spacy/tests/lang/af/test_text.py @@ -0,0 +1,22 @@ +import pytest + + +def test_long_text(af_tokenizer): + # Excerpt: Universal Declaration of Human Rights; “'n” changed to “die” in first sentence + text = """ +Hierdie Universele Verklaring van Menseregte as die algemene standaard vir die verwesenliking deur alle mense en nasies, +om te verseker dat elke individu en elke deel van die gemeenskap hierdie Verklaring in ag sal neem en deur opvoeding, +respek vir hierdie regte en vryhede te bevorder, op nasionale en internasionale vlak, daarna sal strewe om die universele +en effektiewe erkenning en agting van hierdie regte te verseker, nie net vir die mense van die Lidstate nie, maar ook vir +die mense in die gebiede onder hul jurisdiksie. + +""" + tokens = af_tokenizer(text) + assert len(tokens) == 100 + + +@pytest.mark.xfail +def test_indefinite_article(af_tokenizer): + text = "as 'n algemene standaard" + tokens = af_tokenizer(text) + assert len(tokens) == 4 diff --git a/spacy/tests/lang/af/test_tokenizer.py b/spacy/tests/lang/af/test_tokenizer.py new file mode 100644 index 0000000..db52db5 --- /dev/null +++ b/spacy/tests/lang/af/test_tokenizer.py @@ -0,0 +1,29 @@ +import pytest + +AF_BASIC_TOKENIZATION_TESTS = [ + ( + "Elkeen het die reg tot lewe, vryheid en sekuriteit van persoon.", + [ + "Elkeen", + "het", + "die", + "reg", + "tot", + "lewe", + ",", + "vryheid", + "en", + "sekuriteit", + "van", + "persoon", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", AF_BASIC_TOKENIZATION_TESTS) +def test_af_tokenizer_basic(af_tokenizer, text, expected_tokens): + tokens = af_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/am/__init__.py b/spacy/tests/lang/am/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/am/test_exception.py b/spacy/tests/lang/am/test_exception.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/am/test_text.py b/spacy/tests/lang/am/test_text.py new file mode 100644 index 0000000..407f19e --- /dev/null +++ b/spacy/tests/lang/am/test_text.py @@ -0,0 +1,51 @@ +import pytest + + +def test_am_tokenizer_handles_long_text(am_tokenizer): + text = """ሆሴ ሙጂካ በበጋ ወቅት በኦክስፎርድ ንግግር አንድያቀርቡ ሲጋበዙ ጭንቅላታቸው "ፈነዳ"። + +“እጅግ ጥንታዊ” የእንግሊዝኛ ተናጋሪ ዩኒቨርስቲ፣ በአስር ሺዎች የሚቆጠሩ ዩሮዎችን ለተማሪዎች በማስተማር የሚያስከፍለው + +እና ከማርጋሬት ታቸር እስከ ስቲቨን ሆኪንግ በአዳራሾቻቸው ውስጥ ንግግር ያደረጉበት የትምህርት ማዕከል፣ በሞንቴቪዴኦ + +በሚገኘው የመንግስት ትምህርት ቤት የሰለጠኑትን የ81 ዓመቱ አዛውንት አገልግሎት ጠየቁ።""" + tokens = am_tokenizer(text) + + assert len(tokens) == 56 + + +@pytest.mark.parametrize( + "text,length", + [ + ("ሆሴ ሙጂካ ለምን ተመረጠ?", 5), + ("“በፍፁም?”", 4), + ("""አዎ! ሆዜ አርካዲዮ ቡንዲያ “እንሂድ” ሲል መለሰ።""", 11), + ("እነሱ በግምት 10ኪ.ሜ. ሮጡ።", 7), + ("እና ከዚያ ለምን...", 4), + ], +) +def test_am_tokenizer_handles_cnts(am_tokenizer, text, length): + tokens = am_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("999,0", True), + ("አንድ", True), + ("ሁለት", True), + ("ትሪሊዮን", True), + ("ውሻ", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(am_tokenizer, text, match): + tokens = am_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/ar/__init__.py b/spacy/tests/lang/ar/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ar/test_exceptions.py b/spacy/tests/lang/ar/test_exceptions.py new file mode 100644 index 0000000..0129c3a --- /dev/null +++ b/spacy/tests/lang/ar/test_exceptions.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.mark.parametrize("text", ["ق.م", "إلخ", "ص.ب", "ت."]) +def test_ar_tokenizer_handles_abbr(ar_tokenizer, text): + tokens = ar_tokenizer(text) + assert len(tokens) == 1 + + +def test_ar_tokenizer_handles_exc_in_text(ar_tokenizer): + text = "تعود الكتابة الهيروغليفية إلى سنة 3200 ق.م" + tokens = ar_tokenizer(text) + assert len(tokens) == 7 + assert tokens[6].text == "ق.م" + + +def test_ar_tokenizer_handles_exc_in_text_2(ar_tokenizer): + text = "يبلغ طول مضيق طارق 14كم " + tokens = ar_tokenizer(text) + assert len(tokens) == 6 diff --git a/spacy/tests/lang/ar/test_text.py b/spacy/tests/lang/ar/test_text.py new file mode 100644 index 0000000..c5ab376 --- /dev/null +++ b/spacy/tests/lang/ar/test_text.py @@ -0,0 +1,9 @@ +def test_ar_tokenizer_handles_long_text(ar_tokenizer): + text = """نجيب محفوظ مؤلف و كاتب روائي عربي، يعد من أهم الأدباء العرب خلال القرن العشرين. + ولد نجيب محفوظ في مدينة القاهرة، حيث ترعرع و تلقى تعليمه الجامعي في جامعتها، + فتمكن من نيل شهادة في الفلسفة. ألف محفوظ على مدار حياته الكثير من الأعمال الأدبية، و في مقدمتها ثلاثيته الشهيرة. + و قد نجح في الحصول على جائزة نوبل للآداب، ليكون بذلك العربي الوحيد الذي فاز بها.""" + + tokens = ar_tokenizer(text) + assert tokens[3].is_stop is True + assert len(tokens) == 77 diff --git a/spacy/tests/lang/bg/test_text.py b/spacy/tests/lang/bg/test_text.py new file mode 100644 index 0000000..e3a29fe --- /dev/null +++ b/spacy/tests/lang/bg/test_text.py @@ -0,0 +1,30 @@ +import pytest + + +@pytest.mark.parametrize( + "word,match", + [ + ("10", True), + ("1", True), + ("10000", True), + ("1.000", True), + ("бројка", False), + ("999,23", True), + ("едно", True), + ("две", True), + ("цифра", False), + ("единайсет", True), + ("десет", True), + ("сто", True), + ("брой", False), + ("хиляда", True), + ("милион", True), + (",", False), + ("милиарда", True), + ("билион", True), + ], +) +def test_bg_lex_attrs_like_number(bg_tokenizer, word, match): + tokens = bg_tokenizer(word) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/bg/test_tokenizer.py b/spacy/tests/lang/bg/test_tokenizer.py new file mode 100644 index 0000000..b16ef12 --- /dev/null +++ b/spacy/tests/lang/bg/test_tokenizer.py @@ -0,0 +1,5 @@ +def test_bg_tokenizer_handles_final_diacritics(bg_tokenizer): + text = "Ня̀маше яйца̀. Ня̀маше яйца̀." + tokens = bg_tokenizer(text) + assert tokens[1].text == "яйца̀" + assert tokens[2].text == "." diff --git a/spacy/tests/lang/bn/__init__.py b/spacy/tests/lang/bn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/bn/test_tokenizer.py b/spacy/tests/lang/bn/test_tokenizer.py new file mode 100644 index 0000000..e9a4d5e --- /dev/null +++ b/spacy/tests/lang/bn/test_tokenizer.py @@ -0,0 +1,36 @@ +import pytest + +# fmt: off +TESTCASES = [ + # Punctuation tests + ("আমি বাংলায় গান গাই!", ["আমি", "বাংলায়", "গান", "গাই", "!"]), + ("আমি বাংলায় কথা কই।", ["আমি", "বাংলায়", "কথা", "কই", "।"]), + ("বসুন্ধরা জনসম্মুখে দোষ স্বীকার করলো না?", ["বসুন্ধরা", "জনসম্মুখে", "দোষ", "স্বীকার", "করলো", "না", "?"]), + ("টাকা থাকলে কি না হয়!", ["টাকা", "থাকলে", "কি", "না", "হয়", "!"]), + ("সরকারি বিশ্ববিদ্যালয়-এর ছাত্র নই বলেই কি এমন আচরণ?", ["সরকারি", "বিশ্ববিদ্যালয়", "-", "এর", "ছাত্র", "নই", "বলেই", "কি", "এমন", "আচরণ", "?"]), + ('তারা বলে, "ওরা খামারের মুরগি।"', ["তারা", "বলে", ",", '"', "ওরা", "খামারের", "মুরগি", "।", '"']), + ("৩*৩=৬?", ["৩", "*", "৩", "=", "৬", "?"]), + ("কাঁঠাল-এর গন্ধই অন্যরকম", ["কাঁঠাল", "-", "এর", "গন্ধই", "অন্যরকম"]), + # Abbreviations + ("ডঃ খালেদ বললেন ঢাকায় ৩৫ ডিগ্রি সে.।", ["ডঃ", "খালেদ", "বললেন", "ঢাকায়", "৩৫", "ডিগ্রি", "সে.", "।"]), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", TESTCASES) +def test_bn_tokenizer_handles_testcases(bn_tokenizer, text, expected_tokens): + tokens = bn_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +def test_bn_tokenizer_handles_long_text(bn_tokenizer): + text = """নর্থ সাউথ বিশ্ববিদ্যালয়ে সারাবছর কোন না কোন বিষয়ে গবেষণা চলতেই থাকে। \ +অভিজ্ঞ ফ্যাকাল্টি মেম্বারগণ প্রায়ই শিক্ষার্থীদের নিয়ে বিভিন্ন গবেষণা প্রকল্পে কাজ করেন, \ +যার মধ্যে রয়েছে রোবট থেকে মেশিন লার্নিং সিস্টেম ও আর্টিফিশিয়াল ইন্টেলিজেন্স। \ +এসকল প্রকল্পে কাজ করার মাধ্যমে সংশ্লিষ্ট ক্ষেত্রে যথেষ্ঠ পরিমাণ স্পেশালাইজড হওয়া সম্ভব। \ +আর গবেষণার কাজ তোমার ক্যারিয়ারকে ঠেলে নিয়ে যাবে অনেকখানি! \ +কন্টেস্ট প্রোগ্রামার হও, গবেষক কিংবা ডেভেলপার - নর্থ সাউথ ইউনিভার্সিটিতে তোমার প্রতিভা বিকাশের সুযোগ রয়েছেই। \ +নর্থ সাউথের অসাধারণ কমিউনিটিতে তোমাকে সাদর আমন্ত্রণ।""" + tokens = bn_tokenizer(text) + assert len(tokens) == 84 diff --git a/spacy/tests/lang/bo/__init__.py b/spacy/tests/lang/bo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/bo/test_text.py b/spacy/tests/lang/bo/test_text.py new file mode 100644 index 0000000..fb3900d --- /dev/null +++ b/spacy/tests/lang/bo/test_text.py @@ -0,0 +1,21 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("999.0", True), + ("གཅིག་", True), + ("གཉིས་", True), + ("ཀླད་ཀོར་", True), + ("བཅུ་གཅིག་", True), + ("ཁྱི་", False), + (",", False), + ], +) +def test_lex_attrs_like_number(bo_tokenizer, text, match): + tokens = bo_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/ca/__init__.py b/spacy/tests/lang/ca/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ca/test_exception.py b/spacy/tests/lang/ca/test_exception.py new file mode 100644 index 0000000..499027a --- /dev/null +++ b/spacy/tests/lang/ca/test_exception.py @@ -0,0 +1,28 @@ +import pytest + + +@pytest.mark.parametrize( + "text,lemma", + [("aprox.", "aproximadament"), ("pàg.", "pàgina"), ("p.ex.", "per exemple")], +) +def test_ca_tokenizer_handles_abbr(ca_tokenizer, text, lemma): + tokens = ca_tokenizer(text) + assert len(tokens) == 1 + + +def test_ca_tokenizer_handles_exc_in_text(ca_tokenizer): + text = "La Dra. Puig viu a la pl. dels Til·lers." + doc = ca_tokenizer(text) + assert [t.text for t in doc] == [ + "La", + "Dra.", + "Puig", + "viu", + "a", + "la", + "pl.", + "d", + "els", + "Til·lers", + ".", + ] diff --git a/spacy/tests/lang/ca/test_prefix_suffix_infix.py b/spacy/tests/lang/ca/test_prefix_suffix_infix.py new file mode 100644 index 0000000..afbdf36 --- /dev/null +++ b/spacy/tests/lang/ca/test_prefix_suffix_infix.py @@ -0,0 +1,18 @@ +import pytest + + +@pytest.mark.parametrize( + "text,expected_tokens", + [ + ("d'un", ["d'", "un"]), + ("s'ha", ["s'", "ha"]), + ("del", ["d", "el"]), + ("cantar-te", ["cantar", "-te"]), + ("-hola", ["-", "hola"]), + ], +) +def test_contractions(ca_tokenizer, text, expected_tokens): + """Test that the contractions are split into two tokens""" + tokens = ca_tokenizer(text) + assert len(tokens) == 2 + assert [t.text for t in tokens] == expected_tokens diff --git a/spacy/tests/lang/ca/test_text.py b/spacy/tests/lang/ca/test_text.py new file mode 100644 index 0000000..231cc08 --- /dev/null +++ b/spacy/tests/lang/ca/test_text.py @@ -0,0 +1,56 @@ +"""Test that longer and mixed texts are tokenized correctly.""" + +import pytest + + +def test_ca_tokenizer_handles_long_text(ca_tokenizer): + text = """Una taula amb grans gerres de begudes i palles de coloraines com a reclam. Una carta + cridanera amb ofertes de tapes, paelles i sangria. Un cambrer amb un somriure que convida a + seure. La ubicació perfecta: el bell mig de la Rambla. Però és la una del migdia d’un dimecres + de tardor i no hi ha ningú assegut a la terrassa del local. El dia és rúfol, però no fa fred i + a la majoria de terrasses de la Rambla hi ha poca gent. La immensa majoria dels clients -tret + d’alguna excepció com al restaurant Núria- són turistes. I la immensa majoria tenen entre mans + una gerra de cervesa. Ens asseiem -fotògraf i periodista- en una terrassa buida.""" + + tokens = ca_tokenizer(text) + assert len(tokens) == 146 + + +@pytest.mark.parametrize( + "text,length", + [ + ("Perquè va anar-hi?", 5), + ("El cotxe dels veins.", 6), + ("“Ah no?”", 5), + ("""Sí! "Anem", va contestar el Joan Carles""", 11), + ("Van córrer aprox. 10km", 5), + ("Llavors perqué...", 3), + ("Vull parlar-te'n demà al matí", 8), + ("Vull explicar-t'ho demà al matí", 8), + ], +) +def test_ca_tokenizer_handles_cnts(ca_tokenizer, text, length): + tokens = ca_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("999.0", True), + ("un", True), + ("dos", True), + ("bilió", True), + ("gos", False), + (",", False), + ("1/2", True), + ], +) +def test_ca_lex_attrs_like_number(ca_tokenizer, text, match): + tokens = ca_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/cs/__init__.py b/spacy/tests/lang/cs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/cs/test_text.py b/spacy/tests/lang/cs/test_text.py new file mode 100644 index 0000000..b834111 --- /dev/null +++ b/spacy/tests/lang/cs/test_text.py @@ -0,0 +1,23 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("999,0", True), + ("devatenáct", True), + ("osmdesát", True), + ("kvadrilion", True), + ("Pes", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(cs_tokenizer, text, match): + tokens = cs_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/da/__init__.py b/spacy/tests/lang/da/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/da/test_exceptions.py b/spacy/tests/lang/da/test_exceptions.py new file mode 100644 index 0000000..bd9f271 --- /dev/null +++ b/spacy/tests/lang/da/test_exceptions.py @@ -0,0 +1,59 @@ +import pytest + + +@pytest.mark.parametrize("text", ["ca.", "m.a.o.", "Jan.", "Dec.", "kr.", "jf."]) +def test_da_tokenizer_handles_abbr(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["Jul.", "jul.", "Tor.", "Tors."]) +def test_da_tokenizer_handles_ambiguous_abbr(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["1.", "10.", "31."]) +def test_da_tokenizer_handles_dates(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + + +def test_da_tokenizer_handles_exc_in_text(da_tokenizer): + text = "Det er bl.a. ikke meningen" + tokens = da_tokenizer(text) + assert len(tokens) == 5 + assert tokens[2].text == "bl.a." + + +def test_da_tokenizer_handles_custom_base_exc(da_tokenizer): + text = "Her er noget du kan kigge i." + tokens = da_tokenizer(text) + assert len(tokens) == 8 + assert tokens[6].text == "i" + assert tokens[7].text == "." + + +@pytest.mark.parametrize( + "text,n_tokens", + [ + ("Godt og/eller skidt", 3), + ("Kør 4 km/t på vejen", 5), + ("Det blæser 12 m/s.", 5), + ("Det blæser 12 m/sek. på havnen", 6), + ("Windows 8/Windows 10", 5), + ("Billeten virker til bus/tog/metro", 8), + ("26/02/2019", 1), + ("Kristiansen c/o Madsen", 3), + ("Sprogteknologi a/s", 2), + ("De boede i A/B Bellevue", 5), + # note: skipping due to weirdness in UD_Danish-DDT + # ("Rotorhastigheden er 3400 o/m.", 5), + ("Jeg købte billet t/r.", 5), + ("Murerarbejdsmand m/k søges", 3), + ("Netværket kører over TCP/IP", 4), + ], +) +def test_da_tokenizer_slash(da_tokenizer, text, n_tokens): + tokens = da_tokenizer(text) + assert len(tokens) == n_tokens diff --git a/spacy/tests/lang/da/test_noun_chunks.py b/spacy/tests/lang/da/test_noun_chunks.py new file mode 100644 index 0000000..b4d389e --- /dev/null +++ b/spacy/tests/lang/da/test_noun_chunks.py @@ -0,0 +1,71 @@ +import pytest + +from spacy.tokens import Doc + + +def test_noun_chunks_is_parsed(da_tokenizer): + """Test that noun_chunks raises Value Error for 'da' language if Doc is not parsed. + To check this test, we're constructing a Doc + with a new Vocab here and forcing is_parsed to 'False' + to make sure the noun chunks don't run. + """ + doc = da_tokenizer("Det er en sætning") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +DA_NP_TEST_EXAMPLES = [ + ( + "Hun elsker at plukker frugt.", + ["PRON", "VERB", "PART", "VERB", "NOUN", "PUNCT"], + ["nsubj", "ROOT", "mark", "obj", "obj", "punct"], + [1, 0, 1, -2, -1, -4], + ["Hun", "frugt"], + ), + ( + "Påfugle er de smukkeste fugle.", + ["NOUN", "AUX", "DET", "ADJ", "NOUN", "PUNCT"], + ["nsubj", "cop", "det", "amod", "ROOT", "punct"], + [4, 3, 2, 1, 0, -1], + ["Påfugle", "de smukkeste fugle"], + ), + ( + "Rikke og Jacob Jensen glæder sig til en hyggelig skovtur", + [ + "PROPN", + "CCONJ", + "PROPN", + "PROPN", + "VERB", + "PRON", + "ADP", + "DET", + "ADJ", + "NOUN", + ], + ["nsubj", "cc", "conj", "flat", "ROOT", "obj", "case", "det", "amod", "obl"], + [4, 1, -2, -1, 0, -1, 3, 2, 1, -5], + ["Rikke", "Jacob Jensen", "sig", "en hyggelig skovtur"], + ), +] + + +@pytest.mark.parametrize( + "text,pos,deps,heads,expected_noun_chunks", DA_NP_TEST_EXAMPLES +) +def test_da_noun_chunks(da_tokenizer, text, pos, deps, heads, expected_noun_chunks): + tokens = da_tokenizer(text) + + assert len(heads) == len(pos) + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=[head + i for i, head in enumerate(heads)], + deps=deps, + pos=pos, + ) + + noun_chunks = list(doc.noun_chunks) + assert len(noun_chunks) == len(expected_noun_chunks) + for i, np in enumerate(noun_chunks): + assert np.text == expected_noun_chunks[i] diff --git a/spacy/tests/lang/da/test_prefix_suffix_infix.py b/spacy/tests/lang/da/test_prefix_suffix_infix.py new file mode 100644 index 0000000..e36b3cd --- /dev/null +++ b/spacy/tests/lang/da/test_prefix_suffix_infix.py @@ -0,0 +1,168 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(under)"]) +def test_da_tokenizer_splits_no_special(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["ta'r", "Søren's", "Lars'"]) +def test_da_tokenizer_handles_no_punct(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["(ta'r"]) +def test_da_tokenizer_splits_prefix_punct(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "(" + assert tokens[1].text == "ta'r" + + +@pytest.mark.parametrize("text", ["ta'r)"]) +def test_da_tokenizer_splits_suffix_punct(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "ta'r" + assert tokens[1].text == ")" + + +@pytest.mark.parametrize( + "text,expected", [("(ta'r)", ["(", "ta'r", ")"]), ("'ta'r'", ["'", "ta'r", "'"])] +) +def test_da_tokenizer_splits_even_wrap(da_tokenizer, text, expected): + tokens = da_tokenizer(text) + assert len(tokens) == len(expected) + assert [t.text for t in tokens] == expected + + +@pytest.mark.parametrize("text", ["(ta'r?)"]) +def test_da_tokenizer_splits_uneven_wrap(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 4 + assert tokens[0].text == "(" + assert tokens[1].text == "ta'r" + assert tokens[2].text == "?" + assert tokens[3].text == ")" + + +@pytest.mark.parametrize( + "text,expected", + [("f.eks.", ["f.eks."]), ("fe.", ["fe", "."]), ("(f.eks.", ["(", "f.eks."])], +) +def test_da_tokenizer_splits_prefix_interact(da_tokenizer, text, expected): + tokens = da_tokenizer(text) + assert len(tokens) == len(expected) + assert [t.text for t in tokens] == expected + + +@pytest.mark.parametrize("text", ["f.eks.)"]) +def test_da_tokenizer_splits_suffix_interact(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "f.eks." + assert tokens[1].text == ")" + + +@pytest.mark.parametrize("text", ["(f.eks.)"]) +def test_da_tokenizer_splits_even_wrap_interact(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == "(" + assert tokens[1].text == "f.eks." + assert tokens[2].text == ")" + + +@pytest.mark.parametrize("text", ["(f.eks.?)"]) +def test_da_tokenizer_splits_uneven_wrap_interact(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 4 + assert tokens[0].text == "(" + assert tokens[1].text == "f.eks." + assert tokens[2].text == "?" + assert tokens[3].text == ")" + + +@pytest.mark.parametrize("text", ["0,1-13,5", "0,0-0,1", "103,27-300", "1/2-3/4"]) +def test_da_tokenizer_handles_numeric_range(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["sort.Gul", "Hej.Verden"]) +def test_da_tokenizer_splits_period_infix(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Hej,Verden", "en,to"]) +def test_da_tokenizer_splits_comma_infix(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["sort...Gul", "sort...gul"]) +def test_da_tokenizer_splits_ellipsis_infix(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize( + "text", + ["gå-på-mod", "4-hjulstræk", "100-Pfennig-frimærke", "TV-2-spots", "trofæ-vaeggen"], +) +def test_da_tokenizer_keeps_hyphens(da_tokenizer, text): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + + +def test_da_tokenizer_splits_double_hyphen_infix(da_tokenizer): + tokens = da_tokenizer( + "Mange regler--eksempelvis bindestregs-reglerne--er komplicerede." + ) + assert len(tokens) == 9 + assert tokens[0].text == "Mange" + assert tokens[1].text == "regler" + assert tokens[2].text == "--" + assert tokens[3].text == "eksempelvis" + assert tokens[4].text == "bindestregs-reglerne" + assert tokens[5].text == "--" + assert tokens[6].text == "er" + assert tokens[7].text == "komplicerede" + + +def test_da_tokenizer_handles_posessives_and_contractions(da_tokenizer): + tokens = da_tokenizer( + "'DBA's, Lars' og Liz' bil sku' sgu' ik' ha' en bule, det ka' han ik' li' mere', sagde hun." + ) + assert len(tokens) == 25 + assert tokens[0].text == "'" + assert tokens[1].text == "DBA's" + assert tokens[2].text == "," + assert tokens[3].text == "Lars'" + assert tokens[4].text == "og" + assert tokens[5].text == "Liz'" + assert tokens[6].text == "bil" + assert tokens[7].text == "sku'" + assert tokens[8].text == "sgu'" + assert tokens[9].text == "ik'" + assert tokens[10].text == "ha'" + assert tokens[11].text == "en" + assert tokens[12].text == "bule" + assert tokens[13].text == "," + assert tokens[14].text == "det" + assert tokens[15].text == "ka'" + assert tokens[16].text == "han" + assert tokens[17].text == "ik'" + assert tokens[18].text == "li'" + assert tokens[19].text == "mere" + assert tokens[20].text == "'" + assert tokens[21].text == "," + assert tokens[22].text == "sagde" + assert tokens[23].text == "hun" + assert tokens[24].text == "." diff --git a/spacy/tests/lang/da/test_text.py b/spacy/tests/lang/da/test_text.py new file mode 100644 index 0000000..e1f3b96 --- /dev/null +++ b/spacy/tests/lang/da/test_text.py @@ -0,0 +1,41 @@ +import pytest + +from spacy.lang.da.lex_attrs import like_num + + +def test_da_tokenizer_handles_long_text(da_tokenizer): + text = """Der var så dejligt ude på landet. Det var sommer, kornet stod gult, havren grøn, +høet var rejst i stakke nede i de grønne enge, og der gik storken på sine lange, +røde ben og snakkede ægyptisk, for det sprog havde han lært af sin moder. + +Rundt om ager og eng var der store skove, og midt i skovene dybe søer; jo, der var rigtignok dejligt derude på landet!""" + tokens = da_tokenizer(text) + assert len(tokens) == 84 + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("10.00", True), + ("999,0", True), + ("en", True), + ("treoghalvfemsindstyvende", True), + ("hundrede", True), + ("hund", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(da_tokenizer, text, match): + tokens = da_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["elleve", "første"]) +def test_da_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/de/__init__.py b/spacy/tests/lang/de/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/de/test_exceptions.py b/spacy/tests/lang/de/test_exceptions.py new file mode 100644 index 0000000..d51c339 --- /dev/null +++ b/spacy/tests/lang/de/test_exceptions.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.mark.parametrize("text", ["auf'm", "du's", "über'm", "wir's"]) +def test_de_tokenizer_splits_contractions(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["z.B.", "d.h.", "Jan.", "Dez.", "Chr."]) +def test_de_tokenizer_handles_abbr(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 1 + + +def test_de_tokenizer_handles_exc_in_text(de_tokenizer): + text = "Ich bin z.Zt. im Urlaub." + tokens = de_tokenizer(text) + assert len(tokens) == 6 + assert tokens[2].text == "z.Zt." diff --git a/spacy/tests/lang/de/test_noun_chunks.py b/spacy/tests/lang/de/test_noun_chunks.py new file mode 100644 index 0000000..7b8b15b --- /dev/null +++ b/spacy/tests/lang/de/test_noun_chunks.py @@ -0,0 +1,8 @@ +import pytest + + +def test_noun_chunks_is_parsed_de(de_tokenizer): + """Test that noun_chunks raises Value Error for 'de' language if Doc is not parsed.""" + doc = de_tokenizer("Er lag auf seinem") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/de/test_parser.py b/spacy/tests/lang/de/test_parser.py new file mode 100644 index 0000000..8c858a4 --- /dev/null +++ b/spacy/tests/lang/de/test_parser.py @@ -0,0 +1,28 @@ +from spacy.tokens import Doc + + +def test_de_parser_noun_chunks_standard_de(de_vocab): + words = ["Eine", "Tasse", "steht", "auf", "dem", "Tisch", "."] + heads = [1, 2, 2, 2, 5, 3, 2] + pos = ["DET", "NOUN", "VERB", "ADP", "DET", "NOUN", "PUNCT"] + deps = ["nk", "sb", "ROOT", "mo", "nk", "nk", "punct"] + doc = Doc(de_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "Eine Tasse " + assert chunks[1].text_with_ws == "dem Tisch " + + +def test_de_extended_chunk(de_vocab): + # fmt: off + words = ["Die", "Sängerin", "singt", "mit", "einer", "Tasse", "Kaffee", "Arien", "."] + heads = [1, 2, 2, 2, 5, 3, 5, 2, 2] + pos = ["DET", "NOUN", "VERB", "ADP", "DET", "NOUN", "NOUN", "NOUN", "PUNCT"] + deps = ["nk", "sb", "ROOT", "mo", "nk", "nk", "nk", "oa", "punct"] + # fmt: on + doc = Doc(de_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "Die Sängerin " + assert chunks[1].text_with_ws == "einer Tasse Kaffee " + assert chunks[2].text_with_ws == "Arien " diff --git a/spacy/tests/lang/de/test_prefix_suffix_infix.py b/spacy/tests/lang/de/test_prefix_suffix_infix.py new file mode 100644 index 0000000..82bd8ed --- /dev/null +++ b/spacy/tests/lang/de/test_prefix_suffix_infix.py @@ -0,0 +1,108 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(unter)"]) +def test_de_tokenizer_splits_no_special(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["unter'm"]) +def test_de_tokenizer_splits_no_punct(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(unter'm"]) +def test_de_tokenizer_splits_prefix_punct(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["unter'm)"]) +def test_de_tokenizer_splits_suffix_punct(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(unter'm)"]) +def test_de_tokenizer_splits_even_wrap(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["(unter'm?)"]) +def test_de_tokenizer_splits_uneven_wrap(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 5 + + +@pytest.mark.parametrize("text,length", [("z.B.", 1), ("zb.", 2), ("(z.B.", 2)]) +def test_de_tokenizer_splits_prefix_interact(de_tokenizer, text, length): + tokens = de_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["z.B.)"]) +def test_de_tokenizer_splits_suffix_interact(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(z.B.)"]) +def test_de_tokenizer_splits_even_wrap_interact(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(z.B.?)"]) +def test_de_tokenizer_splits_uneven_wrap_interact(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["0.1-13.5", "0.0-0.1", "103.27-300"]) +def test_de_tokenizer_splits_numeric_range(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["blau.Rot", "Hallo.Welt"]) +def test_de_tokenizer_splits_period_infix(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Hallo,Welt", "eins,zwei"]) +def test_de_tokenizer_splits_comma_infix(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["blau...Rot", "blau...rot"]) +def test_de_tokenizer_splits_ellipsis_infix(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Islam-Konferenz", "Ost-West-Konflikt"]) +def test_de_tokenizer_keeps_hyphens(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 1 + + +def test_de_tokenizer_splits_double_hyphen_infix(de_tokenizer): + tokens = de_tokenizer("Viele Regeln--wie die Bindestrich-Regeln--sind kompliziert.") + assert len(tokens) == 10 + assert tokens[0].text == "Viele" + assert tokens[1].text == "Regeln" + assert tokens[2].text == "--" + assert tokens[3].text == "wie" + assert tokens[4].text == "die" + assert tokens[5].text == "Bindestrich-Regeln" + assert tokens[6].text == "--" + assert tokens[7].text == "sind" + assert tokens[8].text == "kompliziert" diff --git a/spacy/tests/lang/de/test_text.py b/spacy/tests/lang/de/test_text.py new file mode 100644 index 0000000..2271176 --- /dev/null +++ b/spacy/tests/lang/de/test_text.py @@ -0,0 +1,45 @@ +import pytest + + +def test_de_tokenizer_handles_long_text(de_tokenizer): + text = """Die Verwandlung + +Als Gregor Samsa eines Morgens aus unruhigen Träumen erwachte, fand er sich in +seinem Bett zu einem ungeheueren Ungeziefer verwandelt. + +Er lag auf seinem panzerartig harten Rücken und sah, wenn er den Kopf ein wenig +hob, seinen gewölbten, braunen, von bogenförmigen Versteifungen geteilten +Bauch, auf dessen Höhe sich die Bettdecke, zum gänzlichen Niedergleiten bereit, +kaum noch erhalten konnte. Seine vielen, im Vergleich zu seinem sonstigen +Umfang kläglich dünnen Beine flimmerten ihm hilflos vor den Augen. + +»Was ist mit mir geschehen?«, dachte er.""" + + tokens = de_tokenizer(text) + assert len(tokens) == 109 + + +@pytest.mark.parametrize( + "text", + [ + "Donaudampfschifffahrtsgesellschaftskapitänsanwärterposten", + "Rindfleischetikettierungsüberwachungsaufgabenübertragungsgesetz", + "Kraftfahrzeug-Haftpflichtversicherung", + "Vakuum-Mittelfrequenz-Induktionsofen", + ], +) +def test_de_tokenizer_handles_long_words(de_tokenizer, text): + tokens = de_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize( + "text,length", + [ + ("»Was ist mit mir geschehen?«, dachte er.", 12), + ("“Dies frühzeitige Aufstehen”, dachte er, “macht einen ganz blödsinnig. ", 15), + ], +) +def test_de_tokenizer_handles_examples(de_tokenizer, text, length): + tokens = de_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/dsb/__init__.py b/spacy/tests/lang/dsb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/dsb/test_text.py b/spacy/tests/lang/dsb/test_text.py new file mode 100644 index 0000000..40f2c15 --- /dev/null +++ b/spacy/tests/lang/dsb/test_text.py @@ -0,0 +1,25 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("jadno", True), + ("dwanassćo", True), + ("milion", True), + ("sto", True), + ("ceła", False), + ("kopica", False), + ("narěcow", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(dsb_tokenizer, text, match): + tokens = dsb_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/dsb/test_tokenizer.py b/spacy/tests/lang/dsb/test_tokenizer.py new file mode 100644 index 0000000..135974f --- /dev/null +++ b/spacy/tests/lang/dsb/test_tokenizer.py @@ -0,0 +1,29 @@ +import pytest + +DSB_BASIC_TOKENIZATION_TESTS = [ + ( + "Ale eksistěrujo mimo togo ceła kopica narěcow, ako na pśikład slěpjańska.", + [ + "Ale", + "eksistěrujo", + "mimo", + "togo", + "ceła", + "kopica", + "narěcow", + ",", + "ako", + "na", + "pśikład", + "slěpjańska", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", DSB_BASIC_TOKENIZATION_TESTS) +def test_dsb_tokenizer_basic(dsb_tokenizer, text, expected_tokens): + tokens = dsb_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/el/__init__.py b/spacy/tests/lang/el/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/el/test_exception.py b/spacy/tests/lang/el/test_exception.py new file mode 100644 index 0000000..a4656ea --- /dev/null +++ b/spacy/tests/lang/el/test_exception.py @@ -0,0 +1,14 @@ +import pytest + + +@pytest.mark.parametrize("text", ["αριθ.", "τρισ.", "δισ.", "σελ."]) +def test_el_tokenizer_handles_abbr(el_tokenizer, text): + tokens = el_tokenizer(text) + assert len(tokens) == 1 + + +def test_el_tokenizer_handles_exc_in_text(el_tokenizer): + text = "Στα 14 τρισ. δολάρια το κόστος από την άνοδο της στάθμης της θάλασσας." + tokens = el_tokenizer(text) + assert len(tokens) == 14 + assert tokens[2].text == "τρισ." diff --git a/spacy/tests/lang/el/test_noun_chunks.py b/spacy/tests/lang/el/test_noun_chunks.py new file mode 100644 index 0000000..2684a5c --- /dev/null +++ b/spacy/tests/lang/el/test_noun_chunks.py @@ -0,0 +1,8 @@ +import pytest + + +def test_noun_chunks_is_parsed_el(el_tokenizer): + """Test that noun_chunks raises Value Error for 'el' language if Doc is not parsed.""" + doc = el_tokenizer("είναι χώρα της νοτιοανατολικής") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/el/test_text.py b/spacy/tests/lang/el/test_text.py new file mode 100644 index 0000000..1b3ef61 --- /dev/null +++ b/spacy/tests/lang/el/test_text.py @@ -0,0 +1,31 @@ +import pytest + + +def test_el_tokenizer_handles_long_text(el_tokenizer): + text = """Η Ελλάδα (παλαιότερα Ελλάς), επίσημα γνωστή ως Ελληνική Δημοκρατία,\ + είναι χώρα της νοτιοανατολικής Ευρώπης στο νοτιότερο άκρο της Βαλκανικής χερσονήσου.\ + Συνορεύει στα βορειοδυτικά με την Αλβανία, στα βόρεια με την πρώην\ + Γιουγκοσλαβική Δημοκρατία της Μακεδονίας και τη Βουλγαρία και στα βορειοανατολικά με την Τουρκία.""" + tokens = el_tokenizer(text) + assert len(tokens) == 54 + + +@pytest.mark.parametrize( + "text,length", + [ + ("Διοικητικά η Ελλάδα διαιρείται σε 13 Περιφέρειες.", 8), + ("Η εκπαίδευση στην Ελλάδα χωρίζεται κυρίως σε τρία επίπεδα.", 10), + ( + "Η Ελλάδα είναι μία από τις χώρες της Ευρωπαϊκής Ένωσης (ΕΕ) που διαθέτει σηµαντικό ορυκτό πλούτο.", + 19, + ), + ( + "Η ναυτιλία αποτέλεσε ένα σημαντικό στοιχείο της Ελληνικής οικονομικής δραστηριότητας από τα αρχαία χρόνια.", + 15, + ), + ("Η Ελλάδα είναι μέλος σε αρκετούς διεθνείς οργανισμούς.", 9), + ], +) +def test_el_tokenizer_handles_cnts(el_tokenizer, text, length): + tokens = el_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/en/__init__.py b/spacy/tests/lang/en/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/en/test_customized_tokenizer.py b/spacy/tests/lang/en/test_customized_tokenizer.py new file mode 100644 index 0000000..8251306 --- /dev/null +++ b/spacy/tests/lang/en/test_customized_tokenizer.py @@ -0,0 +1,154 @@ +import re + +import pytest + +from spacy.lang.en import English +from spacy.tokenizer import Tokenizer +from spacy.util import compile_infix_regex, compile_prefix_regex, compile_suffix_regex + + +@pytest.fixture +def custom_en_tokenizer(en_vocab): + prefix_re = compile_prefix_regex(English.Defaults.prefixes) + suffix_re = compile_suffix_regex(English.Defaults.suffixes) + custom_infixes = [ + r"\.\.\.+", + r"(?<=[0-9])-(?=[0-9])", + r"[0-9]+(,[0-9]+)+", + r"[\[\]!&:,()\*—–\/-]", + ] + infix_re = compile_infix_regex(custom_infixes) + token_match_re = re.compile("a-b") + return Tokenizer( + en_vocab, + English.Defaults.tokenizer_exceptions, + prefix_re.search, + suffix_re.search, + infix_re.finditer, + token_match=token_match_re.match, + ) + + +def test_en_customized_tokenizer_handles_infixes(custom_en_tokenizer): + sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion." + context = [word.text for word in custom_en_tokenizer(sentence)] + assert context == [ + "The", + "8", + "and", + "10", + "-", + "county", + "definitions", + "are", + "not", + "used", + "for", + "the", + "greater", + "Southern", + "California", + "Megaregion", + ".", + ] + # the trailing '-' may cause Assertion Error + sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion." + context = [word.text for word in custom_en_tokenizer(sentence)] + assert context == [ + "The", + "8", + "-", + "and", + "10", + "-", + "county", + "definitions", + "are", + "not", + "used", + "for", + "the", + "greater", + "Southern", + "California", + "Megaregion", + ".", + ] + + +def test_en_customized_tokenizer_handles_token_match(custom_en_tokenizer): + sentence = "The 8 and 10-county definitions a-b not used for the greater Southern California Megaregion." + context = [word.text for word in custom_en_tokenizer(sentence)] + assert context == [ + "The", + "8", + "and", + "10", + "-", + "county", + "definitions", + "a-b", + "not", + "used", + "for", + "the", + "greater", + "Southern", + "California", + "Megaregion", + ".", + ] + + +def test_en_customized_tokenizer_handles_rules(custom_en_tokenizer): + sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)" + context = [word.text for word in custom_en_tokenizer(sentence)] + assert context == [ + "The", + "8", + "and", + "10", + "-", + "county", + "definitions", + "are", + "not", + "used", + "for", + "the", + "greater", + "Southern", + "California", + "Megaregion", + ".", + ":)", + ] + + +def test_en_customized_tokenizer_handles_rules_property(custom_en_tokenizer): + sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)" + rules = custom_en_tokenizer.rules + del rules[":)"] + custom_en_tokenizer.rules = rules + context = [word.text for word in custom_en_tokenizer(sentence)] + assert context == [ + "The", + "8", + "and", + "10", + "-", + "county", + "definitions", + "are", + "not", + "used", + "for", + "the", + "greater", + "Southern", + "California", + "Megaregion", + ".", + ":", + ")", + ] diff --git a/spacy/tests/lang/en/test_exceptions.py b/spacy/tests/lang/en/test_exceptions.py new file mode 100644 index 0000000..02ecaed --- /dev/null +++ b/spacy/tests/lang/en/test_exceptions.py @@ -0,0 +1,127 @@ +import pytest + + +def test_en_tokenizer_handles_basic_contraction(en_tokenizer): + text = "don't giggle" + tokens = en_tokenizer(text) + assert len(tokens) == 3 + assert tokens[1].text == "n't" + text = "i said don't!" + tokens = en_tokenizer(text) + assert len(tokens) == 5 + assert tokens[4].text == "!" + + +@pytest.mark.parametrize("text", ["`ain't", """"isn't""", "can't!"]) +def test_en_tokenizer_handles_basic_contraction_punct(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize( + "text_poss,text", [("Robin's", "Robin"), ("Alexis's", "Alexis")] +) +def test_en_tokenizer_handles_poss_contraction(en_tokenizer, text_poss, text): + tokens = en_tokenizer(text_poss) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == "'s" + + +@pytest.mark.parametrize("text", ["schools'", "Alexis'"]) +def test_en_tokenizer_splits_trailing_apos(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == text.split("'")[0] + assert tokens[1].text == "'" + + +@pytest.mark.parametrize("text", ["'em", "nothin'", "ol'"]) +def test_en_tokenizer_doesnt_split_apos_exc(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].text == text + + +@pytest.mark.parametrize("text", ["we'll", "You'll", "there'll", "this'll", "those'll"]) +def test_en_tokenizer_handles_ll_contraction(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == text.split("'")[0] + assert tokens[1].text == "'ll" + + +@pytest.mark.parametrize( + "text_lower,text_title", [("can't", "Can't"), ("ain't", "Ain't")] +) +def test_en_tokenizer_handles_capitalization(en_tokenizer, text_lower, text_title): + tokens_lower = en_tokenizer(text_lower) + tokens_title = en_tokenizer(text_title) + assert tokens_title[0].text == tokens_lower[0].text.title() + assert tokens_lower[0].text == tokens_title[0].text.lower() + assert tokens_lower[1].text == tokens_title[1].text + + +@pytest.mark.parametrize("pron", ["I", "You", "He", "She", "It", "We", "They"]) +@pytest.mark.parametrize("contraction", ["'ll", "'d"]) +def test_en_tokenizer_keeps_title_case(en_tokenizer, pron, contraction): + tokens = en_tokenizer(pron + contraction) + assert tokens[0].text == pron + assert tokens[1].text == contraction + + +@pytest.mark.parametrize("exc", ["Ill", "ill", "Hell", "hell", "Well", "well"]) +def test_en_tokenizer_excludes_ambiguous(en_tokenizer, exc): + tokens = en_tokenizer(exc) + assert len(tokens) == 1 + + +@pytest.mark.parametrize( + "wo_punct,w_punct", [("We've", "`We've"), ("couldn't", "couldn't)")] +) +def test_en_tokenizer_splits_defined_punct(en_tokenizer, wo_punct, w_punct): + tokens = en_tokenizer(wo_punct) + assert len(tokens) == 2 + tokens = en_tokenizer(w_punct) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["e.g.", "p.m.", "Jan.", "Dec.", "Inc."]) +def test_en_tokenizer_handles_abbr(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 1 + + +def test_en_tokenizer_handles_exc_in_text(en_tokenizer): + text = "It's mediocre i.e. bad." + tokens = en_tokenizer(text) + assert len(tokens) == 6 + assert tokens[3].text == "i.e." + + +@pytest.mark.parametrize("text", ["1am", "12a.m.", "11p.m.", "4pm"]) +def test_en_tokenizer_handles_times(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize( + "text,norms", + [ + ("I'm", ["i", "am"]), + ("shan't", ["shall", "not"]), + ( + "Many factors cause cancer 'cause it is complex", + ["many", "factors", "cause", "cancer", "because", "it", "is", "complex"], + ), + ], +) +def test_en_tokenizer_norm_exceptions(en_tokenizer, text, norms): + tokens = en_tokenizer(text) + assert [token.norm_ for token in tokens] == norms + + +@pytest.mark.parametrize("text,norm", [("Jan.", "January"), ("'cuz", "because")]) +def test_en_lex_attrs_norm_exceptions(en_tokenizer, text, norm): + tokens = en_tokenizer(text) + assert tokens[0].norm_ == norm diff --git a/spacy/tests/lang/en/test_indices.py b/spacy/tests/lang/en/test_indices.py new file mode 100644 index 0000000..93daeec --- /dev/null +++ b/spacy/tests/lang/en/test_indices.py @@ -0,0 +1,25 @@ +def test_en_simple_punct(en_tokenizer): + text = "to walk, do foo" + tokens = en_tokenizer(text) + assert tokens[0].idx == 0 + assert tokens[1].idx == 3 + assert tokens[2].idx == 7 + assert tokens[3].idx == 9 + assert tokens[4].idx == 12 + + +def test_en_complex_punct(en_tokenizer): + text = "Tom (D., Ill.)!" + tokens = en_tokenizer(text) + assert tokens[0].idx == 0 + assert len(tokens[0]) == 3 + assert tokens[1].idx == 4 + assert len(tokens[1]) == 1 + assert tokens[2].idx == 5 + assert len(tokens[2]) == 2 + assert tokens[3].idx == 7 + assert len(tokens[3]) == 1 + assert tokens[4].idx == 9 + assert len(tokens[4]) == 4 + assert tokens[5].idx == 13 + assert tokens[6].idx == 14 diff --git a/spacy/tests/lang/en/test_noun_chunks.py b/spacy/tests/lang/en/test_noun_chunks.py new file mode 100644 index 0000000..bda203b --- /dev/null +++ b/spacy/tests/lang/en/test_noun_chunks.py @@ -0,0 +1,45 @@ +import pytest + +from spacy.tokens import Doc + + +@pytest.fixture +def doc(en_vocab): + words = ["Peter", "has", "chronic", "command", "and", "control", "issues"] + heads = [1, 1, 6, 6, 3, 3, 1] + deps = ["nsubj", "ROOT", "amod", "nmod", "cc", "conj", "dobj"] + pos = ["PROPN", "VERB", "ADJ", "NOUN", "CCONJ", "NOUN", "NOUN"] + return Doc(en_vocab, words=words, heads=heads, deps=deps, pos=pos) + + +def test_noun_chunks_is_parsed(en_tokenizer): + """Test that noun_chunks raises Value Error for 'en' language if Doc is not parsed.""" + doc = en_tokenizer("This is a sentence") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +def test_en_noun_chunks_not_nested(doc, en_vocab): + """Test that each token only appears in one noun chunk at most""" + word_occurred = {} + chunks = list(doc.noun_chunks) + assert len(chunks) > 1 + for chunk in chunks: + for word in chunk: + word_occurred.setdefault(word.text, 0) + word_occurred[word.text] += 1 + assert len(word_occurred) > 0 + for word, freq in word_occurred.items(): + assert freq == 1, (word, [chunk.text for chunk in doc.noun_chunks]) + + +def test_noun_chunks_span(doc, en_tokenizer): + """Test that the span.noun_chunks property works correctly""" + doc_chunks = list(doc.noun_chunks) + span = doc[0:3] + span_chunks = list(span.noun_chunks) + assert 0 < len(span_chunks) < len(doc_chunks) + for chunk in span_chunks: + assert chunk in doc_chunks + assert chunk.start >= 0 + assert chunk.end <= 3 diff --git a/spacy/tests/lang/en/test_parser.py b/spacy/tests/lang/en/test_parser.py new file mode 100644 index 0000000..4266055 --- /dev/null +++ b/spacy/tests/lang/en/test_parser.py @@ -0,0 +1,66 @@ +from spacy.tokens import Doc + + +def test_en_parser_noun_chunks_standard(en_vocab): + words = ["A", "base", "phrase", "should", "be", "recognized", "."] + heads = [2, 2, 5, 5, 5, 5, 5] + pos = ["DET", "ADJ", "NOUN", "AUX", "VERB", "VERB", "PUNCT"] + deps = ["det", "amod", "nsubjpass", "aux", "auxpass", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "A base phrase " + + +def test_en_parser_noun_chunks_coordinated(en_vocab): + # fmt: off + words = ["A", "base", "phrase", "and", "a", "good", "phrase", "are", "often", "the", "same", "."] + heads = [2, 2, 7, 2, 6, 6, 2, 7, 7, 10, 7, 7] + pos = ["DET", "NOUN", "NOUN", "CCONJ", "DET", "ADJ", "NOUN", "VERB", "ADV", "DET", "ADJ", "PUNCT"] + deps = ["det", "compound", "nsubj", "cc", "det", "amod", "conj", "ROOT", "advmod", "det", "attr", "punct"] + # fmt: on + doc = Doc(en_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "A base phrase " + assert chunks[1].text_with_ws == "a good phrase " + + +def test_en_parser_noun_chunks_pp_chunks(en_vocab): + words = ["A", "phrase", "with", "another", "phrase", "occurs", "."] + heads = [1, 5, 1, 4, 2, 5, 5] + pos = ["DET", "NOUN", "ADP", "DET", "NOUN", "VERB", "PUNCT"] + deps = ["det", "nsubj", "prep", "det", "pobj", "ROOT", "punct"] + doc = Doc(en_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "A phrase " + assert chunks[1].text_with_ws == "another phrase " + + +def test_en_parser_noun_chunks_appositional_modifiers(en_vocab): + # fmt: off + words = ["Sam", ",", "my", "brother", ",", "arrived", "to", "the", "house", "."] + heads = [5, 0, 3, 0, 0, 5, 5, 8, 6, 5] + pos = ["PROPN", "PUNCT", "DET", "NOUN", "PUNCT", "VERB", "ADP", "DET", "NOUN", "PUNCT"] + deps = ["nsubj", "punct", "poss", "appos", "punct", "ROOT", "prep", "det", "pobj", "punct"] + # fmt: on + doc = Doc(en_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "Sam " + assert chunks[1].text_with_ws == "my brother " + assert chunks[2].text_with_ws == "the house " + + +def test_en_parser_noun_chunks_dative(en_vocab): + words = ["She", "gave", "Bob", "a", "raise", "."] + heads = [1, 1, 1, 4, 1, 1] + pos = ["PRON", "VERB", "PROPN", "DET", "NOUN", "PUNCT"] + deps = ["nsubj", "ROOT", "dative", "det", "dobj", "punct"] + doc = Doc(en_vocab, words=words, pos=pos, deps=deps, heads=heads) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "She " + assert chunks[1].text_with_ws == "Bob " + assert chunks[2].text_with_ws == "a raise " diff --git a/spacy/tests/lang/en/test_prefix_suffix_infix.py b/spacy/tests/lang/en/test_prefix_suffix_infix.py new file mode 100644 index 0000000..a903496 --- /dev/null +++ b/spacy/tests/lang/en/test_prefix_suffix_infix.py @@ -0,0 +1,137 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(can)"]) +def test_en_tokenizer_splits_no_special(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["can't"]) +def test_en_tokenizer_splits_no_punct(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(can't"]) +def test_en_tokenizer_splits_prefix_punct(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["can't)"]) +def test_en_tokenizer_splits_suffix_punct(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(can't)"]) +def test_en_tokenizer_splits_even_wrap(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["(can't?)"]) +def test_en_tokenizer_splits_uneven_wrap(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 5 + + +@pytest.mark.parametrize("text,length", [("U.S.", 1), ("us.", 2), ("(U.S.", 2)]) +def test_en_tokenizer_splits_prefix_interact(en_tokenizer, text, length): + tokens = en_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["U.S.)"]) +def test_en_tokenizer_splits_suffix_interact(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(U.S.)"]) +def test_en_tokenizer_splits_even_wrap_interact(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(U.S.?)"]) +def test_en_tokenizer_splits_uneven_wrap_interact(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["best-known"]) +def test_en_tokenizer_splits_hyphens(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["0.1-13.5", "0.0-0.1", "103.27-300"]) +def test_en_tokenizer_splits_numeric_range(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["best.Known", "Hello.World"]) +def test_en_tokenizer_splits_period_infix(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Hello,world", "one,two"]) +def test_en_tokenizer_splits_comma_infix(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["best...Known", "best...known"]) +def test_en_tokenizer_splits_ellipsis_infix(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 3 + + +def test_en_tokenizer_splits_double_hyphen_infix(en_tokenizer): + tokens = en_tokenizer("No decent--let alone well-bred--people.") + assert tokens[0].text == "No" + assert tokens[1].text == "decent" + assert tokens[2].text == "--" + assert tokens[3].text == "let" + assert tokens[4].text == "alone" + assert tokens[5].text == "well" + assert tokens[6].text == "-" + assert tokens[7].text == "bred" + assert tokens[8].text == "--" + assert tokens[9].text == "people" + + +def test_en_tokenizer_splits_period_abbr(en_tokenizer): + text = "Today is Tuesday.Mr." + tokens = en_tokenizer(text) + assert len(tokens) == 5 + assert tokens[0].text == "Today" + assert tokens[1].text == "is" + assert tokens[2].text == "Tuesday" + assert tokens[3].text == "." + assert tokens[4].text == "Mr." + + +@pytest.mark.issue(225) +@pytest.mark.xfail(reason="Issue #225 - not yet implemented") +def test_en_tokenizer_splits_em_dash_infix(en_tokenizer): + tokens = en_tokenizer( + """Will this road take me to Puddleton?\u2014No, """ + """you'll have to walk there.\u2014Ariel.""" + ) + assert tokens[6].text == "Puddleton" + assert tokens[7].text == "?" + assert tokens[8].text == "\u2014" + + +@pytest.mark.parametrize("text,length", [("_MATH_", 3), ("_MATH_.", 4)]) +def test_final_period(en_tokenizer, text, length): + tokens = en_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/en/test_punct.py b/spacy/tests/lang/en/test_punct.py new file mode 100644 index 0000000..79d03d2 --- /dev/null +++ b/spacy/tests/lang/en/test_punct.py @@ -0,0 +1,127 @@ +import pytest + +from spacy.lang.punctuation import TOKENIZER_PREFIXES +from spacy.util import compile_prefix_regex + +PUNCT_OPEN = ["(", "[", "{", "*"] +PUNCT_CLOSE = [")", "]", "}", "*"] +PUNCT_PAIRED = [("(", ")"), ("[", "]"), ("{", "}"), ("*", "*")] + + +@pytest.mark.parametrize("text", ["(", "((", "<"]) +def test_en_tokenizer_handles_only_punct(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == len(text) + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_open_punct(en_tokenizer, punct, text): + tokens = en_tokenizer(punct + text) + assert len(tokens) == 2 + assert tokens[0].text == punct + assert tokens[1].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_close_punct(en_tokenizer, punct, text): + tokens = en_tokenizer(text + punct) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_two_diff_open_punct(en_tokenizer, punct, punct_add, text): + tokens = en_tokenizer(punct + punct_add + text) + assert len(tokens) == 3 + assert tokens[0].text == punct + assert tokens[1].text == punct_add + assert tokens[2].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("punct_add", ["'"]) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_two_diff_close_punct(en_tokenizer, punct, punct_add, text): + tokens = en_tokenizer(text + punct + punct_add) + assert len(tokens) == 3 + assert tokens[0].text == text + assert tokens[1].text == punct + assert tokens[2].text == punct_add + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_same_open_punct(en_tokenizer, punct, text): + tokens = en_tokenizer(punct + punct + punct + text) + assert len(tokens) == 4 + assert tokens[0].text == punct + assert tokens[3].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_same_close_punct(en_tokenizer, punct, text): + tokens = en_tokenizer(text + punct + punct + punct) + assert len(tokens) == 4 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("text", ["'The"]) +def test_en_tokenizer_splits_open_appostrophe(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "'" + + +@pytest.mark.parametrize("text", ["Hello''"]) +def test_en_tokenizer_splits_double_end_quote(en_tokenizer, text): + tokens = en_tokenizer(text) + assert len(tokens) == 2 + tokens_punct = en_tokenizer("''") + assert len(tokens_punct) == 1 + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_splits_open_close_punct( + en_tokenizer, punct_open, punct_close, text +): + tokens = en_tokenizer(punct_open + text + punct_close) + assert len(tokens) == 3 + assert tokens[0].text == punct_open + assert tokens[1].text == text + assert tokens[2].text == punct_close + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("punct_open2,punct_close2", [("`", "'")]) +@pytest.mark.parametrize("text", ["Hello"]) +def test_en_tokenizer_two_diff_punct( + en_tokenizer, punct_open, punct_close, punct_open2, punct_close2, text +): + tokens = en_tokenizer(punct_open2 + punct_open + text + punct_close + punct_close2) + assert len(tokens) == 5 + assert tokens[0].text == punct_open2 + assert tokens[1].text == punct_open + assert tokens[2].text == text + assert tokens[3].text == punct_close + assert tokens[4].text == punct_close2 + + +@pytest.mark.parametrize("text,punct", [("(can't", "(")]) +def test_en_tokenizer_splits_pre_punct_regex(text, punct): + en_search_prefixes = compile_prefix_regex(TOKENIZER_PREFIXES).search + match = en_search_prefixes(text) + assert match.group() == punct + + +def test_en_tokenizer_splits_bracket_period(en_tokenizer): + text = "(And a 6a.m. run through Washington Park)." + tokens = en_tokenizer(text) + assert tokens[len(tokens) - 1].text == "." diff --git a/spacy/tests/lang/en/test_sbd.py b/spacy/tests/lang/en/test_sbd.py new file mode 100644 index 0000000..c07c231 --- /dev/null +++ b/spacy/tests/lang/en/test_sbd.py @@ -0,0 +1,46 @@ +import pytest + +from spacy.tokens import Doc + +from ...util import apply_transition_sequence + + +@pytest.mark.issue(309) +def test_issue309(en_vocab): + """Test Issue #309: SBD fails on empty string""" + doc = Doc(en_vocab, words=[" "], heads=[0], deps=["ROOT"]) + assert len(doc) == 1 + sents = list(doc.sents) + assert len(sents) == 1 + + +@pytest.mark.parametrize("words", [["A", "test", "sentence"]]) +@pytest.mark.parametrize("punct", [".", "!", "?", ""]) +def test_en_sbd_single_punct(en_vocab, words, punct): + heads = [2, 2, 2, 2] if punct else [2, 2, 2] + deps = ["dep"] * len(heads) + words = [*words, punct] if punct else words + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert len(doc) == 4 if punct else 3 + assert len(list(doc.sents)) == 1 + assert sum(len(sent) for sent in doc.sents) == len(doc) + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_en_sentence_breaks(en_vocab, en_parser): + # fmt: off + words = ["This", "is", "a", "sentence", ".", "This", "is", "another", "one", "."] + heads = [1, 1, 3, 1, 1, 6, 6, 8, 6, 6] + deps = ["nsubj", "ROOT", "det", "attr", "punct", "nsubj", "ROOT", "det", + "attr", "punct"] + transition = ["L-nsubj", "S", "L-det", "R-attr", "D", "R-punct", "B-ROOT", + "L-nsubj", "S", "L-attr", "R-attr", "D", "R-punct"] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + apply_transition_sequence(en_parser, doc, transition) + assert len(list(doc.sents)) == 2 + for token in doc: + assert token.dep != 0 or token.is_space + assert [token.head.i for token in doc] == [1, 1, 3, 1, 1, 6, 6, 8, 6, 6] diff --git a/spacy/tests/lang/en/test_text.py b/spacy/tests/lang/en/test_text.py new file mode 100644 index 0000000..53cf0cc --- /dev/null +++ b/spacy/tests/lang/en/test_text.py @@ -0,0 +1,70 @@ +import pytest + +from spacy.lang.en.lex_attrs import like_num + + +def test_en_tokenizer_handles_long_text(en_tokenizer): + text = """Tributes pour in for late British Labour Party leader + +Tributes poured in from around the world Thursday +to the late Labour Party leader John Smith, who died earlier from a massive +heart attack aged 55. + +In Washington, the US State Department issued a statement regretting "the +untimely death" of the rapier-tongued Scottish barrister and parliamentarian. + +"Mr. Smith, throughout his distinguished""" + tokens = en_tokenizer(text) + assert len(tokens) == 76 + + +@pytest.mark.parametrize( + "text,length", + [ + ("The U.S. Army likes Shock and Awe.", 8), + ("U.N. regulations are not a part of their concern.", 10), + ("“Isn't it?”", 6), + ("""Yes! "I'd rather have a walk", Ms. Comble sighed. """, 15), + ("""'Me too!', Mr. P. Delaware cried. """, 11), + ("They ran about 10km.", 6), + ("But then the 6,000-year ice age came...", 10), + ], +) +def test_en_tokenizer_handles_cnts(en_tokenizer, text, length): + tokens = en_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("999.0", True), + ("one", True), + ("two", True), + ("billion", True), + ("dog", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(en_tokenizer, text, match): + tokens = en_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize( + "word", ["third", "Millionth", "100th", "Hundredth", "23rd", "52nd"] +) +def test_en_lex_attrs_like_number_for_ordinal(word): + assert like_num(word) + + +@pytest.mark.parametrize("word", ["eleven"]) +def test_en_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/en/test_tokenizer.py b/spacy/tests/lang/en/test_tokenizer.py new file mode 100644 index 0000000..0133d00 --- /dev/null +++ b/spacy/tests/lang/en/test_tokenizer.py @@ -0,0 +1,178 @@ +import pytest + + +@pytest.mark.issue(351) +def test_issue351(en_tokenizer): + doc = en_tokenizer(" This is a cat.") + assert doc[0].idx == 0 + assert len(doc[0]) == 3 + assert doc[1].idx == 3 + + +@pytest.mark.issue(360) +def test_issue360(en_tokenizer): + """Test tokenization of big ellipsis""" + tokens = en_tokenizer("$45...............Asking") + assert len(tokens) > 2 + + +@pytest.mark.issue(736) +@pytest.mark.parametrize("text,number", [("7am", "7"), ("11p.m.", "11")]) +def test_issue736(en_tokenizer, text, number): + """Test that times like "7am" are tokenized correctly and that numbers are + converted to string.""" + tokens = en_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == number + + +@pytest.mark.issue(740) +@pytest.mark.parametrize("text", ["3/4/2012", "01/12/1900"]) +def test_issue740(en_tokenizer, text): + """Test that dates are not split and kept as one token. This behaviour is + currently inconsistent, since dates separated by hyphens are still split. + This will be hard to prevent without causing clashes with numeric ranges.""" + tokens = en_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.issue(744) +@pytest.mark.parametrize("text", ["We were scared", "We Were Scared"]) +def test_issue744(en_tokenizer, text): + """Test that 'were' and 'Were' are excluded from the contractions + generated by the English tokenizer exceptions.""" + tokens = en_tokenizer(text) + assert len(tokens) == 3 + assert tokens[1].text.lower() == "were" + + +@pytest.mark.issue(759) +@pytest.mark.parametrize( + "text,is_num", [("one", True), ("ten", True), ("teneleven", False)] +) +def test_issue759(en_tokenizer, text, is_num): + tokens = en_tokenizer(text) + assert tokens[0].like_num == is_num + + +@pytest.mark.issue(775) +@pytest.mark.parametrize("text", ["Shell", "shell", "Shed", "shed"]) +def test_issue775(en_tokenizer, text): + """Test that 'Shell' and 'shell' are excluded from the contractions + generated by the English tokenizer exceptions.""" + tokens = en_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].text == text + + +@pytest.mark.issue(792) +@pytest.mark.parametrize("text", ["This is a string ", "This is a string\u0020"]) +def test_issue792(en_tokenizer, text): + """Test for Issue #792: Trailing whitespace is removed after tokenization.""" + doc = en_tokenizer(text) + assert "".join([token.text_with_ws for token in doc]) == text + + +@pytest.mark.issue(792) +@pytest.mark.parametrize("text", ["This is a string", "This is a string\n"]) +def test_control_issue792(en_tokenizer, text): + """Test base case for Issue #792: Non-trailing whitespace""" + doc = en_tokenizer(text) + assert "".join([token.text_with_ws for token in doc]) == text + + +@pytest.mark.issue(859) +@pytest.mark.parametrize( + "text", ["aaabbb@ccc.com\nThank you!", "aaabbb@ccc.com \nThank you!"] +) +def test_issue859(en_tokenizer, text): + """Test that no extra space is added in doc.text method.""" + doc = en_tokenizer(text) + assert doc.text == text + + +@pytest.mark.issue(886) +@pytest.mark.parametrize("text", ["Datum:2014-06-02\nDokument:76467"]) +def test_issue886(en_tokenizer, text): + """Test that token.idx matches the original text index for texts with newlines.""" + doc = en_tokenizer(text) + for token in doc: + assert len(token.text) == len(token.text_with_ws) + assert text[token.idx] == token.text[0] + + +@pytest.mark.issue(891) +@pytest.mark.parametrize("text", ["want/need"]) +def test_issue891(en_tokenizer, text): + """Test that / infixes are split correctly.""" + tokens = en_tokenizer(text) + assert len(tokens) == 3 + assert tokens[1].text == "/" + + +@pytest.mark.issue(957) +@pytest.mark.slow +def test_issue957(en_tokenizer): + """Test that spaCy doesn't hang on many punctuation characters. + If this test hangs, check (new) regular expressions for conflicting greedy operators + """ + # Skip test if pytest-timeout is not installed + pytest.importorskip("pytest_timeout") + for punct in [".", ",", "'", '"', ":", "?", "!", ";", "-"]: + string = "0" + for i in range(1, 100): + string += punct + str(i) + doc = en_tokenizer(string) + assert doc + + +@pytest.mark.parametrize("text", ["test@example.com", "john.doe@example.co.uk"]) +@pytest.mark.issue(1698) +def test_issue1698(en_tokenizer, text): + """Test that doc doesn't identify email-addresses as URLs""" + doc = en_tokenizer(text) + assert len(doc) == 1 + assert not doc[0].like_url + + +@pytest.mark.issue(1758) +def test_issue1758(en_tokenizer): + """Test that "would've" is handled by the English tokenizer exceptions.""" + tokens = en_tokenizer("would've") + assert len(tokens) == 2 + + +@pytest.mark.issue(1773) +def test_issue1773(en_tokenizer): + """Test that spaces don't receive a POS but no TAG. This is the root cause + of the serialization issue reported in #1773.""" + doc = en_tokenizer("\n") + if doc[0].pos_ == "SPACE": + assert doc[0].tag_ != "" + + +@pytest.mark.issue(3277) +def test_issue3277(es_tokenizer): + """Test that hyphens are split correctly as prefixes.""" + doc = es_tokenizer("—Yo me llamo... –murmuró el niño– Emilio Sánchez Pérez.") + assert len(doc) == 14 + assert doc[0].text == "\u2014" + assert doc[5].text == "\u2013" + assert doc[9].text == "\u2013" + + +@pytest.mark.parametrize("word", ["don't", "don’t", "I'd", "I’d"]) +@pytest.mark.issue(3521) +def test_issue3521(en_tokenizer, word): + tok = en_tokenizer(word)[1] + # 'not' and 'would' should be stopwords, also in their abbreviated forms + assert tok.is_stop + + +@pytest.mark.issue(10699) +@pytest.mark.parametrize("text", ["theses", "thisre"]) +def test_issue10699(en_tokenizer, text): + """Test that 'theses' and 'thisre' are excluded from the contractions + generated by the English tokenizer exceptions.""" + tokens = en_tokenizer(text) + assert len(tokens) == 1 diff --git a/spacy/tests/lang/es/__init__.py b/spacy/tests/lang/es/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/es/test_exception.py b/spacy/tests/lang/es/test_exception.py new file mode 100644 index 0000000..07df5d6 --- /dev/null +++ b/spacy/tests/lang/es/test_exception.py @@ -0,0 +1,22 @@ +import pytest + + +@pytest.mark.parametrize( + "text,lemma", + [ + ("aprox.", "aproximadamente"), + ("esq.", "esquina"), + ("pág.", "página"), + ("p.ej.", "por ejemplo"), + ], +) +def test_es_tokenizer_handles_abbr(es_tokenizer, text, lemma): + tokens = es_tokenizer(text) + assert len(tokens) == 1 + + +def test_es_tokenizer_handles_exc_in_text(es_tokenizer): + text = "Mariano Rajoy ha corrido aprox. medio kilómetro" + tokens = es_tokenizer(text) + assert len(tokens) == 7 + assert tokens[4].text == "aprox." diff --git a/spacy/tests/lang/es/test_noun_chunks.py b/spacy/tests/lang/es/test_noun_chunks.py new file mode 100644 index 0000000..50d49fc --- /dev/null +++ b/spacy/tests/lang/es/test_noun_chunks.py @@ -0,0 +1,159 @@ +import pytest + +from spacy.tokens import Doc + + +# fmt: off +@pytest.mark.parametrize( + "words,heads,deps,pos,chunk_offsets", + [ + # un gato -> "un gato" + ( + ["un", "gato"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0, 2)], + ), + # la camisa negra -> "la camisa negra" + ( + ["la", "camisa", "negra"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0, 3)], + ), + # un lindo gatito -> "un lindo gatito" + ( + ["Un", "lindo", "gatito"], + [2, 2, 2], + ["det", "amod", "ROOT"], + ["DET", "ADJ", "NOUN"], + [(0,3)] + ), + # una chica hermosa e inteligente -> una chica hermosa e inteligente + ( + ["Una", "chica", "hermosa", "e", "inteligente"], + [1, 1, 1, 4, 2], + ["det", "ROOT", "amod", "cc", "conj"], + ["DET", "NOUN", "ADJ", "CCONJ", "ADJ"], + [(0,5)] + ), + # el fabuloso gato pardo -> "el fabuloso gato pardo" + ( + ["el", "fabuloso", "gato", "pardo"], + [2, 2, 2, 2], + ["det", "amod", "ROOT", "amod"], + ["DET", "ADJ", "NOUN", "ADJ"], + [(0,4)] + ), + # Tengo un gato y un perro -> un gato, un perro + ( + ["Tengo", "un", "gato", "y", "un", "perro"], + [0, 2, 0, 5, 5, 0], + ["ROOT", "det", "obj", "cc", "det", "conj"], + ["VERB", "DET", "NOUN", "CCONJ", "DET", "NOUN"], + [(1,3), (4,6)] + + ), + # Dom Pedro II -> Dom Pedro II + ( + ["Dom", "Pedro", "II"], + [0, 0, 0], + ["ROOT", "flat", "flat"], + ["PROPN", "PROPN", "PROPN"], + [(0,3)] + ), + # los Estados Unidos -> los Estados Unidos + ( + ["los", "Estados", "Unidos"], + [1, 1, 1], + ["det", "ROOT", "flat"], + ["DET", "PROPN", "PROPN"], + [(0,3)] + ), + # Miguel de Cervantes -> Miguel de Cervantes + ( + ["Miguel", "de", "Cervantes"], + [0, 2, 0], + ["ROOT", "case", "flat"], + ["PROPN", "ADP", "PROPN"], + [(0,3)] + ), + ( + ["Rio", "de", "Janeiro"], + [0, 2, 0], + ["ROOT", "case", "flat"], + ["PROPN", "ADP", "PROPN"], + [(0,3)] + ), + # la destrucción de la ciudad -> la destrucción, la ciudad + ( + ["la", "destrucción", "de", "la", "ciudad"], + [1, 1, 4, 4, 1], + ['det', 'ROOT', 'case', 'det', 'nmod'], + ['DET', 'NOUN', 'ADP', 'DET', 'NOUN'], + [(0,2), (3,5)] + ), + # la traducción de Susana del informe -> la traducción, Susana, informe + ( + ['la', 'traducción', 'de', 'Susana', 'del', 'informe'], + [1, 1, 3, 1, 5, 1], + ['det', 'ROOT', 'case', 'nmod', 'case', 'nmod'], + ['DET', 'NOUN', 'ADP', 'PROPN', 'ADP', 'NOUN'], + [(0,2), (3,4), (5,6)] + + ), + # El gato regordete de Susana y su amigo -> el gato regordete, Susana, su amigo + ( + ['El', 'gato', 'regordete', 'de', 'Susana', 'y', 'su', 'amigo'], + [1, 1, 1, 4, 1, 7, 7, 1], + ['det', 'ROOT', 'amod', 'case', 'nmod', 'cc', 'det', 'conj'], + ['DET', 'NOUN', 'ADJ', 'ADP', 'PROPN', 'CCONJ', 'DET', 'NOUN'], + [(0,3), (4,5), (6,8)] + ), + # Afirmó que sigue el criterio europeo y que trata de incentivar el mercado donde no lo hay -> el criterio europeo, el mercado, donde, lo + ( + ['Afirmó', 'que', 'sigue', 'el', 'criterio', 'europeo', 'y', 'que', 'trata', 'de', 'incentivar', 'el', 'mercado', 'donde', 'no', 'lo', 'hay'], + [0, 2, 0, 4, 2, 4, 8, 8, 2, 10, 8, 12, 10, 16, 16, 16, 0], + ['ROOT', 'mark', 'ccomp', 'det', 'obj', 'amod', 'cc', 'mark', 'conj', 'mark', 'xcomp', 'det', 'obj', 'obl', 'advmod', 'obj', 'advcl'], + ['VERB', 'SCONJ', 'VERB', 'DET', 'NOUN', 'ADJ', 'CCONJ', 'SCONJ', 'VERB', 'ADP', 'VERB', 'DET', 'NOUN', 'PRON', 'ADV', 'PRON', 'AUX'], + [(3,6), (11,13), (13,14), (15,16)] + ), + # En este sentido se refirió a la reciente creación del Ministerio de Ciencia y Tecnología y a las primeras declaraciones de su titular, Anna Birulés, sobre el impulso de la investigación, desarrollo e innovación -> este sentido, se, la reciente creación, Ministerio de Ciencia y Tecnología, a las primeras declaraciones, su titular, , Anna Birulés,, el impulso, la investigación, , desarrollo, innovación + ( + ['En', 'este', 'sentido', 'se', 'refirió', 'a', 'la', 'reciente', 'creación', 'del', 'Ministerio', 'de', 'Ciencia', 'y', 'Tecnología', 'y', 'a', 'las', 'primeras', 'declaraciones', 'de', 'su', 'titular', ',', 'Anna', 'Birulés', ',', 'sobre', 'el', 'impulso', 'de', 'la', 'investigación', ',', 'desarrollo', 'e', 'innovación'], + [2, 2, 4, 4, 4, 8, 8, 8, 4, 10, 8, 12, 10, 14, 12, 19, 19, 19, 19, 8, 22, 22, 19, 24, 22, 24, 24, 29, 29, 19, 32, 32, 29, 34, 32, 36, 32], + ['case', 'det', 'obl', 'obj', 'ROOT', 'case', 'det', 'amod', 'obj', 'case', 'nmod', 'case', 'flat', 'cc', 'conj', 'cc', 'case', 'det', 'amod', 'conj', 'case', 'det', 'nmod', 'punct', 'appos', 'flat', 'punct', 'case', 'det', 'nmod', 'case', 'det', 'nmod', 'punct', 'conj', 'cc', 'conj'], + ['ADP', 'DET', 'NOUN', 'PRON', 'VERB', 'ADP', 'DET', 'ADJ', 'NOUN', 'ADP', 'PROPN', 'ADP', 'PROPN', 'CCONJ', 'PROPN', 'CCONJ', 'ADP', 'DET', 'ADJ', 'NOUN', 'ADP', 'DET', 'NOUN', 'PUNCT', 'PROPN', 'PROPN', 'PUNCT', 'ADP', 'DET', 'NOUN', 'ADP', 'DET', 'NOUN', 'PUNCT', 'NOUN', 'CCONJ', 'NOUN'], + [(1, 3), (3, 4), (6, 9), (10, 15), (16, 20), (21, 23), (23, 27), (28, 30), (31, 33), (33, 35), (36, 37)] + ), + # Asimismo defiende la financiación pública de la investigación básica y pone de manifiesto que las empresas se centran más en la investigación y desarrollo con objetivos de mercado. -> la financiación pública, la investigación básica, manifiesto, las empresas, se, la investigación, desarrollo, objetivos, mercado + ( + ['Asimismo', 'defiende', 'la', 'financiación', 'pública', 'de', 'la', 'investigación', 'básica', 'y', 'pone', 'de', 'manifiesto', 'que', 'las', 'empresas', 'se', 'centran', 'más', 'en', 'la', 'investigación', 'y', 'desarrollo', 'con', 'objetivos', 'de', 'mercado'], + [1, 1, 3, 1, 3, 7, 7, 3, 7, 10, 1, 12, 10, 17, 15, 17, 17, 10, 17, 21, 21, 17, 23, 21, 25, 17, 27, 25], + ['advmod', 'ROOT', 'det', 'obj', 'amod', 'case', 'det', 'nmod', 'amod', 'cc', 'conj', 'case', 'obl', 'mark', 'det', 'nsubj', 'obj', 'ccomp', 'obj', 'case', 'det', 'obl', 'cc', 'conj', 'case', 'obl', 'case', 'nmod'], + ['ADV', 'VERB', 'DET', 'NOUN', 'ADJ', 'ADP', 'DET', 'NOUN', 'ADJ', 'CCONJ', 'VERB', 'ADP', 'NOUN', 'SCONJ', 'DET', 'NOUN', 'PRON', 'VERB', 'ADV', 'ADP', 'DET', 'NOUN', 'CCONJ', 'NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN'], + [(2, 5), (6, 9), (12, 13), (14, 16), (16, 17), (20, 22), (23, 24), (25, 26), (27, 28)] + ), + # Tras indicar que la inversión media en investigación en la Unión Europea se sitúa en el 1,8 por ciento del PIB, frente al 2,8 por ciento en Japón y EEUU, Couceiro dijo que España está en "el buen camino" y se está creando un entorno propicio para la innovación empresarial' -> la inversión media, investigación, la Unión Europea, se, PIB, Japón, EEUU, Couceiro, España, se, un entorno propicio para la innovación empresaria + ( + ['Tras', 'indicar', 'que', 'la', 'inversión', 'media', 'en', 'investigación', 'en', 'la', 'Unión', 'Europea', 'se', 'sitúa', 'en', 'el', '1,8', 'por', 'ciento', 'del', 'PIB', ',', 'frente', 'al', '2,8', 'por', 'ciento', 'en', 'Japón', 'y', 'EEUU', ',', 'Couceiro', 'dijo', 'que', 'España', 'está', 'en', '"', 'el', 'buen', 'camino', '"', 'y', 'se', 'está', 'creando', 'un', 'entorno', 'propicio', 'para', 'la', 'innovación', 'empresarial'], + [1, 33, 13, 4, 13, 4, 7, 4, 10, 10, 4, 10, 13, 1, 16, 16, 13, 18, 16, 20, 16, 24, 24, 22, 13, 26, 24, 28, 24, 30, 28, 1, 33, 33, 41, 41, 41, 41, 41, 41, 41, 33, 41, 46, 46, 46, 33, 48, 46, 48, 52, 52, 49, 52], + ['mark', 'advcl', 'mark', 'det', 'nsubj', 'amod', 'case', 'nmod', 'case', 'det', 'nmod', 'flat', 'obj', 'ccomp', 'case', 'det', 'obj', 'case', 'compound', 'case', 'nmod', 'punct', 'case', 'fixed', 'obl', 'case', 'compound', 'case', 'nmod', 'cc', 'conj', 'punct', 'nsubj', 'ROOT', 'mark', 'nsubj', 'cop', 'case', 'punct', 'det', 'amod', 'ccomp', 'punct', 'cc', 'obj', 'aux', 'conj', 'det', 'nsubj', 'amod', 'case', 'det', 'nmod', 'amod'], + ['ADP', 'VERB', 'SCONJ', 'DET', 'NOUN', 'ADJ', 'ADP', 'NOUN', 'ADP', 'DET', 'PROPN', 'PROPN', 'PRON', 'VERB', 'ADP', 'DET', 'NUM', 'ADP', 'NUM', 'ADP', 'PROPN', 'PUNCT', 'NOUN', 'ADP', 'NUM', 'ADP', 'NUM', 'ADP', 'PROPN', 'CCONJ', 'PROPN', 'PUNCT', 'PROPN', 'VERB', 'SCONJ', 'PROPN', 'AUX', 'ADP', 'PUNCT', 'DET', 'ADJ', 'NOUN', 'PUNCT', 'CCONJ', 'PRON', 'AUX', 'VERB', 'DET', 'NOUN', 'ADJ', 'ADP', 'DET', 'NOUN', 'ADJ'], + [(3, 6), (7, 8), (9, 12), (12, 13), (20, 21), (28, 29), (30, 31), (32, 33), (35, 36), (44, 45), (47, 54)] + ), + ], +) +# fmt: on +def test_es_noun_chunks(es_vocab, words, heads, deps, pos, chunk_offsets): + doc = Doc(es_vocab, words=words, heads=heads, deps=deps, pos=pos) + assert [(c.start, c.end) for c in doc.noun_chunks] == chunk_offsets + + +def test_noun_chunks_is_parsed_es(es_tokenizer): + """Test that noun_chunks raises Value Error for 'es' language if Doc is not parsed.""" + doc = es_tokenizer("en Oxford este verano") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/es/test_text.py b/spacy/tests/lang/es/test_text.py new file mode 100644 index 0000000..1d1f7fa --- /dev/null +++ b/spacy/tests/lang/es/test_text.py @@ -0,0 +1,74 @@ +import pytest + +from spacy.lang.es import Spanish +from spacy.lang.es.lex_attrs import like_num + + +@pytest.mark.issue(3803) +def test_issue3803(): + """Test that spanish num-like tokens have True for like_num attribute.""" + nlp = Spanish() + text = "2 dos 1000 mil 12 doce" + doc = nlp(text) + + assert [t.like_num for t in doc] == [True, True, True, True, True, True] + + +def test_es_tokenizer_handles_long_text(es_tokenizer): + text = """Cuando a José Mujica lo invitaron a dar una conferencia + +en Oxford este verano, su cabeza hizo "crac". La "más antigua" universidad de habla + +inglesa, esa que cobra decenas de miles de euros de matrícula a sus alumnos + +y en cuyos salones han disertado desde Margaret Thatcher hasta Stephen Hawking, + +reclamaba los servicios de este viejo de 81 años, formado en un colegio público + +en Montevideo y que pregona las bondades de la vida austera.""" + tokens = es_tokenizer(text) + assert len(tokens) == 90 + + +@pytest.mark.parametrize( + "text,length", + [ + ("¿Por qué José Mujica?", 6), + ("“¿Oh no?”", 6), + ("""¡Sí! "Vámonos", contestó José Arcadio Buendía""", 11), + ("Corrieron aprox. 10km.", 5), + ("Y entonces por qué...", 5), + ], +) +def test_es_tokenizer_handles_cnts(es_tokenizer, text, length): + tokens = es_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("999,0", True), + ("uno", True), + ("dos", True), + ("billón", True), + ("veintiséis", True), + ("perro", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(es_tokenizer, text, match): + tokens = es_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["once"]) +def test_es_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/et/__init__.py b/spacy/tests/lang/et/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/et/test_text.py b/spacy/tests/lang/et/test_text.py new file mode 100644 index 0000000..9515a7c --- /dev/null +++ b/spacy/tests/lang/et/test_text.py @@ -0,0 +1,26 @@ +import pytest + + +def test_long_text(et_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +arvestades, et nimetatud deklaratsiooni eesmärk on tagada selles +kuulutatud õiguste üldine ja tõhus tunnustamine ning järgimine; +arvestades, et Euroopa Nõukogu eesmärk on saavutada tema +liikmete suurem ühtsus ning et üheks selle eesmärgi saavutamise +vahendiks on inimõiguste ja põhivabaduste järgimine ning +elluviimine; +taaskinnitades oma sügavat usku neisse põhivabadustesse, mis +on õigluse ja rahu aluseks maailmas ning mida kõige paremini +tagab ühelt poolt tõhus poliitiline demokraatia ning teiselt poolt +inimõiguste, millest nad sõltuvad, üldine mõistmine ja järgimine; +""" + tokens = et_tokenizer(text) + assert len(tokens) == 94 + + +@pytest.mark.xfail +def test_ordinal_number(et_tokenizer): + text = "10. detsembril 1948" + tokens = et_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/et/test_tokenizer.py b/spacy/tests/lang/et/test_tokenizer.py new file mode 100644 index 0000000..8bee228 --- /dev/null +++ b/spacy/tests/lang/et/test_tokenizer.py @@ -0,0 +1,28 @@ +import pytest + +ET_BASIC_TOKENIZATION_TESTS = [ + ( + "Kedagi ei või piinata ega ebainimlikult või alandavalt kohelda ega karistada.", + [ + "Kedagi", + "ei", + "või", + "piinata", + "ega", + "ebainimlikult", + "või", + "alandavalt", + "kohelda", + "ega", + "karistada", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", ET_BASIC_TOKENIZATION_TESTS) +def test_et_tokenizer_basic(et_tokenizer, text, expected_tokens): + tokens = et_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/eu/__init__.py b/spacy/tests/lang/eu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/eu/test_text.py b/spacy/tests/lang/eu/test_text.py new file mode 100644 index 0000000..94d5ac9 --- /dev/null +++ b/spacy/tests/lang/eu/test_text.py @@ -0,0 +1,19 @@ +import pytest + + +def test_eu_tokenizer_handles_long_text(eu_tokenizer): + text = """ta nere guitarra estrenatu ondoren""" + tokens = eu_tokenizer(text) + assert len(tokens) == 5 + + +@pytest.mark.parametrize( + "text,length", + [ + ("milesker ederra joan zen hitzaldia plazer hutsa", 7), + ("astelehen guztia sofan pasau biot", 5), + ], +) +def test_eu_tokenizer_handles_cnts(eu_tokenizer, text, length): + tokens = eu_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/fa/__init__.py b/spacy/tests/lang/fa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/fa/test_noun_chunks.py b/spacy/tests/lang/fa/test_noun_chunks.py new file mode 100644 index 0000000..d2411e6 --- /dev/null +++ b/spacy/tests/lang/fa/test_noun_chunks.py @@ -0,0 +1,9 @@ +import pytest + + +def test_noun_chunks_is_parsed_fa(fa_tokenizer): + """Test that noun_chunks raises Value Error for 'fa' language if Doc is not parsed.""" + + doc = fa_tokenizer("این یک جمله نمونه می باشد.") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/fi/__init__.py b/spacy/tests/lang/fi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/fi/test_noun_chunks.py b/spacy/tests/lang/fi/test_noun_chunks.py new file mode 100644 index 0000000..37e1b00 --- /dev/null +++ b/spacy/tests/lang/fi/test_noun_chunks.py @@ -0,0 +1,189 @@ +import pytest + +from spacy.tokens import Doc + +FI_NP_TEST_EXAMPLES = [ + ( + "Kaksi tyttöä potkii punaista palloa", + ["NUM", "NOUN", "VERB", "ADJ", "NOUN"], + ["nummod", "nsubj", "ROOT", "amod", "obj"], + [1, 1, 0, 1, -2], + ["Kaksi tyttöä", "punaista palloa"], + ), + ( + "Erittäin vaarallinen leijona karkasi kiertävän sirkuksen eläintenkesyttäjältä", + ["ADV", "ADJ", "NOUN", "VERB", "ADJ", "NOUN", "NOUN"], + ["advmod", "amod", "nsubj", "ROOT", "amod", "nmod:poss", "obl"], + [1, 1, 1, 0, 1, 1, -3], + ["Erittäin vaarallinen leijona", "kiertävän sirkuksen eläintenkesyttäjältä"], + ), + ( + "Leijona raidallisine tassuineen piileksii Porin kaupungin lähellä", + ["NOUN", "ADJ", "NOUN", "VERB", "PROPN", "NOUN", "ADP"], + ["nsubj", "amod", "nmod", "ROOT", "nmod:poss", "obl", "case"], + [3, 1, -2, 0, 1, -2, -1], + ["Leijona raidallisine tassuineen", "Porin kaupungin"], + ), + ( + "Lounaalla nautittiin salaattia, maukasta kanaa ja raikasta vettä", + ["NOUN", "VERB", "NOUN", "PUNCT", "ADJ", "NOUN", "CCONJ", "ADJ", "NOUN"], + ["obl", "ROOT", "obj", "punct", "amod", "conj", "cc", "amod", "conj"], + [1, 0, -1, 2, 1, -3, 2, 1, -6], + ["Lounaalla", "salaattia", "maukasta kanaa", "raikasta vettä"], + ), + ( + "Minua houkuttaa maalle muuttaminen talven jälkeen", + ["PRON", "VERB", "NOUN", "NOUN", "NOUN", "ADP"], + ["obj", "ROOT", "nmod", "nsubj", "obl", "case"], + [1, 0, 1, -2, -3, -1], + ["maalle muuttaminen", "talven"], + ), + ( + "Päivän kohokohta oli vierailu museossa kummilasten kanssa", + ["NOUN", "NOUN", "AUX", "NOUN", "NOUN", "NOUN", "ADP"], + ["nmod:poss", "nsubj:cop", "cop", "ROOT", "nmod", "obl", "case"], + [1, 2, 1, 0, -1, -2, -1], + ["Päivän kohokohta", "vierailu museossa", "kummilasten"], + ), + ( + "Yrittäjät maksoivat tuomioistuimen määräämät korvaukset", + ["NOUN", "VERB", "NOUN", "VERB", "NOUN"], + ["nsubj", "ROOT", "nsubj", "acl", "obj"], + [1, 0, 1, 1, -3], + ["Yrittäjät", "tuomioistuimen", "korvaukset"], + ), + ( + "Julkisoikeudelliset tai niihin rinnastettavat saatavat ovat suoraan ulosottokelpoisia", + ["ADJ", "CCONJ", "PRON", "VERB", "NOUN", "AUX", "ADV", "NOUN"], + ["amod", "cc", "obl", "acl", "nsubj:cop", "cop", "advmod", "ROOT"], + [4, 3, 1, 1, 3, 2, 1, 0], + ["Julkisoikeudelliset tai niihin rinnastettavat saatavat", "ulosottokelpoisia"], + ), + ( + "Se oli ala-arvoista käytöstä kaikilta oppilailta, myös valvojaoppilailta", + ["PRON", "AUX", "ADJ", "NOUN", "PRON", "NOUN", "PUNCT", "ADV", "NOUN"], + ["nsubj:cop", "cop", "amod", "ROOT", "det", "nmod", "punct", "advmod", "appos"], + [3, 2, 1, 0, 1, -2, 2, 1, -3], + ["ala-arvoista käytöstä kaikilta oppilailta", "valvojaoppilailta"], + ), + ( + "Isä souti veneellä, jonka hän oli vuokrannut", + ["NOUN", "VERB", "NOUN", "PUNCT", "PRON", "PRON", "AUX", "VERB"], + ["nsubj", "ROOT", "obl", "punct", "obj", "nsubj", "aux", "acl:relcl"], + [1, 0, -1, 4, 3, 2, 1, -5], + ["Isä", "veneellä"], + ), + ( + "Kirja, jonka poimin hyllystä, kertoo norsuista", + ["NOUN", "PUNCT", "PRON", "VERB", "NOUN", "PUNCT", "VERB", "NOUN"], + ["nsubj", "punct", "obj", "acl:relcl", "obl", "punct", "ROOT", "obl"], + [6, 2, 1, -3, -1, 1, 0, -1], + ["Kirja", "hyllystä", "norsuista"], + ), + ( + "Huomenna on päivä, jota olemme odottaneet", + ["NOUN", "AUX", "NOUN", "PUNCT", "PRON", "AUX", "VERB"], + ["ROOT", "cop", "nsubj:cop", "punct", "obj", "aux", "acl:relcl"], + [0, -1, -2, 3, 2, 1, -4], + ["Huomenna", "päivä"], + ), + ( + "Liikkuvuuden lisääminen on yksi korkeakoulutuksen keskeisistä kehittämiskohteista", + ["NOUN", "NOUN", "AUX", "PRON", "NOUN", "ADJ", "NOUN"], + ["nmod:gobj", "nsubj:cop", "cop", "ROOT", "nmod:poss", "amod", "nmod"], + [1, 2, 1, 0, 2, 1, -3], + [ + "Liikkuvuuden lisääminen", + "korkeakoulutuksen keskeisistä kehittämiskohteista", + ], + ), + ( + "Kaupalliset palvelut jätetään yksityisten palveluntarjoajien tarjottavaksi", + ["ADJ", "NOUN", "VERB", "ADJ", "NOUN", "NOUN"], + ["amod", "obj", "ROOT", "amod", "nmod:gsubj", "obl"], + [1, 1, 0, 1, 1, -3], + ["Kaupalliset palvelut", "yksityisten palveluntarjoajien tarjottavaksi"], + ), + ( + "New York tunnetaan kaupunkina, joka ei koskaan nuku", + ["PROPN", "PROPN", "VERB", "NOUN", "PUNCT", "PRON", "AUX", "ADV", "VERB"], + [ + "obj", + "flat:name", + "ROOT", + "obl", + "punct", + "nsubj", + "aux", + "advmod", + "acl:relcl", + ], + [2, -1, 0, -1, 4, 3, 2, 1, -5], + ["New York", "kaupunkina"], + ), + ( + "Loput vihjeet saat herra Möttöseltä", + ["NOUN", "NOUN", "VERB", "NOUN", "PROPN"], + ["compound:nn", "obj", "ROOT", "compound:nn", "obj"], + [1, 1, 0, 1, -2], + ["Loput vihjeet", "herra Möttöseltä"], + ), + ( + "mahdollisuus tukea muita päivystysyksiköitä", + ["NOUN", "VERB", "PRON", "NOUN"], + ["ROOT", "acl", "det", "obj"], + [0, -1, 1, -2], + ["mahdollisuus", "päivystysyksiköitä"], + ), + ( + "sairaanhoitopiirit harjoittavat leikkaustoimintaa alueellaan useammassa sairaalassa", + ["NOUN", "VERB", "NOUN", "NOUN", "ADJ", "NOUN"], + ["nsubj", "ROOT", "obj", "obl", "amod", "obl"], + [1, 0, -1, -1, 1, -3], + [ + "sairaanhoitopiirit", + "leikkaustoimintaa", + "alueellaan", + "useammassa sairaalassa", + ], + ), + ( + "Lain mukaan varhaiskasvatus on suunnitelmallista toimintaa", + ["NOUN", "ADP", "NOUN", "AUX", "ADJ", "NOUN"], + ["obl", "case", "nsubj:cop", "cop", "amod", "ROOT"], + [5, -1, 3, 2, 1, 0], + ["Lain", "varhaiskasvatus", "suunnitelmallista toimintaa"], + ), +] + + +def test_noun_chunks_is_parsed(fi_tokenizer): + """Test that noun_chunks raises Value Error for 'fi' language if Doc is not parsed. + To check this test, we're constructing a Doc + with a new Vocab here and forcing is_parsed to 'False' + to make sure the noun chunks don't run. + """ + doc = fi_tokenizer("Tämä on testi") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +@pytest.mark.parametrize( + "text,pos,deps,heads,expected_noun_chunks", FI_NP_TEST_EXAMPLES +) +def test_fi_noun_chunks(fi_tokenizer, text, pos, deps, heads, expected_noun_chunks): + tokens = fi_tokenizer(text) + + assert len(heads) == len(pos) + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=[head + i for i, head in enumerate(heads)], + deps=deps, + pos=pos, + ) + + noun_chunks = list(doc.noun_chunks) + assert len(noun_chunks) == len(expected_noun_chunks) + for i, np in enumerate(noun_chunks): + assert np.text == expected_noun_chunks[i] diff --git a/spacy/tests/lang/fi/test_text.py b/spacy/tests/lang/fi/test_text.py new file mode 100644 index 0000000..dbb67ad --- /dev/null +++ b/spacy/tests/lang/fi/test_text.py @@ -0,0 +1,24 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10000", True), + ("10,00", True), + ("-999,0", True), + ("yksi", True), + ("kolmetoista", True), + ("viisikymmentä", True), + ("tuhat", True), + ("1/2", True), + ("hevonen", False), + (",", False), + ], +) +def test_fi_lex_attrs_like_number(fi_tokenizer, text, match): + tokens = fi_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/fi/test_tokenizer.py b/spacy/tests/lang/fi/test_tokenizer.py new file mode 100644 index 0000000..2d9f081 --- /dev/null +++ b/spacy/tests/lang/fi/test_tokenizer.py @@ -0,0 +1,84 @@ +import pytest + +ABBREVIATION_TESTS = [ + ( + "Hyvää uutta vuotta t. siht. Niemelä!", + ["Hyvää", "uutta", "vuotta", "t.", "siht.", "Niemelä", "!"], + ), + ("Paino on n. 2.2 kg", ["Paino", "on", "n.", "2.2", "kg"]), + ( + "Vuonna 1 eaa. tapahtui kauheita.", + ["Vuonna", "1", "eaa.", "tapahtui", "kauheita", "."], + ), +] + +HYPHENATED_TESTS = [ + ( + "1700-luvulle sijoittuva taide-elokuva Wikimedia-säätiön Varsinais-Suomen", + [ + "1700-luvulle", + "sijoittuva", + "taide-elokuva", + "Wikimedia-säätiön", + "Varsinais-Suomen", + ], + ) +] + +ABBREVIATION_INFLECTION_TESTS = [ + ( + "VTT:ssa ennen v:ta 2010 suoritetut mittaukset", + ["VTT:ssa", "ennen", "v:ta", "2010", "suoritetut", "mittaukset"], + ), + ("ALV:n osuus on 24 %.", ["ALV:n", "osuus", "on", "24", "%", "."]), + ("Hiihtäjä oli kilpailun 14:s.", ["Hiihtäjä", "oli", "kilpailun", "14:s", "."]), + ("EU:n toimesta tehtiin jotain.", ["EU:n", "toimesta", "tehtiin", "jotain", "."]), +] + +CONTRACTION_TESTS = [ + ( + "Päätimme ettemme tule.", + ["Päätimme", "ett", "emme", "tule", "."], + ["päätimme", "että", "emme", "tule", "."], + ), + ( + "Miksei puhuttaisi?", + ["Miks", "ei", "puhuttaisi", "?"], + ["miksi", "ei", "puhuttaisi", "?"], + ), + ( + "He tottelivat vaikkeivat halunneet", + ["He", "tottelivat", "vaikk", "eivat", "halunneet"], + ["he", "tottelivat", "vaikka", "eivät", "halunneet"], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", ABBREVIATION_TESTS) +def test_fi_tokenizer_abbreviations(fi_tokenizer, text, expected_tokens): + tokens = fi_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text,expected_tokens", HYPHENATED_TESTS) +def test_fi_tokenizer_hyphenated_words(fi_tokenizer, text, expected_tokens): + tokens = fi_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text,expected_tokens", ABBREVIATION_INFLECTION_TESTS) +def test_fi_tokenizer_abbreviation_inflections(fi_tokenizer, text, expected_tokens): + tokens = fi_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text,expected_tokens,expected_norms", CONTRACTION_TESTS) +def test_fi_tokenizer_contractions(fi_tokenizer, text, expected_tokens, expected_norms): + tokens = fi_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + norm_list = [token.norm_ for token in tokens if not token.is_space] + assert expected_tokens == token_list + assert expected_norms == norm_list diff --git a/spacy/tests/lang/fo/__init__.py b/spacy/tests/lang/fo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/fo/test_tokenizer.py b/spacy/tests/lang/fo/test_tokenizer.py new file mode 100644 index 0000000..e61a62b --- /dev/null +++ b/spacy/tests/lang/fo/test_tokenizer.py @@ -0,0 +1,26 @@ +import pytest + +# examples taken from Basic LAnguage Resource Kit 1.0 for Faroese (https://maltokni.fo/en/resources) licensed with CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/) +# fmt: off +FO_TOKEN_EXCEPTION_TESTS = [ + ( + "Eftir løgtingslóg um samsýning og eftirløn landsstýrismanna v.m., skulu løgmaður og landsstýrismenn vanliga siga frá sær størv í almennari tænastu ella privatum virkjum, samtøkum ella stovnum. ", + [ + "Eftir", "løgtingslóg", "um", "samsýning", "og", "eftirløn", "landsstýrismanna", "v.m.", ",", "skulu", "løgmaður", "og", "landsstýrismenn", "vanliga", "siga", "frá", "sær", "størv", "í", "almennari", "tænastu", "ella", "privatum", "virkjum", ",", "samtøkum", "ella", "stovnum", ".", + ], + ), + ( + "Sambandsflokkurin gongur aftur við 2,7 prosentum í mun til valið í 1994, tá flokkurin fekk undirtøku frá 23,4 prosent av veljarunum.", + [ + "Sambandsflokkurin", "gongur", "aftur", "við", "2,7", "prosentum", "í", "mun", "til", "valið", "í", "1994", ",", "tá", "flokkurin", "fekk", "undirtøku", "frá", "23,4", "prosent", "av", "veljarunum", ".", + ], + ), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", FO_TOKEN_EXCEPTION_TESTS) +def test_fo_tokenizer_handles_exception_cases(fo_tokenizer, text, expected_tokens): + tokens = fo_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/fr/__init__.py b/spacy/tests/lang/fr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/fr/test_exceptions.py b/spacy/tests/lang/fr/test_exceptions.py new file mode 100644 index 0000000..d75c653 --- /dev/null +++ b/spacy/tests/lang/fr/test_exceptions.py @@ -0,0 +1,82 @@ +import pytest + + +@pytest.mark.parametrize( + "text", + [ + "aujourd'hui", + "Aujourd'hui", + "prud'hommes", + "prud’hommal", + "audio-numérique", + "Audio-numérique", + "entr'amis", + "entr'abat", + "rentr'ouvertes", + "grand'hamien", + "Châteauneuf-la-Forêt", + "Château-Guibert", + "refox-trottâmes", + # u"K-POP", + # u"K-Pop", + # u"K-pop", + "z'yeutes", + "black-outeront", + "états-unienne", + "courtes-pattes", + "court-pattes", + "saut-de-ski", + "Écourt-Saint-Quentin", + "Bout-de-l'Îlien", + "pet-en-l'air", + ], +) +def test_fr_tokenizer_infix_exceptions(fr_tokenizer, text): + tokens = fr_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["janv.", "juill.", "Dr.", "av.", "sept."]) +def test_fr_tokenizer_handles_abbr(fr_tokenizer, text): + tokens = fr_tokenizer(text) + assert len(tokens) == 1 + + +def test_fr_tokenizer_handles_exc_in_text(fr_tokenizer): + text = "Je suis allé au mois de janv. aux prud’hommes." + tokens = fr_tokenizer(text) + assert len(tokens) == 10 + assert tokens[6].text == "janv." + assert tokens[8].text == "prud’hommes" + + +def test_fr_tokenizer_handles_exc_in_text_2(fr_tokenizer): + text = "Cette après-midi, je suis allé dans un restaurant italo-mexicain." + tokens = fr_tokenizer(text) + assert len(tokens) == 11 + assert tokens[1].text == "après-midi" + assert tokens[9].text == "italo-mexicain" + + +def test_fr_tokenizer_handles_title(fr_tokenizer): + text = "N'est-ce pas génial?" + tokens = fr_tokenizer(text) + assert len(tokens) == 6 + assert tokens[0].text == "N'" + assert tokens[1].text == "est" + assert tokens[2].text == "-ce" + + +def test_fr_tokenizer_handles_title_2(fr_tokenizer): + text = "Est-ce pas génial?" + tokens = fr_tokenizer(text) + assert len(tokens) == 5 + assert tokens[0].text == "Est" + assert tokens[1].text == "-ce" + + +def test_fr_tokenizer_handles_title_3(fr_tokenizer): + text = "Qu'est-ce que tu fais?" + tokens = fr_tokenizer(text) + assert len(tokens) == 7 + assert tokens[0].text == "Qu'" diff --git a/spacy/tests/lang/fr/test_noun_chunks.py b/spacy/tests/lang/fr/test_noun_chunks.py new file mode 100644 index 0000000..d413f1f --- /dev/null +++ b/spacy/tests/lang/fr/test_noun_chunks.py @@ -0,0 +1,231 @@ +import pytest + +from spacy.tokens import Doc + + +# fmt: off +@pytest.mark.parametrize( + "words,heads,deps,pos,chunk_offsets", + [ + # determiner + noun + # un nom -> un nom + ( + ["un", "nom"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0, 2)], + ), + # determiner + noun starting with vowel + # l'heure -> l'heure + ( + ["l'", "heure"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0, 2)], + ), + # determiner + plural noun + # les romans -> les romans + ( + ["les", "romans"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0, 2)], + ), + # det + adj + noun + # Le vieux Londres -> Le vieux Londres + ( + ['Les', 'vieux', 'Londres'], + [2, 2, 2], + ["det", "amod", "ROOT"], + ["DET", "ADJ", "NOUN"], + [(0,3)] + ), + # det + noun + adj + # le nom propre -> le nom propre a proper noun + ( + ["le", "nom", "propre"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0, 3)], + ), + # det + noun + adj plural + # Les chiens bruns -> les chiens bruns + ( + ["Les", "chiens", "bruns"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0, 3)], + ), + # multiple adjectives: one adj before the noun, one adj after the noun + # un nouveau film intéressant -> un nouveau film intéressant + ( + ["un", "nouveau", "film", "intéressant"], + [2, 2, 2, 2], + ["det", "amod", "ROOT", "amod"], + ["DET", "ADJ", "NOUN", "ADJ"], + [(0,4)] + ), + # multiple adjectives, both adjs after the noun + # une personne intelligente et drôle -> une personne intelligente et drôle + ( + ["une", "personne", "intelligente", "et", "drôle"], + [1, 1, 1, 4, 2], + ["det", "ROOT", "amod", "cc", "conj"], + ["DET", "NOUN", "ADJ", "CCONJ", "ADJ"], + [(0,5)] + ), + # relative pronoun + # un bus qui va au ville -> un bus, qui, ville + ( + ['un', 'bus', 'qui', 'va', 'au', 'ville'], + [1, 1, 3, 1, 5, 3], + ['det', 'ROOT', 'nsubj', 'acl:relcl', 'case', 'obl:arg'], + ['DET', 'NOUN', 'PRON', 'VERB', 'ADP', 'NOUN'], + [(0,2), (2,3), (5,6)] + ), + # relative subclause + # Voilà la maison que nous voulons acheter -> la maison, nous That's the house that we want to buy. + ( + ['Voilà', 'la', 'maison', 'que', 'nous', 'voulons', 'acheter'], + [0, 2, 0, 5, 5, 2, 5], + ['ROOT', 'det', 'obj', 'mark', 'nsubj', 'acl:relcl', 'xcomp'], + ['VERB', 'DET', 'NOUN', 'SCONJ', 'PRON', 'VERB', 'VERB'], + [(1,3), (4,5)] + ), + # Person name and title by flat + # Louis XIV -> Louis XIV + ( + ["Louis", "XIV"], + [0, 0], + ["ROOT", "flat:name"], + ["PROPN", "PROPN"], + [(0,2)] + ), + # Organization name by flat + # Nations Unies -> Nations Unies + ( + ["Nations", "Unies"], + [0, 0], + ["ROOT", "flat:name"], + ["PROPN", "PROPN"], + [(0,2)] + ), + # Noun compound, person name created by two flats + # Louise de Bratagne -> Louise de Bratagne + ( + ["Louise", "de", "Bratagne"], + [0, 0, 0], + ["ROOT", "flat:name", "flat:name"], + ["PROPN", "PROPN", "PROPN"], + [(0,3)] + ), + # Noun compound, person name created by two flats + # Louis François Joseph -> Louis François Joseph + ( + ["Louis", "François", "Joseph"], + [0, 0, 0], + ["ROOT", "flat:name", "flat:name"], + ["PROPN", "PROPN", "PROPN"], + [(0,3)] + ), + # one determiner + one noun + one adjective qualified by an adverb + # quelques agriculteurs très riches -> quelques agriculteurs très riches + ( + ["quelques", "agriculteurs", "très", "riches"], + [1, 1, 3, 1], + ['det', 'ROOT', 'advmod', 'amod'], + ['DET', 'NOUN', 'ADV', 'ADJ'], + [(0,4)] + ), + # Two NPs conjuncted + # Il a un chien et un chat -> Il, un chien, un chat + ( + ['Il', 'a', 'un', 'chien', 'et', 'un', 'chat'], + [1, 1, 3, 1, 6, 6, 3], + ['nsubj', 'ROOT', 'det', 'obj', 'cc', 'det', 'conj'], + ['PRON', 'VERB', 'DET', 'NOUN', 'CCONJ', 'DET', 'NOUN'], + [(0,1), (2,4), (5,7)] + + ), + # Two NPs together + # l'écrivain brésilien Aníbal Machado -> l'écrivain brésilien, Aníbal Machado + ( + ["l'", 'écrivain', 'brésilien', 'Aníbal', 'Machado'], + [1, 1, 1, 1, 3], + ['det', 'ROOT', 'amod', 'appos', 'flat:name'], + ['DET', 'NOUN', 'ADJ', 'PROPN', 'PROPN'], + [(0, 3), (3, 5)] + ), + # nmod relation between NPs + # la destruction de la ville -> la destruction, la ville + ( + ['la', 'destruction', 'de', 'la', 'ville'], + [1, 1, 4, 4, 1], + ['det', 'ROOT', 'case', 'det', 'nmod'], + ['DET', 'NOUN', 'ADP', 'DET', 'NOUN'], + [(0,2), (3,5)] + ), + # nmod relation between NPs + # Archiduchesse d’Autriche -> Archiduchesse, Autriche + ( + ['Archiduchesse', 'd’', 'Autriche'], + [0, 2, 0], + ['ROOT', 'case', 'nmod'], + ['NOUN', 'ADP', 'PROPN'], + [(0,1), (2,3)] + ), + # Compounding by nmod, several NPs chained together + # la première usine de drogue du gouvernement -> la première usine, drogue, gouvernement + ( + ["la", "première", "usine", "de", "drogue", "du", "gouvernement"], + [2, 2, 2, 4, 2, 6, 2], + ['det', 'amod', 'ROOT', 'case', 'nmod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN'], + [(0, 3), (4, 5), (6, 7)] + ), + # several NPs + # Traduction du rapport de Susana -> Traduction, rapport, Susana + ( + ['Traduction', 'du', 'raport', 'de', 'Susana'], + [0, 2, 0, 4, 2], + ['ROOT', 'case', 'nmod', 'case', 'nmod'], + ['NOUN', 'ADP', 'NOUN', 'ADP', 'PROPN'], + [(0,1), (2,3), (4,5)] + + ), + # Several NPs + # Le gros chat de Susana et son amie -> Le gros chat, Susana, son amie + ( + ['Le', 'gros', 'chat', 'de', 'Susana', 'et', 'son', 'amie'], + [2, 2, 2, 4, 2, 7, 7, 2], + ['det', 'amod', 'ROOT', 'case', 'nmod', 'cc', 'det', 'conj'], + ['DET', 'ADJ', 'NOUN', 'ADP', 'PROPN', 'CCONJ', 'DET', 'NOUN'], + [(0,3), (4,5), (6,8)] + ), + # Passive subject + # Les nouvelles dépenses sont alimentées par le grand compte bancaire de Clinton -> Les nouvelles dépenses, le grand compte bancaire, Clinton + ( + ['Les', 'nouvelles', 'dépenses', 'sont', 'alimentées', 'par', 'le', 'grand', 'compte', 'bancaire', 'de', 'Clinton'], + [2, 2, 4, 4, 4, 8, 8, 8, 4, 8, 11, 8], + ['det', 'amod', 'nsubj:pass', 'aux:pass', 'ROOT', 'case', 'det', 'amod', 'obl:agent', 'amod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'AUX', 'VERB', 'ADP', 'DET', 'ADJ', 'NOUN', 'ADJ', 'ADP', 'PROPN'], + [(0, 3), (6, 10), (11, 12)] + ) + ], +) +# fmt: on +def test_fr_noun_chunks(fr_vocab, words, heads, deps, pos, chunk_offsets): + doc = Doc(fr_vocab, words=words, heads=heads, deps=deps, pos=pos) + assert [(c.start, c.end) for c in doc.noun_chunks] == chunk_offsets + + +def test_noun_chunks_is_parsed_fr(fr_tokenizer): + """Test that noun_chunks raises Value Error for 'fr' language if Doc is not parsed.""" + doc = fr_tokenizer("Je suis allé à l'école") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/fr/test_prefix_suffix_infix.py b/spacy/tests/lang/fr/test_prefix_suffix_infix.py new file mode 100644 index 0000000..b81ccbc --- /dev/null +++ b/spacy/tests/lang/fr/test_prefix_suffix_infix.py @@ -0,0 +1,23 @@ +import pytest + +from spacy.lang.char_classes import ALPHA +from spacy.lang.punctuation import TOKENIZER_INFIXES +from spacy.language import BaseDefaults, Language + + +@pytest.mark.issue(768) +@pytest.mark.parametrize( + "text,expected_tokens", [("l'avion", ["l'", "avion"]), ("j'ai", ["j'", "ai"])] +) +def test_issue768(text, expected_tokens): + """Allow zero-width 'infix' token during the tokenization process.""" + SPLIT_INFIX = r"(?<=[{a}]\')(?=[{a}])".format(a=ALPHA) + + class FrenchTest(Language): + class Defaults(BaseDefaults): + infixes = TOKENIZER_INFIXES + [SPLIT_INFIX] + + fr_tokenizer_w_infix = FrenchTest().tokenizer + tokens = fr_tokenizer_w_infix(text) + assert len(tokens) == 2 + assert [t.text for t in tokens] == expected_tokens diff --git a/spacy/tests/lang/fr/test_text.py b/spacy/tests/lang/fr/test_text.py new file mode 100644 index 0000000..2c58a1c --- /dev/null +++ b/spacy/tests/lang/fr/test_text.py @@ -0,0 +1,23 @@ +import pytest + +from spacy.lang.fr.lex_attrs import like_num + + +def test_tokenizer_handles_long_text(fr_tokenizer): + text = """L'histoire du TAL commence dans les années 1950, bien que l'on puisse \ +trouver des travaux antérieurs. En 1950, Alan Turing éditait un article \ +célèbre sous le titre « Computing machinery and intelligence » qui propose ce \ +qu'on appelle à présent le test de Turing comme critère d'intelligence. \ +Ce critère dépend de la capacité d'un programme informatique de personnifier \ +un humain dans une conversation écrite en temps réel, de façon suffisamment \ +convaincante que l'interlocuteur humain ne peut distinguer sûrement — sur la \ +base du seul contenu de la conversation — s'il interagit avec un programme \ +ou avec un autre vrai humain.""" + tokens = fr_tokenizer(text) + assert len(tokens) == 113 + + +@pytest.mark.parametrize("word", ["onze", "onzième"]) +def test_fr_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/ga/__init__.py b/spacy/tests/lang/ga/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ga/test_tokenizer.py b/spacy/tests/lang/ga/test_tokenizer.py new file mode 100644 index 0000000..0c16b27 --- /dev/null +++ b/spacy/tests/lang/ga/test_tokenizer.py @@ -0,0 +1,15 @@ +import pytest + +# fmt: off +GA_TOKEN_EXCEPTION_TESTS = [ + ("Niall Ó Domhnaill, Rialtas na hÉireann 1977 (lch. 600).", ["Niall", "Ó", "Domhnaill", ",", "Rialtas", "na", "hÉireann", "1977", "(", "lch.", "600", ")", "."]), + ("Daoine a bhfuil Gaeilge acu, m.sh. tusa agus mise", ["Daoine", "a", "bhfuil", "Gaeilge", "acu", ",", "m.sh.", "tusa", "agus", "mise"]) +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", GA_TOKEN_EXCEPTION_TESTS) +def test_ga_tokenizer_handles_exception_cases(ga_tokenizer, text, expected_tokens): + tokens = ga_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/grc/__init__.py b/spacy/tests/lang/grc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/grc/test_text.py b/spacy/tests/lang/grc/test_text.py new file mode 100644 index 0000000..5d8317c --- /dev/null +++ b/spacy/tests/lang/grc/test_text.py @@ -0,0 +1,23 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("ι", True), + ("α", True), + ("ϟα", True), + ("ἑκατόν", True), + ("ἐνακόσια", True), + ("δισχίλια", True), + ("μύρια", True), + ("εἷς", True), + ("λόγος", False), + (",", False), + ("λβ", True), + ], +) +def test_lex_attrs_like_number(grc_tokenizer, text, match): + tokens = grc_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/grc/test_tokenizer.py b/spacy/tests/lang/grc/test_tokenizer.py new file mode 100644 index 0000000..9f29b90 --- /dev/null +++ b/spacy/tests/lang/grc/test_tokenizer.py @@ -0,0 +1,17 @@ +import pytest + +# fmt: off +GRC_TOKEN_EXCEPTION_TESTS = [ + ("τὸ 〈τῆς〉 φιλοσοφίας ἔργον ἔνιοί φασιν ἀπὸ ⟦βαρβάρων⟧ ἄρξαι.", ["τὸ", "〈", "τῆς", "〉", "φιλοσοφίας", "ἔργον", "ἔνιοί", "φασιν", "ἀπὸ", "⟦", "βαρβάρων", "⟧", "ἄρξαι", "."]), + ("τὴν δὲ τῶν Αἰγυπτίων φιλοσοφίαν εἶναι τοιαύτην περί τε †θεῶν† καὶ ὑπὲρ δικαιοσύνης.", ["τὴν", "δὲ", "τῶν", "Αἰγυπτίων", "φιλοσοφίαν", "εἶναι", "τοιαύτην", "περί", "τε", "†", "θεῶν", "†", "καὶ", "ὑπὲρ", "δικαιοσύνης", "."]), + ("⸏πόσις δ' Ἐρεχθεύς ἐστί μοι σεσωσμένος⸏", ["⸏", "πόσις", "δ'", "Ἐρεχθεύς", "ἐστί", "μοι", "σεσωσμένος", "⸏"]), + ("⸏ὔπνον ἴδωμεν⸎", ["⸏", "ὔπνον", "ἴδωμεν", "⸎"]), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", GRC_TOKEN_EXCEPTION_TESTS) +def test_grc_tokenizer(grc_tokenizer, text, expected_tokens): + tokens = grc_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/gu/__init__.py b/spacy/tests/lang/gu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/gu/test_text.py b/spacy/tests/lang/gu/test_text.py new file mode 100644 index 0000000..2d25116 --- /dev/null +++ b/spacy/tests/lang/gu/test_text.py @@ -0,0 +1,16 @@ +import pytest + + +def test_gu_tokenizer_handlers_long_text(gu_tokenizer): + text = """પશ્ચિમ ભારતમાં આવેલું ગુજરાત રાજ્ય જે વ્યક્તિઓની માતૃભૂમિ છે""" + tokens = gu_tokenizer(text) + assert len(tokens) == 9 + + +@pytest.mark.parametrize( + "text,length", + [("ગુજરાતીઓ ખાવાના શોખીન માનવામાં આવે છે", 6), ("ખેતરની ખેડ કરવામાં આવે છે.", 5)], +) +def test_gu_tokenizer_handles_cnts(gu_tokenizer, text, length): + tokens = gu_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/he/__init__.py b/spacy/tests/lang/he/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/he/test_tokenizer.py b/spacy/tests/lang/he/test_tokenizer.py new file mode 100644 index 0000000..15d0593 --- /dev/null +++ b/spacy/tests/lang/he/test_tokenizer.py @@ -0,0 +1,70 @@ +import pytest + +from spacy.lang.he.lex_attrs import like_num + + +@pytest.mark.parametrize( + "text,expected_tokens", + [("פייתון היא שפת תכנות דינמית", ["פייתון", "היא", "שפת", "תכנות", "דינמית"])], +) +def test_he_tokenizer_handles_abbreviation(he_tokenizer, text, expected_tokens): + tokens = he_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize( + "text,expected_tokens", + [ + ( + "עקבת אחריו בכל רחבי המדינה.", + ["עקבת", "אחריו", "בכל", "רחבי", "המדינה", "."], + ), + ( + "עקבת אחריו בכל רחבי המדינה?", + ["עקבת", "אחריו", "בכל", "רחבי", "המדינה", "?"], + ), + ( + "עקבת אחריו בכל רחבי המדינה!", + ["עקבת", "אחריו", "בכל", "רחבי", "המדינה", "!"], + ), + ( + "עקבת אחריו בכל רחבי המדינה..", + ["עקבת", "אחריו", "בכל", "רחבי", "המדינה", ".."], + ), + ( + "עקבת אחריו בכל רחבי המדינה...", + ["עקבת", "אחריו", "בכל", "רחבי", "המדינה", "..."], + ), + ], +) +def test_he_tokenizer_handles_punct(he_tokenizer, text, expected_tokens): + tokens = he_tokenizer(text) + assert expected_tokens == [token.text for token in tokens] + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("999.0", True), + ("אחד", True), + ("שתיים", True), + ("מליון", True), + ("כלב", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(he_tokenizer, text, match): + tokens = he_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["שלישי", "מליון", "עשירי", "מאה", "עשר", "אחד עשר"]) +def test_he_lex_attrs_like_number_for_ordinal(word): + assert like_num(word) diff --git a/spacy/tests/lang/hi/__init__.py b/spacy/tests/lang/hi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/hi/test_lex_attrs.py b/spacy/tests/lang/hi/test_lex_attrs.py new file mode 100644 index 0000000..2d8d4a5 --- /dev/null +++ b/spacy/tests/lang/hi/test_lex_attrs.py @@ -0,0 +1,44 @@ +import pytest + +from spacy.lang.hi.lex_attrs import like_num, norm + + +def test_hi_tokenizer_handles_long_text(hi_tokenizer): + text = """ +ये कहानी 1900 के दशक की है। कौशल्या (स्मिता जयकर) को पता चलता है कि उसका +छोटा बेटा, देवदास (शाहरुख खान) वापस घर आ रहा है। देवदास 10 साल पहले कानून की +पढ़ाई करने के लिए इंग्लैंड गया था। उसके लौटने की खुशी में ये बात कौशल्या अपनी पड़ोस +में रहने वाली सुमित्रा (किरण खेर) को भी बता देती है। इस खबर से वो भी खुश हो जाती है। +""" + tokens = hi_tokenizer(text) + assert len(tokens) == 86 + + +@pytest.mark.parametrize( + "word,word_norm", + [ + ("चलता", "चल"), + ("पढ़ाई", "पढ़"), + ("देती", "दे"), + ("जाती", "ज"), + ("मुस्कुराकर", "मुस्कुर"), + ], +) +def test_hi_norm(word, word_norm): + assert norm(word) == word_norm + + +@pytest.mark.parametrize( + "word", + ["१९८७", "1987", "१२,२६७", "उन्नीस", "पाँच", "नवासी", "५/१०"], +) +def test_hi_like_num(word): + assert like_num(word) + + +@pytest.mark.parametrize( + "word", + ["पहला", "तृतीय", "निन्यानवेवाँ", "उन्नीस", "तिहत्तरवाँ", "छत्तीसवाँ"], +) +def test_hi_like_num_ordinal_words(word): + assert like_num(word) diff --git a/spacy/tests/lang/hi/test_text.py b/spacy/tests/lang/hi/test_text.py new file mode 100644 index 0000000..837dc30 --- /dev/null +++ b/spacy/tests/lang/hi/test_text.py @@ -0,0 +1,12 @@ +import pytest + +from spacy.lang.hi import Hindi + + +@pytest.mark.issue(3625) +def test_issue3625(): + """Test that default punctuation rules applies to hindi unicode characters""" + nlp = Hindi() + doc = nlp("hi. how हुए. होटल, होटल") + expected = ["hi", ".", "how", "हुए", ".", "होटल", ",", "होटल"] + assert [token.text for token in doc] == expected diff --git a/spacy/tests/lang/hr/__init__.py b/spacy/tests/lang/hr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/hr/test_text.py b/spacy/tests/lang/hr/test_text.py new file mode 100644 index 0000000..82e65af --- /dev/null +++ b/spacy/tests/lang/hr/test_text.py @@ -0,0 +1,26 @@ +import pytest + + +def test_long_text(hr_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +uzimajući u obzir da ta deklaracija nastoji osigurati opće i djelotvorno +priznanje i poštovanje u njoj proglašenih prava; +uzimajući u obzir da je cilj Vijeća Europe postizanje većeg jedinstva +njegovih članica, i da je jedan od načina postizanja toga cilja +očuvanje i daljnje ostvarivanje ljudskih prava i temeljnih sloboda; +potvrđujući svoju duboku privrženost tim temeljnim slobodama +koje su osnova pravde i mira u svijetu i koje su najbolje zaštićene +istinskom političkom demokracijom s jedne strane te zajedničkim +razumijevanjem i poštovanjem ljudskih prava o kojima te slobode +ovise s druge strane; +""" + tokens = hr_tokenizer(text) + assert len(tokens) == 105 + + +@pytest.mark.xfail +def test_ordinal_number(hr_tokenizer): + text = "10. prosinca 1948" + tokens = hr_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/hr/test_tokenizer.py b/spacy/tests/lang/hr/test_tokenizer.py new file mode 100644 index 0000000..dace33b --- /dev/null +++ b/spacy/tests/lang/hr/test_tokenizer.py @@ -0,0 +1,31 @@ +import pytest + +HR_BASIC_TOKENIZATION_TESTS = [ + ( + "Nitko se ne smije podvrgnuti mučenju ni nečovječnom ili " + "ponižavajućem postupanju ili kazni.", + [ + "Nitko", + "se", + "ne", + "smije", + "podvrgnuti", + "mučenju", + "ni", + "nečovječnom", + "ili", + "ponižavajućem", + "postupanju", + "ili", + "kazni", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", HR_BASIC_TOKENIZATION_TESTS) +def test_hr_tokenizer_basic(hr_tokenizer, text, expected_tokens): + tokens = hr_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/hsb/__init__.py b/spacy/tests/lang/hsb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/hsb/test_text.py b/spacy/tests/lang/hsb/test_text.py new file mode 100644 index 0000000..aaa4984 --- /dev/null +++ b/spacy/tests/lang/hsb/test_text.py @@ -0,0 +1,25 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("jedne", True), + ("dwanaće", True), + ("milion", True), + ("sto", True), + ("załožene", False), + ("wona", False), + ("powšitkownej", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(hsb_tokenizer, text, match): + tokens = hsb_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/hsb/test_tokenizer.py b/spacy/tests/lang/hsb/test_tokenizer.py new file mode 100644 index 0000000..a3ec89b --- /dev/null +++ b/spacy/tests/lang/hsb/test_tokenizer.py @@ -0,0 +1,32 @@ +import pytest + +HSB_BASIC_TOKENIZATION_TESTS = [ + ( + "Hornjoserbšćina wobsteji resp. wobsteješe z wjacorych dialektow, kotrež so zdźěla chětro wot so rozeznawachu.", + [ + "Hornjoserbšćina", + "wobsteji", + "resp.", + "wobsteješe", + "z", + "wjacorych", + "dialektow", + ",", + "kotrež", + "so", + "zdźěla", + "chětro", + "wot", + "so", + "rozeznawachu", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", HSB_BASIC_TOKENIZATION_TESTS) +def test_hsb_tokenizer_basic(hsb_tokenizer, text, expected_tokens): + tokens = hsb_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/ht/__init__.py b/spacy/tests/lang/ht/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ht/test_exceptions.py b/spacy/tests/lang/ht/test_exceptions.py new file mode 100644 index 0000000..ea2e2b2 --- /dev/null +++ b/spacy/tests/lang/ht/test_exceptions.py @@ -0,0 +1,44 @@ +import pytest + + +def test_ht_tokenizer_handles_basic_contraction(ht_tokenizer): + text = "m'ap ri" + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == "m'" + assert tokens[1].text == "ap" + assert tokens[2].text == "ri" + + text = "mwen di'w non!" + tokens = ht_tokenizer(text) + assert len(tokens) == 5 + assert tokens[0].text == "mwen" + assert tokens[1].text == "di" + assert tokens[2].text == "'w" + assert tokens[3].text == "non" + assert tokens[4].text == "!" + + +@pytest.mark.parametrize("text", ["Dr."]) +def test_ht_tokenizer_handles_basic_abbreviation(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].text == text + + +def test_ht_tokenizer_full_sentence(ht_tokenizer): + text = "Si'm ka vini, m'ap pale ak li." + tokens = [t.text for t in ht_tokenizer(text)] + assert tokens == [ + "Si", + "'m", + "ka", + "vini", + ",", + "m'", + "ap", + "pale", + "ak", + "li", + ".", + ] diff --git a/spacy/tests/lang/ht/test_noun_chunks.py b/spacy/tests/lang/ht/test_noun_chunks.py new file mode 100644 index 0000000..fcefd7d --- /dev/null +++ b/spacy/tests/lang/ht/test_noun_chunks.py @@ -0,0 +1,45 @@ +import pytest + +from spacy.tokens import Doc + + +@pytest.fixture +def doc(ht_vocab): + words = ["Pitit", "gen", "gwo", "pwoblèm", "ak", "kontwòl"] + heads = [1, 1, 5, 5, 3, 3] + deps = ["nsubj", "ROOT", "amod", "obj", "case", "nmod"] + pos = ["NOUN", "VERB", "ADJ", "NOUN", "ADP", "NOUN"] + return Doc(ht_vocab, words=words, heads=heads, deps=deps, pos=pos) + + +def test_noun_chunks_is_parsed(ht_tokenizer): + """Test that noun_chunks raises Value Error for 'ht' language if Doc is not parsed.""" + doc = ht_tokenizer("Sa a se yon fraz") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +def test_ht_noun_chunks_not_nested(doc, ht_vocab): + """Test that each token only appears in one noun chunk at most""" + word_occurred = {} + chunks = list(doc.noun_chunks) + assert len(chunks) > 1 + for chunk in chunks: + for word in chunk: + word_occurred.setdefault(word.text, 0) + word_occurred[word.text] += 1 + assert len(word_occurred) > 0 + for word, freq in word_occurred.items(): + assert freq == 1, (word, [chunk.text for chunk in doc.noun_chunks]) + + +def test_noun_chunks_span(doc, ht_tokenizer): + """Test that the span.noun_chunks property works correctly""" + doc_chunks = list(doc.noun_chunks) + span = doc[0:3] + span_chunks = list(span.noun_chunks) + assert 0 < len(span_chunks) < len(doc_chunks) + for chunk in span_chunks: + assert chunk in doc_chunks + assert chunk.start >= 0 + assert chunk.end <= 3 diff --git a/spacy/tests/lang/ht/test_prefix_suffix_infix.py b/spacy/tests/lang/ht/test_prefix_suffix_infix.py new file mode 100644 index 0000000..5ff409c --- /dev/null +++ b/spacy/tests/lang/ht/test_prefix_suffix_infix.py @@ -0,0 +1,132 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(ka)"]) +def test_ht_tokenizer_splits_no_special(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["m'ap"]) +def test_ht_tokenizer_splits_no_punct(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(m'ap"]) +def test_ht_tokenizer_splits_prefix_punct(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["m'ap)"]) +def test_ht_tokenizer_splits_suffix_punct(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(m'ap)"]) +def test_ht_tokenizer_splits_even_wrap(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["(m'ap?)"]) +def test_ht_tokenizer_splits_uneven_wrap(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 5 + + +@pytest.mark.parametrize( + "text,length", [("Ozetazini.", 2), ("Frans.", 2), ("(Ozetazini.", 3)] +) +def test_ht_tokenizer_splits_prefix_interact(ht_tokenizer, text, length): + tokens = ht_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["Ozetazini.)"]) +def test_ht_tokenizer_splits_suffix_interact(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(Ozetazini.)"]) +def test_ht_tokenizer_splits_even_wrap_interact(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["(Ozetazini?)"]) +def test_ht_tokenizer_splits_uneven_wrap_interact(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text", ["pi-bon"]) +def test_ht_tokenizer_splits_hyphens(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["0.1-13.5", "0.0-0.1", "103.27-300"]) +def test_ht_tokenizer_splits_numeric_range(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["pi.Bon", "Bon.Jour"]) +def test_ht_tokenizer_splits_period_infix(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Bonjou,moun", "youn,de"]) +def test_ht_tokenizer_splits_comma_infix(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["pi...Bon", "pi...bon"]) +def test_ht_tokenizer_splits_ellipsis_infix(ht_tokenizer, text): + tokens = ht_tokenizer(text) + assert len(tokens) == 3 + + +def test_ht_tokenizer_splits_double_hyphen_infix(ht_tokenizer): + tokens = ht_tokenizer("Pa vrè--men ou konnen--mwen renmen w.") + assert tokens[0].text == "Pa" + assert tokens[1].text == "vrè" + assert tokens[2].text == "--" + assert tokens[3].text == "men" + assert tokens[4].text == "ou" + assert tokens[5].text == "konnen" + assert tokens[6].text == "--" + assert tokens[7].text == "mwen" + assert tokens[8].text == "renmen" + assert tokens[9].text == "w" + assert tokens[10].text == "." + + +def test_ht_tokenizer_splits_period_abbr(ht_tokenizer): + text = "Jodi a se Madi.Mr." + tokens = ht_tokenizer(text) + assert len(tokens) == 7 + assert tokens[0].text == "Jodi" + assert tokens[1].text == "a" + assert tokens[2].text == "se" + assert tokens[3].text == "Madi" + assert tokens[4].text == "." + assert tokens[5].text == "Mr" + assert tokens[6].text == "." + + +def test_ht_tokenizer_splits_paren_period(ht_tokenizer): + tokens = ht_tokenizer("M ap teste sa (pou kounye a).") + words = [t.text for t in tokens] + assert "a" in words + assert ")" in words + assert "." in words diff --git a/spacy/tests/lang/ht/test_text.py b/spacy/tests/lang/ht/test_text.py new file mode 100644 index 0000000..e63299f --- /dev/null +++ b/spacy/tests/lang/ht/test_text.py @@ -0,0 +1,78 @@ +import pytest + +from spacy.lang.ht.lex_attrs import like_num, norm_custom + + +def test_ht_tokenizer_handles_long_text(ht_tokenizer): + text = """Onè ap fèt pou ansyen lidè Pati Travayè Britanik + +Moun atravè lemond ap voye onè pou ansyen lidè +Pati Travayè a, John Smith, ki mouri pi bonè jodi a apre li te fè yon gwo kriz kadyak a laj 55 an. + +Nan Washington, Depatman Deta Etazini pibliye yon deklarasyon ki eksprime "regre lanmò twò bonè" avoka ak palmantè eskoze a. + +"Misye Smith, pandan tout karyè li ki te make ak distenksyon""" + tokens = ht_tokenizer(text) + assert len(tokens) == 84 + + +@pytest.mark.parametrize( + "text,length", + [ + ("Map manje gato a pandan map gade televizyon lem lakay mwen.", 15), + ("M'ap vini, eske wap la avek lajan'm? Si oui, di'l non pou fre'w.", 22), + ("M ap teste sa (pou kounye a).", 10), + ], +) +def test_ht_tokenizer_handles_cnts(ht_tokenizer, text, length): + tokens = ht_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("999.0", True), + ("en", True), + ("de", True), + ("milya", True), + ("dog", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(ht_tokenizer, text, match): + tokens = ht_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize( + "word", ["ventyèm", "Milyonnyèm", "3yèm", "Santyèm", "25yèm", "52yèm"] +) +def test_ht_lex_attrs_like_number_for_ordinal(word): + assert like_num(word) + + +@pytest.mark.parametrize("word", ["onz"]) +def test_ht_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) + + +@pytest.mark.parametrize( + "word, expected", + [ + ("'m", "mwen"), + ("'n", "nou"), + ("'l", "li"), + ("'y", "yo"), + ("'w", "ou"), + ], +) +def test_ht_lex_attrs_norm_custom(word, expected): + assert norm_custom(word) == expected diff --git a/spacy/tests/lang/hu/__init__.py b/spacy/tests/lang/hu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/hu/test_tokenizer.py b/spacy/tests/lang/hu/test_tokenizer.py new file mode 100644 index 0000000..30f3e94 --- /dev/null +++ b/spacy/tests/lang/hu/test_tokenizer.py @@ -0,0 +1,321 @@ +import pytest + +DEFAULT_TESTS = [ + ("N. kormányzósági\nszékhely.", ["N.", "kormányzósági", "székhely", "."]), + pytest.param( + "A .hu egy tld.", ["A", ".hu", "egy", "tld", "."], marks=pytest.mark.xfail() + ), + ("Az egy.ketto pelda.", ["Az", "egy.ketto", "pelda", "."]), + ("A pl. rovidites.", ["A", "pl.", "rovidites", "."]), + ("A S.M.A.R.T. szo.", ["A", "S.M.A.R.T.", "szo", "."]), + pytest.param("A .hu.", ["A", ".hu", "."], marks=pytest.mark.xfail()), + ("Az egy.ketto.", ["Az", "egy.ketto", "."]), + ("A pl.", ["A", "pl."]), + ("A S.M.A.R.T.", ["A", "S.M.A.R.T."]), + ("Egy..ket.", ["Egy", "..", "ket", "."]), + ("Valami... van.", ["Valami", "...", "van", "."]), + ("Valami ...van...", ["Valami", "...", "van", "..."]), + ("Valami...", ["Valami", "..."]), + ("Valami ...", ["Valami", "..."]), + ("Valami ... más.", ["Valami", "...", "más", "."]), + ("Soha nem lesz!", ["Soha", "nem", "lesz", "!"]), + ("Soha nem lesz?", ["Soha", "nem", "lesz", "?"]), +] + +HYPHEN_TESTS = [ + ( + "Egy -nak, -jaiért, -magyar, bel- van.", + ["Egy", "-nak", ",", "-jaiért", ",", "-magyar", ",", "bel-", "van", "."], + ), + ("Szabolcs-Szatmár-Bereg megye", ["Szabolcs-Szatmár-Bereg", "megye"]), + ("Egy -nak.", ["Egy", "-nak", "."]), + ("Egy bel-.", ["Egy", "bel-", "."]), + ("Dinnye-domb-.", ["Dinnye-domb-", "."]), + ("Ezen -e elcsatangolt.", ["Ezen", "-e", "elcsatangolt", "."]), + ("Lakik-e", ["Lakik", "-e"]), + ("A--B", ["A", "--", "B"]), + ("Lakik-e?", ["Lakik", "-e", "?"]), + ("Lakik-e.", ["Lakik", "-e", "."]), + ("Lakik-e...", ["Lakik", "-e", "..."]), + ("Lakik-e... van.", ["Lakik", "-e", "...", "van", "."]), + ("Lakik-e van?", ["Lakik", "-e", "van", "?"]), + ("Lakik-elem van?", ["Lakik-elem", "van", "?"]), + ("Az életbiztosításáról- egy.", ["Az", "életbiztosításáról-", "egy", "."]), + ("Van lakik-elem.", ["Van", "lakik-elem", "."]), + ("A 7-es busz?", ["A", "7-es", "busz", "?"]), + ("A 7-es?", ["A", "7-es", "?"]), + ("A 7-es.", ["A", "7-es", "."]), + ("Ez (lakik)-e?", ["Ez", "(", "lakik", ")", "-e", "?"]), + ("A %-sal.", ["A", "%-sal", "."]), + ("A $-sal.", ["A", "$-sal", "."]), + ("A CD-ROM-okrol.", ["A", "CD-ROM-okrol", "."]), +] + +NUMBER_TESTS = [ + ("A 2b van.", ["A", "2b", "van", "."]), + ("A 2b-ben van.", ["A", "2b-ben", "van", "."]), + ("A 2b.", ["A", "2b", "."]), + ("A 2b-ben.", ["A", "2b-ben", "."]), + ("A 3.b van.", ["A", "3.b", "van", "."]), + ("A 3.b-ben van.", ["A", "3.b-ben", "van", "."]), + ("A 3.b.", ["A", "3.b", "."]), + ("A 3.b-ben.", ["A", "3.b-ben", "."]), + ("A 1:20:36.7 van.", ["A", "1:20:36.7", "van", "."]), + ("A 1:20:36.7-ben van.", ["A", "1:20:36.7-ben", "van", "."]), + ("A 1:20:36.7-ben.", ["A", "1:20:36.7-ben", "."]), + ("A 1:35 van.", ["A", "1:35", "van", "."]), + ("A 1:35-ben van.", ["A", "1:35-ben", "van", "."]), + ("A 1:35-ben.", ["A", "1:35-ben", "."]), + ("A 1.35 van.", ["A", "1.35", "van", "."]), + ("A 1.35-ben van.", ["A", "1.35-ben", "van", "."]), + ("A 1.35-ben.", ["A", "1.35-ben", "."]), + ("A 4:01,95 van.", ["A", "4:01,95", "van", "."]), + ("A 4:01,95-ben van.", ["A", "4:01,95-ben", "van", "."]), + ("A 4:01,95-ben.", ["A", "4:01,95-ben", "."]), + ("A 10--12 van.", ["A", "10--12", "van", "."]), + ("A 10--12-ben van.", ["A", "10--12-ben", "van", "."]), + ("A 10--12-ben.", ["A", "10--12-ben", "."]), + ("A 10‐12 van.", ["A", "10‐12", "van", "."]), + ("A 10‐12-ben van.", ["A", "10‐12-ben", "van", "."]), + ("A 10‐12-ben.", ["A", "10‐12-ben", "."]), + ("A 10‑12 van.", ["A", "10‑12", "van", "."]), + ("A 10‑12-ben van.", ["A", "10‑12-ben", "van", "."]), + ("A 10‑12-ben.", ["A", "10‑12-ben", "."]), + ("A 10‒12 van.", ["A", "10‒12", "van", "."]), + ("A 10‒12-ben van.", ["A", "10‒12-ben", "van", "."]), + ("A 10‒12-ben.", ["A", "10‒12-ben", "."]), + ("A 10–12 van.", ["A", "10–12", "van", "."]), + ("A 10–12-ben van.", ["A", "10–12-ben", "van", "."]), + ("A 10–12-ben.", ["A", "10–12-ben", "."]), + ("A 10—12 van.", ["A", "10—12", "van", "."]), + ("A 10—12-ben van.", ["A", "10—12-ben", "van", "."]), + ("A 10—12-ben.", ["A", "10—12-ben", "."]), + ("A 10―12 van.", ["A", "10―12", "van", "."]), + ("A 10―12-ben van.", ["A", "10―12-ben", "van", "."]), + ("A 10―12-ben.", ["A", "10―12-ben", "."]), + ("A -23,12 van.", ["A", "-23,12", "van", "."]), + ("A -23,12-ben van.", ["A", "-23,12-ben", "van", "."]), + ("A -23,12-ben.", ["A", "-23,12-ben", "."]), + ("A 2+3 van.", ["A", "2+3", "van", "."]), + ("A 2<3 van.", ["A", "2<3", "van", "."]), + ("A 2=3 van.", ["A", "2=3", "van", "."]), + ("A 2÷3 van.", ["A", "2÷3", "van", "."]), + ("A 1=(2÷3)-2/5 van.", ["A", "1=(2÷3)-2/5", "van", "."]), + ("A 2 +3 van.", ["A", "2", "+3", "van", "."]), + ("A 2+ 3 van.", ["A", "2", "+", "3", "van", "."]), + ("A 2 + 3 van.", ["A", "2", "+", "3", "van", "."]), + ("A 2*3 van.", ["A", "2*3", "van", "."]), + ("A 2 *3 van.", ["A", "2", "*", "3", "van", "."]), + ("A 2* 3 van.", ["A", "2", "*", "3", "van", "."]), + ("A 2 * 3 van.", ["A", "2", "*", "3", "van", "."]), + ("A C++ van.", ["A", "C++", "van", "."]), + ("A C++-ben van.", ["A", "C++-ben", "van", "."]), + ("A C++.", ["A", "C++", "."]), + ("A C++-ben.", ["A", "C++-ben", "."]), + ("A 2003. I. 06. van.", ["A", "2003.", "I.", "06.", "van", "."]), + ("A 2003. I. 06-ben van.", ["A", "2003.", "I.", "06-ben", "van", "."]), + ("A 2003. I. 06.", ["A", "2003.", "I.", "06."]), + ("A 2003. I. 06-ben.", ["A", "2003.", "I.", "06-ben", "."]), + ("A 2003. 01. 06. van.", ["A", "2003.", "01.", "06.", "van", "."]), + ("A 2003. 01. 06-ben van.", ["A", "2003.", "01.", "06-ben", "van", "."]), + ("A 2003. 01. 06.", ["A", "2003.", "01.", "06."]), + ("A 2003. 01. 06-ben.", ["A", "2003.", "01.", "06-ben", "."]), + ("A IV. 12. van.", ["A", "IV.", "12.", "van", "."]), + ("A IV. 12-ben van.", ["A", "IV.", "12-ben", "van", "."]), + ("A IV. 12.", ["A", "IV.", "12."]), + ("A IV. 12-ben.", ["A", "IV.", "12-ben", "."]), + ("A 2003.01.06. van.", ["A", "2003.01.06.", "van", "."]), + ("A 2003.01.06-ben van.", ["A", "2003.01.06-ben", "van", "."]), + ("A 2003.01.06.", ["A", "2003.01.06."]), + ("A 2003.01.06-ben.", ["A", "2003.01.06-ben", "."]), + ("A IV.12. van.", ["A", "IV.12.", "van", "."]), + ("A IV.12-ben van.", ["A", "IV.12-ben", "van", "."]), + ("A IV.12.", ["A", "IV.12."]), + ("A IV.12-ben.", ["A", "IV.12-ben", "."]), + ("A 1.1.2. van.", ["A", "1.1.2.", "van", "."]), + ("A 1.1.2-ben van.", ["A", "1.1.2-ben", "van", "."]), + ("A 1.1.2.", ["A", "1.1.2."]), + ("A 1.1.2-ben.", ["A", "1.1.2-ben", "."]), + ("A 1,5--2,5 van.", ["A", "1,5--2,5", "van", "."]), + ("A 1,5--2,5-ben van.", ["A", "1,5--2,5-ben", "van", "."]), + ("A 1,5--2,5-ben.", ["A", "1,5--2,5-ben", "."]), + ("A 3,14 van.", ["A", "3,14", "van", "."]), + ("A 3,14-ben van.", ["A", "3,14-ben", "van", "."]), + ("A 3,14-ben.", ["A", "3,14-ben", "."]), + ("A 3.14 van.", ["A", "3.14", "van", "."]), + ("A 3.14-ben van.", ["A", "3.14-ben", "van", "."]), + ("A 3.14-ben.", ["A", "3.14-ben", "."]), + ("A 15. van.", ["A", "15.", "van", "."]), + ("A 15-ben van.", ["A", "15-ben", "van", "."]), + ("A 15-ben.", ["A", "15-ben", "."]), + ("A 15.-ben van.", ["A", "15.-ben", "van", "."]), + ("A 15.-ben.", ["A", "15.-ben", "."]), + ("A 2002--2003. van.", ["A", "2002--2003.", "van", "."]), + ("A 2002--2003-ben van.", ["A", "2002--2003-ben", "van", "."]), + ("A 2002-2003-ben.", ["A", "2002-2003-ben", "."]), + ("A +0,99% van.", ["A", "+0,99%", "van", "."]), + ("A -0,99% van.", ["A", "-0,99%", "van", "."]), + ("A -0,99%-ben van.", ["A", "-0,99%-ben", "van", "."]), + ("A -0,99%.", ["A", "-0,99%", "."]), + ("A -0,99%-ben.", ["A", "-0,99%-ben", "."]), + ("A 10--20% van.", ["A", "10--20%", "van", "."]), + ("A 10--20%-ben van.", ["A", "10--20%-ben", "van", "."]), + ("A 10--20%.", ["A", "10--20%", "."]), + ("A 10--20%-ben.", ["A", "10--20%-ben", "."]), + ("A 99§ van.", ["A", "99§", "van", "."]), + ("A 99§-ben van.", ["A", "99§-ben", "van", "."]), + ("A 99§-ben.", ["A", "99§-ben", "."]), + ("A 10--20§ van.", ["A", "10--20§", "van", "."]), + ("A 10--20§-ben van.", ["A", "10--20§-ben", "van", "."]), + ("A 10--20§-ben.", ["A", "10--20§-ben", "."]), + ("A 99° van.", ["A", "99°", "van", "."]), + ("A 99°-ben van.", ["A", "99°-ben", "van", "."]), + ("A 99°-ben.", ["A", "99°-ben", "."]), + ("A 10--20° van.", ["A", "10--20°", "van", "."]), + ("A 10--20°-ben van.", ["A", "10--20°-ben", "van", "."]), + ("A 10--20°-ben.", ["A", "10--20°-ben", "."]), + ("A °C van.", ["A", "°C", "van", "."]), + ("A °C-ben van.", ["A", "°C-ben", "van", "."]), + ("A °C.", ["A", "°C", "."]), + ("A °C-ben.", ["A", "°C-ben", "."]), + ("A 100°C van.", ["A", "100°C", "van", "."]), + ("A 100°C-ben van.", ["A", "100°C-ben", "van", "."]), + ("A 100°C.", ["A", "100°C", "."]), + ("A 100°C-ben.", ["A", "100°C-ben", "."]), + ("A 800x600 van.", ["A", "800x600", "van", "."]), + ("A 800x600-ben van.", ["A", "800x600-ben", "van", "."]), + ("A 800x600-ben.", ["A", "800x600-ben", "."]), + ("A 1x2x3x4 van.", ["A", "1x2x3x4", "van", "."]), + ("A 1x2x3x4-ben van.", ["A", "1x2x3x4-ben", "van", "."]), + ("A 1x2x3x4-ben.", ["A", "1x2x3x4-ben", "."]), + ("A 5/J van.", ["A", "5/J", "van", "."]), + ("A 5/J-ben van.", ["A", "5/J-ben", "van", "."]), + ("A 5/J-ben.", ["A", "5/J-ben", "."]), + ("A 5/J. van.", ["A", "5/J.", "van", "."]), + ("A 5/J.-ben van.", ["A", "5/J.-ben", "van", "."]), + ("A 5/J.-ben.", ["A", "5/J.-ben", "."]), + ("A III/1 van.", ["A", "III/1", "van", "."]), + ("A III/1-ben van.", ["A", "III/1-ben", "van", "."]), + ("A III/1-ben.", ["A", "III/1-ben", "."]), + ("A III/1. van.", ["A", "III/1.", "van", "."]), + ("A III/1.-ben van.", ["A", "III/1.-ben", "van", "."]), + ("A III/1.-ben.", ["A", "III/1.-ben", "."]), + ("A III/c van.", ["A", "III/c", "van", "."]), + ("A III/c-ben van.", ["A", "III/c-ben", "van", "."]), + ("A III/c.", ["A", "III/c", "."]), + ("A III/c-ben.", ["A", "III/c-ben", "."]), + ("A TU–154 van.", ["A", "TU–154", "van", "."]), + ("A TU–154-ben van.", ["A", "TU–154-ben", "van", "."]), + ("A TU–154-ben.", ["A", "TU–154-ben", "."]), + ("A 5cm³", ["A", "5", "cm³"]), + ("A 5 $-ban", ["A", "5", "$-ban"]), + ("A 5$-ban", ["A", "5$-ban"]), + ("A 5$.", ["A", "5", "$", "."]), + ("A 5$", ["A", "5", "$"]), + ("A $5", ["A", "$5"]), + ("A 5km/h", ["A", "5", "km/h"]), + ("A 75%+1-100%-ig", ["A", "75%+1-100%-ig"]), + ("A 5km/h.", ["A", "5", "km/h", "."]), + ("3434/1992. évi elszámolás", ["3434/1992.", "évi", "elszámolás"]), +] + +QUOTE_TESTS = [ + ( + 'Az "Ime, hat"-ban irja.', + ["Az", '"', "Ime", ",", "hat", '"', "-ban", "irja", "."], + ), + ('"Ime, hat"-ban irja.', ['"', "Ime", ",", "hat", '"', "-ban", "irja", "."]), + ('Az "Ime, hat".', ["Az", '"', "Ime", ",", "hat", '"', "."]), + ('Egy 24"-os monitor.', ["Egy", '24"-os', "monitor", "."]), + ("A McDonald's van.", ["A", "McDonald's", "van", "."]), +] + +DOT_TESTS = [ + ("N. kormányzósági\nszékhely.", ["N.", "kormányzósági", "székhely", "."]), + pytest.param( + "A .hu egy tld.", ["A", ".hu", "egy", "tld", "."], marks=pytest.mark.xfail() + ), + ("Az egy.ketto pelda.", ["Az", "egy.ketto", "pelda", "."]), + ("A pl. rövidítés.", ["A", "pl.", "rövidítés", "."]), + ("A S.M.A.R.T. szó.", ["A", "S.M.A.R.T.", "szó", "."]), + pytest.param("A .hu.", ["A", ".hu", "."], marks=pytest.mark.xfail()), + ("Az egy.ketto.", ["Az", "egy.ketto", "."]), + ("A pl.", ["A", "pl."]), + ("A S.M.A.R.T.", ["A", "S.M.A.R.T."]), + ("Egy..ket.", ["Egy", "..", "ket", "."]), + ("Valami... van.", ["Valami", "...", "van", "."]), + ("Valami ...van...", ["Valami", "...", "van", "..."]), + ("Valami...", ["Valami", "..."]), + ("Valami ...", ["Valami", "..."]), + ("Valami ... más.", ["Valami", "...", "más", "."]), +] + +TYPO_TESTS = [ + ( + "Ez egy mondat vége.Ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", ".", "Ez", "egy", "másik", "eleje", "."], + ), + ( + "Ez egy mondat vége .Ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", ".", "Ez", "egy", "másik", "eleje", "."], + ), + ( + "Ez egy mondat vége!ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", "!", "ez", "egy", "másik", "eleje", "."], + ), + ( + "Ez egy mondat vége !ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", "!", "ez", "egy", "másik", "eleje", "."], + ), + ( + "Ez egy mondat vége?Ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", "?", "Ez", "egy", "másik", "eleje", "."], + ), + ( + "Ez egy mondat vége ?Ez egy másik eleje.", + ["Ez", "egy", "mondat", "vége", "?", "Ez", "egy", "másik", "eleje", "."], + ), + ("egy,kettő", ["egy", ",", "kettő"]), + ("egy ,kettő", ["egy", ",", "kettő"]), + ("egy :kettő", ["egy", ":", "kettő"]), +] + +WIKI_TESTS = [ + ('!"', ["!", '"']), + ('lány"a', ["lány", '"', "a"]), + ('lány"a', ["lány", '"', "a"]), + ('!"-lel', ["!", '"', "-lel"]), + ('""-sorozat ', ['"', '"', "-sorozat"]), + ('"(Köszönöm', ['"', "(", "Köszönöm"]), + ("(törvénykönyv)-ben ", ["(", "törvénykönyv", ")", "-ben"]), + ('"(...)"–sokkal ', ['"', "(", "...", ")", '"', "–sokkal"]), + ("cérium(IV)-oxid", ["cérium", "(", "IV", ")", "-oxid"]), +] + +EXTRA_TESTS = ( + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS + TYPO_TESTS # type: ignore[operator] +) + +# normal: default tests + 10% of extra tests +TESTS = DEFAULT_TESTS +TESTS.extend([x for i, x in enumerate(EXTRA_TESTS) if i % 10 == 0]) + +# slow: remaining 90% of extra tests +SLOW_TESTS = [x for i, x in enumerate(EXTRA_TESTS) if i % 10 != 0] +TESTS.extend( + [ + ( + pytest.param(x[0], x[1], marks=pytest.mark.slow()) + if not isinstance(x[0], tuple) + else x + ) + for x in SLOW_TESTS + ] +) + + +@pytest.mark.parametrize("text,expected_tokens", TESTS) +def test_hu_tokenizer_handles_testcases(hu_tokenizer, text, expected_tokens): + tokens = hu_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/hy/__init__.py b/spacy/tests/lang/hy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/hy/test_text.py b/spacy/tests/lang/hy/test_text.py new file mode 100644 index 0000000..7a69c2a --- /dev/null +++ b/spacy/tests/lang/hy/test_text.py @@ -0,0 +1,9 @@ +import pytest + +from spacy.lang.hy.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["հիսուն"]) +def test_hy_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/hy/test_tokenizer.py b/spacy/tests/lang/hy/test_tokenizer.py new file mode 100644 index 0000000..9423cb4 --- /dev/null +++ b/spacy/tests/lang/hy/test_tokenizer.py @@ -0,0 +1,44 @@ +import pytest + +# TODO add test cases with valid punctuation signs. + +hy_tokenize_text_test = [ + ( + "Մետաղագիտությունը պայմանականորեն բաժանվում է տեսականի և կիրառականի (տեխնիկական)", + [ + "Մետաղագիտությունը", + "պայմանականորեն", + "բաժանվում", + "է", + "տեսականի", + "և", + "կիրառականի", + "(", + "տեխնիկական", + ")", + ], + ), + ( + "Գետաբերանը գտնվում է Օմոլոնա գետի ձախ ափից 726 կմ հեռավորության վրա", + [ + "Գետաբերանը", + "գտնվում", + "է", + "Օմոլոնա", + "գետի", + "ձախ", + "ափից", + "726", + "կմ", + "հեռավորության", + "վրա", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", hy_tokenize_text_test) +def test_ga_tokenizer_handles_exception_cases(hy_tokenizer, text, expected_tokens): + tokens = hy_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/id/__init__.py b/spacy/tests/lang/id/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/id/test_noun_chunks.py b/spacy/tests/lang/id/test_noun_chunks.py new file mode 100644 index 0000000..a394565 --- /dev/null +++ b/spacy/tests/lang/id/test_noun_chunks.py @@ -0,0 +1,8 @@ +import pytest + + +def test_noun_chunks_is_parsed_id(id_tokenizer): + """Test that noun_chunks raises Value Error for 'id' language if Doc is not parsed.""" + doc = id_tokenizer("sebelas") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/id/test_prefix_suffix_infix.py b/spacy/tests/lang/id/test_prefix_suffix_infix.py new file mode 100644 index 0000000..2a81dab --- /dev/null +++ b/spacy/tests/lang/id/test_prefix_suffix_infix.py @@ -0,0 +1,111 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(Ma'arif)"]) +def test_id_tokenizer_splits_no_special(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Ma'arif"]) +def test_id_tokenizer_splits_no_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["(Ma'arif"]) +def test_id_tokenizer_splits_prefix_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["Ma'arif)"]) +def test_id_tokenizer_splits_suffix_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(Ma'arif)"]) +def test_id_tokenizer_splits_even_wrap(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(Ma'arif?)"]) +def test_tokenizer_splits_uneven_wrap(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text,length", [("S.Kom.", 1), ("SKom.", 2), ("(S.Kom.", 2)]) +def test_id_tokenizer_splits_prefix_interact(id_tokenizer, text, length): + tokens = id_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["S.Kom.)"]) +def test_id_tokenizer_splits_suffix_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(S.Kom.)"]) +def test_id_tokenizer_splits_even_wrap_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(S.Kom.?)"]) +def test_id_tokenizer_splits_uneven_wrap_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize( + "text,length", [("gara-gara", 1), ("Jokowi-Ahok", 3), ("Sukarno-Hatta", 3)] +) +def test_id_tokenizer_splits_hyphens(id_tokenizer, text, length): + tokens = id_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["0.1-13.5", "0.0-0.1", "103.27-300"]) +def test_id_tokenizer_splits_numeric_range(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["ini.Budi", "Halo.Bandung"]) +def test_id_tokenizer_splits_period_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Halo,Bandung", "satu,dua"]) +def test_id_tokenizer_splits_comma_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["halo...Bandung", "dia...pergi"]) +def test_id_tokenizer_splits_ellipsis_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +def test_id_tokenizer_splits_double_hyphen_infix(id_tokenizer): + tokens = id_tokenizer("Arsene Wenger--manajer Arsenal--melakukan konferensi pers.") + assert len(tokens) == 10 + assert tokens[0].text == "Arsene" + assert tokens[1].text == "Wenger" + assert tokens[2].text == "--" + assert tokens[3].text == "manajer" + assert tokens[4].text == "Arsenal" + assert tokens[5].text == "--" + assert tokens[6].text == "melakukan" + assert tokens[7].text == "konferensi" + assert tokens[8].text == "pers" + assert tokens[9].text == "." diff --git a/spacy/tests/lang/id/test_text.py b/spacy/tests/lang/id/test_text.py new file mode 100644 index 0000000..7397a8c --- /dev/null +++ b/spacy/tests/lang/id/test_text.py @@ -0,0 +1,9 @@ +import pytest + +from spacy.lang.id.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["sebelas"]) +def test_id_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/is/__init__.py b/spacy/tests/lang/is/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/is/test_text.py b/spacy/tests/lang/is/test_text.py new file mode 100644 index 0000000..6e3654a --- /dev/null +++ b/spacy/tests/lang/is/test_text.py @@ -0,0 +1,26 @@ +import pytest + + +def test_long_text(is_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +hafa í huga, að yfirlýsing þessi hefur það markmið að tryggja +almenna og raunhæfa viðurkenningu og vernd þeirra réttinda, +sem þar er lýst; +hafa í huga, að markmið Evrópuráðs er að koma á nánari einingu +aðildarríkjanna og að ein af leiðunum að því marki er sú, að +mannréttindi og mannfrelsi séu í heiðri höfð og efld; +lýsa á ný eindreginni trú sinni á það mannfrelsi, sem er undirstaða +réttlætis og friðar í heiminum og best er tryggt, annars vegar með +virku, lýðræðislegu stjórnarfari og, hins vegar, almennum skilningi +og varðveislu þeirra mannréttinda, sem eru grundvöllur frelsisins; +""" + tokens = is_tokenizer(text) + assert len(tokens) == 120 + + +@pytest.mark.xfail +def test_ordinal_number(is_tokenizer): + text = "10. desember 1948" + tokens = is_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/is/test_tokenizer.py b/spacy/tests/lang/is/test_tokenizer.py new file mode 100644 index 0000000..0c05a60 --- /dev/null +++ b/spacy/tests/lang/is/test_tokenizer.py @@ -0,0 +1,30 @@ +import pytest + +IS_BASIC_TOKENIZATION_TESTS = [ + ( + "Enginn maður skal sæta pyndingum eða ómannlegri eða " + "vanvirðandi meðferð eða refsingu. ", + [ + "Enginn", + "maður", + "skal", + "sæta", + "pyndingum", + "eða", + "ómannlegri", + "eða", + "vanvirðandi", + "meðferð", + "eða", + "refsingu", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", IS_BASIC_TOKENIZATION_TESTS) +def test_is_tokenizer_basic(is_tokenizer, text, expected_tokens): + tokens = is_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/it/__init__.py b/spacy/tests/lang/it/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/it/test_noun_chunks.py b/spacy/tests/lang/it/test_noun_chunks.py new file mode 100644 index 0000000..5fd39ab --- /dev/null +++ b/spacy/tests/lang/it/test_noun_chunks.py @@ -0,0 +1,222 @@ +import pytest + +from spacy.tokens import Doc + + +# fmt: off +@pytest.mark.parametrize( + "words,heads,deps,pos,chunk_offsets", + [ + # determiner + noun + # un pollo -> un pollo + ( + ["un", "pollo"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0,2)], + ), + # two determiners + noun + # il mio cane -> il mio cane + ( + ["il", "mio", "cane"], + [2, 2, 2], + ["det", "det:poss", "ROOT"], + ["DET", "DET", "NOUN"], + [(0,3)], + ), + # two determiners, one is after noun. rare usage but still testing + # il cane mio-> il cane mio + ( + ["il", "cane", "mio"], + [1, 1, 1], + ["det", "ROOT", "det:poss"], + ["DET", "NOUN", "DET"], + [(0,3)], + ), + # relative pronoun + # È molto bello il vestito che hai acquistat -> il vestito, che the dress that you bought is very pretty. + ( + ["È", "molto", "bello", "il", "vestito", "che", "hai", "acquistato"], + [2, 2, 2, 4, 2, 7, 7, 4], + ['cop', 'advmod', 'ROOT', 'det', 'nsubj', 'obj', 'aux', 'acl:relcl'], + ['AUX', 'ADV', 'ADJ', 'DET', 'NOUN', 'PRON', 'AUX', 'VERB'], + [(3,5), (5,6)] + ), + # relative subclause + # il computer che hai comprato -> il computer, che the computer that you bought + ( + ['il', 'computer', 'che', 'hai', 'comprato'], + [1, 1, 4, 4, 1], + ['det', 'ROOT', 'nsubj', 'aux', 'acl:relcl'], + ['DET', 'NOUN', 'PRON', 'AUX', 'VERB'], + [(0,2), (2,3)] + ), + # det + noun + adj + # Una macchina grande -> Una macchina grande + ( + ["Una", "macchina", "grande"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0,3)], + ), + # noun + adj plural + # mucche bianche + ( + ["mucche", "bianche"], + [0, 0], + ["ROOT", "amod"], + ["NOUN", "ADJ"], + [(0,2)], + ), + # det + adj + noun + # Una grande macchina -> Una grande macchina + ( + ['Una', 'grande', 'macchina'], + [2, 2, 2], + ["det", "amod", "ROOT"], + ["DET", "ADJ", "NOUN"], + [(0,3)] + ), + # det + adj + noun, det with apostrophe + # un'importante associazione -> un'importante associazione + ( + ["Un'", 'importante', 'associazione'], + [2, 2, 2], + ["det", "amod", "ROOT"], + ["DET", "ADJ", "NOUN"], + [(0,3)] + ), + # multiple adjectives + # Un cane piccolo e marrone -> Un cane piccolo e marrone + ( + ["Un", "cane", "piccolo", "e", "marrone"], + [1, 1, 1, 4, 2], + ["det", "ROOT", "amod", "cc", "conj"], + ["DET", "NOUN", "ADJ", "CCONJ", "ADJ"], + [(0,5)] + ), + # determiner, adjective, compound created by flat + # le Nazioni Unite -> le Nazioni Unite + ( + ["le", "Nazioni", "Unite"], + [1, 1, 1], + ["det", "ROOT", "flat:name"], + ["DET", "PROPN", "PROPN"], + [(0,3)] + ), + # one determiner + one noun + one adjective qualified by an adverb + # alcuni contadini molto ricchi -> alcuni contadini molto ricchi some very rich farmers + ( + ['alcuni', 'contadini', 'molto', 'ricchi'], + [1, 1, 3, 1], + ['det', 'ROOT', 'advmod', 'amod'], + ['DET', 'NOUN', 'ADV', 'ADJ'], + [(0,4)] + ), + # Two NPs conjuncted + # Ho un cane e un gatto -> un cane, un gatto + ( + ['Ho', 'un', 'cane', 'e', 'un', 'gatto'], + [0, 2, 0, 5, 5, 0], + ['ROOT', 'det', 'obj', 'cc', 'det', 'conj'], + ['VERB', 'DET', 'NOUN', 'CCONJ', 'DET', 'NOUN'], + [(1,3), (4,6)] + + ), + # Two NPs together + # lo scrittore brasiliano Aníbal Machado -> lo scrittore brasiliano, Aníbal Machado + ( + ['lo', 'scrittore', 'brasiliano', 'Aníbal', 'Machado'], + [1, 1, 1, 1, 3], + ['det', 'ROOT', 'amod', 'nmod', 'flat:name'], + ['DET', 'NOUN', 'ADJ', 'PROPN', 'PROPN'], + [(0, 3), (3, 5)] + ), + # Noun compound, person name and titles + # Dom Pedro II -> Dom Pedro II + ( + ["Dom", "Pedro", "II"], + [0, 0, 0], + ["ROOT", "flat:name", "flat:name"], + ["PROPN", "PROPN", "PROPN"], + [(0,3)] + ), + # Noun compound created by flat + # gli Stati Uniti + ( + ["gli", "Stati", "Uniti"], + [1, 1, 1], + ["det", "ROOT", "flat:name"], + ["DET", "PROPN", "PROPN"], + [(0,3)] + ), + # nmod relation between NPs + # la distruzione della città -> la distruzione, città + ( + ['la', 'distruzione', 'della', 'città'], + [1, 1, 3, 1], + ['det', 'ROOT', 'case', 'nmod'], + ['DET', 'NOUN', 'ADP', 'NOUN'], + [(0,2), (3,4)] + ), + # Compounding by nmod, several NPs chained together + # la prima fabbrica di droga del governo -> la prima fabbrica, droga, governo + ( + ["la", "prima", "fabbrica", "di", "droga", "del", "governo"], + [2, 2, 2, 4, 2, 6, 2], + ['det', 'amod', 'ROOT', 'case', 'nmod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN'], + [(0, 3), (4, 5), (6, 7)] + ), + # several NPs + # Traduzione del rapporto di Susana -> Traduzione, rapporto, Susana + ( + ['Traduzione', 'del', 'rapporto', 'di', 'Susana'], + [0, 2, 0, 4, 2], + ['ROOT', 'case', 'nmod', 'case', 'nmod'], + ['NOUN', 'ADP', 'NOUN', 'ADP', 'PROPN'], + [(0,1), (2,3), (4,5)] + + ), + # Several NPs + # Il gatto grasso di Susana e la sua amica -> Il gatto grasso, Susana, sua amica + ( + ['Il', 'gatto', 'grasso', 'di', 'Susana', 'e', 'la', 'sua', 'amica'], + [1, 1, 1, 4, 1, 8, 8, 8, 1], + ['det', 'ROOT', 'amod', 'case', 'nmod', 'cc', 'det', 'det:poss', 'conj'], + ['DET', 'NOUN', 'ADJ', 'ADP', 'PROPN', 'CCONJ', 'DET', 'DET', 'NOUN'], + [(0,3), (4,5), (6,9)] + ), + # Passive subject + # La nuova spesa è alimentata dal grande conto in banca di Clinton -> Le nuova spesa, grande conto, banca, Clinton + ( + ['La', 'nuova', 'spesa', 'è', 'alimentata', 'dal', 'grande', 'conto', 'in', 'banca', 'di', 'Clinton'], + [2, 2, 4, 4, 4, 7, 7, 4, 9, 7, 11, 9], + ['det', 'amod', 'nsubj:pass', 'aux:pass', 'ROOT', 'case', 'amod', 'obl:agent', 'case', 'nmod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'AUX', 'VERB', 'ADP', 'ADJ', 'NOUN', 'ADP', 'NOUN', 'ADP', 'PROPN'], + [(0, 3), (6, 8), (9, 10), (11,12)] + ), + # Misc + # Ma mentre questo prestito possa ora sembrare gestibile, un improvviso cambiamento delle circostanze potrebbe portare a problemi di debiti -> questo prestiti, un provisso cambiento, circostanze, problemi, debiti + ( + ['Ma', 'mentre', 'questo', 'prestito', 'possa', 'ora', 'sembrare', 'gestibile', ',', 'un', 'improvviso', 'cambiamento', 'delle', 'circostanze', 'potrebbe', 'portare', 'a', 'problemi', 'di', 'debitii'], + [15, 6, 3, 6, 6, 6, 15, 6, 6, 11, 11, 15, 13, 11, 15, 15, 17, 15, 19, 17], + ['cc', 'mark', 'det', 'nsubj', 'aux', 'advmod', 'advcl', 'xcomp', 'punct', 'det', 'amod', 'nsubj', 'case', 'nmod', 'aux', 'ROOT', 'case', 'obl', 'case', 'nmod'], + ['CCONJ', 'SCONJ', 'DET', 'NOUN', 'AUX', 'ADV', 'VERB', 'ADJ', 'PUNCT', 'DET', 'ADJ', 'NOUN', 'ADP', 'NOUN', 'AUX', 'VERB', 'ADP', 'NOUN', 'ADP', 'NOUN'], + [(2,4), (9,12), (13,14), (17,18), (19,20)] + ) + ], +) +# fmt: on +def test_it_noun_chunks(it_vocab, words, heads, deps, pos, chunk_offsets): + doc = Doc(it_vocab, words=words, heads=heads, deps=deps, pos=pos) + assert [(c.start, c.end) for c in doc.noun_chunks] == chunk_offsets + + +def test_noun_chunks_is_parsed_it(it_tokenizer): + """Test that noun_chunks raises Value Error for 'it' language if Doc is not parsed.""" + doc = it_tokenizer("Sei andato a Oxford") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/it/test_prefix_suffix_infix.py b/spacy/tests/lang/it/test_prefix_suffix_infix.py new file mode 100644 index 0000000..5834f96 --- /dev/null +++ b/spacy/tests/lang/it/test_prefix_suffix_infix.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.parametrize( + "text,expected_tokens", [("c'è", ["c'", "è"]), ("l'ha", ["l'", "ha"])] +) +def test_contractions(it_tokenizer, text, expected_tokens): + """Test that the contractions are split into two tokens""" + tokens = it_tokenizer(text) + assert len(tokens) == 2 + assert [t.text for t in tokens] == expected_tokens diff --git a/spacy/tests/lang/it/test_stopwords.py b/spacy/tests/lang/it/test_stopwords.py new file mode 100644 index 0000000..9549131 --- /dev/null +++ b/spacy/tests/lang/it/test_stopwords.py @@ -0,0 +1,17 @@ +import pytest + + +@pytest.mark.parametrize( + "word", ["un", "lo", "dell", "dall", "si", "ti", "mi", "quest", "quel", "quello"] +) +def test_stopwords_basic(it_tokenizer, word): + tok = it_tokenizer(word)[0] + assert tok.is_stop + + +@pytest.mark.parametrize( + "word", ["quest'uomo", "l'ho", "un'amica", "dell'olio", "s'arrende", "m'ascolti"] +) +def test_stopwords_elided(it_tokenizer, word): + tok = it_tokenizer(word)[0] + assert tok.is_stop diff --git a/spacy/tests/lang/it/test_text.py b/spacy/tests/lang/it/test_text.py new file mode 100644 index 0000000..6023a20 --- /dev/null +++ b/spacy/tests/lang/it/test_text.py @@ -0,0 +1,14 @@ +import pytest + + +@pytest.mark.issue(2822) +def test_issue2822(it_tokenizer): + """Test that the abbreviation of poco is kept as one word.""" + doc = it_tokenizer("Vuoi un po' di zucchero?") + assert len(doc) == 6 + assert doc[0].text == "Vuoi" + assert doc[1].text == "un" + assert doc[2].text == "po'" + assert doc[3].text == "di" + assert doc[4].text == "zucchero" + assert doc[5].text == "?" diff --git a/spacy/tests/lang/ja/__init__.py b/spacy/tests/lang/ja/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ja/test_lemmatization.py b/spacy/tests/lang/ja/test_lemmatization.py new file mode 100644 index 0000000..523917f --- /dev/null +++ b/spacy/tests/lang/ja/test_lemmatization.py @@ -0,0 +1,30 @@ +import pytest + + +@pytest.mark.parametrize( + "word,lemma", + [ + ("新しく", "新しい"), + ("赤く", "赤い"), + ("すごく", "すごい"), + ("いただきました", "いただく"), + ("なった", "なる"), + ], +) +def test_ja_lemmatizer_assigns(ja_tokenizer, word, lemma): + test_lemma = ja_tokenizer(word)[0].lemma_ + assert test_lemma == lemma + + +@pytest.mark.parametrize( + "word,norm", + [ + ("SUMMER", "サマー"), + ("食べ物", "食べ物"), + ("綜合", "総合"), + ("コンピュータ", "コンピューター"), + ], +) +def test_ja_lemmatizer_norm(ja_tokenizer, word, norm): + test_norm = ja_tokenizer(word)[0].norm_ + assert test_norm == norm diff --git a/spacy/tests/lang/ja/test_morphologizer_factory.py b/spacy/tests/lang/ja/test_morphologizer_factory.py new file mode 100644 index 0000000..d504576 --- /dev/null +++ b/spacy/tests/lang/ja/test_morphologizer_factory.py @@ -0,0 +1,10 @@ +import pytest + +from spacy.lang.ja import Japanese + + +def test_ja_morphologizer_factory(): + pytest.importorskip("sudachipy") + nlp = Japanese() + morphologizer = nlp.add_pipe("morphologizer") + assert morphologizer.cfg["extend"] is True diff --git a/spacy/tests/lang/ja/test_serialize.py b/spacy/tests/lang/ja/test_serialize.py new file mode 100644 index 0000000..f48b257 --- /dev/null +++ b/spacy/tests/lang/ja/test_serialize.py @@ -0,0 +1,42 @@ +import pickle + +from spacy.lang.ja import Japanese + +from ...util import make_tempdir + + +def test_ja_tokenizer_serialize(ja_tokenizer): + tokenizer_bytes = ja_tokenizer.to_bytes() + nlp = Japanese() + nlp.tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + assert nlp.tokenizer.split_mode is None + + with make_tempdir() as d: + file_path = d / "tokenizer" + ja_tokenizer.to_disk(file_path) + nlp = Japanese() + nlp.tokenizer.from_disk(file_path) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + assert nlp.tokenizer.split_mode is None + + # split mode is (de)serialized correctly + nlp = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "B"}}}) + nlp_r = Japanese() + nlp_bytes = nlp.to_bytes() + nlp_r.from_bytes(nlp_bytes) + assert nlp_bytes == nlp_r.to_bytes() + assert nlp_r.tokenizer.split_mode == "B" + + with make_tempdir() as d: + nlp.to_disk(d) + nlp_r = Japanese() + nlp_r.from_disk(d) + assert nlp_bytes == nlp_r.to_bytes() + assert nlp_r.tokenizer.split_mode == "B" + + +def test_ja_tokenizer_pickle(ja_tokenizer): + b = pickle.dumps(ja_tokenizer) + ja_tokenizer_re = pickle.loads(b) + assert ja_tokenizer.to_bytes() == ja_tokenizer_re.to_bytes() diff --git a/spacy/tests/lang/ja/test_tokenizer.py b/spacy/tests/lang/ja/test_tokenizer.py new file mode 100644 index 0000000..36f7e32 --- /dev/null +++ b/spacy/tests/lang/ja/test_tokenizer.py @@ -0,0 +1,173 @@ +import pytest + +from spacy.lang.ja import DetailedToken, Japanese + +from ...tokenizer.test_naughty_strings import NAUGHTY_STRINGS + +# fmt: off +TOKENIZER_TESTS = [ + ("日本語だよ", ['日本', '語', 'だ', 'よ']), + ("東京タワーの近くに住んでいます。", ['東京', 'タワー', 'の', '近く', 'に', '住ん', 'で', 'い', 'ます', '。']), + ("吾輩は猫である。", ['吾輩', 'は', '猫', 'で', 'ある', '。']), + ("月に代わって、お仕置きよ!", ['月', 'に', '代わっ', 'て', '、', 'お', '仕置き', 'よ', '!']), + ("すもももももももものうち", ['すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち']) +] + +TAG_TESTS = [ + ("日本語だよ", ['名詞-固有名詞-地名-国', '名詞-普通名詞-一般', '助動詞', '助詞-終助詞']), + ("東京タワーの近くに住んでいます。", ['名詞-固有名詞-地名-一般', '名詞-普通名詞-一般', '助詞-格助詞', '名詞-普通名詞-副詞可能', '助詞-格助詞', '動詞-一般', '助詞-接続助詞', '動詞-非自立可能', '助動詞', '補助記号-句点']), + ("吾輩は猫である。", ['代名詞', '助詞-係助詞', '名詞-普通名詞-一般', '助動詞', '動詞-非自立可能', '補助記号-句点']), + ("月に代わって、お仕置きよ!", ['名詞-普通名詞-助数詞可能', '助詞-格助詞', '動詞-一般', '助詞-接続助詞', '補助記号-読点', '接頭辞', '名詞-普通名詞-一般', '助詞-終助詞', '補助記号-句点']), + ("すもももももももものうち", ['名詞-普通名詞-一般', '助詞-係助詞', '名詞-普通名詞-一般', '助詞-係助詞', '名詞-普通名詞-一般', '助詞-格助詞', '名詞-普通名詞-副詞可能']) +] + +POS_TESTS = [ + ('日本語だよ', ['PROPN', 'NOUN', 'AUX', 'PART']), + ('東京タワーの近くに住んでいます。', ['PROPN', 'NOUN', 'ADP', 'NOUN', 'ADP', 'VERB', 'SCONJ', 'AUX', 'AUX', 'PUNCT']), + ('吾輩は猫である。', ['PRON', 'ADP', 'NOUN', 'AUX', 'AUX', 'PUNCT']), + ('月に代わって、お仕置きよ!', ['NOUN', 'ADP', 'VERB', 'SCONJ', 'PUNCT', 'NOUN', 'NOUN', 'PART', 'PUNCT']), + ('すもももももももものうち', ['NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN']) +] + +SENTENCE_TESTS = [ + ("あれ。これ。", ["あれ。", "これ。"]), + ("「伝染るんです。」という漫画があります。", ["「伝染るんです。」という漫画があります。"]), +] + +tokens1 = [ + DetailedToken(surface="委員", tag="名詞-普通名詞-一般", inf="", lemma="委員", norm="委員", reading="イイン", sub_tokens=None), + DetailedToken(surface="会", tag="名詞-普通名詞-一般", inf="", lemma="会", norm="会", reading="カイ", sub_tokens=None), +] +tokens2 = [ + DetailedToken(surface="選挙", tag="名詞-普通名詞-サ変可能", inf="", lemma="選挙", norm="選挙", reading="センキョ", sub_tokens=None), + DetailedToken(surface="管理", tag="名詞-普通名詞-サ変可能", inf="", lemma="管理", norm="管理", reading="カンリ", sub_tokens=None), + DetailedToken(surface="委員", tag="名詞-普通名詞-一般", inf="", lemma="委員", norm="委員", reading="イイン", sub_tokens=None), + DetailedToken(surface="会", tag="名詞-普通名詞-一般", inf="", lemma="会", norm="会", reading="カイ", sub_tokens=None), +] +tokens3 = [ + DetailedToken(surface="選挙", tag="名詞-普通名詞-サ変可能", inf="", lemma="選挙", norm="選挙", reading="センキョ", sub_tokens=None), + DetailedToken(surface="管理", tag="名詞-普通名詞-サ変可能", inf="", lemma="管理", norm="管理", reading="カンリ", sub_tokens=None), + DetailedToken(surface="委員会", tag="名詞-普通名詞-一般", inf="", lemma="委員会", norm="委員会", reading="イインカイ", sub_tokens=None), +] +SUB_TOKEN_TESTS = [ + ("選挙管理委員会", [None, None, [tokens1]], [[tokens2, tokens3]]) +] +# fmt: on + + +@pytest.mark.issue(2901) +def test_issue2901(): + """Test that `nlp` doesn't fail.""" + try: + nlp = Japanese() + except ImportError: + pytest.skip() + + doc = nlp("pythonが大好きです") + assert doc + + +@pytest.mark.parametrize("text,expected_tokens", TOKENIZER_TESTS) +def test_ja_tokenizer(ja_tokenizer, text, expected_tokens): + tokens = [token.text for token in ja_tokenizer(text)] + assert tokens == expected_tokens + + +@pytest.mark.parametrize("text,expected_tags", TAG_TESTS) +def test_ja_tokenizer_tags(ja_tokenizer, text, expected_tags): + tags = [token.tag_ for token in ja_tokenizer(text)] + assert tags == expected_tags + + +@pytest.mark.parametrize("text,expected_pos", POS_TESTS) +def test_ja_tokenizer_pos(ja_tokenizer, text, expected_pos): + pos = [token.pos_ for token in ja_tokenizer(text)] + assert pos == expected_pos + + +@pytest.mark.skip(reason="sentence segmentation in tokenizer is buggy") +@pytest.mark.parametrize("text,expected_sents", SENTENCE_TESTS) +def test_ja_tokenizer_sents(ja_tokenizer, text, expected_sents): + sents = [str(sent) for sent in ja_tokenizer(text).sents] + assert sents == expected_sents + + +def test_ja_tokenizer_extra_spaces(ja_tokenizer): + # note: three spaces after "I" + tokens = ja_tokenizer("I like cheese.") + assert tokens[1].orth_ == " " + + +@pytest.mark.parametrize("text", NAUGHTY_STRINGS) +def test_ja_tokenizer_naughty_strings(ja_tokenizer, text): + tokens = ja_tokenizer(text) + assert tokens.text_with_ws == text + + +@pytest.mark.parametrize( + "text,len_a,len_b,len_c", + [ + ("選挙管理委員会", 4, 3, 1), + ("客室乗務員", 3, 2, 1), + ("労働者協同組合", 4, 3, 1), + ("機能性食品", 3, 2, 1), + ], +) +def test_ja_tokenizer_split_modes(ja_tokenizer, text, len_a, len_b, len_c): + nlp_a = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "A"}}}) + nlp_b = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "B"}}}) + nlp_c = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "C"}}}) + + assert len(ja_tokenizer(text)) == len_a + assert len(nlp_a(text)) == len_a + assert len(nlp_b(text)) == len_b + assert len(nlp_c(text)) == len_c + + +@pytest.mark.parametrize("text,sub_tokens_list_b,sub_tokens_list_c", SUB_TOKEN_TESTS) +def test_ja_tokenizer_sub_tokens( + ja_tokenizer, text, sub_tokens_list_b, sub_tokens_list_c +): + nlp_a = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "A"}}}) + nlp_b = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "B"}}}) + nlp_c = Japanese.from_config({"nlp": {"tokenizer": {"split_mode": "C"}}}) + + assert ja_tokenizer(text).user_data.get("sub_tokens") is None + assert nlp_a(text).user_data.get("sub_tokens") is None + assert nlp_b(text).user_data["sub_tokens"] == sub_tokens_list_b + assert nlp_c(text).user_data["sub_tokens"] == sub_tokens_list_c + + +@pytest.mark.parametrize( + "text,inflections,reading_forms", + [ + ( + "取ってつけた", + ( + ["五段-ラ行;連用形-促音便"], + [], + ["下一段-カ行;連用形-一般"], + ["助動詞-タ;終止形-一般"], + ), + (["トッ"], ["テ"], ["ツケ"], ["タ"]), + ), + ("2=3", ([], [], []), (["ニ"], ["_"], ["サン"])), + ], +) +def test_ja_tokenizer_inflections_reading_forms( + ja_tokenizer, text, inflections, reading_forms +): + tokens = ja_tokenizer(text) + test_inflections = [tt.morph.get("Inflection") for tt in tokens] + assert test_inflections == list(inflections) + test_readings = [tt.morph.get("Reading") for tt in tokens] + assert test_readings == list(reading_forms) + + +def test_ja_tokenizer_emptyish_texts(ja_tokenizer): + doc = ja_tokenizer("") + assert len(doc) == 0 + doc = ja_tokenizer(" ") + assert len(doc) == 1 + doc = ja_tokenizer("\n\n\n \t\t \n\n\n") + assert len(doc) == 1 diff --git a/spacy/tests/lang/kmr/__init__.py b/spacy/tests/lang/kmr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/kmr/test_text.py b/spacy/tests/lang/kmr/test_text.py new file mode 100644 index 0000000..405dc28 --- /dev/null +++ b/spacy/tests/lang/kmr/test_text.py @@ -0,0 +1,27 @@ +import pytest + +from spacy.lang.kmr.lex_attrs import like_num + + +@pytest.mark.parametrize( + "word", + [ + "yekem", + "duyemîn", + "100em", + "dehem", + "sedemîn", + "34em", + "30yem", + "20emîn", + "50yemîn", + ], +) +def test_kmr_lex_attrs_like_number_for_ordinal(word): + assert like_num(word) + + +@pytest.mark.parametrize("word", ["deh"]) +def test_kmr_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/ko/__init__.py b/spacy/tests/lang/ko/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ko/test_lemmatization.py b/spacy/tests/lang/ko/test_lemmatization.py new file mode 100644 index 0000000..371e410 --- /dev/null +++ b/spacy/tests/lang/ko/test_lemmatization.py @@ -0,0 +1,16 @@ +import pytest + + +@pytest.mark.parametrize( + "word,lemma", + [ + ("새로운", "새롭"), + ("빨간", "빨갛"), + ("클수록", "크"), + ("뭡니까", "뭣"), + ("됐다", "되"), + ], +) +def test_ko_lemmatizer_assigns(ko_tokenizer, word, lemma): + test_lemma = ko_tokenizer(word)[0].lemma_ + assert test_lemma == lemma diff --git a/spacy/tests/lang/ko/test_serialize.py b/spacy/tests/lang/ko/test_serialize.py new file mode 100644 index 0000000..bba7bce --- /dev/null +++ b/spacy/tests/lang/ko/test_serialize.py @@ -0,0 +1,25 @@ +import pickle + +from spacy.lang.ko import Korean + +from ...util import make_tempdir + + +def test_ko_tokenizer_serialize(ko_tokenizer): + tokenizer_bytes = ko_tokenizer.to_bytes() + nlp = Korean() + nlp.tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + with make_tempdir() as d: + file_path = d / "tokenizer" + ko_tokenizer.to_disk(file_path) + nlp = Korean() + nlp.tokenizer.from_disk(file_path) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + +def test_ko_tokenizer_pickle(ko_tokenizer): + b = pickle.dumps(ko_tokenizer) + ko_tokenizer_re = pickle.loads(b) + assert ko_tokenizer.to_bytes() == ko_tokenizer_re.to_bytes() diff --git a/spacy/tests/lang/ko/test_tokenizer.py b/spacy/tests/lang/ko/test_tokenizer.py new file mode 100644 index 0000000..6e06e40 --- /dev/null +++ b/spacy/tests/lang/ko/test_tokenizer.py @@ -0,0 +1,75 @@ +import pytest + +# fmt: off +TOKENIZER_TESTS = [("서울 타워 근처에 살고 있습니다.", "서울 타워 근처 에 살 고 있 습니다 ."), + ("영등포구에 있는 맛집 좀 알려주세요.", "영등포구 에 있 는 맛집 좀 알려 주 세요 ."), + ("10$ 할인코드를 적용할까요?", "10 $ 할인 코드 를 적용 할까요 ?")] + +TAG_TESTS = [("서울 타워 근처에 살고 있습니다.", + "NNP NNG NNG JKB VV EC VX EF SF"), + ("영등포구에 있는 맛집 좀 알려주세요.", + "NNP JKB VV ETM NNG MAG VV VX EP SF")] + +FULL_TAG_TESTS = [("영등포구에 있는 맛집 좀 알려주세요.", + "NNP JKB VV ETM NNG MAG VV+EC VX EP+EF SF")] + +POS_TESTS = [("서울 타워 근처에 살고 있습니다.", + "PROPN NOUN NOUN ADP VERB X AUX X PUNCT"), + ("영등포구에 있는 맛집 좀 알려주세요.", + "PROPN ADP VERB X NOUN ADV VERB AUX X PUNCT")] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", TOKENIZER_TESTS) +def test_ko_tokenizer(ko_tokenizer, text, expected_tokens): + tokens = [token.text for token in ko_tokenizer(text)] + assert tokens == expected_tokens.split() + + +@pytest.mark.parametrize("text,expected_tags", TAG_TESTS) +def test_ko_tokenizer_tags(ko_tokenizer, text, expected_tags): + tags = [token.tag_ for token in ko_tokenizer(text)] + assert tags == expected_tags.split() + + +@pytest.mark.parametrize("text,expected_tags", FULL_TAG_TESTS) +def test_ko_tokenizer_full_tags(ko_tokenizer, text, expected_tags): + tags = ko_tokenizer(text).user_data["full_tags"] + assert tags == expected_tags.split() + + +@pytest.mark.parametrize("text,expected_pos", POS_TESTS) +def test_ko_tokenizer_pos(ko_tokenizer, text, expected_pos): + pos = [token.pos_ for token in ko_tokenizer(text)] + assert pos == expected_pos.split() + + +def test_ko_empty_doc(ko_tokenizer): + tokens = ko_tokenizer("") + assert len(tokens) == 0 + + +@pytest.mark.issue(10535) +def test_ko_tokenizer_unknown_tag(ko_tokenizer): + tokens = ko_tokenizer("미닛 리피터") + assert tokens[1].pos_ == "X" + + +# fmt: off +SPACY_TOKENIZER_TESTS = [ + ("있다.", "있다 ."), + ("'예'는", "' 예 ' 는"), + ("부 (富) 는", "부 ( 富 ) 는"), + ("부(富)는", "부 ( 富 ) 는"), + ("1982~1983.", "1982 ~ 1983 ."), + ("사과·배·복숭아·수박은 모두 과일이다.", "사과 · 배 · 복숭아 · 수박은 모두 과일이다 ."), + ("그렇구나~", "그렇구나~"), + ("『9시 반의 당구』,", "『 9시 반의 당구 』 ,"), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", SPACY_TOKENIZER_TESTS) +def test_ko_spacy_tokenizer(ko_tokenizer_tokenizer, text, expected_tokens): + tokens = [token.text for token in ko_tokenizer_tokenizer(text)] + assert tokens == expected_tokens.split() diff --git a/spacy/tests/lang/ky/__init__.py b/spacy/tests/lang/ky/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ky/test_tokenizer.py b/spacy/tests/lang/ky/test_tokenizer.py new file mode 100644 index 0000000..b089dd9 --- /dev/null +++ b/spacy/tests/lang/ky/test_tokenizer.py @@ -0,0 +1,85 @@ +import pytest + +INFIX_HYPHEN_TESTS = [ + ("Бала-чака жакшыбы?", "Бала-чака жакшыбы ?".split()), + ("Кыз-келиндер кийими.", "Кыз-келиндер кийими .".split()), +] + +PUNC_INSIDE_WORDS_TESTS = [ + ( + "Пассажир саны - 2,13 млн — киши/күнүнө (2010), 783,9 млн. киши/жылына.", + "Пассажир саны - 2,13 млн — киши / күнүнө ( 2010 ) ," + " 783,9 млн. киши / жылына .".split(), + ), + ('То"кой', 'То " кой'.split()), +] + +MIXED_ORDINAL_NUMS_TESTS = [("Эртең 22-январь...", "Эртең 22 - январь ...".split())] + +ABBREV_TESTS = [ + ("Маселе б-ча эртең келет", "Маселе б-ча эртең келет".split()), + ("Ахунбаев көч. турат.", "Ахунбаев көч. турат .".split()), + ("«3-жылы (б.з.ч.) туулган", "« 3 - жылы ( б.з.ч. ) туулган".split()), + ("Жүгөрү ж.б. дандар колдонулат", "Жүгөрү ж.б. дандар колдонулат".split()), + ("3-4 кк. курулган.", "3 - 4 кк. курулган .".split()), +] + +NAME_ABBREV_TESTS = [ + ("М.Жумаш", "М.Жумаш".split()), + ("М.жумаш", "М.жумаш".split()), + ("м.Жумаш", "м . Жумаш".split()), + ("Жумаш М.Н.", "Жумаш М.Н.".split()), + ("Жумаш.", "Жумаш .".split()), +] + +TYPOS_IN_PUNC_TESTS = [ + ("«3-жылда , туулган", "« 3 - жылда , туулган".split()), + ("«3-жылда,туулган", "« 3 - жылда , туулган".split()), + ("«3-жылда,туулган.", "« 3 - жылда , туулган .".split()), + ("Ал иштейт(качан?)", "Ал иштейт ( качан ? )".split()), + ("Ал (качан?)иштейт", "Ал ( качан ?) иштейт".split()), # "?)" => "?)" or "? )" +] + +LONG_TEXTS_TESTS = [ + ( + "Алыскы өлкөлөргө аздыр-көптүр татаалыраак жүрүштөргө чыккандар " + "азыраак: ал бир топ кымбат жана логистика маселесинин айынан " + "кыйла татаал. Мисалы, январдагы майрамдарда Мароккого үчүнчү " + "категориядагы маршрутка (100 чакырымдан кем эмес) барып " + "келгенге аракет кылдык.", + "Алыскы өлкөлөргө аздыр-көптүр татаалыраак жүрүштөргө чыккандар " + "азыраак : ал бир топ кымбат жана логистика маселесинин айынан " + "кыйла татаал . Мисалы , январдагы майрамдарда Мароккого үчүнчү " + "категориядагы маршрутка ( 100 чакырымдан кем эмес ) барып " + "келгенге аракет кылдык .".split(), + ) +] + +TESTCASES = ( + INFIX_HYPHEN_TESTS + + PUNC_INSIDE_WORDS_TESTS + + MIXED_ORDINAL_NUMS_TESTS + + ABBREV_TESTS + + NAME_ABBREV_TESTS + + LONG_TEXTS_TESTS + + TYPOS_IN_PUNC_TESTS +) + +NORM_TESTCASES = [ + ( + "ит, мышык ж.б.у.с. үй жаныбарлары.", + ["ит", ",", "мышык", "жана башка ушул сыяктуу", "үй", "жаныбарлары", "."], + ) +] + + +@pytest.mark.parametrize("text,expected_tokens", TESTCASES) +def test_ky_tokenizer_handles_testcases(ky_tokenizer, text, expected_tokens): + tokens = [token.text for token in ky_tokenizer(text) if not token.is_space] + assert expected_tokens == tokens + + +@pytest.mark.parametrize("text,norms", NORM_TESTCASES) +def test_ky_tokenizer_handles_norm_exceptions(ky_tokenizer, text, norms): + tokens = ky_tokenizer(text) + assert [token.norm_ for token in tokens] == norms diff --git a/spacy/tests/lang/la/__init__.py b/spacy/tests/lang/la/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/la/test_exception.py b/spacy/tests/lang/la/test_exception.py new file mode 100644 index 0000000..9a6e6a4 --- /dev/null +++ b/spacy/tests/lang/la/test_exception.py @@ -0,0 +1,5 @@ +def test_la_tokenizer_handles_exc_in_text(la_tokenizer): + text = "scio te omnia facturum, ut nobiscum quam primum sis" + tokens = la_tokenizer(text) + assert len(tokens) == 11 + assert tokens[6].text == "nobis" diff --git a/spacy/tests/lang/la/test_noun_chunks.py b/spacy/tests/lang/la/test_noun_chunks.py new file mode 100644 index 0000000..70a3392 --- /dev/null +++ b/spacy/tests/lang/la/test_noun_chunks.py @@ -0,0 +1,53 @@ +import pytest + +from spacy.tokens import Doc + + +def test_noun_chunks_is_parsed(la_tokenizer): + """Test that noun_chunks raises Value Error for 'la' language if Doc is not parsed. + To check this test, we're constructing a Doc + with a new Vocab here and forcing is_parsed to 'False' + to make sure the noun chunks don't run. + """ + doc = la_tokenizer("Haec est sententia.") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +LA_NP_TEST_EXAMPLES = [ + ( + "Haec narrantur a poetis de Perseo.", + ["DET", "VERB", "ADP", "NOUN", "ADP", "PROPN", "PUNCT"], + ["nsubj:pass", "ROOT", "case", "obl", "case", "obl", "punct"], + [1, 0, -1, -1, -3, -1, -5], + ["poetis", "Perseo"], + ), + ( + "Perseus autem in sinu matris dormiebat.", + ["NOUN", "ADV", "ADP", "NOUN", "NOUN", "VERB", "PUNCT"], + ["nsubj", "discourse", "case", "obl", "nmod", "ROOT", "punct"], + [5, 4, 3, -1, -1, 0, -1], + ["Perseus", "sinu matris"], + ), +] + + +@pytest.mark.parametrize( + "text,pos,deps,heads,expected_noun_chunks", LA_NP_TEST_EXAMPLES +) +def test_la_noun_chunks(la_tokenizer, text, pos, deps, heads, expected_noun_chunks): + tokens = la_tokenizer(text) + + assert len(heads) == len(pos) + doc = Doc( + tokens.vocab, + words=[t.text for t in tokens], + heads=[head + i for i, head in enumerate(heads)], + deps=deps, + pos=pos, + ) + + noun_chunks = list(doc.noun_chunks) + assert len(noun_chunks) == len(expected_noun_chunks) + for i, np in enumerate(noun_chunks): + assert np.text == expected_noun_chunks[i] diff --git a/spacy/tests/lang/la/test_text.py b/spacy/tests/lang/la/test_text.py new file mode 100644 index 0000000..74606c4 --- /dev/null +++ b/spacy/tests/lang/la/test_text.py @@ -0,0 +1,36 @@ +import pytest + +from spacy.lang.la.lex_attrs import like_num + + +@pytest.mark.parametrize( + "text,match", + [ + ("IIII", True), + ("VI", True), + ("vi", True), + ("IV", True), + ("iv", True), + ("IX", True), + ("ix", True), + ("MMXXII", True), + ("0", True), + ("1", True), + ("quattuor", True), + ("decem", True), + ("tertius", True), + ("canis", False), + ("MMXX11", False), + (",", False), + ], +) +def test_lex_attrs_like_number(la_tokenizer, text, match): + tokens = la_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["quinque"]) +def test_la_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/lb/__init__.py b/spacy/tests/lang/lb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/lb/test_exceptions.py b/spacy/tests/lang/lb/test_exceptions.py new file mode 100644 index 0000000..fc4b4fa --- /dev/null +++ b/spacy/tests/lang/lb/test_exceptions.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.mark.parametrize("text", ["z.B.", "Jan."]) +def test_lb_tokenizer_handles_abbr(lb_tokenizer, text): + tokens = lb_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["d'Saach", "d'Kanner", "d’Welt", "d’Suen"]) +def test_lb_tokenizer_splits_contractions(lb_tokenizer, text): + tokens = lb_tokenizer(text) + assert len(tokens) == 2 + + +def test_lb_tokenizer_handles_exc_in_text(lb_tokenizer): + text = "Mee 't ass net evident, d'Liewen." + tokens = lb_tokenizer(text) + assert len(tokens) == 9 + assert tokens[1].text == "'t" diff --git a/spacy/tests/lang/lb/test_prefix_suffix_infix.py b/spacy/tests/lang/lb/test_prefix_suffix_infix.py new file mode 100644 index 0000000..3958d15 --- /dev/null +++ b/spacy/tests/lang/lb/test_prefix_suffix_infix.py @@ -0,0 +1,19 @@ +import pytest + + +@pytest.mark.parametrize("text,length", [("z.B.", 1), ("zb.", 2), ("(z.B.", 2)]) +def test_lb_tokenizer_splits_prefix_interact(lb_tokenizer, text, length): + tokens = lb_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["z.B.)"]) +def test_lb_tokenizer_splits_suffix_interact(lb_tokenizer, text): + tokens = lb_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(z.B.)"]) +def test_lb_tokenizer_splits_even_wrap_interact(lb_tokenizer, text): + tokens = lb_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/lb/test_text.py b/spacy/tests/lang/lb/test_text.py new file mode 100644 index 0000000..b0ba76b --- /dev/null +++ b/spacy/tests/lang/lb/test_text.py @@ -0,0 +1,21 @@ +import pytest + + +def test_lb_tokenizer_handles_long_text(lb_tokenizer): + text = """Den Nordwand an d'Sonn An der Zäit hunn sech den Nordwand an d'Sonn gestridden, wie vun hinnen zwee wuel méi staark wier, wéi e Wanderer, deen an ee waarme Mantel agepak war, iwwert de Wee koum. Si goufen sech eens, dass deejéinege fir de Stäerkste gëlle sollt, deen de Wanderer forcéiere géif, säi Mantel auszedoen. Den Nordwand huet mat aller Force geblosen, awer wat e méi geblosen huet, wat de Wanderer sech méi a säi Mantel agewéckelt huet. Um Enn huet den Nordwand säi Kampf opginn. Dunn huet d'Sonn d'Loft mat hire frëndleche Strale gewiermt, a schonn no kuerzer Zäit huet de Wanderer säi Mantel ausgedoen. Do huet den Nordwand missen zouginn, dass d'Sonn vun hinnen zwee de Stäerkste wier.""" + + tokens = lb_tokenizer(text) + assert len(tokens) == 142 + + +@pytest.mark.parametrize( + "text,length", + [ + ("»Wat ass mat mir geschitt?«, huet hie geduecht.", 13), + ("“Dëst fréi Opstoen”, denkt hien, “mécht ee ganz duercherneen. ", 15), + ("Am Grand-Duché ass d'Liewen schéin, mee 't gëtt ze vill Autoen.", 14), + ], +) +def test_lb_tokenizer_handles_examples(lb_tokenizer, text, length): + tokens = lb_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/lg/__init__.py b/spacy/tests/lang/lg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/lg/test_tokenizer.py b/spacy/tests/lang/lg/test_tokenizer.py new file mode 100644 index 0000000..958385a --- /dev/null +++ b/spacy/tests/lang/lg/test_tokenizer.py @@ -0,0 +1,15 @@ +import pytest + +LG_BASIC_TOKENIZATION_TESTS = [ + ( + "Abooluganda ab’emmamba ababiri", + ["Abooluganda", "ab’emmamba", "ababiri"], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", LG_BASIC_TOKENIZATION_TESTS) +def test_lg_tokenizer_basic(lg_tokenizer, text, expected_tokens): + tokens = lg_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/lt/__init__.py b/spacy/tests/lang/lt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/lt/test_text.py b/spacy/tests/lang/lt/test_text.py new file mode 100644 index 0000000..9e2b612 --- /dev/null +++ b/spacy/tests/lang/lt/test_text.py @@ -0,0 +1,53 @@ +import pytest + + +def test_lt_tokenizer_handles_long_text(lt_tokenizer): + text = """Tokios sausros kriterijus atitinka pirmadienį atlikti skaičiavimai, palyginus faktinį ir žemiausią vidutinį daugiametį vandens lygį. Nustatyta, kad iš 48 šalies vandens matavimo stočių 28-iose stotyse vandens lygis yra žemesnis arba lygus žemiausiam vidutiniam daugiamečiam šiltojo laikotarpio vandens lygiui.""" + tokens = lt_tokenizer(text) + assert len(tokens) == 42 + + +@pytest.mark.parametrize( + "text,length", + [ + ( + "177R Parodų rūmai–Ozo g. nuo vasario 18 d. bus skelbiamas interneto tinklalapyje.", + 17, + ), + ( + "ISM universiteto doc. dr. Ieva Augutytė-Kvedaravičienė pastebi, kad tyrimais nustatyti elgesio pokyčiai.", + 18, + ), + ], +) +def test_lt_tokenizer_handles_punct_abbrev(lt_tokenizer, text, length): + tokens = lt_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["km.", "pvz.", "biol."]) +def test_lt_tokenizer_abbrev_exceptions(lt_tokenizer, text): + tokens = lt_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("999.0", True), + ("vienas", True), + ("du", True), + ("milijardas", True), + ("šuo", False), + (",", False), + ("1/2", True), + ], +) +def test_lt_lex_attrs_like_number(lt_tokenizer, text, match): + tokens = lt_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/lv/__init__.py b/spacy/tests/lang/lv/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/lv/test_text.py b/spacy/tests/lang/lv/test_text.py new file mode 100644 index 0000000..5ca5fd0 --- /dev/null +++ b/spacy/tests/lang/lv/test_text.py @@ -0,0 +1,27 @@ +import pytest + + +def test_long_text(lv_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +Ievērodamas, ka šī deklarācija paredz nodrošināt vispārēju un +efektīvu tajā pasludināto tiesību atzīšanu un ievērošanu; +Ievērodamas, ka Eiropas Padomes mērķis ir panākt lielāku vienotību +tās dalībvalstu starpā un ka viens no līdzekļiem, kā šo mērķi +sasniegt, ir cilvēka tiesību un pamatbrīvību ievērošana un turpmāka +īstenošana; +No jauna apliecinādamas patiesu pārliecību, ka šīs pamatbrīvības +ir taisnīguma un miera pamats visā pasaulē un ka tās vislabāk var +nodrošināt patiess demokrātisks politisks režīms no vienas puses un +vispārējo cilvēktiesību, uz kurām tās pamatojas, kopīga izpratne un +ievērošana no otras puses; +""" + tokens = lv_tokenizer(text) + assert len(tokens) == 109 + + +@pytest.mark.xfail +def test_ordinal_number(lv_tokenizer): + text = "10. decembrī" + tokens = lv_tokenizer(text) + assert len(tokens) == 2 diff --git a/spacy/tests/lang/lv/test_tokenizer.py b/spacy/tests/lang/lv/test_tokenizer.py new file mode 100644 index 0000000..3ce7ad5 --- /dev/null +++ b/spacy/tests/lang/lv/test_tokenizer.py @@ -0,0 +1,30 @@ +import pytest + +LV_BASIC_TOKENIZATION_TESTS = [ + ( + "Nevienu nedrīkst spīdzināt vai cietsirdīgi vai pazemojoši ar viņu " + "apieties vai sodīt.", + [ + "Nevienu", + "nedrīkst", + "spīdzināt", + "vai", + "cietsirdīgi", + "vai", + "pazemojoši", + "ar", + "viņu", + "apieties", + "vai", + "sodīt", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", LV_BASIC_TOKENIZATION_TESTS) +def test_lv_tokenizer_basic(lv_tokenizer, text, expected_tokens): + tokens = lv_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/mk/__init__.py b/spacy/tests/lang/mk/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/mk/test_text.py b/spacy/tests/lang/mk/test_text.py new file mode 100644 index 0000000..b3a7ff9 --- /dev/null +++ b/spacy/tests/lang/mk/test_text.py @@ -0,0 +1,78 @@ +import pytest + +from spacy.lang.mk.lex_attrs import like_num + + +def test_tokenizer_handles_long_text(mk_tokenizer): + text = """ + Во организациските работи или на нашите собранија со членството, никој од нас не зборуваше за + организацијата и идеологијата. Работна беше нашата работа, а не идеолошка. Што се однесува до социјализмот на + Делчев, неговата дејност зборува сама за себе - спротивно. Во суштина, водачите си имаа свои основни погледи и + свои разбирања за положбата и работите, коишто стоеја пред нив и ги завршуваа со голема упорност, настојчивост и + насоченост. Значи, идеологија имаше, само што нивната идеологија имаше своја оригиналност. Македонија денеска, + чиста рожба на животот и положбата во Македонија, кои му служеа како база на неговите побуди, беше дејност која + имаше потреба од ум за да си најде своја смисла. Таквата идеологија и заемното дејство на умот и срцето му + помогнаа на Делчев да не се занесе по патот на својата идеологија... Во суштина, Организацијата и нејзините + водачи имаа свои разбирања за работите и положбата во идеен поглед, но тоа беше врската, животот и положбата во + Македонија и го внесуваа во својата идеологија гласот на своето срце, и на крај, прибегнуваа до умот, + за да најдат смисла или да ѝ дадат. Тоа содејство и заемен сооднос на умот и срцето му помогнаа на Делчев да ја + држи својата идеологија во сообразност со положбата на работите... Водачите навистина направија една жртва + бидејќи на населението не му зборуваа за своите мисли и идеи. Тие се одрекоа од секаква субјективност во своите + мисли. Целта беше да не се зголемуваат целите и задачите како и преданоста во работата. Населението не можеше да + ги разбере овие идеи... + """ + tokens = mk_tokenizer(text) + assert len(tokens) == 297 + + +@pytest.mark.parametrize( + "word,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("бројка", False), + ("999,0", True), + ("еден", True), + ("два", True), + ("цифра", False), + ("десет", True), + ("сто", True), + ("број", False), + ("илјада", True), + ("илјади", True), + ("милион", True), + (",", False), + ("милијарда", True), + ("билион", True), + ], +) +def test_mk_lex_attrs_like_number(mk_tokenizer, word, match): + tokens = mk_tokenizer(word) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["двесте", "два-три", "пет-шест"]) +def test_mk_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) + + +@pytest.mark.parametrize( + "word", + [ + "првиот", + "втора", + "четврт", + "четвртата", + "петти", + "петто", + "стоти", + "шеесетите", + "седумдесетите", + ], +) +def test_mk_lex_attrs_like_number_for_ordinal(word): + assert like_num(word) diff --git a/spacy/tests/lang/ml/__init__.py b/spacy/tests/lang/ml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ml/test_text.py b/spacy/tests/lang/ml/test_text.py new file mode 100644 index 0000000..aced784 --- /dev/null +++ b/spacy/tests/lang/ml/test_text.py @@ -0,0 +1,22 @@ +import pytest + + +def test_ml_tokenizer_handles_long_text(ml_tokenizer): + text = """അനാവശ്യമായി കണ്ണിലും മൂക്കിലും വായിലും സ്പർശിക്കാതിരിക്കുക""" + tokens = ml_tokenizer(text) + assert len(tokens) == 5 + + +@pytest.mark.parametrize( + "text,length", + [ + ( + "എന്നാൽ അച്ചടിയുടെ ആവിർഭാവം ലിപിയിൽ കാര്യമായ മാറ്റങ്ങൾ വരുത്തിയത് കൂട്ടക്ഷരങ്ങളെ അണുഅക്ഷരങ്ങളായി പിരിച്ചുകൊണ്ടായിരുന്നു", + 10, + ), + ("പരമ്പരാഗതമായി മലയാളം ഇടത്തുനിന്ന് വലത്തോട്ടാണ് എഴുതുന്നത്", 5), + ], +) +def test_ml_tokenizer_handles_cnts(ml_tokenizer, text, length): + tokens = ml_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/ms/__init__.py b/spacy/tests/lang/ms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ms/test_noun_chunks.py b/spacy/tests/lang/ms/test_noun_chunks.py new file mode 100644 index 0000000..859307d --- /dev/null +++ b/spacy/tests/lang/ms/test_noun_chunks.py @@ -0,0 +1,8 @@ +import pytest + + +def test_noun_chunks_is_parsed_ms(ms_tokenizer): + """Test that noun_chunks raises Value Error for 'ms' language if Doc is not parsed.""" + doc = ms_tokenizer("sebelas") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/ms/test_prefix_suffix_infix.py b/spacy/tests/lang/ms/test_prefix_suffix_infix.py new file mode 100644 index 0000000..0d2b2c5 --- /dev/null +++ b/spacy/tests/lang/ms/test_prefix_suffix_infix.py @@ -0,0 +1,112 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(Ma'arif)"]) +def test_ms_tokenizer_splits_no_special(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Ma'arif"]) +def test_ms_tokenizer_splits_no_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["(Ma'arif"]) +def test_ms_tokenizer_splits_prefix_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["Ma'arif)"]) +def test_ms_tokenizer_splits_suffix_punct(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(Ma'arif)"]) +def test_ms_tokenizer_splits_even_wrap(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(Ma'arif?)"]) +def test_tokenizer_splits_uneven_wrap(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize("text,length", [("S.Kom.", 1), ("SKom.", 2), ("(S.Kom.", 2)]) +def test_ms_tokenizer_splits_prefix_interact(id_tokenizer, text, length): + tokens = id_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["S.Kom.)"]) +def test_ms_tokenizer_splits_suffix_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["(S.Kom.)"]) +def test_ms_tokenizer_splits_even_wrap_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["(S.Kom.?)"]) +def test_ms_tokenizer_splits_uneven_wrap_interact(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 4 + + +@pytest.mark.parametrize( + "text,length", + [("kerana", 1), ("Mahathir-Anwar", 3), ("Tun Dr. Ismail-Abdul Rahman", 6)], +) +def test_my_tokenizer_splits_hyphens(ms_tokenizer, text, length): + tokens = ms_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize("text", ["0.1-13.5", "0.0-0.1", "103.27-300"]) +def test_ms_tokenizer_splits_numeric_range(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["ini.Sani", "Halo.Malaysia"]) +def test_ms_tokenizer_splits_period_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Halo,Malaysia", "satu,dua"]) +def test_ms_tokenizer_splits_comma_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["halo...Malaysia", "dia...pergi"]) +def test_ms_tokenizer_splits_ellipsis_infix(id_tokenizer, text): + tokens = id_tokenizer(text) + assert len(tokens) == 3 + + +def test_ms_tokenizer_splits_double_hyphen_infix(id_tokenizer): + tokens = id_tokenizer("Arsene Wenger--pengurus Arsenal--mengadakan sidang media.") + assert len(tokens) == 10 + assert tokens[0].text == "Arsene" + assert tokens[1].text == "Wenger" + assert tokens[2].text == "--" + assert tokens[3].text == "pengurus" + assert tokens[4].text == "Arsenal" + assert tokens[5].text == "--" + assert tokens[6].text == "mengadakan" + assert tokens[7].text == "sidang" + assert tokens[8].text == "media" + assert tokens[9].text == "." diff --git a/spacy/tests/lang/ms/test_text.py b/spacy/tests/lang/ms/test_text.py new file mode 100644 index 0000000..4b0ac3b --- /dev/null +++ b/spacy/tests/lang/ms/test_text.py @@ -0,0 +1,9 @@ +import pytest + +from spacy.lang.ms.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["sebelas"]) +def test_ms_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/nb/__init__.py b/spacy/tests/lang/nb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/nb/test_noun_chunks.py b/spacy/tests/lang/nb/test_noun_chunks.py new file mode 100644 index 0000000..dd259f2 --- /dev/null +++ b/spacy/tests/lang/nb/test_noun_chunks.py @@ -0,0 +1,8 @@ +import pytest + + +def test_noun_chunks_is_parsed_nb(nb_tokenizer): + """Test that noun_chunks raises Value Error for 'nb' language if Doc is not parsed.""" + doc = nb_tokenizer("Smørsausen brukes bl.a. til") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/nb/test_tokenizer.py b/spacy/tests/lang/nb/test_tokenizer.py new file mode 100644 index 0000000..4f5fd89 --- /dev/null +++ b/spacy/tests/lang/nb/test_tokenizer.py @@ -0,0 +1,19 @@ +import pytest + +NB_TOKEN_EXCEPTION_TESTS = [ + ( + "Smørsausen brukes bl.a. til fisk", + ["Smørsausen", "brukes", "bl.a.", "til", "fisk"], + ), + ( + "Jeg kommer først kl. 13 pga. diverse forsinkelser", + ["Jeg", "kommer", "først", "kl.", "13", "pga.", "diverse", "forsinkelser"], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", NB_TOKEN_EXCEPTION_TESTS) +def test_nb_tokenizer_handles_exception_cases(nb_tokenizer, text, expected_tokens): + tokens = nb_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/ne/__init__.py b/spacy/tests/lang/ne/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ne/test_text.py b/spacy/tests/lang/ne/test_text.py new file mode 100644 index 0000000..e8a6c2e --- /dev/null +++ b/spacy/tests/lang/ne/test_text.py @@ -0,0 +1,15 @@ +import pytest + + +def test_ne_tokenizer_handlers_long_text(ne_tokenizer): + text = """मैले पाएको सर्टिफिकेटलाई म त बोक्रो सम्झन्छु र अभ्यास तब सुरु भयो, जब मैले कलेज पार गरेँ र जीवनको पढाइ सुरु गरेँ ।""" + tokens = ne_tokenizer(text) + assert len(tokens) == 24 + + +@pytest.mark.parametrize( + "text,length", [("समय जान कति पनि बेर लाग्दैन ।", 7), ("म ठूलो हुँदै थिएँ ।", 5)] +) +def test_ne_tokenizer_handles_cnts(ne_tokenizer, text, length): + tokens = ne_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/nl/__init__.py b/spacy/tests/lang/nl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/nl/test_noun_chunks.py b/spacy/tests/lang/nl/test_noun_chunks.py new file mode 100644 index 0000000..6004ac2 --- /dev/null +++ b/spacy/tests/lang/nl/test_noun_chunks.py @@ -0,0 +1,226 @@ +import pytest + +from spacy.tokens import Doc +from spacy.util import filter_spans + + +@pytest.fixture +def nl_sample(nl_vocab): + # TEXT : + # Haar vriend lacht luid. We kregen alweer ruzie toen we de supermarkt ingingen. + # Aan het begin van de supermarkt is al het fruit en de groentes. Uiteindelijk hebben we dan ook + # geen avondeten gekocht. + words = [ + "Haar", + "vriend", + "lacht", + "luid", + ".", + "We", + "kregen", + "alweer", + "ruzie", + "toen", + "we", + "de", + "supermarkt", + "ingingen", + ".", + "Aan", + "het", + "begin", + "van", + "de", + "supermarkt", + "is", + "al", + "het", + "fruit", + "en", + "de", + "groentes", + ".", + "Uiteindelijk", + "hebben", + "we", + "dan", + "ook", + "geen", + "avondeten", + "gekocht", + ".", + ] + heads = [ + 1, + 2, + 2, + 2, + 2, + 6, + 6, + 6, + 6, + 13, + 13, + 12, + 13, + 6, + 6, + 17, + 17, + 24, + 20, + 20, + 17, + 24, + 24, + 24, + 24, + 27, + 27, + 24, + 24, + 36, + 36, + 36, + 36, + 36, + 35, + 36, + 36, + 36, + ] + deps = [ + "nmod:poss", + "nsubj", + "ROOT", + "advmod", + "punct", + "nsubj", + "ROOT", + "advmod", + "obj", + "mark", + "nsubj", + "det", + "obj", + "advcl", + "punct", + "case", + "det", + "obl", + "case", + "det", + "nmod", + "cop", + "advmod", + "det", + "ROOT", + "cc", + "det", + "conj", + "punct", + "advmod", + "aux", + "nsubj", + "advmod", + "advmod", + "det", + "obj", + "ROOT", + "punct", + ] + pos = [ + "PRON", + "NOUN", + "VERB", + "ADJ", + "PUNCT", + "PRON", + "VERB", + "ADV", + "NOUN", + "SCONJ", + "PRON", + "DET", + "NOUN", + "NOUN", + "PUNCT", + "ADP", + "DET", + "NOUN", + "ADP", + "DET", + "NOUN", + "AUX", + "ADV", + "DET", + "NOUN", + "CCONJ", + "DET", + "NOUN", + "PUNCT", + "ADJ", + "AUX", + "PRON", + "ADV", + "ADV", + "DET", + "NOUN", + "VERB", + "PUNCT", + ] + return Doc(nl_vocab, words=words, heads=heads, deps=deps, pos=pos) + + +@pytest.fixture +def nl_reference_chunking(): + # Using frog https://github.com/LanguageMachines/frog/ we obtain the following NOUN-PHRASES: + return [ + "haar vriend", + "we", + "ruzie", + "we", + "de supermarkt", + "het begin", + "de supermarkt", + "het fruit", + "de groentes", + "we", + "geen avondeten", + ] + + +def test_need_dep(nl_tokenizer): + """ + Test that noun_chunks raises Value Error for 'nl' language if Doc is not parsed. + """ + txt = "Haar vriend lacht luid." + doc = nl_tokenizer(txt) + + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +def test_chunking(nl_sample, nl_reference_chunking): + """ + Test the noun chunks of a sample text. Uses a sample. + The sample text simulates a Doc object as would be produced by nl_core_news_md. + """ + chunks = [s.text.lower() for s in nl_sample.noun_chunks] + assert chunks == nl_reference_chunking + + +@pytest.mark.issue(10846) +def test_no_overlapping_chunks(nl_vocab): + # fmt: off + doc = Doc( + nl_vocab, + words=["Dit", "programma", "wordt", "beschouwd", "als", "'s", "werelds", "eerste", "computerprogramma"], + deps=["det", "nsubj:pass", "aux:pass", "ROOT", "mark", "det", "fixed", "amod", "xcomp"], + heads=[1, 3, 3, 3, 8, 8, 5, 8, 3], + pos=["DET", "NOUN", "AUX", "VERB", "SCONJ", "DET", "NOUN", "ADJ", "NOUN"], + ) + # fmt: on + chunks = list(doc.noun_chunks) + assert filter_spans(chunks) == chunks diff --git a/spacy/tests/lang/nl/test_text.py b/spacy/tests/lang/nl/test_text.py new file mode 100644 index 0000000..d6413e0 --- /dev/null +++ b/spacy/tests/lang/nl/test_text.py @@ -0,0 +1,25 @@ +import pytest + +from spacy.lang.nl.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["elf", "elfde"]) +def test_nl_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) + + +@pytest.mark.parametrize( + "text,num_tokens", + [ + ( + "De aftredende minister-president benadrukte al dat zijn partij inhoudelijk weinig gemeen heeft met de groenen.", + 16, + ), + ("Hij is sociaal-cultureel werker.", 5), + ("Er staan een aantal dure auto's in de garage.", 10), + ], +) +def test_tokenizer_doesnt_split_hyphens(nl_tokenizer, text, num_tokens): + tokens = nl_tokenizer(text) + assert len(tokens) == num_tokens diff --git a/spacy/tests/lang/nn/__init__.py b/spacy/tests/lang/nn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/nn/test_tokenizer.py b/spacy/tests/lang/nn/test_tokenizer.py new file mode 100644 index 0000000..74a6937 --- /dev/null +++ b/spacy/tests/lang/nn/test_tokenizer.py @@ -0,0 +1,38 @@ +import pytest + +# examples taken from Omsetjingsminne frå Nynorsk pressekontor 2022 (https://www.nb.no/sprakbanken/en/resource-catalogue/oai-nb-no-sbr-80/) +# fmt: off +NN_TOKEN_EXCEPTION_TESTS = [ + ( + "Målet til direktoratet er at alle skal bli tilbydd jobb i politiet så raskt som mogleg i 2014.", + [ + "Målet", "til", "direktoratet", "er", "at", "alle", "skal", "bli", "tilbydd", "jobb", "i", "politiet", "så", "raskt", "som", "mogleg", "i", "2014", ".", + ], + ), + ( + "Han ønskjer ikkje at staten skal vere med på å finansiere slik undervisning, men dette er rektor på skulen ueinig i.", + [ + "Han", "ønskjer", "ikkje", "at", "staten", "skal", "vere", "med", "på", "å", "finansiere", "slik", "undervisning", ",", "men", "dette", "er", "rektor", "på", "skulen", "ueinig", "i", ".", + ], + ), + ( + "Ifølgje China Daily vart det 8.848 meter høge fjellet flytta 3 centimeter sørvestover under jordskjelvet, som vart målt til 7,8.", + [ + "Ifølgje", "China", "Daily", "vart", "det", "8.848", "meter", "høge", "fjellet", "flytta", "3", "centimeter", "sørvestover", "under", "jordskjelvet", ",", "som", "vart", "målt", "til", "7,8", ".", + ], + ), + ( + "Brukssesongen er frå nov. til mai, med ein topp i mars.", + [ + "Brukssesongen", "er", "frå", "nov.", "til", "mai", ",", "med", "ein", "topp", "i", "mars", ".", + ], + ), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", NN_TOKEN_EXCEPTION_TESTS) +def test_nn_tokenizer_handles_exception_cases(nn_tokenizer, text, expected_tokens): + tokens = nn_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/pl/__init__.py b/spacy/tests/lang/pl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/pl/test_text.py b/spacy/tests/lang/pl/test_text.py new file mode 100644 index 0000000..43e4ab8 --- /dev/null +++ b/spacy/tests/lang/pl/test_text.py @@ -0,0 +1,24 @@ +"""Words like numbers are recognized correctly.""" + +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("jeden", True), + ("dwa", True), + ("milion", True), + ("pies", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(pl_tokenizer, text, match): + tokens = pl_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/pl/test_tokenizer.py b/spacy/tests/lang/pl/test_tokenizer.py new file mode 100644 index 0000000..44b1be9 --- /dev/null +++ b/spacy/tests/lang/pl/test_tokenizer.py @@ -0,0 +1,23 @@ +import pytest + +DOT_TESTS = [ + ("tel.", ["tel", "."]), + ("0 zł 99 gr", ["0", "zł", "99", "gr"]), +] + +HYPHEN_TESTS = [ + ("cztero-", ["cztero-"]), + ("jedno-", ["jedno-"]), + ("dwu-", ["dwu-"]), + ("trzy-", ["trzy-"]), +] + + +TESTCASES = DOT_TESTS + HYPHEN_TESTS + + +@pytest.mark.parametrize("text,expected_tokens", TESTCASES) +def test_tokenizer_handles_testcases(pl_tokenizer, text, expected_tokens): + tokens = pl_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/pt/__init__.py b/spacy/tests/lang/pt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/pt/test_noun_chunks.py b/spacy/tests/lang/pt/test_noun_chunks.py new file mode 100644 index 0000000..5dd7bfd --- /dev/null +++ b/spacy/tests/lang/pt/test_noun_chunks.py @@ -0,0 +1,222 @@ +import pytest + +from spacy.tokens import Doc + + +# fmt: off +@pytest.mark.parametrize( + "words,heads,deps,pos,chunk_offsets", + [ + # determiner + noun + # um cachorro -> um cachorro + ( + ["um", "cachorro"], + [1, 1], + ["det", "ROOT"], + ["DET", "NOUN"], + [(0, 2)], + ), + # two determiners + noun + # meu o pai -> meu o pai + ( + ["meu", "o", "pai"], + [2, 2, 2], + ["det", "det", "ROOT"], + ["DET", "DET", "NOUN"], + [(0, 3)], + ), + # two determiners + noun + # todos essos caros -> todos essos caros + ( + ["todos", "essos", "caros"], + [2, 2, 2], + ["det", "det", "ROOT"], + ["DET", "DET", "NOUN"], + [(0, 3)], + ), + # two determiners, one is after noun + # um irmão meu -> um irmão meu + ( + ["um", "irmão", "meu"], + [1, 1, 1], + ["det", "ROOT", "det"], + ["DET", "NOUN", "DET"], + [(0, 3)], + ), + # two determiners + noun + # o meu pai -> o meu pai + ( + ["o", "meu", "pai"], + [2, 2, 2], + ["det","det", "ROOT"], + ["DET", "DET", "NOUN"], + [(0, 3)], + ), + # relative pronoun + # A bicicleta essa está estragada -> A bicicleta + ( + ['A', 'bicicleta', 'essa', 'está', 'estragada'], + [1, 4, 1, 4, 4], + ['det', 'nsubj', 'det', 'cop', 'ROOT'], + ['DET', 'NOUN', 'PRON', 'AUX', 'ADJ'], + [(0,2)] + ), + # relative subclause + # o computador que comprou -> o computador + ( + ['o', 'computador', 'que', 'comprou'], + [1, 1, 3, 1], + ['det', 'ROOT', 'nsubj', 'acl:relcl'], + ['DET', 'NOUN', 'PRON', 'VERB'], + [(0, 2), (2, 3)] + ), + # det + noun + adj + # O cachorro marrom -> O cachorro marrom + ( + ["O", "cachorro", "marrom"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0, 3)], + ), + # det + noun + adj plural + # As calças baratas -> As calças baratas + ( + ["As", "calças", "baratas"], + [1, 1, 1], + ["det", "ROOT", "amod"], + ["DET", "NOUN", "ADJ"], + [(0, 3)], + ), + # det + adj + noun + # Uma boa ideia -> Uma boa ideia + ( + ['uma', 'boa', 'ideia'], + [2, 2, 2], + ["det", "amod", "ROOT"], + ["DET", "ADJ", "NOUN"], + [(0,3)] + ), + # multiple adjectives + # Uma garota esperta e inteligente -> Uma garota esperta e inteligente + ( + ["Uma", "garota", "esperta", "e", "inteligente"], + [1, 1, 1, 4, 2], + ["det", "ROOT", "amod", "cc", "conj"], + ["DET", "NOUN", "ADJ", "CCONJ", "ADJ"], + [(0,5)] + ), + # determiner, adjective, compound created by flat + # a grande São Paolo -> a grande São Paolo + ( + ["a", "grande", "São", "Paolo"], + [2, 2, 2, 2], + ["det", "amod", "ROOT", "flat:name"], + ["DET", "ADJ", "PROPN", "PROPN"], + [(0,4)] + ), + # one determiner + one noun + one adjective qualified by an adverb + # alguns fazendeiros muito ricos -> alguns fazendeiros muito ricos + ( + ['alguns', 'fazendeiros', 'muito', 'ricos'], + [1, 1, 3, 1], + ['det', 'ROOT', 'advmod', 'amod'], + ['DET', 'NOUN', 'ADV', 'ADJ'], + [(0,4)] + ), + # Two NPs conjuncted + # Eu tenho um cachorro e um gato -> Eu, um cacharo, um gato + ( + ["Eu", "tenho", "um", "cachorro", "e", "um", "gato"], + [1, 1, 3, 1, 6, 6, 3], + ['nsubj', 'ROOT', 'det', 'obj', 'cc', 'det', 'conj'], + ['PRON', 'VERB', 'DET', 'NOUN', 'CCONJ', 'DET', 'NOUN'], + [(0,1), (2,4), (5,7)] + + ), + # Two NPs together + # o escritor brasileiro Aníbal Machado -> o escritor brasileiro, Aníbal Machado + ( + ['o', 'escritor', 'brasileiro', 'Aníbal', 'Machado'], + [1, 1, 1, 1, 3], + ['det', 'ROOT', 'amod', 'appos', 'flat:name'], + ['DET', 'NOUN', 'ADJ', 'PROPN', 'PROPN'], + [(0, 3), (3, 5)] + ), + # Noun compound, person name and titles + # Dom Pedro II -> Dom Pedro II + ( + ["Dom", "Pedro", "II"], + [0, 0, 0], + ["ROOT", "flat:name", "flat:name"], + ["PROPN", "PROPN", "PROPN"], + [(0,3)] + ), + # Noun compound created by flat + # os Estados Unidos -> os Estados Unidos + ( + ["os", "Estados", "Unidos"], + [1, 1, 1], + ["det", "ROOT", "flat:name"], + ["DET", "PROPN", "PROPN"], + [(0,3)] + ), + # nmod relation between NPs + # a destruição da cidade -> a destruição, cidade + ( + ['a', 'destruição', 'da', 'cidade'], + [1, 1, 3, 1], + ['det', 'ROOT', 'case', 'nmod'], + ['DET', 'NOUN', 'ADP', 'NOUN'], + [(0,2), (3,4)] + ), + # Compounding by nmod, several NPs chained together + # a primeira fábrica de medicamentos do governo -> a primeira fábrica, medicamentos, governo + ( + ["a", "primeira", "fábrica", "de", "medicamentos", "do", "governo"], + [2, 2, 2, 4, 2, 6, 2], + ['det', 'amod', 'ROOT', 'case', 'nmod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'ADP', 'NOUN', 'ADP', 'NOUN'], + [(0, 3), (4, 5), (6, 7)] + ), + # several NPs + # Tradução da reportagem de Susana -> Tradução, reportagem, Susana + ( + ['Tradução', 'da', 'reportagem', 'de', 'Susana'], + [0, 2, 0, 4, 2], + ['ROOT', 'case', 'nmod', 'case', 'nmod'], + ['NOUN', 'ADP', 'NOUN', 'ADP', 'PROPN'], + [(0,1), (2,3), (4,5)] + + ), + # Several NPs + # O gato gordo da Susana e seu amigo -> O gato gordo, Susana, seu amigo + ( + ['O', 'gato', 'gordo', 'da', 'Susana', 'e', 'seu', 'amigo'], + [1, 1, 1, 4, 1, 7, 7, 1], + ['det', 'ROOT', 'amod', 'case', 'nmod', 'cc', 'det', 'conj'], + ['DET', 'NOUN', 'ADJ', 'ADP', 'PROPN', 'CCONJ', 'DET', 'NOUN'], + [(0,3), (4,5), (6,8)] + ), + # Passive subject + # Os novos gastos são alimentados pela grande conta bancária de Clinton -> Os novos gastos, grande conta bancária, Clinton + ( + ['Os', 'novos', 'gastos', 'são', 'alimentados', 'pela', 'grande', 'conta', 'bancária', 'de', 'Clinton'], + [2, 2, 4, 4, 4, 7, 7, 4, 7, 10, 7], + ['det', 'amod', 'nsubj:pass', 'aux:pass', 'ROOT', 'case', 'amod', 'obl:agent', 'amod', 'case', 'nmod'], + ['DET', 'ADJ', 'NOUN', 'AUX', 'VERB', 'ADP', 'ADJ', 'NOUN', 'ADJ', 'ADP', 'PROPN'], + [(0, 3), (6, 9), (10, 11)] + ) + ], +) +# fmt: on +def test_pt_noun_chunks(pt_vocab, words, heads, deps, pos, chunk_offsets): + doc = Doc(pt_vocab, words=words, heads=heads, deps=deps, pos=pos) + assert [(c.start, c.end) for c in doc.noun_chunks] == chunk_offsets + + +def test_noun_chunks_is_parsed_pt(pt_tokenizer): + """Test that noun_chunks raises Value Error for 'pt' language if Doc is not parsed.""" + doc = pt_tokenizer("en Oxford este verano") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/pt/test_text.py b/spacy/tests/lang/pt/test_text.py new file mode 100644 index 0000000..cb87239 --- /dev/null +++ b/spacy/tests/lang/pt/test_text.py @@ -0,0 +1,9 @@ +import pytest + +from spacy.lang.pt.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["onze", "quadragésimo"]) +def test_pt_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/ro/__init__.py b/spacy/tests/lang/ro/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ro/test_tokenizer.py b/spacy/tests/lang/ro/test_tokenizer.py new file mode 100644 index 0000000..d2affd6 --- /dev/null +++ b/spacy/tests/lang/ro/test_tokenizer.py @@ -0,0 +1,21 @@ +import pytest + +TEST_CASES = [ + ( + "Adresa este str. Principală nr. 5.", + ["Adresa", "este", "str.", "Principală", "nr.", "5", "."], + ), + ("Teste, etc.", ["Teste", ",", "etc."]), + ("Lista, ș.a.m.d.", ["Lista", ",", "ș.a.m.d."]), + ("Și d.p.d.v. al...", ["Și", "d.p.d.v.", "al", "..."]), + # number tests + ("Clasa a 4-a.", ["Clasa", "a", "4-a", "."]), + ("Al 12-lea ceas.", ["Al", "12-lea", "ceas", "."]), +] + + +@pytest.mark.parametrize("text,expected_tokens", TEST_CASES) +def test_ro_tokenizer_handles_testcases(ro_tokenizer, text, expected_tokens): + tokens = ro_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/ru/__init__.py b/spacy/tests/lang/ru/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ru/test_exceptions.py b/spacy/tests/lang/ru/test_exceptions.py new file mode 100644 index 0000000..4fb417d --- /dev/null +++ b/spacy/tests/lang/ru/test_exceptions.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.parametrize( + "text,norms", + [("пн.", ["понедельник"]), ("пт.", ["пятница"]), ("дек.", ["декабрь"])], +) +def test_ru_tokenizer_abbrev_exceptions(ru_tokenizer, text, norms): + tokens = ru_tokenizer(text) + assert len(tokens) == 1 + assert [token.norm_ for token in tokens] == norms diff --git a/spacy/tests/lang/ru/test_lemmatizer.py b/spacy/tests/lang/ru/test_lemmatizer.py new file mode 100644 index 0000000..66aa7e3 --- /dev/null +++ b/spacy/tests/lang/ru/test_lemmatizer.py @@ -0,0 +1,109 @@ +import pytest + +from spacy.tokens import Doc + +pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") + + +def test_ru_doc_lemmatization(ru_lemmatizer): + words = ["мама", "мыла", "раму"] + pos = ["NOUN", "VERB", "NOUN"] + morphs = [ + "Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing", + "Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act", + "Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing", + ] + doc = Doc(ru_lemmatizer.vocab, words=words, pos=pos, morphs=morphs) + doc = ru_lemmatizer(doc) + lemmas = [token.lemma_ for token in doc] + assert lemmas == ["мама", "мыть", "рама"] + + +@pytest.mark.parametrize( + "text,lemmas", + [ + ("гвоздики", ["гвоздик", "гвоздика"]), + ("люди", ["человек"]), + ("реки", ["река"]), + ("кольцо", ["кольцо"]), + ("пепперони", ["пепперони"]), + ], +) +def test_ru_lemmatizer_noun_lemmas(ru_lemmatizer, text, lemmas): + doc = Doc(ru_lemmatizer.vocab, words=[text], pos=["NOUN"]) + result_lemmas = ru_lemmatizer.pymorphy2_lemmatize(doc[0]) + assert sorted(result_lemmas) == lemmas + + +@pytest.mark.parametrize( + "text,pos,morph,lemma", + [ + ("рой", "NOUN", "", "рой"), + ("рой", "VERB", "", "рыть"), + ("клей", "NOUN", "", "клей"), + ("клей", "VERB", "", "клеить"), + ("три", "NUM", "", "три"), + ("кос", "NOUN", "Number=Sing", "кос"), + ("кос", "NOUN", "Number=Plur", "коса"), + ("кос", "ADJ", "", "косой"), + ("потом", "NOUN", "", "пот"), + ("потом", "ADV", "", "потом"), + ], +) +def test_ru_lemmatizer_works_with_different_pos_homonyms( + ru_lemmatizer, text, pos, morph, lemma +): + doc = Doc(ru_lemmatizer.vocab, words=[text], pos=[pos], morphs=[morph]) + result_lemmas = ru_lemmatizer.pymorphy2_lemmatize(doc[0]) + assert result_lemmas == [lemma] + + +@pytest.mark.parametrize( + "text,morph,lemma", + [ + ("гвоздики", "Gender=Fem", "гвоздика"), + ("гвоздики", "Gender=Masc", "гвоздик"), + ("вина", "Gender=Fem", "вина"), + ("вина", "Gender=Neut", "вино"), + ], +) +def test_ru_lemmatizer_works_with_noun_homonyms(ru_lemmatizer, text, morph, lemma): + doc = Doc(ru_lemmatizer.vocab, words=[text], pos=["NOUN"], morphs=[morph]) + result_lemmas = ru_lemmatizer.pymorphy2_lemmatize(doc[0]) + assert result_lemmas == [lemma] + + +def test_ru_lemmatizer_punct(ru_lemmatizer): + doc = Doc(ru_lemmatizer.vocab, words=["«"], pos=["PUNCT"]) + assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] + doc = Doc(ru_lemmatizer.vocab, words=["»"], pos=["PUNCT"]) + assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] + + +def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer): + assert ru_lookup_lemmatizer.mode == "pymorphy3_lookup" + words = ["мама", "мыла", "раму"] + pos = ["NOUN", "VERB", "NOUN"] + morphs = [ + "Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing", + "Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act", + "Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing", + ] + doc = Doc(ru_lookup_lemmatizer.vocab, words=words, pos=pos, morphs=morphs) + doc = ru_lookup_lemmatizer(doc) + lemmas = [token.lemma_ for token in doc] + assert lemmas == ["мама", "мыла", "раму"] + + +@pytest.mark.parametrize( + "word,lemma", + ( + ("бременем", "бремя"), + ("будешь", "быть"), + ("какая-то", "какой-то"), + ), +) +def test_ru_lookup_lemmatizer(ru_lookup_lemmatizer, word, lemma): + assert ru_lookup_lemmatizer.mode == "pymorphy3_lookup" + doc = Doc(ru_lookup_lemmatizer.vocab, words=[word]) + assert ru_lookup_lemmatizer(doc)[0].lemma_ == lemma diff --git a/spacy/tests/lang/ru/test_text.py b/spacy/tests/lang/ru/test_text.py new file mode 100644 index 0000000..0bbed21 --- /dev/null +++ b/spacy/tests/lang/ru/test_text.py @@ -0,0 +1,9 @@ +import pytest + +from spacy.lang.ru.lex_attrs import like_num + + +@pytest.mark.parametrize("word", ["одиннадцать"]) +def test_ru_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/ru/test_tokenizer.py b/spacy/tests/lang/ru/test_tokenizer.py new file mode 100644 index 0000000..c941e21 --- /dev/null +++ b/spacy/tests/lang/ru/test_tokenizer.py @@ -0,0 +1,158 @@ +from string import punctuation + +import pytest + +PUNCT_OPEN = ["(", "[", "{", "*"] +PUNCT_CLOSE = [")", "]", "}", "*"] +PUNCT_PAIRED = [("(", ")"), ("[", "]"), ("{", "}"), ("*", "*")] + + +@pytest.mark.parametrize("text", ["(", "((", "<"]) +def test_ru_tokenizer_handles_only_punct(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert len(tokens) == len(text) + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_open_punct(ru_tokenizer, punct, text): + tokens = ru_tokenizer(punct + text) + assert len(tokens) == 2 + assert tokens[0].text == punct + assert tokens[1].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_close_punct(ru_tokenizer, punct, text): + tokens = ru_tokenizer(text + punct) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_two_diff_open_punct(ru_tokenizer, punct, punct_add, text): + tokens = ru_tokenizer(punct + punct_add + text) + assert len(tokens) == 3 + assert tokens[0].text == punct + assert tokens[1].text == punct_add + assert tokens[2].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("punct_add", ["'"]) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_two_diff_close_punct(ru_tokenizer, punct, punct_add, text): + tokens = ru_tokenizer(text + punct + punct_add) + assert len(tokens) == 3 + assert tokens[0].text == text + assert tokens[1].text == punct + assert tokens[2].text == punct_add + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_same_open_punct(ru_tokenizer, punct, text): + tokens = ru_tokenizer(punct + punct + punct + text) + assert len(tokens) == 4 + assert tokens[0].text == punct + assert tokens[3].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Привет"]) +def test_ru_tokenizer_splits_same_close_punct(ru_tokenizer, punct, text): + tokens = ru_tokenizer(text + punct + punct + punct) + assert len(tokens) == 4 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("text", ["'Тест"]) +def test_ru_tokenizer_splits_open_appostrophe(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "'" + + +@pytest.mark.parametrize("text", ["Тест''"]) +def test_ru_tokenizer_splits_double_end_quote(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert len(tokens) == 2 + tokens_punct = ru_tokenizer("''") + assert len(tokens_punct) == 1 + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("text", ["Тест"]) +def test_ru_tokenizer_splits_open_close_punct( + ru_tokenizer, punct_open, punct_close, text +): + tokens = ru_tokenizer(punct_open + text + punct_close) + assert len(tokens) == 3 + assert tokens[0].text == punct_open + assert tokens[1].text == text + assert tokens[2].text == punct_close + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("punct_open2,punct_close2", [("`", "'")]) +@pytest.mark.parametrize("text", ["Тест"]) +def test_ru_tokenizer_two_diff_punct( + ru_tokenizer, punct_open, punct_close, punct_open2, punct_close2, text +): + tokens = ru_tokenizer(punct_open2 + punct_open + text + punct_close + punct_close2) + assert len(tokens) == 5 + assert tokens[0].text == punct_open2 + assert tokens[1].text == punct_open + assert tokens[2].text == text + assert tokens[3].text == punct_close + assert tokens[4].text == punct_close2 + + +@pytest.mark.parametrize("text", ["Тест."]) +def test_ru_tokenizer_splits_trailing_dot(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert tokens[1].text == "." + + +def test_ru_tokenizer_splits_bracket_period(ru_tokenizer): + text = "(Раз, два, три, проверка)." + tokens = ru_tokenizer(text) + assert tokens[len(tokens) - 1].text == "." + + +@pytest.mark.parametrize( + "text", + [ + "рекоменду́я подда́ть жару́. Самого́ Баргамота", + "РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́. САМОГО́ БАРГАМОТА", + "рекоменду̍я подда̍ть жару̍.Самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍.'Самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍,самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍:самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍. самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍, самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍: самого̍ Баргамота", + "рекоменду̍я подда̍ть жару̍-самого̍ Баргамота", + ], +) +def test_ru_tokenizer_handles_final_diacritics(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert tokens[2].text in ("жару́", "ЖАРУ́", "жару̍") + assert tokens[3].text in punctuation + + +@pytest.mark.parametrize( + "text", + [ + "РЕКОМЕНДУ́Я ПОДДА́ТЬ ЖАРУ́.САМОГО́ БАРГАМОТА", + "рекоменду̍я подда̍ть жару́.самого́ Баргамота", + ], +) +def test_ru_tokenizer_handles_final_diacritic_and_period(ru_tokenizer, text): + tokens = ru_tokenizer(text) + assert tokens[2].text.lower() == "жару́.самого́" diff --git a/spacy/tests/lang/sa/__init__.py b/spacy/tests/lang/sa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sa/test_text.py b/spacy/tests/lang/sa/test_text.py new file mode 100644 index 0000000..daa8d20 --- /dev/null +++ b/spacy/tests/lang/sa/test_text.py @@ -0,0 +1,42 @@ +import pytest + + +def test_sa_tokenizer_handles_long_text(sa_tokenizer): + text = """नानाविधानि दिव्यानि नानावर्णाकृतीनि च।।""" + tokens = sa_tokenizer(text) + assert len(tokens) == 6 + + +@pytest.mark.parametrize( + "text,length", + [ + ("श्री भगवानुवाच पश्य मे पार्थ रूपाणि शतशोऽथ सहस्रशः।", 9), + ("गुणान् सर्वान् स्वभावो मूर्ध्नि वर्तते ।", 6), + ], +) +def test_sa_tokenizer_handles_cnts(sa_tokenizer, text, length): + tokens = sa_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("999,0", True), + ("एकः ", True), + ("दश", True), + ("पञ्चदश", True), + ("चत्वारिंशत् ", True), + ("कूपे", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(sa_tokenizer, text, match): + tokens = sa_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/sk/__init__.py b/spacy/tests/lang/sk/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sk/test_text.py b/spacy/tests/lang/sk/test_text.py new file mode 100644 index 0000000..62ea2a7 --- /dev/null +++ b/spacy/tests/lang/sk/test_text.py @@ -0,0 +1,48 @@ +import pytest + + +def test_long_text(sk_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +majúc na zreteli, že cieľom tejto deklarácie je zabezpečiť všeobecné +a účinné uznávanie a dodržiavanie práv v nej vyhlásených; +majúc na zreteli, že cieľom Rady Európy je dosiahnutie väčšej +jednoty medzi jej členmi, a že jedným zo spôsobov, ktorým sa +má tento cieľ napĺňať, je ochrana a ďalší rozvoj ľudských práv +a základných slobôd; +znovu potvrdzujúc svoju hlbokú vieru v tie základné slobody, ktoré +sú základom spravodlivosti a mieru vo svete, a ktoré sú najlepšie +zachovávané na jednej strane účinnou politickou demokraciou +a na strane druhej spoločným poňatím a dodržiavaním ľudských +práv, od ktorých závisia; + """ + tokens = sk_tokenizer(text) + assert len(tokens) == 118 + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10,000", True), + ("10,00", True), + ("štyri", True), + ("devätnásť", True), + ("milión", True), + ("pes", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(sk_tokenizer, text, match): + tokens = sk_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.xfail +def test_ordinal_number(sk_tokenizer): + text = "10. decembra 1948" + tokens = sk_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/sk/test_tokenizer.py b/spacy/tests/lang/sk/test_tokenizer.py new file mode 100644 index 0000000..2478472 --- /dev/null +++ b/spacy/tests/lang/sk/test_tokenizer.py @@ -0,0 +1,15 @@ +import pytest + +SK_BASIC_TOKENIZATION_TESTS = [ + ( + "Kedy sa narodil Andrej Kiska?", + ["Kedy", "sa", "narodil", "Andrej", "Kiska", "?"], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", SK_BASIC_TOKENIZATION_TESTS) +def test_sk_tokenizer_basic(sk_tokenizer, text, expected_tokens): + tokens = sk_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/sl/__init__.py b/spacy/tests/lang/sl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sl/test_text.py b/spacy/tests/lang/sl/test_text.py new file mode 100644 index 0000000..4781beb --- /dev/null +++ b/spacy/tests/lang/sl/test_text.py @@ -0,0 +1,23 @@ +def test_long_text(sl_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +upoštevajoč, da si ta deklaracija prizadeva zagotoviti splošno in +učinkovito priznavanje in spoštovanje v njej razglašenih pravic, +upoštevajoč, da je cilj Sveta Evrope doseči večjo enotnost med +njegovimi članicami, in da je eden izmed načinov za zagotavljanje +tega cilja varstvo in nadaljnji razvoj človekovih pravic in temeljnih +svoboščin, +ponovno potrjujoč svojo globoko vero v temeljne svoboščine, na +katerih temeljita pravičnost in mir v svetu, in ki jih je mogoče najbolje +zavarovati na eni strani z dejansko politično demokracijo in na drugi +strani s skupnim razumevanjem in spoštovanjem človekovih pravic, +od katerih so te svoboščine odvisne, +""" + tokens = sl_tokenizer(text) + assert len(tokens) == 116 + + +def test_ordinal_number(sl_tokenizer): + text = "10. decembra 1948" + tokens = sl_tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/lang/sl/test_tokenizer.py b/spacy/tests/lang/sl/test_tokenizer.py new file mode 100644 index 0000000..f2b15b0 --- /dev/null +++ b/spacy/tests/lang/sl/test_tokenizer.py @@ -0,0 +1,32 @@ +import pytest + +SL_BASIC_TOKENIZATION_TESTS = [ + ( + "Vsakdo ima pravico do spoštovanja njegovega zasebnega in " + "družinskega življenja, doma in dopisovanja.", + [ + "Vsakdo", + "ima", + "pravico", + "do", + "spoštovanja", + "njegovega", + "zasebnega", + "in", + "družinskega", + "življenja", + ",", + "doma", + "in", + "dopisovanja", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", SL_BASIC_TOKENIZATION_TESTS) +def test_sl_tokenizer_basic(sl_tokenizer, text, expected_tokens): + tokens = sl_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/sq/__init__.py b/spacy/tests/lang/sq/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sq/test_text.py b/spacy/tests/lang/sq/test_text.py new file mode 100644 index 0000000..24d60af --- /dev/null +++ b/spacy/tests/lang/sq/test_text.py @@ -0,0 +1,22 @@ +def test_long_text(sq_tokenizer): + # Excerpt: European Convention on Human Rights + text = """ +Qeveritë nënshkruese, anëtare të Këshillit të Evropës, +Duke pasur parasysh Deklaratën Universale të të Drejtave të +Njeriut, të shpallur nga Asambleja e Përgjithshme e Kombeve të +Bashkuara më 10 dhjetor 1948; +Duke pasur parasysh, se kjo Deklaratë ka për qëllim të sigurojë +njohjen dhe zbatimin universal dhe efektiv të të drejtave të +shpallura në të; +Duke pasur parasysh se qëllimi i Këshillit të Evropës është që të +realizojë një bashkim më të ngushtë midis anëtarëve të tij dhe +se një nga mjetet për të arritur këtë qëllim është mbrojtja dhe +zhvillimi i të drejtave të njeriut dhe i lirive themelore; +Duke ripohuar besimin e tyre të thellë në këto liri themelore që +përbëjnë themelet e drejtësisë dhe të paqes në botë, ruajtja e të +cilave mbështetet kryesisht mbi një regjim politik demokratik nga +njëra anë, dhe nga ana tjetër mbi një kuptim dhe respektim të +përbashkët të të drejtave të njeriut nga të cilat varen; +""" + tokens = sq_tokenizer(text) + assert len(tokens) == 182 diff --git a/spacy/tests/lang/sq/test_tokenizer.py b/spacy/tests/lang/sq/test_tokenizer.py new file mode 100644 index 0000000..8fd25f5 --- /dev/null +++ b/spacy/tests/lang/sq/test_tokenizer.py @@ -0,0 +1,31 @@ +import pytest + +SQ_BASIC_TOKENIZATION_TESTS = [ + ( + "Askush nuk mund t’i nënshtrohet torturës ose dënimeve ose " + "trajtimeve çnjerëzore ose poshtëruese.", + [ + "Askush", + "nuk", + "mund", + "t’i", + "nënshtrohet", + "torturës", + "ose", + "dënimeve", + "ose", + "trajtimeve", + "çnjerëzore", + "ose", + "poshtëruese", + ".", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", SQ_BASIC_TOKENIZATION_TESTS) +def test_sq_tokenizer_basic(sq_tokenizer, text, expected_tokens): + tokens = sq_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/sr/__init__.py b/spacy/tests/lang/sr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sr/test_exceptions.py b/spacy/tests/lang/sr/test_exceptions.py new file mode 100644 index 0000000..fa92e5e --- /dev/null +++ b/spacy/tests/lang/sr/test_exceptions.py @@ -0,0 +1,16 @@ +import pytest + + +@pytest.mark.parametrize( + "text,norms,lemmas", + [ + ("о.г.", ["ове године"], ["ова година"]), + ("чет.", ["четвртак"], ["четвртак"]), + ("гђа", ["госпођа"], ["госпођа"]), + ("ил'", ["или"], ["или"]), + ], +) +def test_sr_tokenizer_abbrev_exceptions(sr_tokenizer, text, norms, lemmas): + tokens = sr_tokenizer(text) + assert len(tokens) == 1 + assert [token.norm_ for token in tokens] == norms diff --git a/spacy/tests/lang/sr/test_tokenizer.py b/spacy/tests/lang/sr/test_tokenizer.py new file mode 100644 index 0000000..7ecd959 --- /dev/null +++ b/spacy/tests/lang/sr/test_tokenizer.py @@ -0,0 +1,123 @@ +import pytest + +PUNCT_OPEN = ["(", "[", "{", "*"] +PUNCT_CLOSE = [")", "]", "}", "*"] +PUNCT_PAIRED = [("(", ")"), ("[", "]"), ("{", "}"), ("*", "*")] + + +@pytest.mark.parametrize("text", ["(", "((", "<"]) +def test_sr_tokenizer_handles_only_punct(sr_tokenizer, text): + tokens = sr_tokenizer(text) + assert len(tokens) == len(text) + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Здраво"]) +def test_sr_tokenizer_splits_open_punct(sr_tokenizer, punct, text): + tokens = sr_tokenizer(punct + text) + assert len(tokens) == 2 + assert tokens[0].text == punct + assert tokens[1].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Здраво"]) +def test_sr_tokenizer_splits_close_punct(sr_tokenizer, punct, text): + tokens = sr_tokenizer(text + punct) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize("text", ["Ћао"]) +def test_sr_tokenizer_splits_two_diff_open_punct(sr_tokenizer, punct, punct_add, text): + tokens = sr_tokenizer(punct + punct_add + text) + assert len(tokens) == 3 + assert tokens[0].text == punct + assert tokens[1].text == punct_add + assert tokens[2].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("punct_add", ["'"]) +@pytest.mark.parametrize("text", ["Здраво"]) +def test_sr_tokenizer_splits_two_diff_close_punct(sr_tokenizer, punct, punct_add, text): + tokens = sr_tokenizer(text + punct + punct_add) + assert len(tokens) == 3 + assert tokens[0].text == text + assert tokens[1].text == punct + assert tokens[2].text == punct_add + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Здраво"]) +def test_sr_tokenizer_splits_same_open_punct(sr_tokenizer, punct, text): + tokens = sr_tokenizer(punct + punct + punct + text) + assert len(tokens) == 4 + assert tokens[0].text == punct + assert tokens[3].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Здраво"]) +def test_sr_tokenizer_splits_same_close_punct(sr_tokenizer, punct, text): + tokens = sr_tokenizer(text + punct + punct + punct) + assert len(tokens) == 4 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("text", ["'Тест"]) +def test_sr_tokenizer_splits_open_appostrophe(sr_tokenizer, text): + tokens = sr_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "'" + + +@pytest.mark.parametrize("text", ["Тест''"]) +def test_sr_tokenizer_splits_double_end_quote(sr_tokenizer, text): + tokens = sr_tokenizer(text) + assert len(tokens) == 2 + tokens_punct = sr_tokenizer("''") + assert len(tokens_punct) == 1 + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("text", ["Тест"]) +def test_sr_tokenizer_splits_open_close_punct( + sr_tokenizer, punct_open, punct_close, text +): + tokens = sr_tokenizer(punct_open + text + punct_close) + assert len(tokens) == 3 + assert tokens[0].text == punct_open + assert tokens[1].text == text + assert tokens[2].text == punct_close + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("punct_open2,punct_close2", [("`", "'")]) +@pytest.mark.parametrize("text", ["Тест"]) +def test_sr_tokenizer_two_diff_punct( + sr_tokenizer, punct_open, punct_close, punct_open2, punct_close2, text +): + tokens = sr_tokenizer(punct_open2 + punct_open + text + punct_close + punct_close2) + assert len(tokens) == 5 + assert tokens[0].text == punct_open2 + assert tokens[1].text == punct_open + assert tokens[2].text == text + assert tokens[3].text == punct_close + assert tokens[4].text == punct_close2 + + +@pytest.mark.parametrize("text", ["Тест."]) +def test_sr_tokenizer_splits_trailing_dot(sr_tokenizer, text): + tokens = sr_tokenizer(text) + assert tokens[1].text == "." + + +def test_sr_tokenizer_splits_bracket_period(sr_tokenizer): + text = "(Један, два, три, четири, проба)." + tokens = sr_tokenizer(text) + assert tokens[len(tokens) - 1].text == "." diff --git a/spacy/tests/lang/sv/__init__.py b/spacy/tests/lang/sv/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/sv/test_exceptions.py b/spacy/tests/lang/sv/test_exceptions.py new file mode 100644 index 0000000..b49a0c8 --- /dev/null +++ b/spacy/tests/lang/sv/test_exceptions.py @@ -0,0 +1,77 @@ +import pytest + +SV_TOKEN_EXCEPTION_TESTS = [ + ( + "Smörsåsen används bl.a. till fisk", + ["Smörsåsen", "används", "bl.a.", "till", "fisk"], + ), + ( + "Jag kommer först kl. 13 p.g.a. diverse förseningar", + ["Jag", "kommer", "först", "kl.", "13", "p.g.a.", "diverse", "förseningar"], + ), + ( + "Anders I. tycker om ord med i i.", + ["Anders", "I.", "tycker", "om", "ord", "med", "i", "i", "."], + ), +] + + +@pytest.mark.issue(805) +@pytest.mark.parametrize( + "text,expected_tokens", + [ + ( + "Smörsåsen används bl.a. till fisk", + ["Smörsåsen", "används", "bl.a.", "till", "fisk"], + ), + ( + "Jag kommer först kl. 13 p.g.a. diverse förseningar", + ["Jag", "kommer", "först", "kl.", "13", "p.g.a.", "diverse", "förseningar"], + ), + ], +) +def test_issue805(sv_tokenizer, text, expected_tokens): + tokens = sv_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text,expected_tokens", SV_TOKEN_EXCEPTION_TESTS) +def test_sv_tokenizer_handles_exception_cases(sv_tokenizer, text, expected_tokens): + tokens = sv_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text", ["driveru", "hajaru", "Serru", "Fixaru"]) +def test_sv_tokenizer_handles_verb_exceptions(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 2 + assert tokens[1].text == "u" + + +@pytest.mark.parametrize("text", ["bl.a", "m.a.o.", "Jan.", "Dec.", "kr.", "osv."]) +def test_sv_tokenizer_handles_abbr(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["Jul.", "jul.", "sön.", "Sön."]) +def test_sv_tokenizer_handles_ambiguous_abbr(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 2 + + +def test_sv_tokenizer_handles_exc_in_text(sv_tokenizer): + text = "Det är bl.a. inte meningen" + tokens = sv_tokenizer(text) + assert len(tokens) == 5 + assert tokens[2].text == "bl.a." + + +def test_sv_tokenizer_handles_custom_base_exc(sv_tokenizer): + text = "Här är något du kan titta på." + tokens = sv_tokenizer(text) + assert len(tokens) == 8 + assert tokens[6].text == "på" + assert tokens[7].text == "." diff --git a/spacy/tests/lang/sv/test_lex_attrs.py b/spacy/tests/lang/sv/test_lex_attrs.py new file mode 100644 index 0000000..a47b17b --- /dev/null +++ b/spacy/tests/lang/sv/test_lex_attrs.py @@ -0,0 +1,31 @@ +import pytest + +from spacy.lang.sv.lex_attrs import like_num + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("10.00", True), + ("999,0", True), + ("en", True), + ("två", True), + ("miljard", True), + ("hund", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(sv_tokenizer, text, match): + tokens = sv_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["elva"]) +def test_sv_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/sv/test_noun_chunks.py b/spacy/tests/lang/sv/test_noun_chunks.py new file mode 100644 index 0000000..5991483 --- /dev/null +++ b/spacy/tests/lang/sv/test_noun_chunks.py @@ -0,0 +1,49 @@ +import pytest + +from spacy.tokens import Doc + + +def test_noun_chunks_is_parsed_sv(sv_tokenizer): + """Test that noun_chunks raises Value Error for 'sv' language if Doc is not parsed.""" + doc = sv_tokenizer("Studenten läste den bästa boken") + with pytest.raises(ValueError): + list(doc.noun_chunks) + + +SV_NP_TEST_EXAMPLES = [ + ( + "En student läste en bok", # A student read a book + ["DET", "NOUN", "VERB", "DET", "NOUN"], + ["det", "nsubj", "ROOT", "det", "dobj"], + [1, 2, 2, 4, 2], + ["En student", "en bok"], + ), + ( + "Studenten läste den bästa boken.", # The student read the best book + ["NOUN", "VERB", "DET", "ADJ", "NOUN", "PUNCT"], + ["nsubj", "ROOT", "det", "amod", "dobj", "punct"], + [1, 1, 4, 4, 1, 1], + ["Studenten", "den bästa boken"], + ), + ( + "De samvetslösa skurkarna hade stulit de största juvelerna på söndagen", # The remorseless crooks had stolen the largest jewels that sunday + ["DET", "ADJ", "NOUN", "VERB", "VERB", "DET", "ADJ", "NOUN", "ADP", "NOUN"], + ["det", "amod", "nsubj", "aux", "root", "det", "amod", "dobj", "case", "nmod"], + [2, 2, 4, 4, 4, 7, 7, 4, 9, 4], + ["De samvetslösa skurkarna", "de största juvelerna", "på söndagen"], + ), +] + + +@pytest.mark.parametrize( + "text,pos,deps,heads,expected_noun_chunks", SV_NP_TEST_EXAMPLES +) +def test_sv_noun_chunks(sv_tokenizer, text, pos, deps, heads, expected_noun_chunks): + tokens = sv_tokenizer(text) + assert len(heads) == len(pos) + words = [t.text for t in tokens] + doc = Doc(tokens.vocab, words=words, heads=heads, deps=deps, pos=pos) + noun_chunks = list(doc.noun_chunks) + assert len(noun_chunks) == len(expected_noun_chunks) + for i, np in enumerate(noun_chunks): + assert np.text == expected_noun_chunks[i] diff --git a/spacy/tests/lang/sv/test_prefix_suffix_infix.py b/spacy/tests/lang/sv/test_prefix_suffix_infix.py new file mode 100644 index 0000000..0aa4959 --- /dev/null +++ b/spacy/tests/lang/sv/test_prefix_suffix_infix.py @@ -0,0 +1,41 @@ +import pytest + + +@pytest.mark.parametrize("text", ["(under)"]) +def test_tokenizer_splits_no_special(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["gitta'r", "Björn's", "Lars'"]) +def test_tokenizer_handles_no_punct(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["svart.Gul", "Hej.Världen"]) +def test_tokenizer_splits_period_infix(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["Hej,Världen", "en,två"]) +def test_tokenizer_splits_comma_infix(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 3 + assert tokens[0].text == text.split(",")[0] + assert tokens[1].text == "," + assert tokens[2].text == text.split(",")[1] + + +@pytest.mark.parametrize("text", ["svart...Gul", "svart...gul"]) +def test_tokenizer_splits_ellipsis_infix(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.issue(12311) +@pytest.mark.parametrize("text", ["99:e", "c:a", "EU:s", "Maj:t"]) +def test_sv_tokenizer_handles_colon(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 1 diff --git a/spacy/tests/lang/sv/test_text.py b/spacy/tests/lang/sv/test_text.py new file mode 100644 index 0000000..1e26c45 --- /dev/null +++ b/spacy/tests/lang/sv/test_text.py @@ -0,0 +1,14 @@ +def test_sv_tokenizer_handles_long_text(sv_tokenizer): + text = """Det var så härligt ute på landet. Det var sommar, majsen var gul, havren grön, +höet var uppställt i stackar nere vid den gröna ängen, och där gick storken på sina långa, +röda ben och snackade engelska, för det språket hade han lärt sig av sin mor. + +Runt om åkrar och äng låg den stora skogen, och mitt i skogen fanns djupa sjöar; jo, det var verkligen trevligt ute på landet!""" + tokens = sv_tokenizer(text) + assert len(tokens) == 86 + + +def test_sv_tokenizer_handles_trailing_dot_for_i_in_sentence(sv_tokenizer): + text = "Provar att tokenisera en mening med ord i." + tokens = sv_tokenizer(text) + assert len(tokens) == 9 diff --git a/spacy/tests/lang/sv/test_tokenizer.py b/spacy/tests/lang/sv/test_tokenizer.py new file mode 100644 index 0000000..f19c6b6 --- /dev/null +++ b/spacy/tests/lang/sv/test_tokenizer.py @@ -0,0 +1,30 @@ +import pytest + +SV_TOKEN_EXCEPTION_TESTS = [ + ( + "Smörsåsen används bl.a. till fisk", + ["Smörsåsen", "används", "bl.a.", "till", "fisk"], + ), + ( + "Jag kommer först kl. 13 p.g.a. diverse förseningar", + ["Jag", "kommer", "först", "kl.", "13", "p.g.a.", "diverse", "förseningar"], + ), + ( + "Anders I. tycker om ord med i i.", + ["Anders", "I.", "tycker", "om", "ord", "med", "i", "i", "."], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", SV_TOKEN_EXCEPTION_TESTS) +def test_sv_tokenizer_handles_exception_cases(sv_tokenizer, text, expected_tokens): + tokens = sv_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list + + +@pytest.mark.parametrize("text", ["driveru", "hajaru", "Serru", "Fixaru"]) +def test_sv_tokenizer_handles_verb_exceptions(sv_tokenizer, text): + tokens = sv_tokenizer(text) + assert len(tokens) == 2 + assert tokens[1].text == "u" diff --git a/spacy/tests/lang/ta/__init__.py b/spacy/tests/lang/ta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ta/test_text.py b/spacy/tests/lang/ta/test_text.py new file mode 100644 index 0000000..2d15e96 --- /dev/null +++ b/spacy/tests/lang/ta/test_text.py @@ -0,0 +1,26 @@ +import pytest + +from spacy.lang.ta import Tamil + +# Wikipedia excerpt: https://en.wikipedia.org/wiki/Chennai (Tamil Language) +TAMIL_BASIC_TOKENIZER_SENTENCIZER_TEST_TEXT = """சென்னை (Chennai) தமிழ்நாட்டின் தலைநகரமும், இந்தியாவின் நான்காவது பெரிய நகரமும் ஆகும். 1996 ஆம் ஆண்டுக்கு முன்னர் இந்நகரம், மதராசு பட்டினம், மெட்ராஸ் (Madras) மற்றும் சென்னப்பட்டினம் என்றும் அழைக்கப்பட்டு வந்தது. சென்னை, வங்காள விரிகுடாவின் கரையில் அமைந்த துறைமுக நகரங்களுள் ஒன்று. சுமார் 10 மில்லியன் (ஒரு கோடி) மக்கள் வாழும் இந்நகரம், உலகின் 35 பெரிய மாநகரங்களுள் ஒன்று. 17ஆம் நூற்றாண்டில் ஆங்கிலேயர் சென்னையில் கால் பதித்தது முதல், சென்னை நகரம் ஒரு முக்கிய நகரமாக வளர்ந்து வந்திருக்கிறது. சென்னை தென்னிந்தியாவின் வாசலாகக் கருதப்படுகிறது. சென்னை நகரில் உள்ள மெரினா கடற்கரை உலகின் நீளமான கடற்கரைகளுள் ஒன்று. சென்னை கோலிவுட் (Kollywood) என அறியப்படும் தமிழ்த் திரைப்படத் துறையின் தாயகம் ஆகும். பல விளையாட்டு அரங்கங்கள் உள்ள சென்னையில் பல விளையாட்டுப் போட்டிகளும் நடைபெறுகின்றன.""" + + +@pytest.mark.parametrize( + "text, num_tokens", + [(TAMIL_BASIC_TOKENIZER_SENTENCIZER_TEST_TEXT, 23 + 90)], # Punctuation + rest +) +def test_long_text(ta_tokenizer, text, num_tokens): + tokens = ta_tokenizer(text) + assert len(tokens) == num_tokens + + +@pytest.mark.parametrize( + "text, num_sents", [(TAMIL_BASIC_TOKENIZER_SENTENCIZER_TEST_TEXT, 9)] +) +def test_ta_sentencizer(text, num_sents): + nlp = Tamil() + nlp.add_pipe("sentencizer") + + doc = nlp(text) + assert len(list(doc.sents)) == num_sents diff --git a/spacy/tests/lang/ta/test_tokenizer.py b/spacy/tests/lang/ta/test_tokenizer.py new file mode 100644 index 0000000..e668b5a --- /dev/null +++ b/spacy/tests/lang/ta/test_tokenizer.py @@ -0,0 +1,189 @@ +import pytest + +from spacy.lang.ta import Tamil +from spacy.symbols import ORTH + +TA_BASIC_TOKENIZATION_TESTS = [ + ( + "கிறிஸ்துமஸ் மற்றும் இனிய புத்தாண்டு வாழ்த்துக்கள்", + ["கிறிஸ்துமஸ்", "மற்றும்", "இனிய", "புத்தாண்டு", "வாழ்த்துக்கள்"], + ), + ( + "எனக்கு என் குழந்தைப் பருவம் நினைவிருக்கிறது", + ["எனக்கு", "என்", "குழந்தைப்", "பருவம்", "நினைவிருக்கிறது"], + ), + ("உங்கள் பெயர் என்ன?", ["உங்கள்", "பெயர்", "என்ன", "?"]), + ( + "ஏறத்தாழ இலங்கைத் தமிழரில் மூன்றிலொரு பங்கினர் இலங்கையை விட்டு வெளியேறிப் பிற நாடுகளில் வாழ்கின்றனர்", + [ + "ஏறத்தாழ", + "இலங்கைத்", + "தமிழரில்", + "மூன்றிலொரு", + "பங்கினர்", + "இலங்கையை", + "விட்டு", + "வெளியேறிப்", + "பிற", + "நாடுகளில்", + "வாழ்கின்றனர்", + ], + ), + ( + "இந்த ஃபோனுடன் சுமார் ரூ.2,990 மதிப்புள்ள போட் ராக்கர்ஸ் நிறுவனத்தின் ஸ்போர்ட் புளூடூத் ஹெட்போன்ஸ் இலவசமாக வழங்கப்படவுள்ளது.", + [ + "இந்த", + "ஃபோனுடன்", + "சுமார்", + "ரூ.2,990", + "மதிப்புள்ள", + "போட்", + "ராக்கர்ஸ்", + "நிறுவனத்தின்", + "ஸ்போர்ட்", + "புளூடூத்", + "ஹெட்போன்ஸ்", + "இலவசமாக", + "வழங்கப்படவுள்ளது", + ".", + ], + ), + ( + "மட்டக்களப்பில் பல இடங்களில் வீட்டுத் திட்டங்களுக்கு இன்று அடிக்கல் நாட்டல்", + [ + "மட்டக்களப்பில்", + "பல", + "இடங்களில்", + "வீட்டுத்", + "திட்டங்களுக்கு", + "இன்று", + "அடிக்கல்", + "நாட்டல்", + ], + ), + ( + "ஐ போன்க்கு முகத்தை வைத்து அன்லாக் செய்யும் முறை மற்றும் விரலால் தொட்டு அன்லாக் செய்யும் முறையை வாட்ஸ் ஆப் நிறுவனம் இதற்கு முன் கண்டுபிடித்தது", + [ + "ஐ", + "போன்க்கு", + "முகத்தை", + "வைத்து", + "அன்லாக்", + "செய்யும்", + "முறை", + "மற்றும்", + "விரலால்", + "தொட்டு", + "அன்லாக்", + "செய்யும்", + "முறையை", + "வாட்ஸ்", + "ஆப்", + "நிறுவனம்", + "இதற்கு", + "முன்", + "கண்டுபிடித்தது", + ], + ), + ( + "இது ஒரு வாக்கியம்.", + [ + "இது", + "ஒரு", + "வாக்கியம்", + ".", + ], + ), + ( + "தன்னாட்சி கார்கள் காப்பீட்டு பொறுப்பை உற்பத்தியாளரிடம் மாற்றுகின்றன", + [ + "தன்னாட்சி", + "கார்கள்", + "காப்பீட்டு", + "பொறுப்பை", + "உற்பத்தியாளரிடம்", + "மாற்றுகின்றன", + ], + ), + ( + "நடைபாதை விநியோக ரோபோக்களை தடை செய்வதை சான் பிரான்சிஸ்கோ கருதுகிறது", + [ + "நடைபாதை", + "விநியோக", + "ரோபோக்களை", + "தடை", + "செய்வதை", + "சான்", + "பிரான்சிஸ்கோ", + "கருதுகிறது", + ], + ), + ( + "லண்டன் ஐக்கிய இராச்சியத்தில் ஒரு பெரிய நகரம்.", + [ + "லண்டன்", + "ஐக்கிய", + "இராச்சியத்தில்", + "ஒரு", + "பெரிய", + "நகரம்", + ".", + ], + ), + ( + "என்ன வேலை செய்கிறீர்கள்?", + [ + "என்ன", + "வேலை", + "செய்கிறீர்கள்", + "?", + ], + ), + ( + "எந்த கல்லூரியில் படிக்கிறாய்?", + [ + "எந்த", + "கல்லூரியில்", + "படிக்கிறாய்", + "?", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", TA_BASIC_TOKENIZATION_TESTS) +def test_ta_tokenizer_basic(ta_tokenizer, text, expected_tokens): + tokens = ta_tokenizer(text) + token_list = [token.text for token in tokens] + assert expected_tokens == token_list + + +@pytest.mark.parametrize( + "text,expected_tokens", + [ + ( + "ஆப்பிள் நிறுவனம் யு.கே. தொடக்க நிறுவனத்தை ஒரு லட்சம் கோடிக்கு வாங்கப் பார்க்கிறது", + [ + "ஆப்பிள்", + "நிறுவனம்", + "யு.கே.", + "தொடக்க", + "நிறுவனத்தை", + "ஒரு", + "லட்சம்", + "கோடிக்கு", + "வாங்கப்", + "பார்க்கிறது", + ], + ) + ], +) +def test_ta_tokenizer_special_case(text, expected_tokens): + # Add a special rule to tokenize the initialism "யு.கே." (U.K., as + # in the country) as a single token. + nlp = Tamil() + nlp.tokenizer.add_special_case("யு.கே.", [{ORTH: "யு.கே."}]) + tokens = nlp(text) + + token_list = [token.text for token in tokens] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/test_attrs.py b/spacy/tests/lang/test_attrs.py new file mode 100644 index 0000000..fd96e8f --- /dev/null +++ b/spacy/tests/lang/test_attrs.py @@ -0,0 +1,135 @@ +import pytest + +from spacy.attrs import ENT_IOB, IS_ALPHA, LEMMA, NORM, ORTH, intify_attrs +from spacy.lang.en.stop_words import STOP_WORDS +from spacy.lang.lex_attrs import ( + is_ascii, + is_currency, + is_punct, + is_stop, + like_url, + word_shape, +) + + +@pytest.mark.parametrize("word", ["the"]) +@pytest.mark.issue(1889) +def test_issue1889(word): + assert is_stop(word, STOP_WORDS) == is_stop(word.upper(), STOP_WORDS) + + +@pytest.mark.parametrize("text", ["dog"]) +def test_attrs_key(text): + assert intify_attrs({"ORTH": text}) == {ORTH: text} + assert intify_attrs({"NORM": text}) == {NORM: text} + assert intify_attrs({"lemma": text}, strings_map={text: 10}) == {LEMMA: 10} + + +@pytest.mark.parametrize("text", ["dog"]) +def test_attrs_idempotence(text): + int_attrs = intify_attrs({"lemma": text, "is_alpha": True}, strings_map={text: 10}) + assert intify_attrs(int_attrs) == {LEMMA: 10, IS_ALPHA: True} + + +@pytest.mark.parametrize("text", ["dog"]) +def test_attrs_do_deprecated(text): + int_attrs = intify_attrs( + {"F": text, "is_alpha": True}, strings_map={text: 10}, _do_deprecated=True + ) + assert int_attrs == {ORTH: 10, IS_ALPHA: True} + + +def test_attrs_ent_iob_intify(): + int_attrs = intify_attrs({"ENT_IOB": ""}) + assert int_attrs == {ENT_IOB: 0} + + int_attrs = intify_attrs({"ENT_IOB": "I"}) + assert int_attrs == {ENT_IOB: 1} + + int_attrs = intify_attrs({"ENT_IOB": "O"}) + assert int_attrs == {ENT_IOB: 2} + + int_attrs = intify_attrs({"ENT_IOB": "B"}) + assert int_attrs == {ENT_IOB: 3} + + int_attrs = intify_attrs({ENT_IOB: ""}) + assert int_attrs == {ENT_IOB: 0} + + int_attrs = intify_attrs({ENT_IOB: "I"}) + assert int_attrs == {ENT_IOB: 1} + + int_attrs = intify_attrs({ENT_IOB: "O"}) + assert int_attrs == {ENT_IOB: 2} + + int_attrs = intify_attrs({ENT_IOB: "B"}) + assert int_attrs == {ENT_IOB: 3} + + with pytest.raises(ValueError): + int_attrs = intify_attrs({"ENT_IOB": "XX"}) + + with pytest.raises(ValueError): + int_attrs = intify_attrs({ENT_IOB: "XX"}) + + +@pytest.mark.parametrize("text,match", [(",", True), (" ", False), ("a", False)]) +def test_lex_attrs_is_punct(text, match): + assert is_punct(text) == match + + +@pytest.mark.parametrize("text,match", [(",", True), ("£", False), ("♥", False)]) +def test_lex_attrs_is_ascii(text, match): + assert is_ascii(text) == match + + +@pytest.mark.parametrize( + "text,match", + [ + ("$", True), + ("£", True), + ("♥", False), + ("€", True), + ("¥", True), + ("¢", True), + ("a", False), + ("www.google.com", False), + ("dog", False), + ], +) +def test_lex_attrs_is_currency(text, match): + assert is_currency(text) == match + + +@pytest.mark.parametrize( + "text,match", + [ + ("www.google.com", True), + ("google.com", True), + ("sydney.com", True), + ("1abc2def.org", True), + ("http://stupid", True), + ("www.hi", True), + ("example.com/example", True), + ("dog", False), + ("1.2", False), + ("1.a", False), + ("hello.There", False), + ], +) +def test_lex_attrs_like_url(text, match): + assert like_url(text) == match + + +@pytest.mark.parametrize( + "text,shape", + [ + ("Nasa", "Xxxx"), + ("capitalized", "xxxx"), + ("999999999", "dddd"), + ("C3P0", "XdXd"), + (",", ","), + ("\n", "\n"), + ("``,-", "``,-"), + ], +) +def test_lex_attrs_word_shape(text, shape): + assert word_shape(text) == shape diff --git a/spacy/tests/lang/test_initialize.py b/spacy/tests/lang/test_initialize.py new file mode 100644 index 0000000..9b9ca48 --- /dev/null +++ b/spacy/tests/lang/test_initialize.py @@ -0,0 +1,24 @@ +import pytest + +from spacy.util import get_lang_class + +# fmt: off +# Only include languages with no external dependencies +# excluded: ja, ko, th, vi, zh +LANGUAGES = ["af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", + "en", "es", "et", "eu", "fa", "fi", "fr", "ga", "gu", "he", "hi", + "hr", "hu", "hy", "id", "is", "it", "kn", "ky", "lb", "lt", "lv", + "mk", "ml", "mr", "nb", "ne", "nl", "pl", "pt", "ro", "ru", "sa", + "si", "sk", "sl", "sq", "sr", "sv", "ta", "te", "ti", "tl", "tn", + "tr", "tt", "uk", "ur", "xx", "yo", "kmr"] +# fmt: on + + +@pytest.mark.parametrize("lang", LANGUAGES) +def test_lang_initialize(lang, capfd): + """Test that languages can be initialized.""" + nlp = get_lang_class(lang)() + # Check for stray print statements (see #3342) + doc = nlp("test") # noqa: F841 + captured = capfd.readouterr() + assert not captured.out diff --git a/spacy/tests/lang/test_lemmatizers.py b/spacy/tests/lang/test_lemmatizers.py new file mode 100644 index 0000000..ddb3336 --- /dev/null +++ b/spacy/tests/lang/test_lemmatizers.py @@ -0,0 +1,54 @@ +import pytest + +from spacy import registry +from spacy.lookups import Lookups +from spacy.util import get_lang_class + +# fmt: off +# Only include languages with no external dependencies +# excluded: ru, uk +# excluded for custom tables: es, pl +LANGUAGES = ["bn", "ca", "el", "en", "fa", "fr", "nb", "nl", "sv"] +# fmt: on + + +@pytest.mark.parametrize("lang", LANGUAGES) +def test_lemmatizer_initialize(lang, capfd): + @registry.misc("lemmatizer_init_lookups") + def lemmatizer_init_lookups(): + lookups = Lookups() + lookups.add_table("lemma_lookup", {"cope": "cope", "x": "y"}) + lookups.add_table("lemma_index", {"verb": ("cope", "cop")}) + lookups.add_table("lemma_exc", {"verb": {"coping": ("cope",)}}) + lookups.add_table("lemma_rules", {"verb": [["ing", ""]]}) + return lookups + + lang_cls = get_lang_class(lang) + # Test that languages can be initialized + nlp = lang_cls() + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"}) + assert not lemmatizer.lookups.tables + nlp.config["initialize"]["components"]["lemmatizer"] = { + "lookups": {"@misc": "lemmatizer_init_lookups"} + } + with pytest.raises(ValueError): + nlp("x") + nlp.initialize() + assert lemmatizer.lookups.tables + doc = nlp("x") + # Check for stray print statements (see #3342) + captured = capfd.readouterr() + assert not captured.out + assert doc[0].lemma_ == "y" + + # Test initialization by calling .initialize() directly + nlp = lang_cls() + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"}) + lemmatizer.initialize(lookups=lemmatizer_init_lookups()) + assert nlp("x")[0].lemma_ == "y" + + # Test lookups config format + for mode in ("rule", "lookup", "pos_lookup"): + required, optional = lemmatizer.get_lookups_config(mode) + assert isinstance(required, list) + assert isinstance(optional, list) diff --git a/spacy/tests/lang/th/__init__.py b/spacy/tests/lang/th/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/th/test_serialize.py b/spacy/tests/lang/th/test_serialize.py new file mode 100644 index 0000000..57d0f17 --- /dev/null +++ b/spacy/tests/lang/th/test_serialize.py @@ -0,0 +1,25 @@ +import pickle + +from spacy.lang.th import Thai + +from ...util import make_tempdir + + +def test_th_tokenizer_serialize(th_tokenizer): + tokenizer_bytes = th_tokenizer.to_bytes() + nlp = Thai() + nlp.tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + with make_tempdir() as d: + file_path = d / "tokenizer" + th_tokenizer.to_disk(file_path) + nlp = Thai() + nlp.tokenizer.from_disk(file_path) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + +def test_th_tokenizer_pickle(th_tokenizer): + b = pickle.dumps(th_tokenizer) + th_tokenizer_re = pickle.loads(b) + assert th_tokenizer.to_bytes() == th_tokenizer_re.to_bytes() diff --git a/spacy/tests/lang/th/test_tokenizer.py b/spacy/tests/lang/th/test_tokenizer.py new file mode 100644 index 0000000..1e1ba52 --- /dev/null +++ b/spacy/tests/lang/th/test_tokenizer.py @@ -0,0 +1,9 @@ +import pytest + + +@pytest.mark.parametrize( + "text,expected_tokens", [("คุณรักผมไหม", ["คุณ", "รัก", "ผม", "ไหม"])] +) +def test_th_tokenizer(th_tokenizer, text, expected_tokens): + tokens = [token.text for token in th_tokenizer(text)] + assert tokens == expected_tokens diff --git a/spacy/tests/lang/ti/__init__.py b/spacy/tests/lang/ti/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ti/test_exception.py b/spacy/tests/lang/ti/test_exception.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ti/test_text.py b/spacy/tests/lang/ti/test_text.py new file mode 100644 index 0000000..d210056 --- /dev/null +++ b/spacy/tests/lang/ti/test_text.py @@ -0,0 +1,51 @@ +import pytest + + +def test_ti_tokenizer_handles_long_text(ti_tokenizer): + text = """ቻንስለር ጀርመን ኣንገላ መርከል ኣብታ ሃገር ቁጽሪ መትሓዝቲ ኮቪድ መዓልታዊ ክብረ መዝገብ ድሕሪ ምህራሙ- ጽኑዕ እገዳ ክግበር ጸዊዓ። + +መርከል ሎሚ ንታሕታዋይ ባይቶ ሃገራ ክትገልጽ ከላ፡ ኣብ ወሳኒ ምዕራፍ ቃልሲ ኢና ዘለና-ዳሕራዋይ ማዕበል ካብቲ ቀዳማይ ክገድድ ይኽእል`ዩ ኢላ። + +ትካል ምክልኻል ተላገብቲ ሕማማት ጀርመን፡ ኣብ ዝሓለፈ 24 ሰዓታት ኣብ ምልእቲ ጀርመር 590 ሰባት ብኮቪድ19 ምሟቶም ኣፍሊጡ`ሎ። + +ቻንስለር ኣንጀላ መርከል ኣብ እዋን በዓላት ልደት ስድራቤታት ክተኣኻኸባ ዝፍቀደለን`ኳ እንተኾነ ድሕሪኡ ኣብ ዘሎ ግዜ ግን እቲ እገዳታት ክትግበር ትደሊ።""" + tokens = ti_tokenizer(text) + + assert len(tokens) == 85 + + +@pytest.mark.parametrize( + "text,length", + [ + ("ቻንስለር ጀርመን ኣንገላ መርከል፧", 5), + ("“ስድራቤታት፧”", 4), + ("""ኣብ እዋን በዓላት ልደት ስድራቤታት ክተኣኻኸባ ዝፍቀደለን`ኳ እንተኾነ።""", 9), + ("ብግምት 10ኪ.ሜ. ጎይዩ።", 6), + ("ኣብ ዝሓለፈ 24 ሰዓታት...", 5), + ], +) +def test_ti_tokenizer_handles_cnts(ti_tokenizer, text, length): + tokens = ti_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("10.000", True), + ("1000", True), + ("999,0", True), + ("ሓደ", True), + ("ክልተ", True), + ("ትሪልዮን", True), + ("ከልቢ", False), + (",", False), + ("1/2", True), + ], +) +def test_lex_attrs_like_number(ti_tokenizer, text, match): + tokens = ti_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/tl/__init__.py b/spacy/tests/lang/tl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/tl/test_indices.py b/spacy/tests/lang/tl/test_indices.py new file mode 100644 index 0000000..7c99ae5 --- /dev/null +++ b/spacy/tests/lang/tl/test_indices.py @@ -0,0 +1,8 @@ +def test_tl_simple_punct(tl_tokenizer): + text = "Sige, punta ka dito" + tokens = tl_tokenizer(text) + assert tokens[0].idx == 0 + assert tokens[1].idx == 4 + assert tokens[2].idx == 6 + assert tokens[3].idx == 12 + assert tokens[4].idx == 15 diff --git a/spacy/tests/lang/tl/test_punct.py b/spacy/tests/lang/tl/test_punct.py new file mode 100644 index 0000000..e2c93bf --- /dev/null +++ b/spacy/tests/lang/tl/test_punct.py @@ -0,0 +1,127 @@ +import pytest + +from spacy.lang.punctuation import TOKENIZER_PREFIXES +from spacy.util import compile_prefix_regex + +PUNCT_OPEN = ["(", "[", "{", "*"] +PUNCT_CLOSE = [")", "]", "}", "*"] +PUNCT_PAIRED = [("(", ")"), ("[", "]"), ("{", "}"), ("*", "*")] + + +@pytest.mark.parametrize("text", ["(", "((", "<"]) +def test_tl_tokenizer_handles_only_punct(tl_tokenizer, text): + tokens = tl_tokenizer(text) + assert len(tokens) == len(text) + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_split_open_punct(tl_tokenizer, punct, text): + tokens = tl_tokenizer(punct + text) + assert len(tokens) == 2 + assert tokens[0].text == punct + assert tokens[1].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_close_punct(tl_tokenizer, punct, text): + tokens = tl_tokenizer(text + punct) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_two_diff_open_punct(tl_tokenizer, punct, punct_add, text): + tokens = tl_tokenizer(punct + punct_add + text) + assert len(tokens) == 3 + assert tokens[0].text == punct + assert tokens[1].text == punct_add + assert tokens[2].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_two_diff_close_punct(tl_tokenizer, punct, punct_add, text): + tokens = tl_tokenizer(text + punct + punct_add) + assert len(tokens) == 3 + assert tokens[0].text == text + assert tokens[1].text == punct + assert tokens[2].text == punct_add + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_same_open_punct(tl_tokenizer, punct, text): + tokens = tl_tokenizer(punct + punct + punct + text) + assert len(tokens) == 4 + assert tokens[0].text == punct + assert tokens[3].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_same_close_punct(tl_tokenizer, punct, text): + tokens = tl_tokenizer(text + punct + punct + punct) + assert len(tokens) == 4 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("text", ["'Ang"]) +def test_tl_tokenizer_splits_open_apostrophe(tl_tokenizer, text): + tokens = tl_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "'" + + +@pytest.mark.parametrize("text", ["Mabuhay''"]) +def test_tl_tokenizer_splits_double_end_quote(tl_tokenizer, text): + tokens = tl_tokenizer(text) + assert len(tokens) == 2 + tokens_punct = tl_tokenizer("''") + assert len(tokens_punct) == 1 + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_splits_open_close_punct( + tl_tokenizer, punct_open, punct_close, text +): + tokens = tl_tokenizer(punct_open + text + punct_close) + assert len(tokens) == 3 + assert tokens[0].text == punct_open + assert tokens[1].text == text + assert tokens[2].text == punct_close + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("punct_open2,punct_close2", [("`", "'")]) +@pytest.mark.parametrize("text", ["Mabuhay"]) +def test_tl_tokenizer_two_diff_punct( + tl_tokenizer, punct_open, punct_close, punct_open2, punct_close2, text +): + tokens = tl_tokenizer(punct_open2 + punct_open + text + punct_close + punct_close2) + assert len(tokens) == 5 + assert tokens[0].text == punct_open2 + assert tokens[1].text == punct_open + assert tokens[2].text == text + assert tokens[3].text == punct_close + assert tokens[4].text == punct_close2 + + +@pytest.mark.parametrize("text,punct", [("(sa'yo", "(")]) +def test_tl_tokenizer_splits_pre_punct_regex(text, punct): + tl_search_prefixes = compile_prefix_regex(TOKENIZER_PREFIXES).search + match = tl_search_prefixes(text) + assert match.group() == punct + + +def test_tl_tokenizer_splits_bracket_period(tl_tokenizer): + text = "(Dumating siya kahapon)." + tokens = tl_tokenizer(text) + assert tokens[len(tokens) - 1].text == "." diff --git a/spacy/tests/lang/tl/test_text.py b/spacy/tests/lang/tl/test_text.py new file mode 100644 index 0000000..26635ca --- /dev/null +++ b/spacy/tests/lang/tl/test_text.py @@ -0,0 +1,74 @@ +import pytest + +from spacy.lang.tl.lex_attrs import like_num + +# https://github.com/explosion/spaCy/blob/master/spacy/tests/lang/en/test_text.py + + +def test_tl_tokenizer_handles_long_text(tl_tokenizer): + # Excerpt: "Sapagkat ang Pilosopiya ay Ginagawa" by Padre Roque Ferriols + text = """ + Tingin tayo nang tingin. Kailangan lamang nating dumilat at + marami tayong makikita. At ang pagtingin ay isang gawain na ako lamang ang + makagagawa, kung ako nga ang makakita. Kahit na napanood na ng aking + matalik na kaibigan ang isang sine, kailangan ko pa ring panoorin, kung + ako nga ang may gustong makakita. Kahit na gaano kadikit ang aming + pagkabuklod, hindi siya maaaring tumingin sa isang paraan na ako ang + nakakakita. Kung ako ang makakita, ako lamang ang makatitingin. + """ + tokens = tl_tokenizer(text) + assert len(tokens) == 97 + + +@pytest.mark.parametrize( + "text,length", + [ + ("Huwag mo nang itanong sa akin.", 7), + ("Nasubukan mo na bang hulihin ang hangin?", 8), + ("Hindi ba?", 3), + ("Nagbukas ang DFA ng 1,000 appointment slots para sa pasaporte.", 11), + ("'Wala raw pasok bukas kasi may bagyo!' sabi ni Micah.", 14), + ("'Ingat,' aniya. 'Maingay sila pag malayo at tahimik kung malapit.'", 17), + ], +) +def test_tl_tokenizer_handles_cnts(tl_tokenizer, text, length): + tokens = tl_tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("isa", True), + ("dalawa", True), + ("tatlumpu", True), + pytest.param( + "isang daan", + True, + marks=pytest.mark.xfail(reason="Not yet implemented (means 100)"), + ), + pytest.param( + "kalahati", + True, + marks=pytest.mark.xfail(reason="Not yet implemented (means 1/2)"), + ), + pytest.param( + "isa't kalahati", + True, + marks=pytest.mark.xfail( + reason="Not yet implemented (means one-and-a-half)" + ), + ), + ], +) +def test_lex_attrs_like_number(tl_tokenizer, text, match): + tokens = tl_tokenizer(text) + assert all([token.like_num for token in tokens]) == match + + +@pytest.mark.xfail(reason="Not yet implemented, fails when capitalized.") +@pytest.mark.parametrize("word", ["isa", "dalawa", "tatlo"]) +def test_tl_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/tr/__init__.py b/spacy/tests/lang/tr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/tr/test_noun_chunks.py b/spacy/tests/lang/tr/test_noun_chunks.py new file mode 100644 index 0000000..003e4f0 --- /dev/null +++ b/spacy/tests/lang/tr/test_noun_chunks.py @@ -0,0 +1,12 @@ +import pytest + + +def test_noun_chunks_is_parsed(tr_tokenizer): + """Test that noun_chunks raises Value Error for 'tr' language if Doc is not parsed. + To check this test, we're constructing a Doc + with a new Vocab here and forcing is_parsed to 'False' + to make sure the noun chunks don't run. + """ + doc = tr_tokenizer("Dün seni gördüm.") + with pytest.raises(ValueError): + list(doc.noun_chunks) diff --git a/spacy/tests/lang/tr/test_parser.py b/spacy/tests/lang/tr/test_parser.py new file mode 100644 index 0000000..b23d086 --- /dev/null +++ b/spacy/tests/lang/tr/test_parser.py @@ -0,0 +1,575 @@ +from spacy.tokens import Doc + + +def test_tr_noun_chunks_amod_simple(tr_tokenizer): + text = "sarı kedi" + heads = [1, 1] + deps = ["amod", "ROOT"] + pos = ["ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "sarı kedi " + + +def test_tr_noun_chunks_nmod_simple(tr_tokenizer): + text = "arkadaşımın kedisi" # my friend's cat + heads = [1, 1] + deps = ["nmod", "ROOT"] + pos = ["NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "arkadaşımın kedisi " + + +def test_tr_noun_chunks_determiner_simple(tr_tokenizer): + text = "O kedi" # that cat + heads = [1, 1] + deps = ["det", "ROOT"] + pos = ["DET", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "O kedi " + + +def test_tr_noun_chunks_nmod_amod(tr_tokenizer): + text = "okulun eski müdürü" + heads = [2, 2, 2] + deps = ["nmod", "amod", "ROOT"] + pos = ["NOUN", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "okulun eski müdürü " + + +def test_tr_noun_chunks_one_det_one_adj_simple(tr_tokenizer): + text = "O sarı kedi" + heads = [2, 2, 2] + deps = ["det", "amod", "ROOT"] + pos = ["DET", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "O sarı kedi " + + +def test_tr_noun_chunks_two_adjs_simple(tr_tokenizer): + text = "beyaz tombik kedi" + heads = [2, 2, 2] + deps = ["amod", "amod", "ROOT"] + pos = ["ADJ", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "beyaz tombik kedi " + + +def test_tr_noun_chunks_one_det_two_adjs_simple(tr_tokenizer): + text = "o beyaz tombik kedi" + heads = [3, 3, 3, 3] + deps = ["det", "amod", "amod", "ROOT"] + pos = ["DET", "ADJ", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "o beyaz tombik kedi " + + +def test_tr_noun_chunks_nmod_two(tr_tokenizer): + text = "kızın saçının rengi" + heads = [1, 2, 2] + deps = ["nmod", "nmod", "ROOT"] + pos = ["NOUN", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "kızın saçının rengi " + + +def test_tr_noun_chunks_chain_nmod_with_adj(tr_tokenizer): + text = "ev sahibinin tatlı köpeği" + heads = [1, 3, 3, 3] + deps = ["nmod", "nmod", "amod", "ROOT"] + pos = ["NOUN", "NOUN", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "ev sahibinin tatlı köpeği " + + +def test_tr_noun_chunks_chain_nmod_with_acl(tr_tokenizer): + text = "ev sahibinin gelen köpeği" + heads = [1, 3, 3, 3] + deps = ["nmod", "nmod", "acl", "ROOT"] + pos = ["NOUN", "NOUN", "VERB", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "ev sahibinin gelen köpeği " + + +def test_tr_noun_chunks_chain_nmod_head_with_amod_acl(tr_tokenizer): + text = "arabanın kırdığım sol aynası" + heads = [3, 3, 3, 3] + deps = ["nmod", "acl", "amod", "ROOT"] + pos = ["NOUN", "VERB", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "arabanın kırdığım sol aynası " + + +def test_tr_noun_chunks_nmod_three(tr_tokenizer): + text = "güney Afrika ülkelerinden Mozambik" + heads = [1, 2, 3, 3] + deps = ["nmod", "nmod", "nmod", "ROOT"] + pos = ["NOUN", "PROPN", "NOUN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "güney Afrika ülkelerinden Mozambik " + + +def test_tr_noun_chunks_det_amod_nmod(tr_tokenizer): + text = "bazı eski oyun kuralları" + heads = [3, 3, 3, 3] + deps = ["det", "nmod", "nmod", "ROOT"] + pos = ["DET", "ADJ", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "bazı eski oyun kuralları " + + +def test_tr_noun_chunks_acl_simple(tr_tokenizer): + text = "bahçesi olan okul" + heads = [2, 0, 2] + deps = ["acl", "cop", "ROOT"] + pos = ["NOUN", "AUX", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "bahçesi olan okul " + + +def test_tr_noun_chunks_acl_verb(tr_tokenizer): + text = "sevdiğim sanatçılar" + heads = [1, 1] + deps = ["acl", "ROOT"] + pos = ["VERB", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "sevdiğim sanatçılar " + + +def test_tr_noun_chunks_acl_nmod(tr_tokenizer): + text = "en sevdiğim ses sanatçısı" + heads = [1, 3, 3, 3] + deps = ["advmod", "acl", "nmod", "ROOT"] + pos = ["ADV", "VERB", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "en sevdiğim ses sanatçısı " + + +def test_tr_noun_chunks_acl_nmod2(tr_tokenizer): + text = "bildiğim bir turizm şirketi" + heads = [3, 3, 3, 3] + deps = ["acl", "det", "nmod", "ROOT"] + pos = ["VERB", "DET", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "bildiğim bir turizm şirketi " + + +def test_tr_noun_chunks_np_recursive_nsubj_to_root(tr_tokenizer): + text = "Simge'nin okuduğu kitap" + heads = [1, 2, 2] + deps = ["nsubj", "acl", "ROOT"] + pos = ["PROPN", "VERB", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Simge'nin okuduğu kitap " + + +def test_tr_noun_chunks_np_recursive_nsubj_attached_to_pron_root(tr_tokenizer): + text = "Simge'nin konuşabileceği birisi" + heads = [1, 2, 2] + deps = ["nsubj", "acl", "ROOT"] + pos = ["PROPN", "VERB", "PRON"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Simge'nin konuşabileceği birisi " + + +def test_tr_noun_chunks_np_recursive_nsubj_in_subnp(tr_tokenizer): + text = "Simge'nin yarın gideceği yer" + heads = [2, 2, 3, 3] + deps = ["nsubj", "obl", "acl", "ROOT"] + pos = ["PROPN", "NOUN", "VERB", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Simge'nin yarın gideceği yer " + + +def test_tr_noun_chunks_np_recursive_two_nmods(tr_tokenizer): + text = "ustanın kapısını degiştireceği çamasır makinası" + heads = [2, 2, 4, 4, 4] + deps = ["nsubj", "obj", "acl", "nmod", "ROOT"] + pos = ["NOUN", "NOUN", "VERB", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "ustanın kapısını degiştireceği çamasır makinası " + + +def test_tr_noun_chunks_np_recursive_four_nouns(tr_tokenizer): + text = "kızına piyano dersi verdiğim hanım" + heads = [3, 2, 3, 4, 4] + deps = ["obl", "nmod", "obj", "acl", "ROOT"] + pos = ["NOUN", "NOUN", "NOUN", "VERB", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "kızına piyano dersi verdiğim hanım " + + +def test_tr_noun_chunks_np_recursive_no_nmod(tr_tokenizer): + text = "içine birkaç çiçek konmuş olan bir vazo" + heads = [3, 2, 3, 6, 3, 6, 6] + deps = ["obl", "det", "nsubj", "acl", "aux", "det", "ROOT"] + pos = ["ADP", "DET", "NOUN", "VERB", "AUX", "DET", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "içine birkaç çiçek konmuş olan bir vazo " + + +def test_tr_noun_chunks_np_recursive_long_two_acls(tr_tokenizer): + text = "içine Simge'nin bahçesinden toplanmış birkaç çiçeğin konmuş olduğu bir vazo" + heads = [6, 2, 3, 5, 5, 6, 9, 6, 9, 9] + deps = ["obl", "nmod", "obl", "acl", "det", "nsubj", "acl", "aux", "det", "ROOT"] + pos = ["ADP", "PROPN", "NOUN", "VERB", "DET", "NOUN", "VERB", "AUX", "DET", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert ( + chunks[0].text_with_ws + == "içine Simge'nin bahçesinden toplanmış birkaç çiçeğin konmuş olduğu bir vazo " + ) + + +def test_tr_noun_chunks_two_nouns_in_nmod(tr_tokenizer): + text = "kız ve erkek çocuklar" + heads = [3, 2, 0, 3] + deps = ["nmod", "cc", "conj", "ROOT"] + pos = ["NOUN", "CCONJ", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "kız ve erkek çocuklar " + + +def test_tr_noun_chunks_two_nouns_in_nmod2(tr_tokenizer): + text = "tatlı ve gürbüz çocuklar" + heads = [3, 2, 0, 3] + deps = ["amod", "cc", "conj", "ROOT"] + pos = ["ADJ", "CCONJ", "NOUN", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "tatlı ve gürbüz çocuklar " + + +def test_tr_noun_chunks_conj_simple(tr_tokenizer): + text = "Sen ya da ben" + heads = [0, 3, 1, 0] + deps = ["ROOT", "cc", "fixed", "conj"] + pos = ["PRON", "CCONJ", "CCONJ", "PRON"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "ben " + assert chunks[1].text_with_ws == "Sen " + + +def test_tr_noun_chunks_conj_three(tr_tokenizer): + text = "sen, ben ve ondan" + heads = [0, 2, 0, 4, 0] + deps = ["ROOT", "punct", "conj", "cc", "conj"] + pos = ["PRON", "PUNCT", "PRON", "CCONJ", "PRON"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "ondan " + assert chunks[1].text_with_ws == "ben " + assert chunks[2].text_with_ws == "sen " + + +def test_tr_noun_chunks_conj_three2(tr_tokenizer): + text = "ben ya da sen ya da onlar" + heads = [0, 3, 1, 0, 6, 4, 3] + deps = ["ROOT", "cc", "fixed", "conj", "cc", "fixed", "conj"] + pos = ["PRON", "CCONJ", "CCONJ", "PRON", "CCONJ", "CCONJ", "PRON"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "onlar " + assert chunks[1].text_with_ws == "sen " + assert chunks[2].text_with_ws == "ben " + + +def test_tr_noun_chunks_conj_and_adj_phrase(tr_tokenizer): + text = "ben ve akıllı çocuk" + heads = [0, 3, 3, 0] + deps = ["ROOT", "cc", "amod", "conj"] + pos = ["PRON", "CCONJ", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "akıllı çocuk " + assert chunks[1].text_with_ws == "ben " + + +def test_tr_noun_chunks_conj_fixed_adj_phrase(tr_tokenizer): + text = "ben ya da akıllı çocuk" + heads = [0, 4, 1, 4, 0] + deps = ["ROOT", "cc", "fixed", "amod", "conj"] + pos = ["PRON", "CCONJ", "CCONJ", "ADJ", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "akıllı çocuk " + assert chunks[1].text_with_ws == "ben " + + +def test_tr_noun_chunks_conj_subject(tr_tokenizer): + text = "Sen ve ben iyi anlaşıyoruz" + heads = [4, 2, 0, 2, 4] + deps = ["nsubj", "cc", "conj", "adv", "ROOT"] + pos = ["PRON", "CCONJ", "PRON", "ADV", "VERB"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "ben " + assert chunks[1].text_with_ws == "Sen " + + +def test_tr_noun_chunks_conj_noun_head_verb(tr_tokenizer): + text = "Simge babasını görmüyormuş, annesini değil" + heads = [2, 2, 2, 4, 2, 4] + deps = ["nsubj", "obj", "ROOT", "punct", "conj", "aux"] + pos = ["PROPN", "NOUN", "VERB", "PUNCT", "NOUN", "AUX"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 3 + assert chunks[0].text_with_ws == "annesini " + assert chunks[1].text_with_ws == "babasını " + assert chunks[2].text_with_ws == "Simge " + + +def test_tr_noun_chunks_flat_simple(tr_tokenizer): + text = "New York" + heads = [0, 0] + deps = ["ROOT", "flat"] + pos = ["PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "New York " + + +def test_tr_noun_chunks_flat_names_and_title(tr_tokenizer): + text = "Gazi Mustafa Kemal" + heads = [1, 1, 1] + deps = ["nmod", "ROOT", "flat"] + pos = ["PROPN", "PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Gazi Mustafa Kemal " + + +def test_tr_noun_chunks_flat_names_and_title2(tr_tokenizer): + text = "Ahmet Vefik Paşa" + heads = [2, 0, 2] + deps = ["nmod", "flat", "ROOT"] + pos = ["PROPN", "PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Ahmet Vefik Paşa " + + +def test_tr_noun_chunks_flat_name_lastname_and_title(tr_tokenizer): + text = "Cumhurbaşkanı Ahmet Necdet Sezer" + heads = [1, 1, 1, 1] + deps = ["nmod", "ROOT", "flat", "flat"] + pos = ["NOUN", "PROPN", "PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Cumhurbaşkanı Ahmet Necdet Sezer " + + +def test_tr_noun_chunks_flat_in_nmod(tr_tokenizer): + text = "Ahmet Sezer adında bir ögrenci" + heads = [2, 0, 4, 4, 4] + deps = ["nmod", "flat", "nmod", "det", "ROOT"] + pos = ["PROPN", "PROPN", "NOUN", "DET", "NOUN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Ahmet Sezer adında bir ögrenci " + + +def test_tr_noun_chunks_flat_and_chain_nmod(tr_tokenizer): + text = "Batı Afrika ülkelerinden Sierra Leone" + heads = [1, 2, 3, 3, 3] + deps = ["nmod", "nmod", "nmod", "ROOT", "flat"] + pos = ["NOUN", "PROPN", "NOUN", "PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 1 + assert chunks[0].text_with_ws == "Batı Afrika ülkelerinden Sierra Leone " + + +def test_tr_noun_chunks_two_flats_conjed(tr_tokenizer): + text = "New York ve Sierra Leone" + heads = [0, 0, 3, 0, 3] + deps = ["ROOT", "flat", "cc", "conj", "flat"] + pos = ["PROPN", "PROPN", "CCONJ", "PROPN", "PROPN"] + tokens = tr_tokenizer(text) + doc = Doc( + tokens.vocab, words=[t.text for t in tokens], pos=pos, heads=heads, deps=deps + ) + chunks = list(doc.noun_chunks) + assert len(chunks) == 2 + assert chunks[0].text_with_ws == "Sierra Leone " + assert chunks[1].text_with_ws == "New York " diff --git a/spacy/tests/lang/tr/test_text.py b/spacy/tests/lang/tr/test_text.py new file mode 100644 index 0000000..b4d84da --- /dev/null +++ b/spacy/tests/lang/tr/test_text.py @@ -0,0 +1,48 @@ +import pytest + +from spacy.lang.tr.lex_attrs import like_num + + +def test_tr_tokenizer_handles_long_text(tr_tokenizer): + text = """Pamuk nasıl ipliğe dönüştürülür? + +Sıkıştırılmış balyalar halindeki pamuk, iplik fabrikasına getirildiğinde hem +lifleri birbirine dolaşmıştır, hem de tarladan toplanırken araya bitkinin +parçaları karışmıştır. Üstelik balyalardaki pamuğun cinsi aynı olsa bile kalitesi +değişeceğinden, önce bütün balyaların birbirine karıştırılarak harmanlanması gerekir. + +Daha sonra pamuk yığınları, liflerin açılıp temizlenmesi için tek bir birim halinde +birleştirilmiş çeşitli makinelerden geçirilir.Bunlardan biri, dönen tokmaklarıyla +pamuğu dövüp kabartarak dağınık yumaklar haline getiren ve liflerin arasındaki yabancı +maddeleri temizleyen hallaç makinesidir. Daha sonra tarak makinesine giren pamuk demetleri, +herbirinin yüzeyinde yüzbinlerce incecik iğne bulunan döner silindirlerin arasından geçerek lif lif ayrılır +ve tül inceliğinde gevşek bir örtüye dönüşür. Ama bir sonraki makine bu lifleri dağınık +ve gevşek bir biçimde birbirine yaklaştırarak 2 cm eninde bir pamuk şeridi haline getirir.""" + tokens = tr_tokenizer(text) + assert len(tokens) == 146 + + +@pytest.mark.parametrize( + "word", + [ + "bir", + "iki", + "dört", + "altı", + "milyon", + "100", + "birinci", + "üçüncü", + "beşinci", + "100üncü", + "8inci", + ], +) +def test_tr_lex_attrs_like_number_cardinal_ordinal(word): + assert like_num(word) + + +@pytest.mark.parametrize("word", ["beş", "yedi", "yedinci", "birinci", "milyonuncu"]) +def test_tr_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/tr/test_tokenizer.py b/spacy/tests/lang/tr/test_tokenizer.py new file mode 100644 index 0000000..b07c985 --- /dev/null +++ b/spacy/tests/lang/tr/test_tokenizer.py @@ -0,0 +1,696 @@ +import pytest + +ABBREV_TESTS = [ + ("Dr. Murat Bey ile görüştüm.", ["Dr.", "Murat", "Bey", "ile", "görüştüm", "."]), + ("Dr.la görüştüm.", ["Dr.la", "görüştüm", "."]), + ("Dr.'la görüştüm.", ["Dr.'la", "görüştüm", "."]), + ("TBMM'de çalışıyormuş.", ["TBMM'de", "çalışıyormuş", "."]), + ( + "Hem İst. hem Ank. bu konuda gayet iyi durumda.", + ["Hem", "İst.", "hem", "Ank.", "bu", "konuda", "gayet", "iyi", "durumda", "."], + ), + ( + "Hem İst. hem Ank.'da yağış var.", + ["Hem", "İst.", "hem", "Ank.'da", "yağış", "var", "."], + ), + ("Dr.", ["Dr."]), + ("Yrd.Doç.", ["Yrd.Doç."]), + ("Prof.'un", ["Prof.'un"]), + ("Böl.'nde", ["Böl.'nde"]), +] + + +URL_TESTS = [ + ( + "Bizler de www.duygu.com.tr adında bir websitesi kurduk.", + [ + "Bizler", + "de", + "www.duygu.com.tr", + "adında", + "bir", + "websitesi", + "kurduk", + ".", + ], + ), + ( + "Bizler de https://www.duygu.com.tr adında bir websitesi kurduk.", + [ + "Bizler", + "de", + "https://www.duygu.com.tr", + "adında", + "bir", + "websitesi", + "kurduk", + ".", + ], + ), + ( + "Bizler de www.duygu.com.tr'dan satın aldık.", + ["Bizler", "de", "www.duygu.com.tr'dan", "satın", "aldık", "."], + ), + ( + "Bizler de https://www.duygu.com.tr'dan satın aldık.", + ["Bizler", "de", "https://www.duygu.com.tr'dan", "satın", "aldık", "."], + ), +] + + +NUMBER_TESTS = [ + ("Rakamla 6 yazılıydı.", ["Rakamla", "6", "yazılıydı", "."]), + ("Hava -4 dereceydi.", ["Hava", "-4", "dereceydi", "."]), + ( + "Hava sıcaklığı -4ten +6ya yükseldi.", + ["Hava", "sıcaklığı", "-4ten", "+6ya", "yükseldi", "."], + ), + ( + "Hava sıcaklığı -4'ten +6'ya yükseldi.", + ["Hava", "sıcaklığı", "-4'ten", "+6'ya", "yükseldi", "."], + ), + ("Yarışta 6. oldum.", ["Yarışta", "6.", "oldum", "."]), + ("Yarışta 438547745. oldum.", ["Yarışta", "438547745.", "oldum", "."]), + ("Kitap IV. Murat hakkında.", ["Kitap", "IV.", "Murat", "hakkında", "."]), + # ("Bana söylediği sayı 6.", ["Bana", "söylediği", "sayı", "6", "."]), + ("Saat 6'da buluşalım.", ["Saat", "6'da", "buluşalım", "."]), + ("Saat 6dan sonra buluşalım.", ["Saat", "6dan", "sonra", "buluşalım", "."]), + ("6.dan sonra saymadım.", ["6.dan", "sonra", "saymadım", "."]), + ("6.'dan sonra saymadım.", ["6.'dan", "sonra", "saymadım", "."]), + ("Saat 6'ydı.", ["Saat", "6'ydı", "."]), + ("5'te", ["5'te"]), + ("6'da", ["6'da"]), + ("9dan", ["9dan"]), + ("19'da", ["19'da"]), + ("VI'da", ["VI'da"]), + ("5.", ["5."]), + ("72.", ["72."]), + ("VI.", ["VI."]), + ("6.'dan", ["6.'dan"]), + ("19.'dan", ["19.'dan"]), + ("6.dan", ["6.dan"]), + ("16.dan", ["16.dan"]), + ("VI.'dan", ["VI.'dan"]), + ("VI.dan", ["VI.dan"]), + ("Hepsi 1994 yılında oldu.", ["Hepsi", "1994", "yılında", "oldu", "."]), + ("Hepsi 1994'te oldu.", ["Hepsi", "1994'te", "oldu", "."]), + ( + "2/3 tarihli faturayı bulamadım.", + ["2/3", "tarihli", "faturayı", "bulamadım", "."], + ), + ( + "2.3 tarihli faturayı bulamadım.", + ["2.3", "tarihli", "faturayı", "bulamadım", "."], + ), + ( + "2.3. tarihli faturayı bulamadım.", + ["2.3.", "tarihli", "faturayı", "bulamadım", "."], + ), + ( + "2/3/2020 tarihli faturayı bulamadm.", + ["2/3/2020", "tarihli", "faturayı", "bulamadm", "."], + ), + ( + "2/3/1987 tarihinden beri burda yaşıyorum.", + ["2/3/1987", "tarihinden", "beri", "burda", "yaşıyorum", "."], + ), + ( + "2-3-1987 tarihinden beri burdayım.", + ["2-3-1987", "tarihinden", "beri", "burdayım", "."], + ), + ( + "2.3.1987 tarihinden beri burdayım.", + ["2.3.1987", "tarihinden", "beri", "burdayım", "."], + ), + ( + "Bu olay 2005-2006 tarihleri arasında oldu.", + ["Bu", "olay", "2005", "-", "2006", "tarihleri", "arasında", "oldu", "."], + ), + ( + "Bu olay 4/12/2005-21/3/2006 tarihleri arasında oldu.", + [ + "Bu", + "olay", + "4/12/2005", + "-", + "21/3/2006", + "tarihleri", + "arasında", + "oldu", + ".", + ], + ), + ( + "Ek fıkra: 5/11/2003-4999/3 maddesine göre uygundur.", + [ + "Ek", + "fıkra", + ":", + "5/11/2003", + "-", + "4999/3", + "maddesine", + "göre", + "uygundur", + ".", + ], + ), + ( + "2/A alanları: 6831 sayılı Kanunun 2nci maddesinin birinci fıkrasının (A) bendine göre", + [ + "2/A", + "alanları", + ":", + "6831", + "sayılı", + "Kanunun", + "2nci", + "maddesinin", + "birinci", + "fıkrasının", + "(", + "A", + ")", + "bendine", + "göre", + ], + ), + ( + "ŞEHİTTEĞMENKALMAZ Cad. No: 2/311", + ["ŞEHİTTEĞMENKALMAZ", "Cad.", "No", ":", "2/311"], + ), + ( + "2-3-2025", + [ + "2-3-2025", + ], + ), + ("2/3/2025", ["2/3/2025"]), + ("Yıllardır 0.5 uç kullanıyorum.", ["Yıllardır", "0.5", "uç", "kullanıyorum", "."]), + ( + "Kan değerlerim 0.5-0.7 arasıydı.", + ["Kan", "değerlerim", "0.5", "-", "0.7", "arasıydı", "."], + ), + ("0.5", ["0.5"]), + ("1/2", ["1/2"]), + ("%1", ["%", "1"]), + ("%1lik", ["%", "1lik"]), + ("%1'lik", ["%", "1'lik"]), + ("%1lik dilim", ["%", "1lik", "dilim"]), + ("%1'lik dilim", ["%", "1'lik", "dilim"]), + ("%1.5", ["%", "1.5"]), + # ("%1-%2 arası büyüme bekleniyor.", ["%", "1", "-", "%", "2", "arası", "büyüme", "bekleniyor", "."]), + ( + "%1-2 arası büyüme bekliyoruz.", + ["%", "1", "-", "2", "arası", "büyüme", "bekliyoruz", "."], + ), + ( + "%11-12 arası büyüme bekliyoruz.", + ["%", "11", "-", "12", "arası", "büyüme", "bekliyoruz", "."], + ), + ("%1.5luk büyüme bekliyoruz.", ["%", "1.5luk", "büyüme", "bekliyoruz", "."]), + ( + "Saat 1-2 arası gelin lütfen.", + ["Saat", "1", "-", "2", "arası", "gelin", "lütfen", "."], + ), + ("Saat 15:30 gibi buluşalım.", ["Saat", "15:30", "gibi", "buluşalım", "."]), + ("Saat 15:30'da buluşalım.", ["Saat", "15:30'da", "buluşalım", "."]), + ("Saat 15.30'da buluşalım.", ["Saat", "15.30'da", "buluşalım", "."]), + ("Saat 15.30da buluşalım.", ["Saat", "15.30da", "buluşalım", "."]), + ("Saat 15 civarı buluşalım.", ["Saat", "15", "civarı", "buluşalım", "."]), + ("9’daki otobüse binsek mi?", ["9’daki", "otobüse", "binsek", "mi", "?"]), + ("Okulumuz 3-B şubesi", ["Okulumuz", "3-B", "şubesi"]), + ("Okulumuz 3/B şubesi", ["Okulumuz", "3/B", "şubesi"]), + ("Okulumuz 3B şubesi", ["Okulumuz", "3B", "şubesi"]), + ("Okulumuz 3b şubesi", ["Okulumuz", "3b", "şubesi"]), + ( + "Antonio Gaudí 20. yüzyılda, 1904-1914 yılları arasında on yıl süren bir reform süreci getirmiştir.", + [ + "Antonio", + "Gaudí", + "20.", + "yüzyılda", + ",", + "1904", + "-", + "1914", + "yılları", + "arasında", + "on", + "yıl", + "süren", + "bir", + "reform", + "süreci", + "getirmiştir", + ".", + ], + ), + ( + "Dizel yakıtın avro bölgesi ortalaması olan 1,165 avroya kıyasla litre başına 1,335 avroya mal olduğunu gösteriyor.", + [ + "Dizel", + "yakıtın", + "avro", + "bölgesi", + "ortalaması", + "olan", + "1,165", + "avroya", + "kıyasla", + "litre", + "başına", + "1,335", + "avroya", + "mal", + "olduğunu", + "gösteriyor", + ".", + ], + ), + ( + "Marcus Antonius M.Ö. 1 Ocak 49'da, Sezar'dan Vali'nin kendisini barış dostu ilan ettiği bir bildiri yayınlamıştır.", + [ + "Marcus", + "Antonius", + "M.Ö.", + "1", + "Ocak", + "49'da", + ",", + "Sezar'dan", + "Vali'nin", + "kendisini", + "barış", + "dostu", + "ilan", + "ettiği", + "bir", + "bildiri", + "yayınlamıştır", + ".", + ], + ), +] + + +PUNCT_TESTS = [ + ("Gitmedim dedim ya!", ["Gitmedim", "dedim", "ya", "!"]), + ("Gitmedim dedim ya!!", ["Gitmedim", "dedim", "ya", "!", "!"]), + ("Gitsek mi?", ["Gitsek", "mi", "?"]), + ("Gitsek mi??", ["Gitsek", "mi", "?", "?"]), + ("Gitsek mi?!?", ["Gitsek", "mi", "?", "!", "?"]), + ( + "Ankara - Antalya arası otobüs işliyor.", + ["Ankara", "-", "Antalya", "arası", "otobüs", "işliyor", "."], + ), + ( + "Ankara-Antalya arası otobüs işliyor.", + ["Ankara", "-", "Antalya", "arası", "otobüs", "işliyor", "."], + ), + ("Sen--ben, ya da onlar.", ["Sen", "--", "ben", ",", "ya", "da", "onlar", "."]), + ( + "Senden, benden, bizden şarkısını biliyor musun?", + ["Senden", ",", "benden", ",", "bizden", "şarkısını", "biliyor", "musun", "?"], + ), + ( + "Akif'le geldik, sonra da o ayrıldı.", + ["Akif'le", "geldik", ",", "sonra", "da", "o", "ayrıldı", "."], + ), + ("Bu adam ne dedi şimdi???", ["Bu", "adam", "ne", "dedi", "şimdi", "?", "?", "?"]), + ( + "Yok hasta olmuş, yok annesi hastaymış, bahaneler işte...", + [ + "Yok", + "hasta", + "olmuş", + ",", + "yok", + "annesi", + "hastaymış", + ",", + "bahaneler", + "işte", + "...", + ], + ), + ( + "Ankara'dan İstanbul'a ... bir aşk hikayesi.", + ["Ankara'dan", "İstanbul'a", "...", "bir", "aşk", "hikayesi", "."], + ), + ("Ahmet'te", ["Ahmet'te"]), + ("İstanbul'da", ["İstanbul'da"]), +] + +GENERAL_TESTS = [ + ( + "1914'teki Endurance seferinde, Sir Ernest Shackleton'ın kaptanlığını yaptığı İngiliz Endurance gemisi yirmi sekiz kişi ile Antarktika'yı geçmek üzere yelken açtı.", + [ + "1914'teki", + "Endurance", + "seferinde", + ",", + "Sir", + "Ernest", + "Shackleton'ın", + "kaptanlığını", + "yaptığı", + "İngiliz", + "Endurance", + "gemisi", + "yirmi", + "sekiz", + "kişi", + "ile", + "Antarktika'yı", + "geçmek", + "üzere", + "yelken", + "açtı", + ".", + ], + ), + ( + 'Danışılan "%100 Cospedal" olduğunu belirtti.', + ["Danışılan", '"', "%", "100", "Cospedal", '"', "olduğunu", "belirtti", "."], + ), + ( + "1976'da parkur artık kullanılmıyordu; 1990'da ise bir yangın, daha sonraları ahırlarla birlikte yıkılacak olan tahta tribünlerden geri kalanları da yok etmişti.", + [ + "1976'da", + "parkur", + "artık", + "kullanılmıyordu", + ";", + "1990'da", + "ise", + "bir", + "yangın", + ",", + "daha", + "sonraları", + "ahırlarla", + "birlikte", + "yıkılacak", + "olan", + "tahta", + "tribünlerden", + "geri", + "kalanları", + "da", + "yok", + "etmişti", + ".", + ], + ), + ( + "Dahiyane bir ameliyat ve zorlu bir rehabilitasyon sürecinden sonra, tamamen iyileştim.", + [ + "Dahiyane", + "bir", + "ameliyat", + "ve", + "zorlu", + "bir", + "rehabilitasyon", + "sürecinden", + "sonra", + ",", + "tamamen", + "iyileştim", + ".", + ], + ), + ( + "Yaklaşık iki hafta süren bireysel erken oy kullanma döneminin ardından 5,7 milyondan fazla Floridalı sandık başına gitti.", + [ + "Yaklaşık", + "iki", + "hafta", + "süren", + "bireysel", + "erken", + "oy", + "kullanma", + "döneminin", + "ardından", + "5,7", + "milyondan", + "fazla", + "Floridalı", + "sandık", + "başına", + "gitti", + ".", + ], + ), + ( + "Ancak, bu ABD Çevre Koruma Ajansı'nın dünyayı bu konularda uyarmasının ardından ortaya çıktı.", + [ + "Ancak", + ",", + "bu", + "ABD", + "Çevre", + "Koruma", + "Ajansı'nın", + "dünyayı", + "bu", + "konularda", + "uyarmasının", + "ardından", + "ortaya", + "çıktı", + ".", + ], + ), + ( + "Ortalama şansa ve 10.000 Sterlin değerinde tahvillere sahip bir yatırımcı yılda 125 Sterlin ikramiye kazanabilir.", + [ + "Ortalama", + "şansa", + "ve", + "10.000", + "Sterlin", + "değerinde", + "tahvillere", + "sahip", + "bir", + "yatırımcı", + "yılda", + "125", + "Sterlin", + "ikramiye", + "kazanabilir", + ".", + ], + ), + ( + "Granit adaları; Seyşeller ve Tioman ile Saint Helena gibi volkanik adaları kapsar.", + [ + "Granit", + "adaları", + ";", + "Seyşeller", + "ve", + "Tioman", + "ile", + "Saint", + "Helena", + "gibi", + "volkanik", + "adaları", + "kapsar", + ".", + ], + ), + ( + "Barış antlaşmasıyla İspanya, Amerika'ya Porto Riko, Guam ve Filipinler kolonilerini devretti.", + [ + "Barış", + "antlaşmasıyla", + "İspanya", + ",", + "Amerika'ya", + "Porto", + "Riko", + ",", + "Guam", + "ve", + "Filipinler", + "kolonilerini", + "devretti", + ".", + ], + ), + ( + "Makedonya'nın sınır bölgelerini güvence altına alan Philip, büyük bir Makedon ordusu kurdu ve uzun bir fetih seferi için Trakya'ya doğru yürüdü.", + [ + "Makedonya'nın", + "sınır", + "bölgelerini", + "güvence", + "altına", + "alan", + "Philip", + ",", + "büyük", + "bir", + "Makedon", + "ordusu", + "kurdu", + "ve", + "uzun", + "bir", + "fetih", + "seferi", + "için", + "Trakya'ya", + "doğru", + "yürüdü", + ".", + ], + ), + ( + "Fransız gazetesi Le Figaro'ya göre bu hükumet planı sayesinde 42 milyon Euro kazanç sağlanabilir ve elde edilen paranın 15.5 milyonu ulusal güvenlik için kullanılabilir.", + [ + "Fransız", + "gazetesi", + "Le", + "Figaro'ya", + "göre", + "bu", + "hükumet", + "planı", + "sayesinde", + "42", + "milyon", + "Euro", + "kazanç", + "sağlanabilir", + "ve", + "elde", + "edilen", + "paranın", + "15.5", + "milyonu", + "ulusal", + "güvenlik", + "için", + "kullanılabilir", + ".", + ], + ), + ( + "Ortalama şansa ve 10.000 Sterlin değerinde tahvillere sahip bir yatırımcı yılda 125 Sterlin ikramiye kazanabilir.", + [ + "Ortalama", + "şansa", + "ve", + "10.000", + "Sterlin", + "değerinde", + "tahvillere", + "sahip", + "bir", + "yatırımcı", + "yılda", + "125", + "Sterlin", + "ikramiye", + "kazanabilir", + ".", + ], + ), + ( + "3 Kasım Salı günü, Ankara Belediye Başkanı 2014'te hükümetle birlikte oluşturulan kentsel gelişim anlaşmasını askıya alma kararı verdi.", + [ + "3", + "Kasım", + "Salı", + "günü", + ",", + "Ankara", + "Belediye", + "Başkanı", + "2014'te", + "hükümetle", + "birlikte", + "oluşturulan", + "kentsel", + "gelişim", + "anlaşmasını", + "askıya", + "alma", + "kararı", + "verdi", + ".", + ], + ), + ( + "Stalin, Abakumov'u Beria'nın enerji bakanlıkları üzerindeki baskınlığına karşı MGB içinde kendi ağını kurmaya teşvik etmeye başlamıştı.", + [ + "Stalin", + ",", + "Abakumov'u", + "Beria'nın", + "enerji", + "bakanlıkları", + "üzerindeki", + "baskınlığına", + "karşı", + "MGB", + "içinde", + "kendi", + "ağını", + "kurmaya", + "teşvik", + "etmeye", + "başlamıştı", + ".", + ], + ), + ( + "Güney Avrupa'daki kazı alanlarının çoğunluğu gibi, bu bulgu M.Ö. 5. yüzyılın başlar", + [ + "Güney", + "Avrupa'daki", + "kazı", + "alanlarının", + "çoğunluğu", + "gibi", + ",", + "bu", + "bulgu", + "M.Ö.", + "5.", + "yüzyılın", + "başlar", + ], + ), + ( + "Sağlığın bozulması Hitchcock hayatının son yirmi yılında üretimini azalttı.", + [ + "Sağlığın", + "bozulması", + "Hitchcock", + "hayatının", + "son", + "yirmi", + "yılında", + "üretimini", + "azalttı", + ".", + ], + ), +] + + +TESTS = ABBREV_TESTS + URL_TESTS + NUMBER_TESTS + PUNCT_TESTS + GENERAL_TESTS + + +@pytest.mark.parametrize("text,expected_tokens", TESTS) +def test_tr_tokenizer_handles_allcases(tr_tokenizer, text, expected_tokens): + tokens = tr_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/tt/__init__.py b/spacy/tests/lang/tt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/tt/test_tokenizer.py b/spacy/tests/lang/tt/test_tokenizer.py new file mode 100644 index 0000000..0bb241f --- /dev/null +++ b/spacy/tests/lang/tt/test_tokenizer.py @@ -0,0 +1,83 @@ +import pytest + +INFIX_HYPHEN_TESTS = [ + ("Явым-төшем күләме.", "Явым-төшем күләме .".split()), + ("Хатын-кыз киеме.", "Хатын-кыз киеме .".split()), +] + +PUNC_INSIDE_WORDS_TESTS = [ + ( + "Пассаҗир саны - 2,13 млн — кеше/көндә (2010), 783,9 млн. кеше/елда.", + "Пассаҗир саны - 2,13 млн — кеше / көндә ( 2010 ) ," + " 783,9 млн. кеше / елда .".split(), + ), + ('Ту"кай', 'Ту " кай'.split()), +] + +MIXED_ORDINAL_NUMS_TESTS = [ + ("Иртәгә 22нче гыйнвар...", "Иртәгә 22нче гыйнвар ...".split()) +] + +ABBREV_TESTS = [ + ("«3 елда (б.э.к.) туган", "« 3 елда ( б.э.к. ) туган".split()), + ("тукымадан һ.б.ш. тегелгән.", "тукымадан һ.б.ш. тегелгән .".split()), +] + +NAME_ABBREV_TESTS = [ + ("Ә.Тукай", "Ә.Тукай".split()), + ("Ә.тукай", "Ә.тукай".split()), + ("ә.Тукай", "ә . Тукай".split()), + ("Миләүшә.", "Миләүшә .".split()), +] + +TYPOS_IN_PUNC_TESTS = [ + ("«3 елда , туган", "« 3 елда , туган".split()), + ("«3 елда,туган", "« 3 елда , туган".split()), + ("«3 елда,туган.", "« 3 елда , туган .".split()), + ("Ул эшли(кайчан?)", "Ул эшли ( кайчан ? )".split()), + ("Ул (кайчан?)эшли", "Ул ( кайчан ?) эшли".split()), # "?)" => "?)" or "? )" +] + +LONG_TEXTS_TESTS = [ + ( + "Иң борынгы кешеләр суыклар һәм салкын кышлар булмый торган җылы " + "якларда яшәгәннәр, шуңа күрә аларга кием кирәк булмаган.Йөз " + "меңнәрчә еллар үткән, борынгы кешеләр акрынлап Европа һәм Азиянең " + "салкын илләрендә дә яши башлаганнар. Алар кырыс һәм салкын " + "кышлардан саклану өчен кием-салым уйлап тапканнар - итәк.", + "Иң борынгы кешеләр суыклар һәм салкын кышлар булмый торган җылы " + "якларда яшәгәннәр , шуңа күрә аларга кием кирәк булмаган . Йөз " + "меңнәрчә еллар үткән , борынгы кешеләр акрынлап Европа һәм Азиянең " + "салкын илләрендә дә яши башлаганнар . Алар кырыс һәм салкын " + "кышлардан саклану өчен кием-салым уйлап тапканнар - итәк .".split(), + ) +] + +TESTCASES = ( + INFIX_HYPHEN_TESTS + + PUNC_INSIDE_WORDS_TESTS + + MIXED_ORDINAL_NUMS_TESTS + + ABBREV_TESTS + + NAME_ABBREV_TESTS + + LONG_TEXTS_TESTS + + TYPOS_IN_PUNC_TESTS +) + +NORM_TESTCASES = [ + ( + "тукымадан һ.б.ш. тегелгән.", + ["тукымадан", "һәм башка шундыйлар", "тегелгән", "."], + ) +] + + +@pytest.mark.parametrize("text,expected_tokens", TESTCASES) +def test_tt_tokenizer_handles_testcases(tt_tokenizer, text, expected_tokens): + tokens = [token.text for token in tt_tokenizer(text) if not token.is_space] + assert expected_tokens == tokens + + +@pytest.mark.parametrize("text,norms", NORM_TESTCASES) +def test_tt_tokenizer_handles_norm_exceptions(tt_tokenizer, text, norms): + tokens = tt_tokenizer(text) + assert [token.norm_ for token in tokens] == norms diff --git a/spacy/tests/lang/uk/__init__.py b/spacy/tests/lang/uk/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/uk/test_lemmatizer.py b/spacy/tests/lang/uk/test_lemmatizer.py new file mode 100644 index 0000000..060114c --- /dev/null +++ b/spacy/tests/lang/uk/test_lemmatizer.py @@ -0,0 +1,27 @@ +import pytest + +from spacy.tokens import Doc + +pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") + + +def test_uk_lemmatizer(uk_lemmatizer): + """Check that the default uk lemmatizer runs.""" + doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"]) + assert uk_lemmatizer.mode == "pymorphy3" + uk_lemmatizer(doc) + assert [token.lemma for token in doc] + + +@pytest.mark.parametrize( + "word,lemma", + ( + ("якийсь", "якийсь"), + ("розповідають", "розповідати"), + ("розповіси", "розповісти"), + ), +) +def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer, word, lemma): + assert uk_lookup_lemmatizer.mode == "pymorphy3_lookup" + doc = Doc(uk_lookup_lemmatizer.vocab, words=[word]) + assert uk_lookup_lemmatizer(doc)[0].lemma_ == lemma diff --git a/spacy/tests/lang/uk/test_tokenizer.py b/spacy/tests/lang/uk/test_tokenizer.py new file mode 100644 index 0000000..7960a30 --- /dev/null +++ b/spacy/tests/lang/uk/test_tokenizer.py @@ -0,0 +1,148 @@ +import pytest + +PUNCT_OPEN = ["(", "[", "{", "*"] +PUNCT_CLOSE = [")", "]", "}", "*"] +PUNCT_PAIRED = [("(", ")"), ("[", "]"), ("{", "}"), ("*", "*")] + + +@pytest.mark.parametrize("text", ["(", "((", "<"]) +def test_uk_tokenizer_handles_only_punct(uk_tokenizer, text): + tokens = uk_tokenizer(text) + assert len(tokens) == len(text) + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_open_punct(uk_tokenizer, punct, text): + tokens = uk_tokenizer(punct + text) + assert len(tokens) == 2 + assert tokens[0].text == punct + assert tokens[1].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_close_punct(uk_tokenizer, punct, text): + tokens = uk_tokenizer(text + punct) + assert len(tokens) == 2 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize("punct_add", ["`"]) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_two_diff_open_punct(uk_tokenizer, punct, punct_add, text): + tokens = uk_tokenizer(punct + punct_add + text) + assert len(tokens) == 3 + assert tokens[0].text == punct + assert tokens[1].text == punct_add + assert tokens[2].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize("punct_add", ["'"]) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_two_diff_close_punct(uk_tokenizer, punct, punct_add, text): + tokens = uk_tokenizer(text + punct + punct_add) + assert len(tokens) == 3 + assert tokens[0].text == text + assert tokens[1].text == punct + assert tokens[2].text == punct_add + + +@pytest.mark.parametrize("punct", PUNCT_OPEN) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_same_open_punct(uk_tokenizer, punct, text): + tokens = uk_tokenizer(punct + punct + punct + text) + assert len(tokens) == 4 + assert tokens[0].text == punct + assert tokens[3].text == text + + +@pytest.mark.parametrize("punct", PUNCT_CLOSE) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_same_close_punct(uk_tokenizer, punct, text): + tokens = uk_tokenizer(text + punct + punct + punct) + assert len(tokens) == 4 + assert tokens[0].text == text + assert tokens[1].text == punct + + +@pytest.mark.parametrize("text", ["'Тест"]) +def test_uk_tokenizer_splits_open_appostrophe(uk_tokenizer, text): + tokens = uk_tokenizer(text) + assert len(tokens) == 2 + assert tokens[0].text == "'" + + +@pytest.mark.parametrize("text", ["Тест''"]) +def test_uk_tokenizer_splits_double_end_quote(uk_tokenizer, text): + tokens = uk_tokenizer(text) + assert len(tokens) == 2 + tokens_punct = uk_tokenizer("''") + assert len(tokens_punct) == 1 + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_splits_open_close_punct( + uk_tokenizer, punct_open, punct_close, text +): + tokens = uk_tokenizer(punct_open + text + punct_close) + assert len(tokens) == 3 + assert tokens[0].text == punct_open + assert tokens[1].text == text + assert tokens[2].text == punct_close + + +@pytest.mark.parametrize("punct_open,punct_close", PUNCT_PAIRED) +@pytest.mark.parametrize("punct_open2,punct_close2", [("`", "'")]) +@pytest.mark.parametrize( + "text", ["Привет", "Привіт", "Ґелґотати", "З'єднання", "Єдність", "їхні"] +) +def test_uk_tokenizer_two_diff_punct( + uk_tokenizer, punct_open, punct_close, punct_open2, punct_close2, text +): + tokens = uk_tokenizer(punct_open2 + punct_open + text + punct_close + punct_close2) + assert len(tokens) == 5 + assert tokens[0].text == punct_open2 + assert tokens[1].text == punct_open + assert tokens[2].text == text + assert tokens[3].text == punct_close + assert tokens[4].text == punct_close2 + + +@pytest.mark.parametrize( + "text", ["Привет.", "Привіт.", "Ґелґотати.", "З'єднання.", "Єдність.", "їхні."] +) +def test_uk_tokenizer_splits_trailing_dot(uk_tokenizer, text): + tokens = uk_tokenizer(text) + assert tokens[1].text == "." + + +def test_uk_tokenizer_splits_bracket_period(uk_tokenizer): + text = "(Раз, два, три, проверка)." + tokens = uk_tokenizer(text) + assert tokens[len(tokens) - 1].text == "." + + +def test_uk_tokenizer_handles_final_diacritics(uk_tokenizer): + text = "Хлібі́в не було́. Хлібі́в не було́." + tokens = uk_tokenizer(text) + assert tokens[2].text == "було́" + assert tokens[3].text == "." diff --git a/spacy/tests/lang/uk/test_tokenizer_exc.py b/spacy/tests/lang/uk/test_tokenizer_exc.py new file mode 100644 index 0000000..4fb4a6b --- /dev/null +++ b/spacy/tests/lang/uk/test_tokenizer_exc.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.parametrize( + "text,norms,lemmas", + [("ім.", ["імені"], ["ім'я"]), ("проф.", ["професор"], ["професор"])], +) +def test_uk_tokenizer_abbrev_exceptions(uk_tokenizer, text, norms, lemmas): + tokens = uk_tokenizer(text) + assert len(tokens) == 1 + assert [token.norm_ for token in tokens] == norms diff --git a/spacy/tests/lang/ur/__init__.py b/spacy/tests/lang/ur/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/ur/test_prefix_suffix_infix.py b/spacy/tests/lang/ur/test_prefix_suffix_infix.py new file mode 100644 index 0000000..e9f3272 --- /dev/null +++ b/spacy/tests/lang/ur/test_prefix_suffix_infix.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.mark.parametrize("text", ["ہےں۔", "کیا۔"]) +def test_contractions(ur_tokenizer, text): + """Test specific Urdu punctuation character""" + tokens = ur_tokenizer(text) + assert len(tokens) == 2 diff --git a/spacy/tests/lang/ur/test_text.py b/spacy/tests/lang/ur/test_text.py new file mode 100644 index 0000000..5da831c --- /dev/null +++ b/spacy/tests/lang/ur/test_text.py @@ -0,0 +1,13 @@ +import pytest + + +def test_ur_tokenizer_handles_long_text(ur_tokenizer): + text = """اصل میں، رسوا ہونے کی ہمیں کچھ عادت سی ہو گئی ہے۔""" + tokens = ur_tokenizer(text) + assert len(tokens) == 14 + + +@pytest.mark.parametrize("text,length", [("تحریر باسط حبیب", 3), ("میرا پاکستان", 2)]) +def test_ur_tokenizer_handles_cnts(ur_tokenizer, text, length): + tokens = ur_tokenizer(text) + assert len(tokens) == length diff --git a/spacy/tests/lang/vi/__init__.py b/spacy/tests/lang/vi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/vi/test_serialize.py b/spacy/tests/lang/vi/test_serialize.py new file mode 100644 index 0000000..20bfd20 --- /dev/null +++ b/spacy/tests/lang/vi/test_serialize.py @@ -0,0 +1,42 @@ +import pickle + +from spacy.lang.vi import Vietnamese + +from ...util import make_tempdir + + +def test_vi_tokenizer_serialize(vi_tokenizer): + tokenizer_bytes = vi_tokenizer.to_bytes() + nlp = Vietnamese() + nlp.tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + assert nlp.tokenizer.use_pyvi is True + + with make_tempdir() as d: + file_path = d / "tokenizer" + vi_tokenizer.to_disk(file_path) + nlp = Vietnamese() + nlp.tokenizer.from_disk(file_path) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + assert nlp.tokenizer.use_pyvi is True + + # mode is (de)serialized correctly + nlp = Vietnamese.from_config({"nlp": {"tokenizer": {"use_pyvi": False}}}) + nlp_bytes = nlp.to_bytes() + nlp_r = Vietnamese() + nlp_r.from_bytes(nlp_bytes) + assert nlp_bytes == nlp_r.to_bytes() + assert nlp_r.tokenizer.use_pyvi is False + + with make_tempdir() as d: + nlp.to_disk(d) + nlp_r = Vietnamese() + nlp_r.from_disk(d) + assert nlp_bytes == nlp_r.to_bytes() + assert nlp_r.tokenizer.use_pyvi is False + + +def test_vi_tokenizer_pickle(vi_tokenizer): + b = pickle.dumps(vi_tokenizer) + vi_tokenizer_re = pickle.loads(b) + assert vi_tokenizer.to_bytes() == vi_tokenizer_re.to_bytes() diff --git a/spacy/tests/lang/vi/test_tokenizer.py b/spacy/tests/lang/vi/test_tokenizer.py new file mode 100644 index 0000000..ca6dee9 --- /dev/null +++ b/spacy/tests/lang/vi/test_tokenizer.py @@ -0,0 +1,47 @@ +import pytest + +from spacy.lang.vi import Vietnamese + +from ...tokenizer.test_naughty_strings import NAUGHTY_STRINGS + +# fmt: off +TOKENIZER_TESTS = [ + ("Đây là một văn bản bằng tiếng Việt Sau đó, đây là một văn bản khác bằng ngôn ngữ này", ['Đây', 'là', 'một', 'văn bản', 'bằng', 'tiếng', 'Việt', 'Sau', 'đó', ',', 'đây', 'là', 'một', 'văn bản', 'khác', 'bằng', 'ngôn ngữ', 'này']), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", TOKENIZER_TESTS) +def test_vi_tokenizer(vi_tokenizer, text, expected_tokens): + tokens = [token.text for token in vi_tokenizer(text)] + assert tokens == expected_tokens + + +def test_vi_tokenizer_extra_spaces(vi_tokenizer): + # note: three spaces after "I" + tokens = vi_tokenizer("I like cheese.") + assert tokens[1].orth_ == " " + + +@pytest.mark.parametrize("text", NAUGHTY_STRINGS) +def test_vi_tokenizer_naughty_strings(vi_tokenizer, text): + tokens = vi_tokenizer(text) + assert tokens.text_with_ws == text + + +def test_vi_tokenizer_emptyish_texts(vi_tokenizer): + doc = vi_tokenizer("") + assert len(doc) == 0 + doc = vi_tokenizer(" ") + assert len(doc) == 1 + doc = vi_tokenizer("\n\n\n \t\t \n\n\n") + assert len(doc) == 1 + + +def test_vi_tokenizer_no_pyvi(): + """Test for whitespace tokenization without pyvi""" + nlp = Vietnamese.from_config({"nlp": {"tokenizer": {"use_pyvi": False}}}) + text = "Đây là một văn bản bằng tiếng Việt Sau đó, đây là một văn bản khác bằng ngôn ngữ này" + doc = nlp(text) + assert [t.text for t in doc if not t.is_space] == text.split() + assert doc[4].text == " " diff --git a/spacy/tests/lang/xx/__init__.py b/spacy/tests/lang/xx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/xx/test_text.py b/spacy/tests/lang/xx/test_text.py new file mode 100644 index 0000000..a4eafdc --- /dev/null +++ b/spacy/tests/lang/xx/test_text.py @@ -0,0 +1,21 @@ +def test_long_text(xx_tokenizer): + # Excerpt: Text in Skolt Sami taken from https://www.samediggi.fi + text = """ +Säʹmmla lie Euroopp unioon oʹdinakai alggmeer. Säʹmmlai alggmeerstatus lij raʹvvjum Lääʹddjânnam vuâđđlääʹjjest. +Alggmeer kriteeʹr vuâđđâʹvve meeraikõskksaž tuâjjorganisaatio, ILO, suåppmõʹšše nââmar 169. +Suåppmõõžž mieʹldd jiõččvälddsaž jânnmin jälsteei meeraid ââʹnet alggmeeran, +ko sij puõlvvâʹvve naroodâst, kååʹtt jânnam välddmõõžž leʹbe aazztummuž leʹbe ânnʼjõž riikkraaʹji šõddâm ääiʹj jälste +jânnmest leʹbe tõn mäddtiõđlaž vuuʹdest, koozz jânnam kooll. Alggmeer ij leäkku mieʹrreei sââʹjest jiiʹjjes jälstemvuuʹdest. +Alggmeer âlgg jiõčč ââʹnned jiiʹjjes alggmeeran leʹbe leeʹd tõn miõlâst, što sij lie alggmeer. +Alggmeer lij õlggâm seeilted vuõiggâdvuõđlaž sââʹjest huõlǩâni obbnes leʹbe vueʹzzi jiiʹjjes sosiaalʼlaž, täälʼlaž, +kulttuurlaž da poliittlaž instituutioid. + +Säʹmmlai statuuzz ǩeeʹrjteš Lääʹddjânnam vuâđđläkka eeʹjj 1995. Säʹmmlain alggmeeran lij vuõiggâdvuõtt tuõʹllʼjed da +ooudâsviikkâd ǩiõlâz da kulttuurâz di tõõzz kuulli ääʹrbvuâlaž jieʹllemvueʹjjeez. Sääʹmǩiõl ââʹnnmest veʹrǧǧniiʹǩǩi +åʹrnn lij šiõttuum jiiʹjjes lääʹǩǩ. Säʹmmlain lij leämmaž eeʹjjest 1996 vueʹljeeʹl dommvuuʹdsteez ǩiõlâz da kulttuurâz kuõskki +vuâđđlääʹjj meâldlaž jiõččvaaldâšm. Säʹmmlai jiõččvaldšma kuulli tuâjaid håidd säʹmmlai vaalin vaʹlljääm parlameʹntt, +Sääʹmteʹǧǧ. +""" + + tokens = xx_tokenizer(text) + assert len(tokens) == 179 diff --git a/spacy/tests/lang/xx/test_tokenizer.py b/spacy/tests/lang/xx/test_tokenizer.py new file mode 100644 index 0000000..15c760a --- /dev/null +++ b/spacy/tests/lang/xx/test_tokenizer.py @@ -0,0 +1,25 @@ +import pytest + +XX_BASIC_TOKENIZATION_TESTS = [ + ( + "Lääʹddjânnmest lie nuʹtt 10 000 säʹmmliʹžžed. Seeʹst pâʹjjel", + [ + "Lääʹddjânnmest", + "lie", + "nuʹtt", + "10", + "000", + "säʹmmliʹžžed", + ".", + "Seeʹst", + "pâʹjjel", + ], + ), +] + + +@pytest.mark.parametrize("text,expected_tokens", XX_BASIC_TOKENIZATION_TESTS) +def test_xx_tokenizer_basic(xx_tokenizer, text, expected_tokens): + tokens = xx_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list diff --git a/spacy/tests/lang/yo/__init__.py b/spacy/tests/lang/yo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/yo/test_text.py b/spacy/tests/lang/yo/test_text.py new file mode 100644 index 0000000..a1bbc38 --- /dev/null +++ b/spacy/tests/lang/yo/test_text.py @@ -0,0 +1,30 @@ +import pytest + +from spacy.lang.yo.lex_attrs import like_num + + +def test_yo_tokenizer_handles_long_text(yo_tokenizer): + text = """Àwọn ọmọ ìlú tí wọ́n ń ṣàmúlò ayélujára ti bẹ̀rẹ̀ ìkọkúkọ sórí àwòrán ààrẹ Nkurunziza nínú ìfẹ̀hónúhàn pẹ̀lú àmì ìdámọ̀: Nkurunziza àti Burundi: + Ọmọ ilé ẹ̀kọ́ gíga ní ẹ̀wọ̀n fún kíkọ ìkọkúkọ sí orí àwòrán Ààrẹ . + Bí mo bá ṣe èyí ní Burundi , ó ṣe é ṣe kí a fi mí sí àtìmọ́lé + Ìjọba Burundi fi akẹ́kọ̀ọ́bìnrin sí àtìmọ́lé látàrí ẹ̀sùn ìkọkúkọ sí orí àwòrán ààrẹ. A túwíìtì àwòrán ìkọkúkọ wa ní ìbánikẹ́dùn ìṣẹ̀lẹ̀ náà. + Wọ́n ní kí a dán an wò, kí a kọ nǹkan sí orí àwòrán ààrẹ mo sì ṣe bẹ́ẹ̀. Mo ní ìgbóyà wípé ẹnikẹ́ni kò ní mú mi níbí. + Ìfòfinlíle mú àtakò""" + tokens = yo_tokenizer(text) + assert len(tokens) == 121 + + +@pytest.mark.parametrize( + "text,match", + [("ení", True), ("ogun", True), ("mewadinlogun", True), ("ten", False)], +) +def test_lex_attrs_like_number(yo_tokenizer, text, match): + tokens = yo_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + + +@pytest.mark.parametrize("word", ["eji", "ejila", "ogun", "aárùn"]) +def test_yo_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/spacy/tests/lang/zh/__init__.py b/spacy/tests/lang/zh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/lang/zh/test_serialize.py b/spacy/tests/lang/zh/test_serialize.py new file mode 100644 index 0000000..4b014d7 --- /dev/null +++ b/spacy/tests/lang/zh/test_serialize.py @@ -0,0 +1,47 @@ +import pytest + +from spacy.lang.zh import Chinese + +from ...util import make_tempdir + + +def zh_tokenizer_serialize(zh_tokenizer): + tokenizer_bytes = zh_tokenizer.to_bytes() + nlp = Chinese() + nlp.tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + with make_tempdir() as d: + file_path = d / "tokenizer" + zh_tokenizer.to_disk(file_path) + nlp = Chinese() + nlp.tokenizer.from_disk(file_path) + assert tokenizer_bytes == nlp.tokenizer.to_bytes() + + +def test_zh_tokenizer_serialize_char(zh_tokenizer_char): + zh_tokenizer_serialize(zh_tokenizer_char) + + +def test_zh_tokenizer_serialize_jieba(zh_tokenizer_jieba): + zh_tokenizer_serialize(zh_tokenizer_jieba) + + +@pytest.mark.slow +def test_zh_tokenizer_serialize_pkuseg_with_processors(zh_tokenizer_pkuseg): + config = { + "nlp": { + "tokenizer": { + "@tokenizers": "spacy.zh.ChineseTokenizer", + "segmenter": "pkuseg", + } + }, + "initialize": { + "tokenizer": { + "pkuseg_model": "medicine", + } + }, + } + nlp = Chinese.from_config(config) + nlp.initialize() + zh_tokenizer_serialize(nlp.tokenizer) diff --git a/spacy/tests/lang/zh/test_text.py b/spacy/tests/lang/zh/test_text.py new file mode 100644 index 0000000..1482573 --- /dev/null +++ b/spacy/tests/lang/zh/test_text.py @@ -0,0 +1,21 @@ +import pytest + + +@pytest.mark.parametrize( + "text,match", + [ + ("10", True), + ("1", True), + ("999.0", True), + ("一", True), + ("二", True), + ("〇", True), + ("十一", True), + ("狗", False), + (",", False), + ], +) +def test_lex_attrs_like_number(zh_tokenizer_jieba, text, match): + tokens = zh_tokenizer_jieba(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match diff --git a/spacy/tests/lang/zh/test_tokenizer.py b/spacy/tests/lang/zh/test_tokenizer.py new file mode 100644 index 0000000..cb9b4ec --- /dev/null +++ b/spacy/tests/lang/zh/test_tokenizer.py @@ -0,0 +1,78 @@ +import pytest +from confection import ConfigValidationError + +from spacy.lang.zh import Chinese, _get_pkuseg_trie_data + +# fmt: off +TEXTS = ("作为语言而言，为世界使用人数最多的语言，目前世界有五分之一人口做为母语。",) +JIEBA_TOKENIZER_TESTS = [ + (TEXTS[0], + ['作为', '语言', '而言', '，', '为', '世界', '使用', '人', '数最多', + '的', '语言', '，', '目前', '世界', '有', '五分之一', '人口', '做', + '为', '母语', '。']), +] +PKUSEG_TOKENIZER_TESTS = [ + (TEXTS[0], + ['作为', '语言', '而言', '，', '为', '世界', '使用', '人数', '最多', + '的', '语言', '，', '目前', '世界', '有', '五分之一', '人口', '做为', + '母语', '。']), +] +# fmt: on + + +@pytest.mark.parametrize("text", TEXTS) +def test_zh_tokenizer_char(zh_tokenizer_char, text): + tokens = [token.text for token in zh_tokenizer_char(text)] + assert tokens == list(text) + + +@pytest.mark.parametrize("text,expected_tokens", JIEBA_TOKENIZER_TESTS) +def test_zh_tokenizer_jieba(zh_tokenizer_jieba, text, expected_tokens): + tokens = [token.text for token in zh_tokenizer_jieba(text)] + assert tokens == expected_tokens + + +@pytest.mark.parametrize("text,expected_tokens", PKUSEG_TOKENIZER_TESTS) +def test_zh_tokenizer_pkuseg(zh_tokenizer_pkuseg, text, expected_tokens): + tokens = [token.text for token in zh_tokenizer_pkuseg(text)] + assert tokens == expected_tokens + + +def test_zh_tokenizer_pkuseg_user_dict(zh_tokenizer_pkuseg, zh_tokenizer_char): + user_dict = _get_pkuseg_trie_data(zh_tokenizer_pkuseg.pkuseg_seg.preprocesser.trie) + zh_tokenizer_pkuseg.pkuseg_update_user_dict(["nonsense_asdf"]) + updated_user_dict = _get_pkuseg_trie_data( + zh_tokenizer_pkuseg.pkuseg_seg.preprocesser.trie + ) + assert len(user_dict) == len(updated_user_dict) - 1 + + # reset user dict + zh_tokenizer_pkuseg.pkuseg_update_user_dict([], reset=True) + reset_user_dict = _get_pkuseg_trie_data( + zh_tokenizer_pkuseg.pkuseg_seg.preprocesser.trie + ) + assert len(reset_user_dict) == 0 + + # warn if not relevant + with pytest.warns(UserWarning): + zh_tokenizer_char.pkuseg_update_user_dict(["nonsense_asdf"]) + + +def test_zh_extra_spaces(zh_tokenizer_char): + # note: three spaces after "I" + tokens = zh_tokenizer_char("I like cheese.") + assert tokens[1].orth_ == " " + + +def test_zh_unsupported_segmenter(): + config = {"nlp": {"tokenizer": {"segmenter": "unk"}}} + with pytest.raises(ConfigValidationError): + Chinese.from_config(config) + + +def test_zh_uninitialized_pkuseg(): + config = {"nlp": {"tokenizer": {"segmenter": "char"}}} + nlp = Chinese.from_config(config) + nlp.tokenizer.segmenter = "pkuseg" + with pytest.raises(ValueError): + nlp("test") diff --git a/spacy/tests/matcher/__init__.py b/spacy/tests/matcher/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/matcher/test_dependency_matcher.py b/spacy/tests/matcher/test_dependency_matcher.py new file mode 100644 index 0000000..be33f90 --- /dev/null +++ b/spacy/tests/matcher/test_dependency_matcher.py @@ -0,0 +1,493 @@ +import copy +import pickle +import re + +import pytest +from mock import Mock + +from spacy.matcher import DependencyMatcher +from spacy.tokens import Doc, Token + +from ..doc.test_underscore import clean_underscore # noqa: F401 + + +@pytest.fixture +def doc(en_vocab): + words = ["The", "quick", "brown", "fox", "jumped", "over", "the", "lazy", "fox"] + heads = [3, 3, 3, 4, 4, 4, 8, 8, 5] + deps = ["det", "amod", "amod", "nsubj", "ROOT", "prep", "pobj", "det", "amod"] + return Doc(en_vocab, words=words, heads=heads, deps=deps) + + +@pytest.fixture +def patterns(en_vocab): + def is_brown_yellow(text): + return bool(re.compile(r"brown|yellow").match(text)) + + IS_BROWN_YELLOW = en_vocab.add_flag(is_brown_yellow) + + pattern1 = [ + {"RIGHT_ID": "fox", "RIGHT_ATTRS": {"ORTH": "fox"}}, + { + "LEFT_ID": "fox", + "REL_OP": ">", + "RIGHT_ID": "q", + "RIGHT_ATTRS": {"ORTH": "quick", "DEP": "amod"}, + }, + { + "LEFT_ID": "fox", + "REL_OP": ">", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {IS_BROWN_YELLOW: True}, + }, + ] + + pattern2 = [ + {"RIGHT_ID": "jumped", "RIGHT_ATTRS": {"ORTH": "jumped"}}, + { + "LEFT_ID": "jumped", + "REL_OP": ">", + "RIGHT_ID": "fox1", + "RIGHT_ATTRS": {"ORTH": "fox"}, + }, + { + "LEFT_ID": "jumped", + "REL_OP": ".", + "RIGHT_ID": "over", + "RIGHT_ATTRS": {"ORTH": "over"}, + }, + ] + + pattern3 = [ + {"RIGHT_ID": "jumped", "RIGHT_ATTRS": {"ORTH": "jumped"}}, + { + "LEFT_ID": "jumped", + "REL_OP": ">", + "RIGHT_ID": "fox", + "RIGHT_ATTRS": {"ORTH": "fox"}, + }, + { + "LEFT_ID": "fox", + "REL_OP": ">>", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "brown"}, + }, + ] + + pattern4 = [ + {"RIGHT_ID": "jumped", "RIGHT_ATTRS": {"ORTH": "jumped"}}, + { + "LEFT_ID": "jumped", + "REL_OP": ">", + "RIGHT_ID": "fox", + "RIGHT_ATTRS": {"ORTH": "fox"}, + }, + ] + + pattern5 = [ + {"RIGHT_ID": "jumped", "RIGHT_ATTRS": {"ORTH": "jumped"}}, + { + "LEFT_ID": "jumped", + "REL_OP": ">>", + "RIGHT_ID": "fox", + "RIGHT_ATTRS": {"ORTH": "fox"}, + }, + ] + + return [pattern1, pattern2, pattern3, pattern4, pattern5] + + +@pytest.fixture +def dependency_matcher(en_vocab, patterns, doc): + matcher = DependencyMatcher(en_vocab) + mock = Mock() + for i in range(1, len(patterns) + 1): + if i == 1: + matcher.add("pattern1", [patterns[0]], on_match=mock) + else: + matcher.add("pattern" + str(i), [patterns[i - 1]]) + + return matcher + + +def test_dependency_matcher(dependency_matcher, doc, patterns): + assert len(dependency_matcher) == 5 + assert "pattern3" in dependency_matcher + assert dependency_matcher.get("pattern3") == (None, [patterns[2]]) + matches = dependency_matcher(doc) + assert len(matches) == 6 + assert matches[0][1] == [3, 1, 2] + assert matches[1][1] == [4, 3, 5] + assert matches[2][1] == [4, 3, 2] + assert matches[3][1] == [4, 3] + assert matches[4][1] == [4, 3] + assert matches[5][1] == [4, 8] + + span = doc[0:6] + matches = dependency_matcher(span) + assert len(matches) == 5 + assert matches[0][1] == [3, 1, 2] + assert matches[1][1] == [4, 3, 5] + assert matches[2][1] == [4, 3, 2] + assert matches[3][1] == [4, 3] + assert matches[4][1] == [4, 3] + + +def test_dependency_matcher_pickle(en_vocab, patterns, doc): + matcher = DependencyMatcher(en_vocab) + for i in range(1, len(patterns) + 1): + matcher.add("pattern" + str(i), [patterns[i - 1]]) + + matches = matcher(doc) + assert matches[0][1] == [3, 1, 2] + assert matches[1][1] == [4, 3, 5] + assert matches[2][1] == [4, 3, 2] + assert matches[3][1] == [4, 3] + assert matches[4][1] == [4, 3] + assert matches[5][1] == [4, 8] + + b = pickle.dumps(matcher) + matcher_r = pickle.loads(b) + + assert len(matcher) == len(matcher_r) + matches = matcher_r(doc) + assert matches[0][1] == [3, 1, 2] + assert matches[1][1] == [4, 3, 5] + assert matches[2][1] == [4, 3, 2] + assert matches[3][1] == [4, 3] + assert matches[4][1] == [4, 3] + assert matches[5][1] == [4, 8] + + +def test_dependency_matcher_pattern_validation(en_vocab): + pattern = [ + {"RIGHT_ID": "fox", "RIGHT_ATTRS": {"ORTH": "fox"}}, + { + "LEFT_ID": "fox", + "REL_OP": ">", + "RIGHT_ID": "q", + "RIGHT_ATTRS": {"ORTH": "quick", "DEP": "amod"}, + }, + { + "LEFT_ID": "fox", + "REL_OP": ">", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "brown"}, + }, + ] + + matcher = DependencyMatcher(en_vocab) + # original pattern is valid + matcher.add("FOUNDED", [pattern]) + # individual pattern not wrapped in a list + with pytest.raises(ValueError): + matcher.add("FOUNDED", pattern) + # no anchor node + with pytest.raises(ValueError): + matcher.add("FOUNDED", [pattern[1:]]) + # required keys missing + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + del pattern2[0]["RIGHT_ID"] + matcher.add("FOUNDED", [pattern2]) + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + del pattern2[1]["RIGHT_ID"] + matcher.add("FOUNDED", [pattern2]) + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + del pattern2[1]["RIGHT_ATTRS"] + matcher.add("FOUNDED", [pattern2]) + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + del pattern2[1]["LEFT_ID"] + matcher.add("FOUNDED", [pattern2]) + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + del pattern2[1]["REL_OP"] + matcher.add("FOUNDED", [pattern2]) + # invalid operator + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + pattern2[1]["REL_OP"] = "!!!" + matcher.add("FOUNDED", [pattern2]) + # duplicate node name + with pytest.raises(ValueError): + pattern2 = copy.deepcopy(pattern) + pattern2[1]["RIGHT_ID"] = "fox" + matcher.add("FOUNDED", [pattern2]) + # invalid key + with pytest.warns(UserWarning): + pattern2 = copy.deepcopy(pattern) + pattern2[1]["FOO"] = "BAR" + matcher.add("FOUNDED", [pattern2]) + + +def test_dependency_matcher_callback(en_vocab, doc): + pattern = [ + {"RIGHT_ID": "quick", "RIGHT_ATTRS": {"ORTH": "quick"}}, + ] + nomatch_pattern = [ + {"RIGHT_ID": "quick", "RIGHT_ATTRS": {"ORTH": "NOMATCH"}}, + ] + + matcher = DependencyMatcher(en_vocab) + mock = Mock() + matcher.add("pattern", [pattern], on_match=mock) + matcher.add("nomatch_pattern", [nomatch_pattern], on_match=mock) + matches = matcher(doc) + assert len(matches) == 1 + mock.assert_called_once_with(matcher, doc, 0, matches) + + # check that matches with and without callback are the same (#4590) + matcher2 = DependencyMatcher(en_vocab) + matcher2.add("pattern", [pattern]) + matches2 = matcher2(doc) + assert matches == matches2 + + +@pytest.mark.parametrize("op,num_matches", [(".", 8), (".*", 20), (";", 8), (";*", 20)]) +def test_dependency_matcher_precedence_ops(en_vocab, op, num_matches): + # two sentences to test that all matches are within the same sentence + doc = Doc( + en_vocab, + words=["a", "b", "c", "d", "e"] * 2, + heads=[0, 0, 0, 0, 0, 5, 5, 5, 5, 5], + deps=["dep"] * 10, + ) + match_count = 0 + for text in ["a", "b", "c", "d", "e"]: + pattern = [ + {"RIGHT_ID": "1", "RIGHT_ATTRS": {"ORTH": text}}, + {"LEFT_ID": "1", "REL_OP": op, "RIGHT_ID": "2", "RIGHT_ATTRS": {}}, + ] + matcher = DependencyMatcher(en_vocab) + matcher.add("A", [pattern]) + matches = matcher(doc) + match_count += len(matches) + for match in matches: + match_id, token_ids = match + # token_ids[0] op token_ids[1] + if op == ".": + assert token_ids[0] == token_ids[1] - 1 + elif op == ";": + assert token_ids[0] == token_ids[1] + 1 + elif op == ".*": + assert token_ids[0] < token_ids[1] + elif op == ";*": + assert token_ids[0] > token_ids[1] + # all tokens are within the same sentence + assert doc[token_ids[0]].sent == doc[token_ids[1]].sent + assert match_count == num_matches + + +@pytest.mark.parametrize( + "left,right,op,num_matches", + [ + ("fox", "jumped", "<", 1), + ("the", "lazy", "<", 0), + ("jumped", "jumped", "<", 0), + ("fox", "jumped", ">", 0), + ("fox", "lazy", ">", 1), + ("lazy", "lazy", ">", 0), + ("fox", "jumped", "<<", 2), + ("jumped", "fox", "<<", 0), + ("the", "fox", "<<", 2), + ("fox", "jumped", ">>", 0), + ("over", "the", ">>", 1), + ("fox", "the", ">>", 2), + ("fox", "jumped", ".", 1), + ("lazy", "fox", ".", 1), + ("the", "fox", ".", 0), + ("the", "the", ".", 0), + ("fox", "jumped", ";", 0), + ("lazy", "fox", ";", 0), + ("the", "fox", ";", 0), + ("the", "the", ";", 0), + ("quick", "fox", ".*", 2), + ("the", "fox", ".*", 3), + ("the", "the", ".*", 1), + ("fox", "jumped", ";*", 1), + ("quick", "fox", ";*", 0), + ("the", "fox", ";*", 1), + ("the", "the", ";*", 1), + ("quick", "brown", "$+", 1), + ("brown", "quick", "$+", 0), + ("brown", "brown", "$+", 0), + ("quick", "brown", "$-", 0), + ("brown", "quick", "$-", 1), + ("brown", "brown", "$-", 0), + ("the", "brown", "$++", 1), + ("brown", "the", "$++", 0), + ("brown", "brown", "$++", 0), + ("the", "brown", "$--", 0), + ("brown", "the", "$--", 1), + ("brown", "brown", "$--", 0), + ("over", "jumped", "<+", 0), + ("quick", "fox", "<+", 0), + ("the", "quick", "<+", 0), + ("brown", "fox", "<+", 1), + ("quick", "fox", "<++", 1), + ("quick", "over", "<++", 0), + ("over", "jumped", "<++", 0), + ("the", "fox", "<++", 2), + ("brown", "fox", "<-", 0), + ("fox", "over", "<-", 0), + ("the", "over", "<-", 0), + ("over", "jumped", "<-", 1), + ("brown", "fox", "<--", 0), + ("fox", "jumped", "<--", 0), + ("fox", "over", "<--", 1), + ("fox", "brown", ">+", 0), + ("over", "fox", ">+", 0), + ("over", "the", ">+", 0), + ("jumped", "over", ">+", 1), + ("jumped", "over", ">++", 1), + ("fox", "lazy", ">++", 0), + ("over", "the", ">++", 0), + ("jumped", "over", ">-", 0), + ("fox", "quick", ">-", 0), + ("brown", "quick", ">-", 0), + ("fox", "brown", ">-", 1), + ("brown", "fox", ">--", 0), + ("fox", "brown", ">--", 1), + ("jumped", "fox", ">--", 1), + ("fox", "the", ">--", 2), + ], +) +def test_dependency_matcher_ops(en_vocab, doc, left, right, op, num_matches): + right_id = right + if left == right: + right_id = right + "2" + pattern = [ + {"RIGHT_ID": left, "RIGHT_ATTRS": {"LOWER": left}}, + { + "LEFT_ID": left, + "REL_OP": op, + "RIGHT_ID": right_id, + "RIGHT_ATTRS": {"LOWER": right}, + }, + ] + + matcher = DependencyMatcher(en_vocab) + matcher.add("pattern", [pattern]) + matches = matcher(doc) + assert len(matches) == num_matches + + +def test_dependency_matcher_long_matches(en_vocab, doc): + pattern = [ + {"RIGHT_ID": "quick", "RIGHT_ATTRS": {"DEP": "amod", "OP": "+"}}, + ] + + matcher = DependencyMatcher(en_vocab) + with pytest.raises(ValueError): + matcher.add("pattern", [pattern]) + + +@pytest.mark.usefixtures("clean_underscore") +def test_dependency_matcher_span_user_data(en_tokenizer): + doc = en_tokenizer("a b c d e") + for token in doc: + token.head = doc[0] + token.dep_ = "a" + Token.set_extension("is_c", default=False) + doc[2]._.is_c = True + pattern = [ + {"RIGHT_ID": "c", "RIGHT_ATTRS": {"_": {"is_c": True}}}, + ] + matcher = DependencyMatcher(en_tokenizer.vocab) + matcher.add("C", [pattern]) + doc_matches = matcher(doc) + offset = 1 + span_matches = matcher(doc[offset:]) + for doc_match, span_match in zip(sorted(doc_matches), sorted(span_matches)): + assert doc_match[0] == span_match[0] + for doc_t_i, span_t_i in zip(doc_match[1], span_match[1]): + assert doc_t_i == span_t_i + offset + + +@pytest.mark.issue(9263) +def test_dependency_matcher_order_issue(en_tokenizer): + # issue from #9263 + doc = en_tokenizer("I like text") + doc[2].head = doc[1] + + # this matches on attrs but not rel op + pattern1 = [ + {"RIGHT_ID": "root", "RIGHT_ATTRS": {"ORTH": "like"}}, + { + "LEFT_ID": "root", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "text"}, + "REL_OP": "<", + }, + ] + + # this matches on rel op but not attrs + pattern2 = [ + {"RIGHT_ID": "root", "RIGHT_ATTRS": {"ORTH": "like"}}, + { + "LEFT_ID": "root", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "fish"}, + "REL_OP": ">", + }, + ] + + matcher = DependencyMatcher(en_tokenizer.vocab) + + # This should behave the same as the next pattern + matcher.add("check", [pattern1, pattern2]) + matches = matcher(doc) + + assert matches == [] + + # use a new matcher + matcher = DependencyMatcher(en_tokenizer.vocab) + # adding one at a time under same label gets a match + matcher.add("check", [pattern1]) + matcher.add("check", [pattern2]) + matches = matcher(doc) + + assert matches == [] + + +@pytest.mark.issue(9263) +def test_dependency_matcher_remove(en_tokenizer): + # issue from #9263 + doc = en_tokenizer("The red book") + doc[1].head = doc[2] + + # this matches + pattern1 = [ + {"RIGHT_ID": "root", "RIGHT_ATTRS": {"ORTH": "book"}}, + { + "LEFT_ID": "root", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "red"}, + "REL_OP": ">", + }, + ] + + # add and then remove it + matcher = DependencyMatcher(en_tokenizer.vocab) + matcher.add("check", [pattern1]) + matcher.remove("check") + + # this matches on rel op but not attrs + pattern2 = [ + {"RIGHT_ID": "root", "RIGHT_ATTRS": {"ORTH": "flag"}}, + { + "LEFT_ID": "root", + "RIGHT_ID": "r", + "RIGHT_ATTRS": {"ORTH": "blue"}, + "REL_OP": ">", + }, + ] + + # Adding this new pattern with the same label, which should not match + matcher.add("check", [pattern2]) + matches = matcher(doc) + + assert matches == [] diff --git a/spacy/tests/matcher/test_levenshtein.py b/spacy/tests/matcher/test_levenshtein.py new file mode 100644 index 0000000..fd85579 --- /dev/null +++ b/spacy/tests/matcher/test_levenshtein.py @@ -0,0 +1,74 @@ +import pytest + +from spacy.matcher import levenshtein +from spacy.matcher.levenshtein import levenshtein_compare + + +# empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests +# from polyleven +@pytest.mark.parametrize( + "dist,a,b", + [ + (0, "", ""), + (4, "bbcb", "caba"), + (3, "abcb", "cacc"), + (3, "aa", "ccc"), + (1, "cca", "ccac"), + (1, "aba", "aa"), + (4, "bcbb", "abac"), + (3, "acbc", "bba"), + (3, "cbba", "a"), + (2, "bcc", "ba"), + (4, "aaa", "ccbb"), + (3, "うあい", "いいうい"), + (2, "あううい", "うあい"), + (3, "いういい", "うううあ"), + (2, "うい", "あいあ"), + (2, "いあい", "いう"), + (1, "いい", "あいい"), + (3, "あうあ", "いいああ"), + (4, "いあうう", "ううああ"), + (3, "いあいい", "ういああ"), + (3, "いいああ", "ううあう"), + ( + 166, + "TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC", + "ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC", + ), + ( + 111, + "GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG", + "CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT", + ), + ], +) +def test_levenshtein(dist, a, b): + assert levenshtein(a, b) == dist + + +@pytest.mark.parametrize( + "a,b,fuzzy,expected", + [ + ("a", "a", 1, True), + ("a", "a", 0, True), + ("a", "a", -1, True), + ("a", "ab", 1, True), + ("a", "ab", 0, False), + ("a", "ab", -1, True), + ("ab", "ac", 1, True), + ("ab", "ac", -1, True), + ("abc", "cde", 4, True), + ("abc", "cde", -1, False), + ("abcdef", "cdefgh", 4, True), + ("abcdef", "cdefgh", 3, False), + ("abcdef", "cdefgh", -1, False), # default (2 for length 6) + ("abcdefgh", "cdefghijk", 5, True), + ("abcdefgh", "cdefghijk", 4, False), + ("abcdefgh", "cdefghijk", -1, False), # default (2) + ("abcdefgh", "cdefghijkl", 6, True), + ("abcdefgh", "cdefghijkl", 5, False), + ("abcdefgh", "cdefghijkl", -1, False), # default (2) + ], +) +def test_levenshtein_compare(a, b, fuzzy, expected): + assert levenshtein_compare(a, b, fuzzy) == expected diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py new file mode 100644 index 0000000..c824ca3 --- /dev/null +++ b/spacy/tests/matcher/test_matcher_api.py @@ -0,0 +1,909 @@ +import pytest +from mock import Mock + +from spacy.matcher import Matcher +from spacy.tokens import Doc, Span, Token + +from ..doc.test_underscore import clean_underscore # noqa: F401 + + +@pytest.fixture +def matcher(en_vocab): + rules = { + "JS": [[{"ORTH": "JavaScript"}]], + "GoogleNow": [[{"ORTH": "Google"}, {"ORTH": "Now"}]], + "Java": [[{"LOWER": "java"}]], + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns) + return matcher + + +def test_matcher_from_api_docs(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "test"}] + assert len(matcher) == 0 + matcher.add("Rule", [pattern]) + assert len(matcher) == 1 + matcher.remove("Rule") + assert "Rule" not in matcher + matcher.add("Rule", [pattern]) + assert "Rule" in matcher + on_match, patterns = matcher.get("Rule") + assert len(patterns[0]) + + +def test_matcher_empty_patterns_warns(en_vocab): + matcher = Matcher(en_vocab) + assert len(matcher) == 0 + doc = Doc(en_vocab, words=["This", "is", "quite", "something"]) + with pytest.warns(UserWarning): + matcher(doc) + assert len(doc.ents) == 0 + + +def test_matcher_from_usage_docs(en_vocab): + text = "Wow 😀 This is really cool! 😂 😂" + doc = Doc(en_vocab, words=text.split(" ")) + pos_emoji = ["😀", "😃", "😂", "🤣", "😊", "😍"] + pos_patterns = [[{"ORTH": emoji}] for emoji in pos_emoji] + + def label_sentiment(matcher, doc, i, matches): + match_id, start, end = matches[i] + if doc.vocab.strings[match_id] == "HAPPY": + doc.sentiment += 0.1 + span = doc[start:end] + with doc.retokenize() as retokenizer: + retokenizer.merge(span) + token = doc[start] + token.vocab[token.text].norm_ = "happy emoji" + + matcher = Matcher(en_vocab) + matcher.add("HAPPY", pos_patterns, on_match=label_sentiment) + matcher(doc) + assert doc.sentiment != 0 + assert doc[1].norm_ == "happy emoji" + + +def test_matcher_len_contains(matcher): + assert len(matcher) == 3 + matcher.add("TEST", [[{"ORTH": "test"}]]) + assert "TEST" in matcher + assert "TEST2" not in matcher + + +def test_matcher_add_new_api(en_vocab): + doc = Doc(en_vocab, words=["a", "b"]) + patterns = [[{"TEXT": "a"}], [{"TEXT": "a"}, {"TEXT": "b"}]] + matcher = Matcher(en_vocab) + on_match = Mock() + matcher = Matcher(en_vocab) + matcher.add("NEW_API", patterns) + assert len(matcher(doc)) == 2 + matcher = Matcher(en_vocab) + on_match = Mock() + matcher.add("NEW_API_CALLBACK", patterns, on_match=on_match) + assert len(matcher(doc)) == 2 + assert on_match.call_count == 2 + + +def test_matcher_no_match(matcher): + doc = Doc(matcher.vocab, words=["I", "like", "cheese", "."]) + assert matcher(doc) == [] + + +def test_matcher_match_start(matcher): + doc = Doc(matcher.vocab, words=["JavaScript", "is", "good"]) + assert matcher(doc) == [(matcher.vocab.strings["JS"], 0, 1)] + + +def test_matcher_match_end(matcher): + words = ["I", "like", "java"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [(doc.vocab.strings["Java"], 2, 3)] + + +def test_matcher_match_middle(matcher): + words = ["I", "like", "Google", "Now", "best"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [(doc.vocab.strings["GoogleNow"], 2, 4)] + + +def test_matcher_match_multi(matcher): + words = ["I", "like", "Google", "Now", "and", "java", "best"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 2, 4), + (doc.vocab.strings["Java"], 5, 6), + ] + + +@pytest.mark.parametrize( + "rules,match_locs", + [ + ( + { + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + }, + [(2, 4)], + ), + ( + { + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(5, 6)], + ), + ( + { + "JS": [[{"ORTH": {"FUZZY": "JavaScript"}}]], + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(2, 4), (5, 6), (8, 9)], + ), + # only the second pattern matches (check that predicate keys used for + # caching don't collide) + ( + { + "A": [[{"ORTH": {"FUZZY": "Javascripts"}}]], + "B": [[{"ORTH": {"FUZZY5": "Javascripts"}}]], + }, + [(8, 9)], + ), + ], +) +def test_matcher_match_fuzzy(en_vocab, rules, match_locs): + words = ["They", "like", "Goggle", "Now", "and", "Jav", "but", "not", "JvvaScrpt"] + doc = Doc(en_vocab, words=words) + + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns) + assert match_locs == [(start, end) for m_id, start, end in matcher(doc)] + + +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzy_set_op_longest(en_vocab, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(en_vocab, words=words) + assert len(matcher(doc)) == 1 + + +def test_matcher_match_fuzzy_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + +@pytest.mark.parametrize("fuzzyn", range(1, 10)) +def test_matcher_match_fuzzyn_all_insertions(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance + words = ["GoogleNow" + "a" * i for i in range(0, 10)] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("fuzzyn", range(1, 6)) +def test_matcher_match_fuzzyn_various_edits(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance of different edit types + words = [ + "GoogleNow", + "GoogleNuw", + "GoogleNuew", + "GoogleNoweee", + "GiggleNuw3", + "gouggle5New", + ] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("greedy", ["FIRST", "LONGEST"]) +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzyn_set_op_longest(en_vocab, greedy, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY2": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy=greedy) + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + spans = matcher(doc, as_spans=True) + assert len(spans) == 1 + if set_op == "IN": + assert spans[0].text == "Goggle Noo" + else: + assert spans[0].text == "They like" + + +def test_matcher_match_fuzzyn_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY1": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + +def test_matcher_empty_dict(en_vocab): + """Test matcher allows empty token specs, meaning match on any token.""" + matcher = Matcher(en_vocab) + doc = Doc(matcher.vocab, words=["a", "b", "c"]) + matcher.add("A.C", [[{"ORTH": "a"}, {}, {"ORTH": "c"}]]) + matches = matcher(doc) + assert len(matches) == 1 + assert matches[0][1:] == (0, 3) + matcher = Matcher(en_vocab) + matcher.add("A.", [[{"ORTH": "a"}, {}]]) + matches = matcher(doc) + assert matches[0][1:] == (0, 2) + + +def test_matcher_operator_shadow(en_vocab): + matcher = Matcher(en_vocab) + doc = Doc(matcher.vocab, words=["a", "b", "c"]) + pattern = [{"ORTH": "a"}, {"IS_ALPHA": True, "OP": "+"}, {"ORTH": "c"}] + matcher.add("A.C", [pattern]) + matches = matcher(doc) + assert len(matches) == 1 + assert matches[0][1:] == (0, 3) + + +def test_matcher_match_zero(matcher): + words1 = 'He said , " some words " ...'.split() + words2 = 'He said , " some three words " ...'.split() + pattern1 = [ + {"ORTH": '"'}, + {"OP": "!", "IS_PUNCT": True}, + {"OP": "!", "IS_PUNCT": True}, + {"ORTH": '"'}, + ] + pattern2 = [ + {"ORTH": '"'}, + {"IS_PUNCT": True}, + {"IS_PUNCT": True}, + {"IS_PUNCT": True}, + {"ORTH": '"'}, + ] + matcher.add("Quote", [pattern1]) + doc = Doc(matcher.vocab, words=words1) + assert len(matcher(doc)) == 1 + doc = Doc(matcher.vocab, words=words2) + assert len(matcher(doc)) == 0 + matcher.add("Quote", [pattern2]) + assert len(matcher(doc)) == 0 + + +def test_matcher_match_zero_plus(matcher): + words = 'He said , " some words " ...'.split() + pattern = [{"ORTH": '"'}, {"OP": "*", "IS_PUNCT": False}, {"ORTH": '"'}] + matcher = Matcher(matcher.vocab) + matcher.add("Quote", [pattern]) + doc = Doc(matcher.vocab, words=words) + assert len(matcher(doc)) == 1 + + +def test_matcher_match_one_plus(matcher): + control = Matcher(matcher.vocab) + control.add("BasicPhilippe", [[{"ORTH": "Philippe"}]]) + doc = Doc(control.vocab, words=["Philippe", "Philippe"]) + m = control(doc) + assert len(m) == 2 + pattern = [{"ORTH": "Philippe"}, {"ORTH": "Philippe", "OP": "+"}] + matcher.add("KleenePhilippe", [pattern]) + m = matcher(doc) + assert len(m) == 1 + + +def test_matcher_any_token_operator(en_vocab): + """Test that patterns with "any token" {} work with operators.""" + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"ORTH": "test"}, {"OP": "*"}]]) + doc = Doc(en_vocab, words=["test", "hello", "world"]) + matches = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches) == 3 + assert matches[0] == "test" + assert matches[1] == "test hello" + assert matches[2] == "test hello world" + + +@pytest.mark.usefixtures("clean_underscore") +def test_matcher_extension_attribute(en_vocab): + matcher = Matcher(en_vocab) + get_is_fruit = lambda token: token.text in ("apple", "banana") + Token.set_extension("is_fruit", getter=get_is_fruit, force=True) + pattern = [{"ORTH": "an"}, {"_": {"is_fruit": True}}] + matcher.add("HAVING_FRUIT", [pattern]) + doc = Doc(en_vocab, words=["an", "apple"]) + matches = matcher(doc) + assert len(matches) == 1 + doc = Doc(en_vocab, words=["an", "aardvark"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_set_value(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"IN": ["an", "a"]}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "apple"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["aardvark"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_set_value_operator(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"IN": ["a", "the"]}, "OP": "?"}, {"ORTH": "house"}] + matcher.add("DET_HOUSE", [pattern]) + doc = Doc(en_vocab, words=["In", "a", "house"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["my", "house"]) + matches = matcher(doc) + assert len(matches) == 1 + + +def test_matcher_subset_value_operator(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"MORPH": {"IS_SUBSET": ["Feat=Val", "Feat2=Val2"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc)) == 3 + doc[0].set_morph("Feat=Val") + assert len(matcher(doc)) == 3 + doc[0].set_morph("Feat=Val|Feat2=Val2") + assert len(matcher(doc)) == 3 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3") + assert len(matcher(doc)) == 2 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3|Feat4=Val4") + assert len(matcher(doc)) == 2 + + # IS_SUBSET acts like "IN" for attrs other than MORPH + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"IS_SUBSET": ["A", "B"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 1 + + # IS_SUBSET with an empty list matches nothing + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"IS_SUBSET": []}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 0 + + # IS_SUBSET with a list value + Token.set_extension("ext", default=[]) + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"IS_SUBSET": ["A", "B"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = ["A"] + doc[1]._.ext = ["C", "D"] + assert len(matcher(doc)) == 2 + + +def test_matcher_superset_value_operator(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"MORPH": {"IS_SUPERSET": ["Feat=Val", "Feat2=Val2", "Feat3=Val3"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc)) == 0 + doc[0].set_morph("Feat=Val|Feat2=Val2") + assert len(matcher(doc)) == 0 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3") + assert len(matcher(doc)) == 1 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3|Feat4=Val4") + assert len(matcher(doc)) == 1 + + # IS_SUPERSET with more than one value only matches for MORPH + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"IS_SUPERSET": ["A", "B"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 0 + + # IS_SUPERSET with one value is the same as == + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"IS_SUPERSET": ["A"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 1 + + # IS_SUPERSET with an empty value matches everything + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"IS_SUPERSET": []}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 3 + + # IS_SUPERSET with a list value + Token.set_extension("ext", default=[]) + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"IS_SUPERSET": ["A"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 1 + + +def test_matcher_intersect_value_operator(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"MORPH": {"INTERSECTS": ["Feat=Val", "Feat2=Val2", "Feat3=Val3"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc)) == 0 + doc[0].set_morph("Feat=Val") + assert len(matcher(doc)) == 1 + doc[0].set_morph("Feat=Val|Feat2=Val2") + assert len(matcher(doc)) == 1 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3") + assert len(matcher(doc)) == 1 + doc[0].set_morph("Feat=Val|Feat2=Val2|Feat3=Val3|Feat4=Val4") + assert len(matcher(doc)) == 1 + + # INTERSECTS with a single value is the same as IN + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"INTERSECTS": ["A", "B"]}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 1 + + # INTERSECTS with an empty pattern list matches nothing + matcher = Matcher(en_vocab) + pattern = [{"TAG": {"INTERSECTS": []}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0].tag_ = "A" + assert len(matcher(doc)) == 0 + + # INTERSECTS with a list value + Token.set_extension("ext", default=[]) + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"INTERSECTS": ["A", "C"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 1 + + # INTERSECTS matches nothing for iterables that aren't all str or int + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"INTERSECTS": ["Abx", "C"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = [["Abx"], "B"] + assert len(matcher(doc)) == 0 + doc[0]._.ext = ["Abx", "B"] + assert len(matcher(doc)) == 1 + + # INTERSECTS with an empty pattern list matches nothing + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"INTERSECTS": []}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 0 + + # INTERSECTS with an empty value matches nothing + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"INTERSECTS": ["A", "B"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = [] + assert len(matcher(doc)) == 0 + + +def test_matcher_morph_handling(en_vocab): + # order of features in pattern doesn't matter + matcher = Matcher(en_vocab) + pattern1 = [{"MORPH": {"IN": ["Feat1=Val1|Feat2=Val2"]}}] + pattern2 = [{"MORPH": {"IN": ["Feat2=Val2|Feat1=Val1"]}}] + matcher.add("M", [pattern1]) + matcher.add("N", [pattern2]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc)) == 0 + + doc[0].set_morph("Feat2=Val2|Feat1=Val1") + assert len(matcher(doc)) == 2 + doc[0].set_morph("Feat1=Val1|Feat2=Val2") + assert len(matcher(doc)) == 2 + + # multiple values are split + matcher = Matcher(en_vocab) + pattern1 = [{"MORPH": {"IS_SUPERSET": ["Feat1=Val1", "Feat2=Val2"]}}] + pattern2 = [{"MORPH": {"IS_SUPERSET": ["Feat1=Val1", "Feat1=Val3", "Feat2=Val2"]}}] + matcher.add("M", [pattern1]) + matcher.add("N", [pattern2]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc)) == 0 + + doc[0].set_morph("Feat2=Val2,Val3|Feat1=Val1") + assert len(matcher(doc)) == 1 + doc[0].set_morph("Feat1=Val1,Val3|Feat2=Val2") + assert len(matcher(doc)) == 2 + + +def test_matcher_regex(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": r"(?:a|an)"}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_regex_set_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_regex_set_not_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"NOT_IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 1 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 1 + + +def test_matcher_regex_shape(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"SHAPE": {"REGEX": r"^[^x]+$"}}] + matcher.add("NON_ALPHA", [pattern]) + doc = Doc(en_vocab, words=["99", "problems", "!"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 0 + + +@pytest.mark.parametrize( + "cmp, bad", + [ + ("==", ["a", "aaa"]), + ("!=", ["aa"]), + (">=", ["a"]), + ("<=", ["aaa"]), + (">", ["a", "aa"]), + ("<", ["aa", "aaa"]), + ], +) +def test_matcher_compare_length(en_vocab, cmp, bad): + matcher = Matcher(en_vocab) + pattern = [{"LENGTH": {cmp: 2}}] + matcher.add("LENGTH_COMPARE", [pattern]) + doc = Doc(en_vocab, words=["a", "aa", "aaa"]) + matches = matcher(doc) + assert len(matches) == len(doc) - len(bad) + doc = Doc(en_vocab, words=bad) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_extension_set_membership(en_vocab): + matcher = Matcher(en_vocab) + get_reversed = lambda token: "".join(reversed(token.text)) + Token.set_extension("reversed", getter=get_reversed, force=True) + pattern = [{"_": {"reversed": {"IN": ["eyb", "ih"]}}}] + matcher.add("REVERSED", [pattern]) + doc = Doc(en_vocab, words=["hi", "bye", "hello"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["aardvark"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_extension_in_set_predicate(en_vocab): + matcher = Matcher(en_vocab) + Token.set_extension("ext", default=[]) + pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + + # The IN predicate expects an exact match between the + # extension value and one of the pattern's values. + doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 0 + + doc[0]._.ext = ["A"] + assert len(matcher(doc)) == 0 + + doc[0]._.ext = "A" + assert len(matcher(doc)) == 1 + + +def test_matcher_basic_check(en_vocab): + matcher = Matcher(en_vocab) + # Potential mistake: pass in pattern instead of list of patterns + pattern = [{"TEXT": "hello"}, {"TEXT": "world"}] + with pytest.raises(ValueError): + matcher.add("TEST", pattern) + + +def test_attr_pipeline_checks(en_vocab): + doc1 = Doc(en_vocab, words=["Test"]) + doc1[0].dep_ = "ROOT" + doc2 = Doc(en_vocab, words=["Test"]) + doc2[0].tag_ = "TAG" + doc2[0].pos_ = "X" + doc2[0].set_morph("Feat=Val") + doc2[0].lemma_ = "LEMMA" + doc3 = Doc(en_vocab, words=["Test"]) + # DEP requires DEP + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"DEP": "a"}]]) + matcher(doc1) + with pytest.raises(ValueError): + matcher(doc2) + with pytest.raises(ValueError): + matcher(doc3) + # errors can be suppressed if desired + matcher(doc2, allow_missing=True) + matcher(doc3, allow_missing=True) + # TAG, POS, LEMMA require those values + for attr in ("TAG", "POS", "LEMMA"): + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{attr: "a"}]]) + matcher(doc2) + with pytest.raises(ValueError): + matcher(doc1) + with pytest.raises(ValueError): + matcher(doc3) + # TEXT/ORTH only require tokens + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"ORTH": "a"}]]) + matcher(doc1) + matcher(doc2) + matcher(doc3) + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"TEXT": "a"}]]) + matcher(doc1) + matcher(doc2) + matcher(doc3) + + +@pytest.mark.parametrize( + "pattern,text", + [ + ([{"IS_ALPHA": True}], "a"), + ([{"IS_ASCII": True}], "a"), + ([{"IS_DIGIT": True}], "1"), + ([{"IS_LOWER": True}], "a"), + ([{"IS_UPPER": True}], "A"), + ([{"IS_TITLE": True}], "Aaaa"), + ([{"IS_PUNCT": True}], "."), + ([{"IS_SPACE": True}], "\n"), + ([{"IS_BRACKET": True}], "["), + ([{"IS_QUOTE": True}], '"'), + ([{"IS_LEFT_PUNCT": True}], "``"), + ([{"IS_RIGHT_PUNCT": True}], "''"), + ([{"IS_STOP": True}], "the"), + ([{"SPACY": True}], "the"), + ([{"LIKE_NUM": True}], "1"), + ([{"LIKE_URL": True}], "http://example.com"), + ([{"LIKE_EMAIL": True}], "mail@example.com"), + ], +) +def test_matcher_schema_token_attributes(en_vocab, pattern, text): + matcher = Matcher(en_vocab) + doc = Doc(en_vocab, words=text.split(" ")) + matcher.add("Rule", [pattern]) + assert len(matcher) == 1 + matches = matcher(doc) + assert len(matches) == 1 + + +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_matcher_valid_callback(en_vocab): + """Test that on_match can only be None or callable.""" + matcher = Matcher(en_vocab) + with pytest.raises(ValueError): + matcher.add("TEST", [[{"TEXT": "test"}]], on_match=[]) + matcher(Doc(en_vocab, words=["test"])) + + +def test_matcher_callback(en_vocab): + mock = Mock() + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "test"}] + matcher.add("Rule", [pattern], on_match=mock) + doc = Doc(en_vocab, words=["This", "is", "a", "test", "."]) + matches = matcher(doc) + mock.assert_called_once_with(matcher, doc, 0, matches) + + +def test_matcher_callback_with_alignments(en_vocab): + mock = Mock() + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "test"}] + matcher.add("Rule", [pattern], on_match=mock) + doc = Doc(en_vocab, words=["This", "is", "a", "test", "."]) + matches = matcher(doc, with_alignments=True) + mock.assert_called_once_with(matcher, doc, 0, matches) + + +def test_matcher_span(matcher): + text = "JavaScript is good but Java is better" + doc = Doc(matcher.vocab, words=text.split()) + span_js = doc[:3] + span_java = doc[4:] + assert len(matcher(doc)) == 2 + assert len(matcher(span_js)) == 1 + assert len(matcher(span_java)) == 1 + + +def test_matcher_as_spans(matcher): + """Test the new as_spans=True API.""" + text = "JavaScript is good but Java is better" + doc = Doc(matcher.vocab, words=text.split()) + matches = matcher(doc, as_spans=True) + assert len(matches) == 2 + assert isinstance(matches[0], Span) + assert matches[0].text == "JavaScript" + assert matches[0].label_ == "JS" + assert isinstance(matches[1], Span) + assert matches[1].text == "Java" + assert matches[1].label_ == "Java" + + matches = matcher(doc[1:], as_spans=True) + assert len(matches) == 1 + assert isinstance(matches[0], Span) + assert matches[0].text == "Java" + assert matches[0].label_ == "Java" + + +def test_matcher_deprecated(matcher): + doc = Doc(matcher.vocab, words=["hello", "world"]) + with pytest.warns(DeprecationWarning) as record: + for _ in matcher.pipe([doc]): + pass + assert record.list + assert "spaCy v3.0" in str(record.list[0].message) + + +def test_matcher_remove_zero_operator(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"OP": "!"}] + matcher.add("Rule", [pattern]) + doc = Doc(en_vocab, words=["This", "is", "a", "test", "."]) + matches = matcher(doc) + assert len(matches) == 0 + assert "Rule" in matcher + matcher.remove("Rule") + assert "Rule" not in matcher + + +def test_matcher_no_zero_length(en_vocab): + doc = Doc(en_vocab, words=["a", "b"], tags=["A", "B"]) + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"TAG": "C", "OP": "?"}]]) + assert len(matcher(doc)) == 0 + + +def test_matcher_ent_iob_key(en_vocab): + """Test that patterns with ent_iob works correctly.""" + matcher = Matcher(en_vocab) + matcher.add("Rule", [[{"ENT_IOB": "I"}]]) + doc1 = Doc(en_vocab, words=["I", "visited", "New", "York", "and", "California"]) + doc1.ents = [Span(doc1, 2, 4, label="GPE"), Span(doc1, 5, 6, label="GPE")] + doc2 = Doc(en_vocab, words=["I", "visited", "my", "friend", "Alicia"]) + doc2.ents = [Span(doc2, 4, 5, label="PERSON")] + matches1 = [doc1[start:end].text for _, start, end in matcher(doc1)] + matches2 = [doc2[start:end].text for _, start, end in matcher(doc2)] + assert len(matches1) == 1 + assert matches1[0] == "York" + assert len(matches2) == 0 + + matcher = Matcher(en_vocab) # Test iob pattern with operators + matcher.add("Rule", [[{"ENT_IOB": "I", "OP": "+"}]]) + doc = Doc( + en_vocab, words=["I", "visited", "my", "friend", "Anna", "Maria", "Esperanza"] + ) + doc.ents = [Span(doc, 4, 7, label="PERSON")] + matches = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches) == 3 + assert matches[0] == "Maria" + assert matches[1] == "Maria Esperanza" + assert matches[2] == "Esperanza" + + +def test_matcher_min_max_operator(en_vocab): + # Exactly n matches {n} + doc = Doc( + en_vocab, + words=["foo", "bar", "foo", "foo", "bar", "foo", "foo", "foo", "bar", "bar"], + ) + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{3}"}] + matcher.add("TEST", [pattern]) + + matches1 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches1) == 1 + + # At least n matches {n,} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{2,}"}] + matcher.add("TEST", [pattern]) + matches2 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches2) == 4 + + # At most m matches {,m} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{,2}"}] + matcher.add("TEST", [pattern]) + matches3 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches3) == 9 + + # At least n matches and most m matches {n,m} + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "foo", "OP": "{2,3}"}] + matcher.add("TEST", [pattern]) + matches4 = [doc[start:end].text for _, start, end in matcher(doc)] + assert len(matches4) == 4 diff --git a/spacy/tests/matcher/test_matcher_logic.py b/spacy/tests/matcher/test_matcher_logic.py new file mode 100644 index 0000000..1109766 --- /dev/null +++ b/spacy/tests/matcher/test_matcher_logic.py @@ -0,0 +1,788 @@ +import re + +import pytest + +from spacy.attrs import IS_PUNCT, LOWER, ORTH +from spacy.errors import MatchPatternError +from spacy.lang.en import English +from spacy.lang.lex_attrs import LEX_ATTRS +from spacy.matcher import Matcher +from spacy.tokens import Doc, Span, Token +from spacy.vocab import Vocab + +pattern1 = [{"ORTH": "A"}, {"ORTH": "A", "OP": "*"}] +pattern2 = [{"ORTH": "A", "OP": "*"}, {"ORTH": "A"}] +pattern3 = [{"ORTH": "A"}, {"ORTH": "A"}] +pattern4 = [{"ORTH": "B"}, {"ORTH": "A", "OP": "*"}, {"ORTH": "B"}] +pattern5 = [{"ORTH": "B", "OP": "*"}, {"ORTH": "A", "OP": "*"}, {"ORTH": "B"}] + +re_pattern1 = "AA*" +re_pattern2 = "A*A" +re_pattern3 = "AA" +re_pattern4 = "BA*B" +re_pattern5 = "B*A*B" + +longest1 = "A A A A A" +longest2 = "A A A A A" +longest3 = "A A" +longest4 = "B A A A A A B" # "FIRST" would be "B B" +longest5 = "B B A A A A A B" + + +@pytest.fixture +def text(): + return "(BBAAAAAB)." + + +@pytest.fixture +def doc(en_tokenizer, text): + doc = en_tokenizer(" ".join(text)) + return doc + + +@pytest.mark.issue(118) +@pytest.mark.parametrize( + "patterns", + [ + [[{"LOWER": "celtics"}], [{"LOWER": "boston"}, {"LOWER": "celtics"}]], + [[{"LOWER": "boston"}, {"LOWER": "celtics"}], [{"LOWER": "celtics"}]], + ], +) +def test_issue118(en_tokenizer, patterns): + """Test a bug that arose from having overlapping matches""" + text = ( + "how many points did lebron james score against the boston celtics last night" + ) + doc = en_tokenizer(text) + ORG = doc.vocab.strings["ORG"] + matcher = Matcher(doc.vocab) + matcher.add("BostonCeltics", patterns) + assert len(list(doc.ents)) == 0 + matches = [(ORG, start, end) for _, start, end in matcher(doc)] + assert matches == [(ORG, 9, 11), (ORG, 10, 11)] + doc.ents = matches[:1] + ents = list(doc.ents) + assert len(ents) == 1 + assert ents[0].label == ORG + assert ents[0].start == 9 + assert ents[0].end == 11 + + +@pytest.mark.issue(118) +@pytest.mark.parametrize( + "patterns", + [ + [[{"LOWER": "boston"}], [{"LOWER": "boston"}, {"LOWER": "celtics"}]], + [[{"LOWER": "boston"}, {"LOWER": "celtics"}], [{"LOWER": "boston"}]], + ], +) +def test_issue118_prefix_reorder(en_tokenizer, patterns): + """Test a bug that arose from having overlapping matches""" + text = ( + "how many points did lebron james score against the boston celtics last night" + ) + doc = en_tokenizer(text) + ORG = doc.vocab.strings["ORG"] + matcher = Matcher(doc.vocab) + matcher.add("BostonCeltics", patterns) + assert len(list(doc.ents)) == 0 + matches = [(ORG, start, end) for _, start, end in matcher(doc)] + doc.ents += tuple(matches)[1:] + assert matches == [(ORG, 9, 10), (ORG, 9, 11)] + ents = doc.ents + assert len(ents) == 1 + assert ents[0].label == ORG + assert ents[0].start == 9 + assert ents[0].end == 11 + + +@pytest.mark.issue(242) +def test_issue242(en_tokenizer): + """Test overlapping multi-word phrases.""" + text = "There are different food safety standards in different countries." + patterns = [ + [{"LOWER": "food"}, {"LOWER": "safety"}], + [{"LOWER": "safety"}, {"LOWER": "standards"}], + ] + doc = en_tokenizer(text) + matcher = Matcher(doc.vocab) + matcher.add("FOOD", patterns) + matches = [(ent_type, start, end) for ent_type, start, end in matcher(doc)] + match1, match2 = matches + assert match1[1] == 3 + assert match1[2] == 5 + assert match2[1] == 4 + assert match2[2] == 6 + with pytest.raises(ValueError): + # One token can only be part of one entity, so test that the matches + # can't be added as entities + doc.ents += tuple(matches) + + +@pytest.mark.issue(587) +def test_issue587(en_tokenizer): + """Test that Matcher doesn't segfault on particular input""" + doc = en_tokenizer("a b; c") + matcher = Matcher(doc.vocab) + matcher.add("TEST1", [[{ORTH: "a"}, {ORTH: "b"}]]) + matches = matcher(doc) + assert len(matches) == 1 + matcher.add("TEST2", [[{ORTH: "a"}, {ORTH: "b"}, {IS_PUNCT: True}, {ORTH: "c"}]]) + matches = matcher(doc) + assert len(matches) == 2 + matcher.add("TEST3", [[{ORTH: "a"}, {ORTH: "b"}, {IS_PUNCT: True}, {ORTH: "d"}]]) + matches = matcher(doc) + assert len(matches) == 2 + + +@pytest.mark.issue(588) +def test_issue588(en_vocab): + """Test if empty specs still cause an error when adding patterns""" + matcher = Matcher(en_vocab) + with pytest.raises(ValueError): + matcher.add("TEST", [[]]) + + +@pytest.mark.issue(590) +def test_issue590(en_vocab): + """Test overlapping matches""" + doc = Doc(en_vocab, words=["n", "=", "1", ";", "a", ":", "5", "%"]) + matcher = Matcher(en_vocab) + matcher.add( + "ab", [[{"IS_ALPHA": True}, {"ORTH": ":"}, {"LIKE_NUM": True}, {"ORTH": "%"}]] + ) + matcher.add("ab", [[{"IS_ALPHA": True}, {"ORTH": "="}, {"LIKE_NUM": True}]]) + matches = matcher(doc) + assert len(matches) == 2 + + +@pytest.mark.issue(615) +def test_issue615(en_tokenizer): + def merge_phrases(matcher, doc, i, matches): + """Merge a phrase. We have to be careful here because we'll change the + token indices. To avoid problems, merge all the phrases once we're called + on the last match.""" + if i != len(matches) - 1: + return None + spans = [Span(doc, start, end, label=label) for label, start, end in matches] + with doc.retokenize() as retokenizer: + for span in spans: + tag = "NNP" if span.label_ else span.root.tag_ + attrs = {"tag": tag, "lemma": span.text} + retokenizer.merge(span, attrs=attrs) + doc.ents = doc.ents + (span,) + + text = "The golf club is broken" + pattern = [{"ORTH": "golf"}, {"ORTH": "club"}] + label = "Sport_Equipment" + doc = en_tokenizer(text) + matcher = Matcher(doc.vocab) + matcher.add(label, [pattern], on_match=merge_phrases) + matcher(doc) + entities = list(doc.ents) + assert entities != [] + assert entities[0].label != 0 + + +@pytest.mark.issue(850) +def test_issue850(): + """The variable-length pattern matches the succeeding token. Check we + handle the ambiguity correctly.""" + vocab = Vocab(lex_attr_getters={LOWER: lambda string: string.lower()}) + matcher = Matcher(vocab) + pattern = [{"LOWER": "bob"}, {"OP": "*"}, {"LOWER": "frank"}] + matcher.add("FarAway", [pattern]) + doc = Doc(matcher.vocab, words=["bob", "and", "and", "frank"]) + match = matcher(doc) + assert len(match) == 1 + ent_id, start, end = match[0] + assert start == 0 + assert end == 4 + + +@pytest.mark.issue(850) +def test_issue850_basic(): + """Test Matcher matches with '*' operator and Boolean flag""" + vocab = Vocab(lex_attr_getters={LOWER: lambda string: string.lower()}) + matcher = Matcher(vocab) + pattern = [{"LOWER": "bob"}, {"OP": "*", "LOWER": "and"}, {"LOWER": "frank"}] + matcher.add("FarAway", [pattern]) + doc = Doc(matcher.vocab, words=["bob", "and", "and", "frank"]) + match = matcher(doc) + assert len(match) == 1 + ent_id, start, end = match[0] + assert start == 0 + assert end == 4 + + +@pytest.mark.issue(1434) +def test_issue1434(): + """Test matches occur when optional element at end of short doc.""" + pattern = [{"ORTH": "Hello"}, {"IS_ALPHA": True, "OP": "?"}] + vocab = Vocab(lex_attr_getters=LEX_ATTRS) + hello_world = Doc(vocab, words=["Hello", "World"]) + hello = Doc(vocab, words=["Hello"]) + matcher = Matcher(vocab) + matcher.add("MyMatcher", [pattern]) + matches = matcher(hello_world) + assert matches + matches = matcher(hello) + assert matches + + +@pytest.mark.parametrize( + "string,start,end", + [ + ("a", 0, 1), + ("a b", 0, 2), + ("a c", 0, 1), + ("a b c", 0, 2), + ("a b b c", 0, 3), + ("a b b", 0, 3), + ], +) +@pytest.mark.issue(1450) +def test_issue1450(string, start, end): + """Test matcher works when patterns end with * operator.""" + pattern = [{"ORTH": "a"}, {"ORTH": "b", "OP": "*"}] + matcher = Matcher(Vocab()) + matcher.add("TSTEND", [pattern]) + doc = Doc(Vocab(), words=string.split()) + matches = matcher(doc) + if start is None or end is None: + assert matches == [] + assert matches[-1][1] == start + assert matches[-1][2] == end + + +@pytest.mark.issue(1945) +def test_issue1945(): + """Test regression in Matcher introduced in v2.0.6.""" + matcher = Matcher(Vocab()) + matcher.add("MWE", [[{"orth": "a"}, {"orth": "a"}]]) + doc = Doc(matcher.vocab, words=["a", "a", "a"]) + matches = matcher(doc) # we should see two overlapping matches here + assert len(matches) == 2 + assert matches[0][1:] == (0, 2) + assert matches[1][1:] == (1, 3) + + +@pytest.mark.issue(1971) +def test_issue1971(en_vocab): + # Possibly related to #2675 and #2671? + matcher = Matcher(en_vocab) + pattern = [ + {"ORTH": "Doe"}, + {"ORTH": "!", "OP": "?"}, + {"_": {"optional": True}, "OP": "?"}, + {"ORTH": "!", "OP": "?"}, + ] + Token.set_extension("optional", default=False) + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=["Hello", "John", "Doe", "!"]) + # We could also assert length 1 here, but this is more conclusive, because + # the real problem here is that it returns a duplicate match for a match_id + # that's not actually in the vocab! + matches = matcher(doc) + assert all([match_id in en_vocab.strings for match_id, start, end in matches]) + + +@pytest.mark.issue(1971) +def test_issue_1971_2(en_vocab): + matcher = Matcher(en_vocab) + pattern1 = [{"ORTH": "EUR", "LOWER": {"IN": ["eur"]}}, {"LIKE_NUM": True}] + pattern2 = [{"LIKE_NUM": True}, {"ORTH": "EUR"}] # {"IN": ["EUR"]}}] + doc = Doc(en_vocab, words=["EUR", "10", "is", "10", "EUR"]) + matcher.add("TEST1", [pattern1, pattern2]) + matches = matcher(doc) + assert len(matches) == 2 + + +@pytest.mark.issue(1971) +def test_issue_1971_3(en_vocab): + """Test that pattern matches correctly for multiple extension attributes.""" + Token.set_extension("a", default=1, force=True) + Token.set_extension("b", default=2, force=True) + doc = Doc(en_vocab, words=["hello", "world"]) + matcher = Matcher(en_vocab) + matcher.add("A", [[{"_": {"a": 1}}]]) + matcher.add("B", [[{"_": {"b": 2}}]]) + matches = sorted((en_vocab.strings[m_id], s, e) for m_id, s, e in matcher(doc)) + assert len(matches) == 4 + assert matches == sorted([("A", 0, 1), ("A", 1, 2), ("B", 0, 1), ("B", 1, 2)]) + + +@pytest.mark.issue(1971) +def test_issue_1971_4(en_vocab): + """Test that pattern matches correctly with multiple extension attribute + values on a single token. + """ + Token.set_extension("ext_a", default="str_a", force=True) + Token.set_extension("ext_b", default="str_b", force=True) + matcher = Matcher(en_vocab) + doc = Doc(en_vocab, words=["this", "is", "text"]) + pattern = [{"_": {"ext_a": "str_a", "ext_b": "str_b"}}] * 3 + matcher.add("TEST", [pattern]) + matches = matcher(doc) + # Uncommenting this caused a segmentation fault + assert len(matches) == 1 + assert matches[0] == (en_vocab.strings["TEST"], 0, 3) + + +@pytest.mark.issue(2464) +def test_issue2464(en_vocab): + """Test problem with successive ?. This is the same bug, so putting it here.""" + matcher = Matcher(en_vocab) + doc = Doc(en_vocab, words=["a", "b"]) + matcher.add("4", [[{"OP": "?"}, {"OP": "?"}]]) + matches = matcher(doc) + assert len(matches) == 3 + + +@pytest.mark.issue(2569) +def test_issue2569(en_tokenizer): + """Test that operator + is greedy.""" + doc = en_tokenizer("It is May 15, 1993.") + doc.ents = [Span(doc, 2, 6, label=doc.vocab.strings["DATE"])] + matcher = Matcher(doc.vocab) + matcher.add("RULE", [[{"ENT_TYPE": "DATE", "OP": "+"}]]) + matched = [doc[start:end] for _, start, end in matcher(doc)] + matched = sorted(matched, key=len, reverse=True) + assert len(matched) == 10 + assert len(matched[0]) == 4 + assert matched[0].text == "May 15, 1993" + + +@pytest.mark.issue(2671) +def test_issue2671(): + """Ensure the correct entity ID is returned for matches with quantifiers. + See also #2675 + """ + nlp = English() + matcher = Matcher(nlp.vocab) + pattern_id = "test_pattern" + pattern = [ + {"LOWER": "high"}, + {"IS_PUNCT": True, "OP": "?"}, + {"LOWER": "adrenaline"}, + ] + matcher.add(pattern_id, [pattern]) + doc1 = nlp("This is a high-adrenaline situation.") + doc2 = nlp("This is a high adrenaline situation.") + matches1 = matcher(doc1) + for match_id, start, end in matches1: + assert nlp.vocab.strings[match_id] == pattern_id + matches2 = matcher(doc2) + for match_id, start, end in matches2: + assert nlp.vocab.strings[match_id] == pattern_id + + +@pytest.mark.issue(3009) +def test_issue3009(en_vocab): + """Test problem with matcher quantifiers""" + patterns = [ + [{"ORTH": "has"}, {"LOWER": "to"}, {"LOWER": "do"}, {"TAG": "IN"}], + [ + {"ORTH": "has"}, + {"IS_ASCII": True, "IS_PUNCT": False, "OP": "*"}, + {"LOWER": "to"}, + {"LOWER": "do"}, + {"TAG": "IN"}, + ], + [ + {"ORTH": "has"}, + {"IS_ASCII": True, "IS_PUNCT": False, "OP": "?"}, + {"LOWER": "to"}, + {"LOWER": "do"}, + {"TAG": "IN"}, + ], + ] + words = ["also", "has", "to", "do", "with"] + tags = ["RB", "VBZ", "TO", "VB", "IN"] + pos = ["ADV", "VERB", "ADP", "VERB", "ADP"] + doc = Doc(en_vocab, words=words, tags=tags, pos=pos) + matcher = Matcher(en_vocab) + for i, pattern in enumerate(patterns): + matcher.add(str(i), [pattern]) + matches = matcher(doc) + assert matches + + +@pytest.mark.issue(3328) +def test_issue3328(en_vocab): + doc = Doc(en_vocab, words=["Hello", ",", "how", "are", "you", "doing", "?"]) + matcher = Matcher(en_vocab) + patterns = [ + [{"LOWER": {"IN": ["hello", "how"]}}], + [{"LOWER": {"IN": ["you", "doing"]}}], + ] + matcher.add("TEST", patterns) + matches = matcher(doc) + assert len(matches) == 4 + matched_texts = [doc[start:end].text for _, start, end in matches] + assert matched_texts == ["Hello", "how", "you", "doing"] + + +@pytest.mark.issue(3549) +def test_issue3549(en_vocab): + """Test that match pattern validation doesn't raise on empty errors.""" + matcher = Matcher(en_vocab, validate=True) + pattern = [{"LOWER": "hello"}, {"LOWER": "world"}] + matcher.add("GOOD", [pattern]) + with pytest.raises(MatchPatternError): + matcher.add("BAD", [[{"X": "Y"}]]) + + +@pytest.mark.skip("Matching currently only works on strings and integers") +@pytest.mark.issue(3555) +def test_issue3555(en_vocab): + """Test that custom extensions with default None don't break matcher.""" + Token.set_extension("issue3555", default=None) + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "have"}, {"_": {"issue3555": True}}] + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=["have", "apple"]) + matcher(doc) + + +@pytest.mark.issue(3839) +def test_issue3839(en_vocab): + """Test that match IDs returned by the matcher are correct, are in the string""" + doc = Doc(en_vocab, words=["terrific", "group", "of", "people"]) + matcher = Matcher(en_vocab) + match_id = "PATTERN" + pattern1 = [{"LOWER": "terrific"}, {"OP": "?"}, {"LOWER": "group"}] + pattern2 = [{"LOWER": "terrific"}, {"OP": "?"}, {"OP": "?"}, {"LOWER": "group"}] + matcher.add(match_id, [pattern1]) + matches = matcher(doc) + assert matches[0][0] == en_vocab.strings[match_id] + matcher = Matcher(en_vocab) + matcher.add(match_id, [pattern2]) + matches = matcher(doc) + assert matches[0][0] == en_vocab.strings[match_id] + + +@pytest.mark.issue(3879) +def test_issue3879(en_vocab): + doc = Doc(en_vocab, words=["This", "is", "a", "test", "."]) + assert len(doc) == 5 + pattern = [{"ORTH": "This", "OP": "?"}, {"OP": "?"}, {"ORTH": "test"}] + matcher = Matcher(en_vocab) + matcher.add("TEST", [pattern]) + assert len(matcher(doc)) == 2 # fails because of a FP match 'is a test' + + +@pytest.mark.issue(3951) +def test_issue3951(en_vocab): + """Test that combinations of optional rules are matched correctly.""" + matcher = Matcher(en_vocab) + pattern = [ + {"LOWER": "hello"}, + {"LOWER": "this", "OP": "?"}, + {"OP": "?"}, + {"LOWER": "world"}, + ] + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=["Hello", "my", "new", "world"]) + matches = matcher(doc) + assert len(matches) == 0 + + +@pytest.mark.issue(4120) +def test_issue4120(en_vocab): + """Test that matches without a final {OP: ?} token are returned.""" + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"ORTH": "a"}, {"OP": "?"}]]) + doc1 = Doc(en_vocab, words=["a"]) + assert len(matcher(doc1)) == 1 # works + doc2 = Doc(en_vocab, words=["a", "b", "c"]) + assert len(matcher(doc2)) == 2 # fixed + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"ORTH": "a"}, {"OP": "?"}, {"ORTH": "b"}]]) + doc3 = Doc(en_vocab, words=["a", "b", "b", "c"]) + assert len(matcher(doc3)) == 2 # works + matcher = Matcher(en_vocab) + matcher.add("TEST", [[{"ORTH": "a"}, {"OP": "?"}, {"ORTH": "b", "OP": "?"}]]) + doc4 = Doc(en_vocab, words=["a", "b", "b", "c"]) + assert len(matcher(doc4)) == 3 # fixed + + +@pytest.mark.parametrize( + "pattern,re_pattern", + [ + (pattern1, re_pattern1), + (pattern2, re_pattern2), + (pattern3, re_pattern3), + (pattern4, re_pattern4), + (pattern5, re_pattern5), + ], +) +def test_greedy_matching_first(doc, text, pattern, re_pattern): + """Test that the greedy matching behavior "FIRST" is consistent with + other re implementations.""" + matcher = Matcher(doc.vocab) + matcher.add(re_pattern, [pattern], greedy="FIRST") + matches = matcher(doc) + re_matches = [m.span() for m in re.finditer(re_pattern, text)] + for (key, m_s, m_e), (re_s, re_e) in zip(matches, re_matches): + # matching the string, not the exact position + assert doc[m_s:m_e].text == doc[re_s:re_e].text + + +@pytest.mark.parametrize( + "pattern,longest", + [ + (pattern1, longest1), + (pattern2, longest2), + (pattern3, longest3), + (pattern4, longest4), + (pattern5, longest5), + ], +) +def test_greedy_matching_longest(doc, text, pattern, longest): + """Test the "LONGEST" greedy matching behavior""" + matcher = Matcher(doc.vocab) + matcher.add("RULE", [pattern], greedy="LONGEST") + matches = matcher(doc) + for key, s, e in matches: + assert doc[s:e].text == longest + + +def test_greedy_matching_longest_first(en_tokenizer): + """Test that "LONGEST" matching prefers the first of two equally long matches""" + doc = en_tokenizer(" ".join("CCC")) + matcher = Matcher(doc.vocab) + pattern = [{"ORTH": "C"}, {"ORTH": "C"}] + matcher.add("RULE", [pattern], greedy="LONGEST") + matches = matcher(doc) + # out of 0-2 and 1-3, the first should be picked + assert len(matches) == 1 + assert matches[0][1] == 0 + assert matches[0][2] == 2 + + +def test_invalid_greediness(doc, text): + matcher = Matcher(doc.vocab) + with pytest.raises(ValueError): + matcher.add("RULE", [pattern1], greedy="GREEDY") + + +@pytest.mark.parametrize( + "pattern,re_pattern", + [ + (pattern1, re_pattern1), + (pattern2, re_pattern2), + (pattern3, re_pattern3), + (pattern4, re_pattern4), + (pattern5, re_pattern5), + ], +) +def test_match_consuming(doc, text, pattern, re_pattern): + """Test that matcher.__call__ consumes tokens on a match similar to + re.findall.""" + matcher = Matcher(doc.vocab) + matcher.add(re_pattern, [pattern], greedy="FIRST") + matches = matcher(doc) + re_matches = [m.span() for m in re.finditer(re_pattern, text)] + assert len(matches) == len(re_matches) + + +def test_operator_combos(en_vocab): + cases = [ + ("aaab", "a a a b", True), + ("aaab", "a+ b", True), + ("aaab", "a+ a+ b", True), + ("aaab", "a+ a+ a b", True), + ("aaab", "a+ a+ a+ b", True), + ("aaab", "a+ a a b", True), + ("aaab", "a+ a a", True), + ("aaab", "a+", True), + ("aaa", "a+ b", False), + ("aaa", "a+ a+ b", False), + ("aaa", "a+ a+ a+ b", False), + ("aaa", "a+ a b", False), + ("aaa", "a+ a a b", False), + ("aaab", "a+ a a", True), + ("aaab", "a+", True), + ("aaab", "a+ a b", True), + ] + for string, pattern_str, result in cases: + matcher = Matcher(en_vocab) + doc = Doc(matcher.vocab, words=list(string)) + pattern = [] + for part in pattern_str.split(): + if part.endswith("+"): + pattern.append({"ORTH": part[0], "OP": "+"}) + else: + pattern.append({"ORTH": part}) + matcher.add("PATTERN", [pattern]) + matches = matcher(doc) + if result: + assert matches, (string, pattern_str) + else: + assert not matches, (string, pattern_str) + + +@pytest.mark.issue(1450) +def test_matcher_end_zero_plus(en_vocab): + """Test matcher works when patterns end with * operator. (issue 1450)""" + matcher = Matcher(en_vocab) + pattern = [{"ORTH": "a"}, {"ORTH": "b", "OP": "*"}] + matcher.add("TSTEND", [pattern]) + nlp = lambda string: Doc(matcher.vocab, words=string.split()) + assert len(matcher(nlp("a"))) == 1 + assert len(matcher(nlp("a b"))) == 2 + assert len(matcher(nlp("a c"))) == 1 + assert len(matcher(nlp("a b c"))) == 2 + assert len(matcher(nlp("a b b c"))) == 3 + assert len(matcher(nlp("a b b"))) == 3 + + +def test_matcher_sets_return_correct_tokens(en_vocab): + matcher = Matcher(en_vocab) + patterns = [ + [{"LOWER": {"IN": ["zero"]}}], + [{"LOWER": {"IN": ["one"]}}], + [{"LOWER": {"IN": ["two"]}}], + ] + matcher.add("TEST", patterns) + doc = Doc(en_vocab, words="zero one two three".split()) + matches = matcher(doc) + texts = [Span(doc, s, e, label=L).text for L, s, e in matches] + assert texts == ["zero", "one", "two"] + + +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_matcher_remove(): + nlp = English() + matcher = Matcher(nlp.vocab) + text = "This is a test case." + + pattern = [{"ORTH": "test"}, {"OP": "?"}] + assert len(matcher) == 0 + matcher.add("Rule", [pattern]) + assert "Rule" in matcher + + # should give two matches + results1 = matcher(nlp(text)) + assert len(results1) == 2 + + # removing once should work + matcher.remove("Rule") + + # should not return any maches anymore + results2 = matcher(nlp(text)) + assert len(results2) == 0 + + # removing again should throw an error + with pytest.raises(ValueError): + matcher.remove("Rule") + + +def test_matcher_with_alignments_greedy_longest(en_vocab): + cases = [ + ("aaab", "a* b", [0, 0, 0, 1]), + ("baab", "b a* b", [0, 1, 1, 2]), + ("aaab", "a a a b", [0, 1, 2, 3]), + ("aaab", "a+ b", [0, 0, 0, 1]), + ("aaba", "a+ b a+", [0, 0, 1, 2]), + ("aabaa", "a+ b a+", [0, 0, 1, 2, 2]), + ("aaba", "a+ b a*", [0, 0, 1, 2]), + ("aaaa", "a*", [0, 0, 0, 0]), + ("baab", "b a* b b*", [0, 1, 1, 2]), + ("aabb", "a* b* a*", [0, 0, 1, 1]), + ("aaab", "a+ a+ a b", [0, 1, 2, 3]), + ("aaab", "a+ a+ a+ b", [0, 1, 2, 3]), + ("aaab", "a+ a a b", [0, 1, 2, 3]), + ("aaab", "a+ a a", [0, 1, 2]), + ("aaab", "a+ a a?", [0, 1, 2]), + ("aaaa", "a a a a a?", [0, 1, 2, 3]), + ("aaab", "a+ a b", [0, 0, 1, 2]), + ("aaab", "a+ a+ b", [0, 0, 1, 2]), + ("aaab", "a{2,} b", [0, 0, 0, 1]), + ("aaab", "a{,3} b", [0, 0, 0, 1]), + ("aaab", "a{2} b", [0, 0, 1]), + ("aaab", "a{2,3} b", [0, 0, 0, 1]), + ] + for string, pattern_str, result in cases: + matcher = Matcher(en_vocab) + doc = Doc(matcher.vocab, words=list(string)) + pattern = [] + for part in pattern_str.split(): + if part.endswith("+"): + pattern.append({"ORTH": part[0], "OP": "+"}) + elif part.endswith("*"): + pattern.append({"ORTH": part[0], "OP": "*"}) + elif part.endswith("?"): + pattern.append({"ORTH": part[0], "OP": "?"}) + elif part.endswith("}"): + pattern.append({"ORTH": part[0], "OP": part[1:]}) + else: + pattern.append({"ORTH": part}) + matcher.add("PATTERN", [pattern], greedy="LONGEST") + matches = matcher(doc, with_alignments=True) + n_matches = len(matches) + + _, s, e, expected = matches[0] + + assert expected == result, (string, pattern_str, s, e, n_matches) + + +def test_matcher_with_alignments_non_greedy(en_vocab): + cases = [ + (0, "aaab", "a* b", [[0, 1], [0, 0, 1], [0, 0, 0, 1], [1]]), + (1, "baab", "b a* b", [[0, 1, 1, 2]]), + (2, "aaab", "a a a b", [[0, 1, 2, 3]]), + (3, "aaab", "a+ b", [[0, 1], [0, 0, 1], [0, 0, 0, 1]]), + (4, "aaba", "a+ b a+", [[0, 1, 2], [0, 0, 1, 2]]), + ( + 5, + "aabaa", + "a+ b a+", + [[0, 1, 2], [0, 0, 1, 2], [0, 0, 1, 2, 2], [0, 1, 2, 2]], + ), + (6, "aaba", "a+ b a*", [[0, 1], [0, 0, 1], [0, 0, 1, 2], [0, 1, 2]]), + (7, "aaaa", "a*", [[0], [0, 0], [0, 0, 0], [0, 0, 0, 0]]), + (8, "baab", "b a* b b*", [[0, 1, 1, 2]]), + ( + 9, + "aabb", + "a* b* a*", + [[1], [2], [2, 2], [0, 1], [0, 0, 1], [0, 0, 1, 1], [0, 1, 1], [1, 1]], + ), + (10, "aaab", "a+ a+ a b", [[0, 1, 2, 3]]), + (11, "aaab", "a+ a+ a+ b", [[0, 1, 2, 3]]), + (12, "aaab", "a+ a a b", [[0, 1, 2, 3]]), + (13, "aaab", "a+ a a", [[0, 1, 2]]), + (14, "aaab", "a+ a a?", [[0, 1], [0, 1, 2]]), + (15, "aaaa", "a a a a a?", [[0, 1, 2, 3]]), + (16, "aaab", "a+ a b", [[0, 1, 2], [0, 0, 1, 2]]), + (17, "aaab", "a+ a+ b", [[0, 1, 2], [0, 0, 1, 2]]), + (18, "aaab", "a{2,} b", [[0, 0, 1], [0, 0, 0, 1]]), + (19, "aaab", "a{3} b", [[0, 0, 0, 1]]), + (20, "aaab", "a{2} b", [[0, 0, 1]]), + (21, "aaab", "a{2,3} b", [[0, 0, 1], [0, 0, 0, 1]]), + ] + for case_id, string, pattern_str, results in cases: + matcher = Matcher(en_vocab) + doc = Doc(matcher.vocab, words=list(string)) + pattern = [] + for part in pattern_str.split(): + if part.endswith("+"): + pattern.append({"ORTH": part[0], "OP": "+"}) + elif part.endswith("*"): + pattern.append({"ORTH": part[0], "OP": "*"}) + elif part.endswith("?"): + pattern.append({"ORTH": part[0], "OP": "?"}) + elif part.endswith("}"): + pattern.append({"ORTH": part[0], "OP": part[1:]}) + else: + pattern.append({"ORTH": part}) + + matcher.add("PATTERN", [pattern]) + matches = matcher(doc, with_alignments=True) + n_matches = len(matches) + + for _, s, e, expected in matches: + assert expected in results, (case_id, string, pattern_str, s, e, n_matches) + assert len(expected) == e - s diff --git a/spacy/tests/matcher/test_pattern_validation.py b/spacy/tests/matcher/test_pattern_validation.py new file mode 100644 index 0000000..e0dc7d5 --- /dev/null +++ b/spacy/tests/matcher/test_pattern_validation.py @@ -0,0 +1,102 @@ +import pytest + +from spacy.errors import MatchPatternError +from spacy.matcher import Matcher +from spacy.schemas import validate_token_pattern + +# (pattern, num errors with validation, num errors identified with minimal +# checks) +TEST_PATTERNS = [ + # Bad patterns flagged in all cases + ([{"XX": "foo"}], 1, 1), + ([{"IS_ALPHA": {"==": True}}, {"LIKE_NUM": None}], 2, 1), + ( + [{"IS_PUNCT": True, "OP": "$"}], + 2, + 1, + ), # v2: union reports 2 errors (enum + pattern) + ([{"_": "foo"}], 1, 1), + ('[{"TEXT": "foo"}, {"LOWER": "bar"}]', 1, 1), + ([{"ENT_IOB": "foo"}], 1, 1), + ([1, 2, 3], 3, 1), + ([{"TEXT": "foo", "OP": "{,}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{,4}4"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{a,3}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{a}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{,a}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{1,2,3}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{1, 3}"}], 2, 1), # v2: union reports 2 errors + ([{"TEXT": "foo", "OP": "{-2}"}], 2, 1), # v2: union reports 2 errors + # Bad patterns flagged outside of Matcher + ( + [{"_": {"foo": "bar", "baz": {"IN": "foo"}}}], + 7, + 0, + ), # v2: more detailed union errors + # Bad patterns not flagged with minimal checks + ( + [{"LENGTH": "2", "TEXT": 2}, {"LOWER": "test"}], + 5, + 0, + ), # v2: more detailed union errors + ( + [{"LENGTH": {"IN": [1, 2, "3"]}}, {"POS": {"IN": "VERB"}}], + 5, + 0, + ), # v2: more detailed union errors + ([{"LENGTH": {"VALUE": 5}}], 3, 0), # v2: more detailed union errors + ([{"TEXT": {"VALUE": "foo"}}], 2, 0), + ([{"IS_DIGIT": -1}], 1, 0), + ([{"ORTH": -1}], 2, 0), # v2: union reports 2 errors + ([{"ENT_ID": -1}], 2, 0), # v2: union reports 2 errors + ([{"ENT_KB_ID": -1}], 2, 0), # v2: union reports 2 errors + # Good patterns + ([{"TEXT": "foo"}, {"LOWER": "bar"}], 0, 0), + ([{"LEMMA": {"IN": ["love", "like"]}}, {"POS": "DET", "OP": "?"}], 0, 0), + ([{"LIKE_NUM": True, "LENGTH": {">=": 5}}], 0, 0), + ([{"LENGTH": 2}], 0, 0), + ([{"LOWER": {"REGEX": "^X", "NOT_IN": ["XXX", "XY"]}}], 0, 0), + ([{"NORM": "a"}, {"POS": {"IN": ["NOUN"]}}], 0, 0), + ([{"_": {"foo": {"NOT_IN": ["bar", "baz"]}, "a": 5, "b": {">": 10}}}], 0, 0), + ([{"orth": "foo"}], 0, 0), # prev: xfail + ([{"IS_SENT_START": True}], 0, 0), + ([{"SENT_START": True}], 0, 0), + ([{"ENT_ID": "STRING"}], 0, 0), + ([{"ENT_KB_ID": "STRING"}], 0, 0), + ([{"TEXT": "ha", "OP": "{3}"}], 0, 0), +] + + +@pytest.mark.parametrize( + "pattern", + [[{"XX": "y"}], [{"LENGTH": "2"}], [{"TEXT": {"IN": 5}}], [{"text": {"in": 6}}]], +) +def test_matcher_pattern_validation(en_vocab, pattern): + matcher = Matcher(en_vocab, validate=True) + with pytest.raises(MatchPatternError): + matcher.add("TEST", [pattern]) + + +@pytest.mark.parametrize("pattern,n_errors,_", TEST_PATTERNS) +def test_pattern_validation(pattern, n_errors, _): + errors = validate_token_pattern(pattern) + assert len(errors) == n_errors + + +@pytest.mark.parametrize("pattern,n_errors,n_min_errors", TEST_PATTERNS) +def test_minimal_pattern_validation(en_vocab, pattern, n_errors, n_min_errors): + matcher = Matcher(en_vocab) + if n_min_errors > 0: + with pytest.raises(ValueError): + matcher.add("TEST", [pattern]) + elif n_errors == 0: + matcher.add("TEST", [pattern]) + + +def test_pattern_errors(en_vocab): + matcher = Matcher(en_vocab) + # normalize "regex" to upper like "text" + matcher.add("TEST1", [[{"text": {"regex": "regex"}}]]) + # error if subpattern attribute isn't recognized and processed + with pytest.raises(MatchPatternError): + matcher.add("TEST2", [[{"TEXT": {"XX": "xx"}}]]) diff --git a/spacy/tests/matcher/test_phrase_matcher.py b/spacy/tests/matcher/test_phrase_matcher.py new file mode 100644 index 0000000..f09f51a --- /dev/null +++ b/spacy/tests/matcher/test_phrase_matcher.py @@ -0,0 +1,509 @@ +import warnings + +import pytest +import srsly +from mock import Mock + +from spacy.lang.en import English +from spacy.matcher import Matcher, PhraseMatcher +from spacy.tokens import Doc, Span +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +@pytest.mark.issue(3248) +def test_issue3248_1(): + """Test that the PhraseMatcher correctly reports its number of rules, not + total number of patterns.""" + nlp = English() + matcher = PhraseMatcher(nlp.vocab) + matcher.add("TEST1", [nlp("a"), nlp("b"), nlp("c")]) + matcher.add("TEST2", [nlp("d")]) + assert len(matcher) == 2 + + +@pytest.mark.issue(3331) +def test_issue3331(en_vocab): + """Test that duplicate patterns for different rules result in multiple + matches, one per rule. + """ + matcher = PhraseMatcher(en_vocab) + matcher.add("A", [Doc(en_vocab, words=["Barack", "Obama"])]) + matcher.add("B", [Doc(en_vocab, words=["Barack", "Obama"])]) + doc = Doc(en_vocab, words=["Barack", "Obama", "lifts", "America"]) + matches = matcher(doc) + assert len(matches) == 2 + match_ids = [en_vocab.strings[matches[0][0]], en_vocab.strings[matches[1][0]]] + assert sorted(match_ids) == ["A", "B"] + + +@pytest.mark.issue(3972) +def test_issue3972(en_vocab): + """Test that the PhraseMatcher returns duplicates for duplicate match IDs.""" + matcher = PhraseMatcher(en_vocab) + matcher.add("A", [Doc(en_vocab, words=["New", "York"])]) + matcher.add("B", [Doc(en_vocab, words=["New", "York"])]) + doc = Doc(en_vocab, words=["I", "live", "in", "New", "York"]) + matches = matcher(doc) + + assert len(matches) == 2 + + # We should have a match for each of the two rules + found_ids = [en_vocab.strings[ent_id] for (ent_id, _, _) in matches] + assert "A" in found_ids + assert "B" in found_ids + + +@pytest.mark.issue(4002) +def test_issue4002(en_vocab): + """Test that the PhraseMatcher can match on overwritten NORM attributes.""" + matcher = PhraseMatcher(en_vocab, attr="NORM") + pattern1 = Doc(en_vocab, words=["c", "d"]) + assert [t.norm_ for t in pattern1] == ["c", "d"] + matcher.add("TEST", [pattern1]) + doc = Doc(en_vocab, words=["a", "b", "c", "d"]) + assert [t.norm_ for t in doc] == ["a", "b", "c", "d"] + matches = matcher(doc) + assert len(matches) == 1 + matcher = PhraseMatcher(en_vocab, attr="NORM") + pattern2 = Doc(en_vocab, words=["1", "2"]) + pattern2[0].norm_ = "c" + pattern2[1].norm_ = "d" + assert [t.norm_ for t in pattern2] == ["c", "d"] + matcher.add("TEST", [pattern2]) + matches = matcher(doc) + assert len(matches) == 1 + + +@pytest.mark.issue(4373) +def test_issue4373(): + """Test that PhraseMatcher.vocab can be accessed (like Matcher.vocab).""" + matcher = Matcher(Vocab()) + assert isinstance(matcher.vocab, Vocab) + matcher = PhraseMatcher(Vocab()) + assert isinstance(matcher.vocab, Vocab) + + +@pytest.mark.issue(4651) +def test_issue4651_with_phrase_matcher_attr(): + """Test that the EntityRuler PhraseMatcher is deserialized correctly using + the method from_disk when the EntityRuler argument phrase_matcher_attr is + specified. + """ + text = "Spacy is a python library for nlp" + nlp = English() + patterns = [{"label": "PYTHON_LIB", "pattern": "spacy", "id": "spaCy"}] + ruler = nlp.add_pipe("entity_ruler", config={"phrase_matcher_attr": "LOWER"}) + ruler.add_patterns(patterns) + doc = nlp(text) + res = [(ent.text, ent.label_, ent.ent_id_) for ent in doc.ents] + nlp_reloaded = English() + with make_tempdir() as d: + file_path = d / "entityruler" + ruler.to_disk(file_path) + nlp_reloaded.add_pipe("entity_ruler").from_disk(file_path) + doc_reloaded = nlp_reloaded(text) + res_reloaded = [(ent.text, ent.label_, ent.ent_id_) for ent in doc_reloaded.ents] + assert res == res_reloaded + + +@pytest.mark.issue(6839) +def test_issue6839(en_vocab): + """Ensure that PhraseMatcher accepts Span as input""" + # fmt: off + words = ["I", "like", "Spans", "and", "Docs", "in", "my", "input", ",", "and", "nothing", "else", "."] + # fmt: on + doc = Doc(en_vocab, words=words) + span = doc[:8] + pattern = Doc(en_vocab, words=["Spans", "and", "Docs"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("SPACY", [pattern]) + matches = matcher(span) + assert matches + + +@pytest.mark.issue(10643) +def test_issue10643(en_vocab): + """Ensure overlapping terms can be removed from PhraseMatcher""" + + # fmt: off + words = ["Only", "save", "out", "the", "binary", "data", "for", "the", "individual", "components", "."] + # fmt: on + doc = Doc(en_vocab, words=words) + terms = { + "0": Doc(en_vocab, words=["binary"]), + "1": Doc(en_vocab, words=["binary", "data"]), + } + matcher = PhraseMatcher(en_vocab) + for match_id, term in terms.items(): + matcher.add(match_id, [term]) + + matches = matcher(doc) + assert matches == [(en_vocab.strings["0"], 4, 5), (en_vocab.strings["1"], 4, 6)] + + matcher.remove("0") + assert len(matcher) == 1 + new_matches = matcher(doc) + assert new_matches == [(en_vocab.strings["1"], 4, 6)] + + matcher.remove("1") + assert len(matcher) == 0 + no_matches = matcher(doc) + assert not no_matches + + +def test_matcher_phrase_matcher(en_vocab): + doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) + # intermediate phrase + pattern = Doc(en_vocab, words=["Google", "Now"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("COMPANY", [pattern]) + assert len(matcher(doc)) == 1 + # initial token + pattern = Doc(en_vocab, words=["I"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("I", [pattern]) + assert len(matcher(doc)) == 1 + # initial phrase + pattern = Doc(en_vocab, words=["I", "like"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("ILIKE", [pattern]) + assert len(matcher(doc)) == 1 + # final token + pattern = Doc(en_vocab, words=["best"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("BEST", [pattern]) + assert len(matcher(doc)) == 1 + # final phrase + pattern = Doc(en_vocab, words=["Now", "best"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("NOWBEST", [pattern]) + assert len(matcher(doc)) == 1 + + +def test_phrase_matcher_length(en_vocab): + matcher = PhraseMatcher(en_vocab) + assert len(matcher) == 0 + matcher.add("TEST", [Doc(en_vocab, words=["test"])]) + assert len(matcher) == 1 + matcher.add("TEST2", [Doc(en_vocab, words=["test2"])]) + assert len(matcher) == 2 + + +def test_phrase_matcher_contains(en_vocab): + matcher = PhraseMatcher(en_vocab) + matcher.add("TEST", [Doc(en_vocab, words=["test"])]) + assert "TEST" in matcher + assert "TEST2" not in matcher + + +def test_phrase_matcher_add_new_api(en_vocab): + doc = Doc(en_vocab, words=["a", "b"]) + patterns = [Doc(en_vocab, words=["a"]), Doc(en_vocab, words=["a", "b"])] + matcher = PhraseMatcher(en_vocab) + matcher.add("OLD_API", None, *patterns) + assert len(matcher(doc)) == 2 + matcher = PhraseMatcher(en_vocab) + on_match = Mock() + matcher.add("OLD_API_CALLBACK", on_match, *patterns) + assert len(matcher(doc)) == 2 + assert on_match.call_count == 2 + # New API: add(key: str, patterns: List[List[dict]], on_match: Callable) + matcher = PhraseMatcher(en_vocab) + matcher.add("NEW_API", patterns) + assert len(matcher(doc)) == 2 + matcher = PhraseMatcher(en_vocab) + on_match = Mock() + matcher.add("NEW_API_CALLBACK", patterns, on_match=on_match) + assert len(matcher(doc)) == 2 + assert on_match.call_count == 2 + + +def test_phrase_matcher_repeated_add(en_vocab): + matcher = PhraseMatcher(en_vocab) + # match ID only gets added once + matcher.add("TEST", [Doc(en_vocab, words=["like"])]) + matcher.add("TEST", [Doc(en_vocab, words=["like"])]) + matcher.add("TEST", [Doc(en_vocab, words=["like"])]) + matcher.add("TEST", [Doc(en_vocab, words=["like"])]) + doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) + assert "TEST" in matcher + assert "TEST2" not in matcher + assert len(matcher(doc)) == 1 + + +def test_phrase_matcher_remove(en_vocab): + matcher = PhraseMatcher(en_vocab) + matcher.add("TEST1", [Doc(en_vocab, words=["like"])]) + matcher.add("TEST2", [Doc(en_vocab, words=["best"])]) + doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) + assert "TEST1" in matcher + assert "TEST2" in matcher + assert "TEST3" not in matcher + assert len(matcher(doc)) == 2 + matcher.remove("TEST1") + assert "TEST1" not in matcher + assert "TEST2" in matcher + assert "TEST3" not in matcher + assert len(matcher(doc)) == 1 + matcher.remove("TEST2") + assert "TEST1" not in matcher + assert "TEST2" not in matcher + assert "TEST3" not in matcher + assert len(matcher(doc)) == 0 + with pytest.raises(KeyError): + matcher.remove("TEST3") + assert "TEST1" not in matcher + assert "TEST2" not in matcher + assert "TEST3" not in matcher + assert len(matcher(doc)) == 0 + + +def test_phrase_matcher_overlapping_with_remove(en_vocab): + matcher = PhraseMatcher(en_vocab) + matcher.add("TEST", [Doc(en_vocab, words=["like"])]) + # TEST2 is added alongside TEST + matcher.add("TEST2", [Doc(en_vocab, words=["like"])]) + doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) + assert "TEST" in matcher + assert len(matcher) == 2 + assert len(matcher(doc)) == 2 + # removing TEST does not remove the entry for TEST2 + matcher.remove("TEST") + assert "TEST" not in matcher + assert len(matcher) == 1 + assert len(matcher(doc)) == 1 + assert matcher(doc)[0][0] == en_vocab.strings["TEST2"] + # removing TEST2 removes all + matcher.remove("TEST2") + assert "TEST2" not in matcher + assert len(matcher) == 0 + assert len(matcher(doc)) == 0 + + +def test_phrase_matcher_string_attrs(en_vocab): + words1 = ["I", "like", "cats"] + pos1 = ["PRON", "VERB", "NOUN"] + words2 = ["Yes", ",", "you", "hate", "dogs", "very", "much"] + pos2 = ["INTJ", "PUNCT", "PRON", "VERB", "NOUN", "ADV", "ADV"] + pattern = Doc(en_vocab, words=words1, pos=pos1) + matcher = PhraseMatcher(en_vocab, attr="POS") + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=words2, pos=pos2) + matches = matcher(doc) + assert len(matches) == 1 + match_id, start, end = matches[0] + assert match_id == en_vocab.strings["TEST"] + assert start == 2 + assert end == 5 + + +def test_phrase_matcher_string_attrs_negative(en_vocab): + """Test that token with the control codes as ORTH are *not* matched.""" + words1 = ["I", "like", "cats"] + pos1 = ["PRON", "VERB", "NOUN"] + words2 = ["matcher:POS-PRON", "matcher:POS-VERB", "matcher:POS-NOUN"] + pos2 = ["X", "X", "X"] + pattern = Doc(en_vocab, words=words1, pos=pos1) + matcher = PhraseMatcher(en_vocab, attr="POS") + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=words2, pos=pos2) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_phrase_matcher_bool_attrs(en_vocab): + words1 = ["Hello", "world", "!"] + words2 = ["No", "problem", ",", "he", "said", "."] + pattern = Doc(en_vocab, words=words1) + matcher = PhraseMatcher(en_vocab, attr="IS_PUNCT") + matcher.add("TEST", [pattern]) + doc = Doc(en_vocab, words=words2) + matches = matcher(doc) + assert len(matches) == 2 + match_id1, start1, end1 = matches[0] + match_id2, start2, end2 = matches[1] + assert match_id1 == en_vocab.strings["TEST"] + assert match_id2 == en_vocab.strings["TEST"] + assert start1 == 0 + assert end1 == 3 + assert start2 == 3 + assert end2 == 6 + + +def test_phrase_matcher_validation(en_vocab): + doc1 = Doc(en_vocab, words=["Test"]) + doc1[0].dep_ = "ROOT" + doc2 = Doc(en_vocab, words=["Test"]) + doc2[0].tag_ = "TAG" + doc2[0].pos_ = "X" + doc2[0].set_morph("Feat=Val") + doc3 = Doc(en_vocab, words=["Test"]) + matcher = PhraseMatcher(en_vocab, validate=True) + with pytest.warns(UserWarning): + matcher.add("TEST1", [doc1]) + with pytest.warns(UserWarning): + matcher.add("TEST2", [doc2]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + matcher.add("TEST3", [doc3]) + matcher = PhraseMatcher(en_vocab, attr="POS", validate=True) + with warnings.catch_warnings(): + warnings.simplefilter("error") + matcher.add("TEST4", [doc2]) + + +def test_attr_validation(en_vocab): + with pytest.raises(ValueError): + PhraseMatcher(en_vocab, attr="UNSUPPORTED") + + +def test_attr_pipeline_checks(en_vocab): + doc1 = Doc(en_vocab, words=["Test"]) + doc1[0].dep_ = "ROOT" + doc2 = Doc(en_vocab, words=["Test"]) + doc2[0].tag_ = "TAG" + doc2[0].pos_ = "X" + doc2[0].set_morph("Feat=Val") + doc2[0].lemma_ = "LEMMA" + doc3 = Doc(en_vocab, words=["Test"]) + # DEP requires DEP + matcher = PhraseMatcher(en_vocab, attr="DEP") + matcher.add("TEST1", [doc1]) + with pytest.raises(ValueError): + matcher.add("TEST2", [doc2]) + with pytest.raises(ValueError): + matcher.add("TEST3", [doc3]) + # TAG, POS, LEMMA require those values + for attr in ("TAG", "POS", "LEMMA"): + matcher = PhraseMatcher(en_vocab, attr=attr) + matcher.add("TEST2", [doc2]) + with pytest.raises(ValueError): + matcher.add("TEST1", [doc1]) + with pytest.raises(ValueError): + matcher.add("TEST3", [doc3]) + # TEXT/ORTH only require tokens + matcher = PhraseMatcher(en_vocab, attr="ORTH") + matcher.add("TEST3", [doc3]) + matcher = PhraseMatcher(en_vocab, attr="TEXT") + matcher.add("TEST3", [doc3]) + + +def test_phrase_matcher_callback(en_vocab): + mock = Mock() + doc = Doc(en_vocab, words=["I", "like", "Google", "Now", "best"]) + pattern = Doc(en_vocab, words=["Google", "Now"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("COMPANY", [pattern], on_match=mock) + matches = matcher(doc) + mock.assert_called_once_with(matcher, doc, 0, matches) + + +def test_phrase_matcher_remove_overlapping_patterns(en_vocab): + matcher = PhraseMatcher(en_vocab) + pattern1 = Doc(en_vocab, words=["this"]) + pattern2 = Doc(en_vocab, words=["this", "is"]) + pattern3 = Doc(en_vocab, words=["this", "is", "a"]) + pattern4 = Doc(en_vocab, words=["this", "is", "a", "word"]) + matcher.add("THIS", [pattern1, pattern2, pattern3, pattern4]) + matcher.remove("THIS") + + +def test_phrase_matcher_basic_check(en_vocab): + matcher = PhraseMatcher(en_vocab) + # Potential mistake: pass in pattern instead of list of patterns + pattern = Doc(en_vocab, words=["hello", "world"]) + with pytest.raises(ValueError): + matcher.add("TEST", pattern) + + +def test_phrase_matcher_pickle(en_vocab): + matcher = PhraseMatcher(en_vocab) + mock = Mock() + matcher.add("TEST", [Doc(en_vocab, words=["test"])]) + matcher.add("TEST2", [Doc(en_vocab, words=["test2"])], on_match=mock) + doc = Doc(en_vocab, words=["these", "are", "tests", ":", "test", "test2"]) + assert len(matcher) == 2 + + b = srsly.pickle_dumps(matcher) + matcher_unpickled = srsly.pickle_loads(b) + + # call after pickling to avoid recursion error related to mock + matches = matcher(doc) + matches_unpickled = matcher_unpickled(doc) + + assert len(matcher) == len(matcher_unpickled) + assert matches == matches_unpickled + + # clunky way to vaguely check that callback is unpickled + vocab, docs, callbacks, attr = matcher_unpickled.__reduce__()[1] + assert isinstance(callbacks.get("TEST2"), Mock) + + +def test_phrase_matcher_as_spans(en_vocab): + """Test the new as_spans=True API.""" + matcher = PhraseMatcher(en_vocab) + matcher.add("A", [Doc(en_vocab, words=["hello", "world"])]) + matcher.add("B", [Doc(en_vocab, words=["test"])]) + doc = Doc(en_vocab, words=["...", "hello", "world", "this", "is", "a", "test"]) + matches = matcher(doc, as_spans=True) + assert len(matches) == 2 + assert isinstance(matches[0], Span) + assert matches[0].text == "hello world" + assert matches[0].label_ == "A" + assert isinstance(matches[1], Span) + assert matches[1].text == "test" + assert matches[1].label_ == "B" + + +def test_phrase_matcher_deprecated(en_vocab): + matcher = PhraseMatcher(en_vocab) + matcher.add("TEST", [Doc(en_vocab, words=["helllo"])]) + doc = Doc(en_vocab, words=["hello", "world"]) + with pytest.warns(DeprecationWarning) as record: + for _ in matcher.pipe([doc]): + pass + assert record.list + assert "spaCy v3.0" in str(record.list[0].message) + + +@pytest.mark.parametrize("attr", ["SENT_START", "IS_SENT_START"]) +def test_phrase_matcher_sent_start(en_vocab, attr): + _ = PhraseMatcher(en_vocab, attr=attr) # noqa: F841 + + +def test_span_in_phrasematcher(en_vocab): + """Ensure that PhraseMatcher accepts Span and Doc as input""" + # fmt: off + words = ["I", "like", "Spans", "and", "Docs", "in", "my", "input", ",", "and", "nothing", "else", "."] + # fmt: on + doc = Doc(en_vocab, words=words) + span = doc[:8] + pattern = Doc(en_vocab, words=["Spans", "and", "Docs"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("SPACY", [pattern]) + matches_doc = matcher(doc) + matches_span = matcher(span) + assert len(matches_doc) == 1 + assert len(matches_span) == 1 + + +def test_span_v_doc_in_phrasematcher(en_vocab): + """Ensure that PhraseMatcher only returns matches in input Span and not in entire Doc""" + # fmt: off + words = [ + "I", "like", "Spans", "and", "Docs", "in", "my", "input", ",", "Spans", + "and", "Docs", "in", "my", "matchers", "," "and", "Spans", "and", "Docs", + "everywhere", "." + ] + # fmt: on + doc = Doc(en_vocab, words=words) + span = doc[9:15] # second clause + pattern = Doc(en_vocab, words=["Spans", "and", "Docs"]) + matcher = PhraseMatcher(en_vocab) + matcher.add("SPACY", [pattern]) + matches_doc = matcher(doc) + matches_span = matcher(span) + assert len(matches_doc) == 3 + assert len(matches_span) == 1 diff --git a/spacy/tests/morphology/__init__.py b/spacy/tests/morphology/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/morphology/test_morph_converters.py b/spacy/tests/morphology/test_morph_converters.py new file mode 100644 index 0000000..6973bf7 --- /dev/null +++ b/spacy/tests/morphology/test_morph_converters.py @@ -0,0 +1,21 @@ +from spacy.morphology import Morphology + + +def test_feats_converters(): + feats = "Case=dat,gen|Number=sing" + feats_dict = {"Case": "dat,gen", "Number": "sing"} + + # simple conversions + assert Morphology.dict_to_feats(feats_dict) == feats + assert Morphology.feats_to_dict(feats) == feats_dict + + # roundtrips + assert Morphology.dict_to_feats(Morphology.feats_to_dict(feats)) == feats + assert Morphology.feats_to_dict(Morphology.dict_to_feats(feats_dict)) == feats_dict + + # unsorted input is normalized + unsorted_feats = "Number=sing|Case=gen,dat" + unsorted_feats_dict = {"Case": "gen,dat", "Number": "sing"} + assert Morphology.feats_to_dict(unsorted_feats) == feats_dict + assert Morphology.dict_to_feats(unsorted_feats_dict) == feats + assert Morphology.dict_to_feats(Morphology.feats_to_dict(unsorted_feats)) == feats diff --git a/spacy/tests/morphology/test_morph_features.py b/spacy/tests/morphology/test_morph_features.py new file mode 100644 index 0000000..ae20f9b --- /dev/null +++ b/spacy/tests/morphology/test_morph_features.py @@ -0,0 +1,50 @@ +import pytest + +from spacy.morphology import Morphology +from spacy.strings import StringStore, get_string_id + + +@pytest.fixture +def morphology(): + return Morphology(StringStore()) + + +def test_init(morphology): + pass + + +def test_add_morphology_with_string_names(morphology): + morphology.add({"Case": "gen", "Number": "sing"}) + + +def test_add_morphology_with_int_ids(morphology): + morphology.strings.add("Case") + morphology.strings.add("gen") + morphology.strings.add("Number") + morphology.strings.add("sing") + morphology.add( + { + get_string_id("Case"): get_string_id("gen"), + get_string_id("Number"): get_string_id("sing"), + } + ) + + +def test_add_morphology_with_mix_strings_and_ints(morphology): + morphology.strings.add("PunctSide") + morphology.strings.add("ini") + morphology.add( + {get_string_id("PunctSide"): get_string_id("ini"), "VerbType": "aux"} + ) + + +def test_morphology_tags_hash_distinctly(morphology): + tag1 = morphology.add({"PunctSide": "ini", "VerbType": "aux"}) + tag2 = morphology.add({"Case": "gen", "Number": "sing"}) + assert tag1 != tag2 + + +def test_morphology_tags_hash_independent_of_order(morphology): + tag1 = morphology.add({"Case": "gen", "Number": "sing"}) + tag2 = morphology.add({"Number": "sing", "Case": "gen"}) + assert tag1 == tag2 diff --git a/spacy/tests/morphology/test_morph_pickle.py b/spacy/tests/morphology/test_morph_pickle.py new file mode 100644 index 0000000..5c1a8a3 --- /dev/null +++ b/spacy/tests/morphology/test_morph_pickle.py @@ -0,0 +1,23 @@ +import pickle + +import pytest + +from spacy.morphology import Morphology +from spacy.strings import StringStore + + +@pytest.fixture +def morphology(): + morphology = Morphology(StringStore()) + morphology.add("Feat1=Val1|Feat2=Val2") + morphology.add("Feat3=Val3|Feat4=Val4") + return morphology + + +def test_morphology_pickle_roundtrip(morphology): + b = pickle.dumps(morphology) + reloaded_morphology = pickle.loads(b) + feat = reloaded_morphology.get(morphology.strings["Feat1=Val1|Feat2=Val2"]) + assert feat == "Feat1=Val1|Feat2=Val2" + feat = reloaded_morphology.get(morphology.strings["Feat3=Val3|Feat4=Val4"]) + assert feat == "Feat3=Val3|Feat4=Val4" diff --git a/spacy/tests/package/__init__.py b/spacy/tests/package/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/package/test_requirements.py b/spacy/tests/package/test_requirements.py new file mode 100644 index 0000000..f4c6f05 --- /dev/null +++ b/spacy/tests/package/test_requirements.py @@ -0,0 +1,97 @@ +import re +from pathlib import Path + + +def test_build_dependencies(): + # Check that library requirements are pinned exactly the same across different setup files. + libs_ignore_requirements = [ + "numpy", + "pytest", + "pytest-timeout", + "mock", + "flake8", + "hypothesis", + "pre-commit", + "cython-lint", + "black", + "isort", + "ruff", + "mypy", + "types-dataclasses", + "types-mock", + "types-requests", + "types-setuptools", + ] + # ignore language-specific packages that shouldn't be installed by all + libs_ignore_setup = [ + "numpy", + "fugashi", + "natto-py", + "pythainlp", + "sudachipy", + "sudachidict_core", + "spacy-pkuseg", + "thinc-apple-ops", + ] + + # check requirements.txt + req_dict = {} + + root_dir = Path(__file__).parent + req_file = root_dir / "test.txt" + with req_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_dict[lib] = v + # check setup.cfg and compare to requirements.txt + # also fails when there are missing or additional libs + setup_file = root_dir / "test.cfg" + with setup_file.open() as f: + lines = f.readlines() + + setup_keys = set() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: + req_v = req_dict.get(lib, None) + assert req_v is not None, ( + "{} in setup.cfg but not in requirements.txt".format(lib) + ) + assert (lib + v) == (lib + req_v), ( + "{} has different version in setup.cfg and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + setup_keys.add(lib) + assert sorted(setup_keys) == sorted( + req_dict.keys() + ) # if fail: requirements.txt contains a lib not in setup.cfg + + # check pyproject.toml and compare the versions of the libs to requirements.txt + # does not fail when there are missing or additional libs + toml_file = root_dir / "test.toml" + with toml_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip().strip(",").strip('"') + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_v = req_dict.get(lib, None) + assert (lib + v) == (lib + req_v), ( + "{} has different version in pyproject.toml and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + + +def _parse_req(line): + lib = re.match(r"^[a-z0-9\-]*", line).group(0) + v = line.replace(lib, "").strip() + if not re.match(r"^[<>=][<>=].*", v): + return None, None + return lib, v diff --git a/spacy/tests/parser/__init__.py b/spacy/tests/parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/parser/test_add_label.py b/spacy/tests/parser/test_add_label.py new file mode 100644 index 0000000..8962659 --- /dev/null +++ b/spacy/tests/parser/test_add_label.py @@ -0,0 +1,159 @@ +import pytest +from thinc.api import Adam, fix_random_seed + +from spacy import registry +from spacy.attrs import NORM +from spacy.language import Language +from spacy.pipeline import DependencyParser, EntityRecognizer +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.ner import DEFAULT_NER_MODEL +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +@pytest.fixture +def vocab(): + return Vocab(lex_attr_getters={NORM: lambda s: s}) + + +@pytest.fixture +def parser(vocab): + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(vocab, model) + return parser + + +def test_init_parser(parser): + pass + + +def _train_parser(parser): + fix_random_seed(1) + parser.add_label("left") + parser.initialize(lambda: [_parser_example(parser)]) + sgd = Adam(0.001) + + for i in range(5): + losses = {} + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + gold = {"heads": [1, 1, 3, 3], "deps": ["left", "ROOT", "left", "ROOT"]} + example = Example.from_dict(doc, gold) + parser.update([example], sgd=sgd, losses=losses) + return parser + + +def _parser_example(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + gold = {"heads": [1, 1, 3, 3], "deps": ["right", "ROOT", "left", "ROOT"]} + return Example.from_dict(doc, gold) + + +def _ner_example(ner): + doc = Doc( + ner.vocab, + words=["Joe", "loves", "visiting", "London", "during", "the", "weekend"], + ) + gold = {"entities": [(0, 3, "PERSON"), (19, 25, "LOC")]} + return Example.from_dict(doc, gold) + + +def test_add_label(parser): + parser = _train_parser(parser) + parser.add_label("right") + sgd = Adam(0.001) + for i in range(100): + losses = {} + parser.update([_parser_example(parser)], sgd=sgd, losses=losses) + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc = parser(doc) + assert doc[0].dep_ == "right" + assert doc[2].dep_ == "left" + + +def test_add_label_deserializes_correctly(): + cfg = {"model": DEFAULT_NER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + ner1 = EntityRecognizer(Vocab(), model) + ner1.add_label("C") + ner1.add_label("B") + ner1.add_label("A") + ner1.initialize(lambda: [_ner_example(ner1)]) + ner2 = EntityRecognizer(Vocab(), model) + + # the second model needs to be resized before we can call from_bytes + ner2.model.attrs["resize_output"](ner2.model, ner1.moves.n_moves) + ner2.from_bytes(ner1.to_bytes()) + assert ner1.moves.n_moves == ner2.moves.n_moves + for i in range(ner1.moves.n_moves): + assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i) + + +@pytest.mark.parametrize( + "pipe_cls,n_moves,model_config", + [ + (DependencyParser, 5, DEFAULT_PARSER_MODEL), + (EntityRecognizer, 4, DEFAULT_NER_MODEL), + ], +) +def test_add_label_get_label(pipe_cls, n_moves, model_config): + """Test that added labels are returned correctly. This test was added to + test for a bug in DependencyParser.labels that'd cause it to fail when + splitting the move names. + """ + labels = ["A", "B", "C"] + model = registry.resolve({"model": model_config}, validate=True)["model"] + pipe = pipe_cls(Vocab(), model) + for label in labels: + pipe.add_label(label) + assert len(pipe.move_names) == len(labels) * n_moves + pipe_labels = sorted(list(pipe.labels)) + assert pipe_labels == labels + + +def test_ner_labels_added_implicitly_on_predict(): + nlp = Language() + ner = nlp.add_pipe("ner") + for label in ["A", "B", "C"]: + ner.add_label(label) + nlp.initialize() + doc = Doc(nlp.vocab, words=["hello", "world"], ents=["B-D", "O"]) + ner(doc) + assert [t.ent_type_ for t in doc] == ["D", ""] + assert "D" in ner.labels + + +def test_ner_labels_added_implicitly_on_beam_parse(): + nlp = Language() + ner = nlp.add_pipe("beam_ner") + for label in ["A", "B", "C"]: + ner.add_label(label) + nlp.initialize() + doc = Doc(nlp.vocab, words=["hello", "world"], ents=["B-D", "O"]) + ner.beam_parse([doc], beam_width=32) + assert "D" in ner.labels + + +def test_ner_labels_added_implicitly_on_greedy_parse(): + nlp = Language() + ner = nlp.add_pipe("beam_ner") + for label in ["A", "B", "C"]: + ner.add_label(label) + nlp.initialize() + doc = Doc(nlp.vocab, words=["hello", "world"], ents=["B-D", "O"]) + ner.greedy_parse([doc]) + assert "D" in ner.labels + + +def test_ner_labels_added_implicitly_on_update(): + nlp = Language() + ner = nlp.add_pipe("ner") + for label in ["A", "B", "C"]: + ner.add_label(label) + nlp.initialize() + doc = Doc(nlp.vocab, words=["hello", "world"], ents=["B-D", "O"]) + example = Example(nlp.make_doc(doc.text), doc) + assert "D" not in ner.labels + nlp.update([example]) + assert "D" in ner.labels diff --git a/spacy/tests/parser/test_arc_eager_oracle.py b/spacy/tests/parser/test_arc_eager_oracle.py new file mode 100644 index 0000000..fafd232 --- /dev/null +++ b/spacy/tests/parser/test_arc_eager_oracle.py @@ -0,0 +1,294 @@ +import pytest + +from spacy import registry +from spacy.pipeline import DependencyParser +from spacy.pipeline._parser_internals.arc_eager import ArcEager +from spacy.pipeline._parser_internals.nonproj import projectivize +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +def get_sequence_costs(M, words, heads, deps, transitions): + doc = Doc(Vocab(), words=words) + example = Example.from_dict(doc, {"heads": heads, "deps": deps}) + states, golds, _ = M.init_gold_batch([example]) + state = states[0] + gold = golds[0] + cost_history = [] + for gold_action in transitions: + gold.update(state) + state_costs = {} + for i in range(M.n_moves): + name = M.class_name(i) + state_costs[name] = M.get_cost(state, gold, i) + M.transition(state, gold_action) + cost_history.append(state_costs) + return state, cost_history + + +@pytest.fixture +def vocab(): + return Vocab() + + +@pytest.fixture +def arc_eager(vocab): + moves = ArcEager(vocab.strings, ArcEager.get_actions()) + moves.add_action(2, "left") + moves.add_action(3, "right") + return moves + + +@pytest.mark.issue(7056) +def test_issue7056(): + """Test that the Unshift transition works properly, and doesn't cause + sentence segmentation errors.""" + vocab = Vocab() + ae = ArcEager( + vocab.strings, ArcEager.get_actions(left_labels=["amod"], right_labels=["pobj"]) + ) + doc = Doc(vocab, words="Severe pain , after trauma".split()) + state = ae.init_batch([doc])[0] + ae.apply_transition(state, "S") + ae.apply_transition(state, "L-amod") + ae.apply_transition(state, "S") + ae.apply_transition(state, "S") + ae.apply_transition(state, "S") + ae.apply_transition(state, "R-pobj") + ae.apply_transition(state, "D") + ae.apply_transition(state, "D") + ae.apply_transition(state, "D") + assert not state.eol() + + +def test_oracle_four_words(arc_eager, vocab): + words = ["a", "b", "c", "d"] + heads = [1, 1, 3, 3] + deps = ["left", "ROOT", "left", "ROOT"] + for dep in deps: + arc_eager.add_action(2, dep) # Left + arc_eager.add_action(3, dep) # Right + actions = ["S", "L-left", "B-ROOT", "S", "D", "S", "L-left", "S", "D"] + state, cost_history = get_sequence_costs(arc_eager, words, heads, deps, actions) + expected_gold = [ + ["S"], + ["B-ROOT", "L-left"], + ["B-ROOT"], + ["S"], + ["D"], + ["S"], + ["L-left"], + ["S"], + ["D"], + ] + assert state.is_final() + for i, state_costs in enumerate(cost_history): + # Check gold moves is 0 cost + golds = [act for act, cost in state_costs.items() if cost < 1] + assert golds == expected_gold[i], (i, golds, expected_gold[i]) + + +annot_tuples = [ + (0, "When", "WRB", 11, "advmod", "O"), + (1, "Walter", "NNP", 2, "compound", "B-PERSON"), + (2, "Rodgers", "NNP", 11, "nsubj", "L-PERSON"), + (3, ",", ",", 2, "punct", "O"), + (4, "our", "PRP$", 6, "poss", "O"), + (5, "embedded", "VBN", 6, "amod", "O"), + (6, "reporter", "NN", 2, "appos", "O"), + (7, "with", "IN", 6, "prep", "O"), + (8, "the", "DT", 10, "det", "B-ORG"), + (9, "3rd", "NNP", 10, "compound", "I-ORG"), + (10, "Cavalry", "NNP", 7, "pobj", "L-ORG"), + (11, "says", "VBZ", 44, "advcl", "O"), + (12, "three", "CD", 13, "nummod", "U-CARDINAL"), + (13, "battalions", "NNS", 16, "nsubj", "O"), + (14, "of", "IN", 13, "prep", "O"), + (15, "troops", "NNS", 14, "pobj", "O"), + (16, "are", "VBP", 11, "ccomp", "O"), + (17, "on", "IN", 16, "prep", "O"), + (18, "the", "DT", 19, "det", "O"), + (19, "ground", "NN", 17, "pobj", "O"), + (20, ",", ",", 17, "punct", "O"), + (21, "inside", "IN", 17, "prep", "O"), + (22, "Baghdad", "NNP", 21, "pobj", "U-GPE"), + (23, "itself", "PRP", 22, "appos", "O"), + (24, ",", ",", 16, "punct", "O"), + (25, "have", "VBP", 26, "aux", "O"), + (26, "taken", "VBN", 16, "dep", "O"), + (27, "up", "RP", 26, "prt", "O"), + (28, "positions", "NNS", 26, "dobj", "O"), + (29, "they", "PRP", 31, "nsubj", "O"), + (30, "'re", "VBP", 31, "aux", "O"), + (31, "going", "VBG", 26, "parataxis", "O"), + (32, "to", "TO", 33, "aux", "O"), + (33, "spend", "VB", 31, "xcomp", "O"), + (34, "the", "DT", 35, "det", "B-TIME"), + (35, "night", "NN", 33, "dobj", "L-TIME"), + (36, "there", "RB", 33, "advmod", "O"), + (37, "presumably", "RB", 33, "advmod", "O"), + (38, ",", ",", 44, "punct", "O"), + (39, "how", "WRB", 40, "advmod", "O"), + (40, "many", "JJ", 41, "amod", "O"), + (41, "soldiers", "NNS", 44, "pobj", "O"), + (42, "are", "VBP", 44, "aux", "O"), + (43, "we", "PRP", 44, "nsubj", "O"), + (44, "talking", "VBG", 44, "ROOT", "O"), + (45, "about", "IN", 44, "prep", "O"), + (46, "right", "RB", 47, "advmod", "O"), + (47, "now", "RB", 44, "advmod", "O"), + (48, "?", ".", 44, "punct", "O"), +] + + +def test_get_oracle_actions(): + ids, words, tags, heads, deps, ents = [], [], [], [], [], [] + for id_, word, tag, head, dep, ent in annot_tuples: + ids.append(id_) + words.append(word) + tags.append(tag) + heads.append(head) + deps.append(dep) + ents.append(ent) + doc = Doc(Vocab(), words=[t[1] for t in annot_tuples]) + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(doc.vocab, model) + parser.moves.add_action(0, "") + parser.moves.add_action(1, "") + parser.moves.add_action(1, "") + parser.moves.add_action(4, "ROOT") + heads, deps = projectivize(heads, deps) + for i, (head, dep) in enumerate(zip(heads, deps)): + if head > i: + parser.moves.add_action(2, dep) + elif head < i: + parser.moves.add_action(3, dep) + example = Example.from_dict( + doc, {"words": words, "tags": tags, "heads": heads, "deps": deps} + ) + parser.moves.get_oracle_sequence(example) + + +def test_oracle_dev_sentence(vocab, arc_eager): + words_deps_heads = """ + Rolls-Royce nn Inc. + Motor nn Inc. + Cars nn Inc. + Inc. nsubj said + said ROOT said + it nsubj expects + expects ccomp said + its poss sales + U.S. nn sales + sales nsubj steady + to aux steady + remain cop steady + steady xcomp expects + at prep steady + about quantmod 1,200 + 1,200 num cars + cars pobj at + in prep steady + 1990 pobj in + . punct said + """ + expected_transitions = [ + "S", # Shift "Rolls-Royce" + "S", # Shift 'Motor' + "S", # Shift 'Cars' + "L-nn", # Attach 'Cars' to 'Inc.' + "L-nn", # Attach 'Motor' to 'Inc.' + "L-nn", # Attach 'Rolls-Royce' to 'Inc.' + "S", # Shift "Inc." + "L-nsubj", # Attach 'Inc.' to 'said' + "S", # Shift 'said' + "S", # Shift 'it' + "L-nsubj", # Attach 'it.' to 'expects' + "R-ccomp", # Attach 'expects' to 'said' + "S", # Shift 'its' + "S", # Shift 'U.S.' + "L-nn", # Attach 'U.S.' to 'sales' + "L-poss", # Attach 'its' to 'sales' + "S", # Shift 'sales' + "S", # Shift 'to' + "S", # Shift 'remain' + "L-cop", # Attach 'remain' to 'steady' + "L-aux", # Attach 'to' to 'steady' + "L-nsubj", # Attach 'sales' to 'steady' + "R-xcomp", # Attach 'steady' to 'expects' + "R-prep", # Attach 'at' to 'steady' + "S", # Shift 'about' + "L-quantmod", # Attach "about" to "1,200" + "S", # Shift "1,200" + "L-num", # Attach "1,200" to "cars" + "R-pobj", # Attach "cars" to "at" + "D", # Reduce "cars" + "D", # Reduce "at" + "R-prep", # Attach "in" to "steady" + "R-pobj", # Attach "1990" to "in" + "D", # Reduce "1990" + "D", # Reduce "in" + "D", # Reduce "steady" + "D", # Reduce "expects" + "R-punct", # Attach "." to "said" + "D", # Reduce "." + "D", # Reduce "said" + ] + + gold_words = [] + gold_deps = [] + gold_heads = [] + for line in words_deps_heads.strip().split("\n"): + line = line.strip() + if not line: + continue + word, dep, head = line.split() + gold_words.append(word) + gold_deps.append(dep) + gold_heads.append(head) + gold_heads = [gold_words.index(head) for head in gold_heads] + for dep in gold_deps: + arc_eager.add_action(2, dep) # Left + arc_eager.add_action(3, dep) # Right + doc = Doc(Vocab(), words=gold_words) + example = Example.from_dict(doc, {"heads": gold_heads, "deps": gold_deps}) + ae_oracle_actions = arc_eager.get_oracle_sequence(example, _debug=False) + ae_oracle_actions = [arc_eager.get_class_name(i) for i in ae_oracle_actions] + assert ae_oracle_actions == expected_transitions + + +def test_oracle_bad_tokenization(vocab, arc_eager): + words_deps_heads = """ + [catalase] dep is + : punct is + that nsubj is + is root is + bad comp is + """ + + gold_words = [] + gold_deps = [] + gold_heads = [] + for line in words_deps_heads.strip().split("\n"): + line = line.strip() + if not line: + continue + word, dep, head = line.split() + gold_words.append(word) + gold_deps.append(dep) + gold_heads.append(head) + gold_heads = [gold_words.index(head) for head in gold_heads] + for dep in gold_deps: + arc_eager.add_action(2, dep) # Left + arc_eager.add_action(3, dep) # Right + reference = Doc(Vocab(), words=gold_words, deps=gold_deps, heads=gold_heads) + predicted = Doc( + reference.vocab, words=["[", "catalase", "]", ":", "that", "is", "bad"] + ) + example = Example(predicted=predicted, reference=reference) + ae_oracle_actions = arc_eager.get_oracle_sequence(example, _debug=False) + ae_oracle_actions = [arc_eager.get_class_name(i) for i in ae_oracle_actions] + assert ae_oracle_actions diff --git a/spacy/tests/parser/test_ner.py b/spacy/tests/parser/test_ner.py new file mode 100644 index 0000000..1509c31 --- /dev/null +++ b/spacy/tests/parser/test_ner.py @@ -0,0 +1,828 @@ +import logging +import random + +import pytest +from numpy.testing import assert_equal + +from spacy import registry, util +from spacy.attrs import ENT_IOB +from spacy.lang.en import English +from spacy.lang.it import Italian +from spacy.language import Language +from spacy.lookups import Lookups +from spacy.pipeline import EntityRecognizer +from spacy.pipeline._parser_internals.ner import BiluoPushDown +from spacy.pipeline.ner import DEFAULT_NER_MODEL +from spacy.tokens import Doc, Span +from spacy.training import Example, iob_to_biluo, split_bilu_label +from spacy.vocab import Vocab + +from ..util import make_tempdir + +TRAIN_DATA = [ + ("Who is Shaka Khan?", {"entities": [(7, 17, "PERSON")]}), + ("I like London and Berlin.", {"entities": [(7, 13, "LOC"), (18, 24, "LOC")]}), +] + + +@pytest.fixture +def neg_key(): + return "non_entities" + + +@pytest.fixture +def vocab(): + return Vocab() + + +@pytest.fixture +def doc(vocab): + return Doc(vocab, words=["Casey", "went", "to", "New", "York", "."]) + + +@pytest.fixture +def entity_annots(doc): + casey = doc[0:1] + ny = doc[3:5] + return [ + (casey.start_char, casey.end_char, "PERSON"), + (ny.start_char, ny.end_char, "GPE"), + ] + + +@pytest.fixture +def entity_types(entity_annots): + return sorted(set([label for (s, e, label) in entity_annots])) + + +@pytest.fixture +def tsys(vocab, entity_types): + actions = BiluoPushDown.get_actions(entity_types=entity_types) + return BiluoPushDown(vocab.strings, actions) + + +@pytest.mark.parametrize("label", ["U-JOB-NAME"]) +@pytest.mark.issue(1967) +def test_issue1967(label): + nlp = Language() + config = {} + ner = nlp.create_pipe("ner", config=config) + example = Example.from_dict( + Doc(ner.vocab, words=["word"]), + { + "ids": [0], + "words": ["word"], + "tags": ["tag"], + "heads": [0], + "deps": ["dep"], + "entities": [label], + }, + ) + assert "JOB-NAME" in ner.moves.get_actions(examples=[example])[1] + + +@pytest.mark.issue(2179) +def test_issue2179(): + """Test that spurious 'extra_labels' aren't created when initializing NER.""" + nlp = Italian() + ner = nlp.add_pipe("ner") + ner.add_label("CITIZENSHIP") + nlp.initialize() + nlp2 = Italian() + nlp2.add_pipe("ner") + assert len(nlp2.get_pipe("ner").labels) == 0 + model = nlp2.get_pipe("ner").model + model.attrs["resize_output"](model, nlp.get_pipe("ner").moves.n_moves) + nlp2.from_bytes(nlp.to_bytes()) + assert "extra_labels" not in nlp2.get_pipe("ner").cfg + assert nlp2.get_pipe("ner").labels == ("CITIZENSHIP",) + + +@pytest.mark.issue(2385) +def test_issue2385(): + """Test that IOB tags are correctly converted to BILUO tags.""" + # fix bug in labels with a 'b' character + tags1 = ("B-BRAWLER", "I-BRAWLER", "I-BRAWLER") + assert iob_to_biluo(tags1) == ["B-BRAWLER", "I-BRAWLER", "L-BRAWLER"] + # maintain support for iob1 format + tags2 = ("I-ORG", "I-ORG", "B-ORG") + assert iob_to_biluo(tags2) == ["B-ORG", "L-ORG", "U-ORG"] + # maintain support for iob2 format + tags3 = ("B-PERSON", "I-PERSON", "B-PERSON") + assert iob_to_biluo(tags3) == ["B-PERSON", "L-PERSON", "U-PERSON"] + # ensure it works with hyphens in the name + tags4 = ("B-MULTI-PERSON", "I-MULTI-PERSON", "B-MULTI-PERSON") + assert iob_to_biluo(tags4) == ["B-MULTI-PERSON", "L-MULTI-PERSON", "U-MULTI-PERSON"] + + +@pytest.mark.issue(2800) +def test_issue2800(): + """Test issue that arises when too many labels are added to NER model. + Used to cause segfault. + """ + nlp = English() + train_data = [] + train_data.extend( + [Example.from_dict(nlp.make_doc("One sentence"), {"entities": []})] + ) + entity_types = [str(i) for i in range(1000)] + ner = nlp.add_pipe("ner") + for entity_type in list(entity_types): + ner.add_label(entity_type) + optimizer = nlp.initialize() + for i in range(20): + losses = {} + random.shuffle(train_data) + for example in train_data: + nlp.update([example], sgd=optimizer, losses=losses, drop=0.5) + + +@pytest.mark.issue(3209) +def test_issue3209(): + """Test issue that occurred in spaCy nightly where NER labels were being + mapped to classes incorrectly after loading the model, when the labels + were added using ner.add_label(). + """ + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("ANIMAL") + nlp.initialize() + move_names = ["O", "B-ANIMAL", "I-ANIMAL", "L-ANIMAL", "U-ANIMAL"] + assert ner.move_names == move_names + nlp2 = English() + ner2 = nlp2.add_pipe("ner") + model = ner2.model + model.attrs["resize_output"](model, ner.moves.n_moves) + nlp2.from_bytes(nlp.to_bytes()) + assert ner2.move_names == move_names + + +def test_labels_from_BILUO(): + """Test that labels are inferred correctly when there's a - in label.""" + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("LARGE-ANIMAL") + nlp.initialize() + move_names = [ + "O", + "B-LARGE-ANIMAL", + "I-LARGE-ANIMAL", + "L-LARGE-ANIMAL", + "U-LARGE-ANIMAL", + ] + labels = {"LARGE-ANIMAL"} + assert ner.move_names == move_names + assert set(ner.labels) == labels + + +@pytest.mark.issue(4267) +def test_issue4267(): + """Test that running an entity_ruler after ner gives consistent results""" + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("PEOPLE") + nlp.initialize() + assert "ner" in nlp.pipe_names + # assert that we have correct IOB annotations + doc1 = nlp("hi") + assert doc1.has_annotation("ENT_IOB") + for token in doc1: + assert token.ent_iob == 2 + # add entity ruler and run again + patterns = [{"label": "SOFTWARE", "pattern": "spacy"}] + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + assert "entity_ruler" in nlp.pipe_names + assert "ner" in nlp.pipe_names + # assert that we still have correct IOB annotations + doc2 = nlp("hi") + assert doc2.has_annotation("ENT_IOB") + for token in doc2: + assert token.ent_iob == 2 + + +@pytest.mark.issue(4313) +def test_issue4313(): + """This should not crash or exit with some strange error code""" + beam_width = 16 + beam_density = 0.0001 + nlp = English() + config = { + "beam_width": beam_width, + "beam_density": beam_density, + } + ner = nlp.add_pipe("beam_ner", config=config) + ner.add_label("SOME_LABEL") + nlp.initialize() + # add a new label to the doc + doc = nlp("What do you think about Apple ?") + assert len(ner.labels) == 1 + assert "SOME_LABEL" in ner.labels + apple_ent = Span(doc, 5, 6, label="MY_ORG") + doc.ents = list(doc.ents) + [apple_ent] + + # ensure the beam_parse still works with the new label + docs = [doc] + ner.beam_parse(docs, drop=0.0, beam_width=beam_width, beam_density=beam_density) + assert len(ner.labels) == 2 + assert "MY_ORG" in ner.labels + + +def test_get_oracle_moves(tsys, doc, entity_annots): + example = Example.from_dict(doc, {"entities": entity_annots}) + act_classes = tsys.get_oracle_sequence(example, _debug=False) + names = [tsys.get_class_name(act) for act in act_classes] + assert names == ["U-PERSON", "O", "O", "B-GPE", "L-GPE", "O"] + + +def test_negative_samples_two_word_input(tsys, vocab, neg_key): + """Test that we don't get stuck in a two word input when we have a negative + span. This could happen if we don't have the right check on the B action. + """ + tsys.cfg["neg_key"] = neg_key + doc = Doc(vocab, words=["A", "B"]) + entity_annots = [None, None] + example = Example.from_dict(doc, {"entities": entity_annots}) + # These mean that the oracle sequence shouldn't have O for the first + # word, and it shouldn't analyse it as B-PERSON, L-PERSON + example.y.spans[neg_key] = [ + Span(example.y, 0, 1, label="O"), + Span(example.y, 0, 2, label="PERSON"), + ] + act_classes = tsys.get_oracle_sequence(example) + names = [tsys.get_class_name(act) for act in act_classes] + assert names + assert names[0] != "O" + assert names[0] != "B-PERSON" + assert names[1] != "L-PERSON" + + +def test_negative_samples_three_word_input(tsys, vocab, neg_key): + """Test that we exclude a 2-word entity correctly using a negative example.""" + tsys.cfg["neg_key"] = neg_key + doc = Doc(vocab, words=["A", "B", "C"]) + entity_annots = [None, None, None] + example = Example.from_dict(doc, {"entities": entity_annots}) + # These mean that the oracle sequence shouldn't have O for the first + # word, and it shouldn't analyse it as B-PERSON, L-PERSON + example.y.spans[neg_key] = [ + Span(example.y, 0, 1, label="O"), + Span(example.y, 0, 2, label="PERSON"), + ] + act_classes = tsys.get_oracle_sequence(example) + names = [tsys.get_class_name(act) for act in act_classes] + assert names + assert names[0] != "O" + assert names[1] != "B-PERSON" + + +def test_negative_samples_U_entity(tsys, vocab, neg_key): + """Test that we exclude a 2-word entity correctly using a negative example.""" + tsys.cfg["neg_key"] = neg_key + doc = Doc(vocab, words=["A"]) + entity_annots = [None] + example = Example.from_dict(doc, {"entities": entity_annots}) + # These mean that the oracle sequence shouldn't have O for the first + # word, and it shouldn't analyse it as B-PERSON, L-PERSON + example.y.spans[neg_key] = [ + Span(example.y, 0, 1, label="O"), + Span(example.y, 0, 1, label="PERSON"), + ] + act_classes = tsys.get_oracle_sequence(example) + names = [tsys.get_class_name(act) for act in act_classes] + assert names + assert names[0] != "O" + assert names[0] != "U-PERSON" + + +def test_negative_sample_key_is_in_config(vocab, entity_types): + actions = BiluoPushDown.get_actions(entity_types=entity_types) + tsys = BiluoPushDown(vocab.strings, actions, incorrect_spans_key="non_entities") + assert tsys.cfg["neg_key"] == "non_entities" + + +# We can't easily represent this on a Doc object. Not sure what the best solution +# would be, but I don't think it's an important use case? +@pytest.mark.skip(reason="No longer supported") +def test_oracle_moves_missing_B(en_vocab): + words = ["B", "52", "Bomber"] + biluo_tags = [None, None, "L-PRODUCT"] + + doc = Doc(en_vocab, words=words) + example = Example.from_dict(doc, {"words": words, "entities": biluo_tags}) + + moves = BiluoPushDown(en_vocab.strings) + move_types = ("M", "B", "I", "L", "U", "O") + for tag in biluo_tags: + if tag is None: + continue + elif tag == "O": + moves.add_action(move_types.index("O"), "") + else: + action, label = split_bilu_label(tag) + moves.add_action(move_types.index("B"), label) + moves.add_action(move_types.index("I"), label) + moves.add_action(move_types.index("L"), label) + moves.add_action(move_types.index("U"), label) + moves.get_oracle_sequence(example) + + +# We can't easily represent this on a Doc object. Not sure what the best solution +# would be, but I don't think it's an important use case? +@pytest.mark.skip(reason="No longer supported") +def test_oracle_moves_whitespace(en_vocab): + words = ["production", "\n", "of", "Northrop", "\n", "Corp.", "\n", "'s", "radar"] + biluo_tags = ["O", "O", "O", "B-ORG", None, "I-ORG", "L-ORG", "O", "O"] + + doc = Doc(en_vocab, words=words) + example = Example.from_dict(doc, {"entities": biluo_tags}) + + moves = BiluoPushDown(en_vocab.strings) + move_types = ("M", "B", "I", "L", "U", "O") + for tag in biluo_tags: + if tag is None: + continue + elif tag == "O": + moves.add_action(move_types.index("O"), "") + else: + action, label = split_bilu_label(tag) + moves.add_action(move_types.index(action), label) + moves.get_oracle_sequence(example) + + +def test_accept_blocked_token(): + """Test succesful blocking of tokens to be in an entity.""" + # 1. test normal behaviour + nlp1 = English() + doc1 = nlp1("I live in New York") + config = {} + ner1 = nlp1.create_pipe("ner", config=config) + assert [token.ent_iob_ for token in doc1] == ["", "", "", "", ""] + assert [token.ent_type_ for token in doc1] == ["", "", "", "", ""] + + # Add the OUT action + ner1.moves.add_action(5, "") + ner1.add_label("GPE") + # Get into the state just before "New" + state1 = ner1.moves.init_batch([doc1])[0] + ner1.moves.apply_transition(state1, "O") + ner1.moves.apply_transition(state1, "O") + ner1.moves.apply_transition(state1, "O") + # Check that B-GPE is valid. + assert ner1.moves.is_valid(state1, "B-GPE") + + # 2. test blocking behaviour + nlp2 = English() + doc2 = nlp2("I live in New York") + config = {} + ner2 = nlp2.create_pipe("ner", config=config) + + # set "New York" to a blocked entity + doc2.set_ents([], blocked=[doc2[3:5]], default="unmodified") + assert [token.ent_iob_ for token in doc2] == ["", "", "", "B", "B"] + assert [token.ent_type_ for token in doc2] == ["", "", "", "", ""] + + # Check that B-GPE is now invalid. + ner2.moves.add_action(4, "") + ner2.moves.add_action(5, "") + ner2.add_label("GPE") + state2 = ner2.moves.init_batch([doc2])[0] + ner2.moves.apply_transition(state2, "O") + ner2.moves.apply_transition(state2, "O") + ner2.moves.apply_transition(state2, "O") + # we can only use U- for "New" + assert not ner2.moves.is_valid(state2, "B-GPE") + assert ner2.moves.is_valid(state2, "U-") + ner2.moves.apply_transition(state2, "U-") + # we can only use U- for "York" + assert not ner2.moves.is_valid(state2, "B-GPE") + assert ner2.moves.is_valid(state2, "U-") + + +def test_train_empty(): + """Test that training an empty text does not throw errors.""" + train_data = [ + ("Who is Shaka Khan?", {"entities": [(7, 17, "PERSON")]}), + ("", {"entities": []}), + ] + + nlp = English() + train_examples = [] + for t in train_data: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + ner = nlp.add_pipe("ner", last=True) + ner.add_label("PERSON") + nlp.initialize() + for itn in range(2): + losses = {} + batches = util.minibatch(train_examples, size=8) + for batch in batches: + nlp.update(batch, losses=losses) + + +def test_train_negative_deprecated(): + """Test that the deprecated negative entity format raises a custom error.""" + train_data = [ + ("Who is Shaka Khan?", {"entities": [(7, 17, "!PERSON")]}), + ] + + nlp = English() + train_examples = [] + for t in train_data: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + ner = nlp.add_pipe("ner", last=True) + ner.add_label("PERSON") + nlp.initialize() + for itn in range(2): + losses = {} + batches = util.minibatch(train_examples, size=8) + for batch in batches: + with pytest.raises(ValueError): + nlp.update(batch, losses=losses) + + +def test_overwrite_token(): + nlp = English() + nlp.add_pipe("ner") + nlp.initialize() + # The untrained NER will predict O for each token + doc = nlp("I live in New York") + assert [token.ent_iob_ for token in doc] == ["O", "O", "O", "O", "O"] + assert [token.ent_type_ for token in doc] == ["", "", "", "", ""] + # Check that a new ner can overwrite O + config = {} + ner2 = nlp.create_pipe("ner", config=config) + ner2.moves.add_action(5, "") + ner2.add_label("GPE") + state = ner2.moves.init_batch([doc])[0] + assert ner2.moves.is_valid(state, "B-GPE") + assert ner2.moves.is_valid(state, "U-GPE") + ner2.moves.apply_transition(state, "B-GPE") + assert ner2.moves.is_valid(state, "I-GPE") + assert ner2.moves.is_valid(state, "L-GPE") + + +def test_empty_ner(): + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("MY_LABEL") + nlp.initialize() + doc = nlp("John is watching the news about Croatia's elections") + # if this goes wrong, the initialization of the parser's upper layer is probably broken + result = ["O", "O", "O", "O", "O", "O", "O", "O", "O"] + assert [token.ent_iob_ for token in doc] == result + + +def test_ruler_before_ner(): + """Test that an NER works after an entity_ruler: the second can add annotations""" + nlp = English() + + # 1 : Entity Ruler - should set "this" to B and everything else to empty + patterns = [{"label": "THING", "pattern": "This"}] + ruler = nlp.add_pipe("entity_ruler") + + # 2: untrained NER - should set everything else to O + untrained_ner = nlp.add_pipe("ner") + untrained_ner.add_label("MY_LABEL") + nlp.initialize() + ruler.add_patterns(patterns) + doc = nlp("This is Antti Korhonen speaking in Finland") + expected_iobs = ["B", "O", "O", "O", "O", "O", "O"] + expected_types = ["THING", "", "", "", "", "", ""] + assert [token.ent_iob_ for token in doc] == expected_iobs + assert [token.ent_type_ for token in doc] == expected_types + + +def test_ner_constructor(en_vocab): + config = { + "update_with_oracle_cut_size": 100, + } + cfg = {"model": DEFAULT_NER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + EntityRecognizer(en_vocab, model, **config) + EntityRecognizer(en_vocab, model) + + +def test_ner_before_ruler(): + """Test that an entity_ruler works after an NER: the second can overwrite O annotations""" + nlp = English() + + # 1: untrained NER - should set everything to O + untrained_ner = nlp.add_pipe("ner", name="uner") + untrained_ner.add_label("MY_LABEL") + nlp.initialize() + + # 2 : Entity Ruler - should set "this" to B and keep everything else O + patterns = [{"label": "THING", "pattern": "This"}] + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + + doc = nlp("This is Antti Korhonen speaking in Finland") + expected_iobs = ["B", "O", "O", "O", "O", "O", "O"] + expected_types = ["THING", "", "", "", "", "", ""] + assert [token.ent_iob_ for token in doc] == expected_iobs + assert [token.ent_type_ for token in doc] == expected_types + + +def test_block_ner(): + """Test functionality for blocking tokens so they can't be in a named entity""" + # block "Antti L Korhonen" from being a named entity + nlp = English() + nlp.add_pipe("blocker", config={"start": 2, "end": 5}) + untrained_ner = nlp.add_pipe("ner") + untrained_ner.add_label("MY_LABEL") + nlp.initialize() + doc = nlp("This is Antti L Korhonen speaking in Finland") + expected_iobs = ["O", "O", "B", "B", "B", "O", "O", "O"] + expected_types = ["", "", "", "", "", "", "", ""] + assert [token.ent_iob_ for token in doc] == expected_iobs + assert [token.ent_type_ for token in doc] == expected_types + + +@pytest.mark.parametrize("use_upper", [True, False]) +def test_overfitting_IO(use_upper): + # Simple test to try and quickly overfit the NER component + nlp = English() + ner = nlp.add_pipe("ner", config={"model": {"use_upper": use_upper}}) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for ent in annotations.get("entities"): + ner.add_label(ent[2]) + optimizer = nlp.initialize() + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["ner"] < 0.00001 + + # test the trained model + test_text = "I like London." + doc = nlp(test_text) + ents = doc.ents + assert len(ents) == 1 + assert ents[0].text == "London" + assert ents[0].label_ == "LOC" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + ents2 = doc2.ents + assert len(ents2) == 1 + assert ents2[0].text == "London" + assert ents2[0].label_ == "LOC" + # Ensure that the predictions are still the same, even after adding a new label + ner2 = nlp2.get_pipe("ner") + assert ner2.model.attrs["has_upper"] == use_upper + ner2.add_label("RANDOM_NEW_LABEL") + doc3 = nlp2(test_text) + ents3 = doc3.ents + assert len(ents3) == 1 + assert ents3[0].text == "London" + assert ents3[0].label_ == "LOC" + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Just a sentence.", + "Then one more sentence about London.", + "Here is another one.", + "I like London.", + ] + batch_deps_1 = [doc.to_array([ENT_IOB]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([ENT_IOB]) for doc in nlp.pipe(texts)] + no_batch_deps = [doc.to_array([ENT_IOB]) for doc in [nlp(text) for text in texts]] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + # test that kb_id is preserved + test_text = "I like London and London." + doc = nlp.make_doc(test_text) + doc.ents = [Span(doc, 2, 3, label="LOC", kb_id=1234)] + ents = doc.ents + assert len(ents) == 1 + assert ents[0].text == "London" + assert ents[0].label_ == "LOC" + assert ents[0].kb_id == 1234 + doc = nlp.get_pipe("ner")(doc) + ents = doc.ents + assert len(ents) == 2 + assert ents[0].text == "London" + assert ents[0].label_ == "LOC" + assert ents[0].kb_id == 1234 + # ent added by ner has kb_id == 0 + assert ents[1].text == "London" + assert ents[1].label_ == "LOC" + assert ents[1].kb_id == 0 + + +def test_beam_ner_scores(): + # Test that we can get confidence values out of the beam_ner pipe + beam_width = 16 + beam_density = 0.0001 + nlp = English() + config = { + "beam_width": beam_width, + "beam_density": beam_density, + } + ner = nlp.add_pipe("beam_ner", config=config) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for ent in annotations.get("entities"): + ner.add_label(ent[2]) + optimizer = nlp.initialize() + + # update once + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # test the scores from the beam + test_text = "I like London." + doc = nlp.make_doc(test_text) + docs = [doc] + beams = ner.predict(docs) + entity_scores = ner.scored_ents(beams)[0] + + for j in range(len(doc)): + for label in ner.labels: + score = entity_scores[(j, j + 1, label)] + eps = 0.00001 + assert 0 - eps <= score <= 1 + eps + + +def test_beam_overfitting_IO(neg_key): + # Simple test to try and quickly overfit the Beam NER component + nlp = English() + beam_width = 16 + beam_density = 0.0001 + config = { + "beam_width": beam_width, + "beam_density": beam_density, + "incorrect_spans_key": neg_key, + } + ner = nlp.add_pipe("beam_ner", config=config) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for ent in annotations.get("entities"): + ner.add_label(ent[2]) + optimizer = nlp.initialize() + + # run overfitting + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["beam_ner"] < 0.0001 + + # test the scores from the beam + test_text = "I like London" + docs = [nlp.make_doc(test_text)] + beams = ner.predict(docs) + entity_scores = ner.scored_ents(beams)[0] + assert entity_scores[(2, 3, "LOC")] == 1.0 + assert entity_scores[(2, 3, "PERSON")] == 0.0 + assert len(nlp(test_text).ents) == 1 + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + docs2 = [nlp2.make_doc(test_text)] + ner2 = nlp2.get_pipe("beam_ner") + beams2 = ner2.predict(docs2) + entity_scores2 = ner2.scored_ents(beams2)[0] + assert entity_scores2[(2, 3, "LOC")] == 1.0 + assert entity_scores2[(2, 3, "PERSON")] == 0.0 + + # Try to unlearn the entity by using negative annotations + neg_doc = nlp.make_doc(test_text) + neg_ex = Example(neg_doc, neg_doc) + neg_ex.reference.spans[neg_key] = [Span(neg_doc, 2, 3, "LOC")] + neg_train_examples = [neg_ex] + + for i in range(20): + losses = {} + nlp.update(neg_train_examples, sgd=optimizer, losses=losses) + + # test the "untrained" model + assert len(nlp(test_text).ents) == 0 + + +def test_neg_annotation(neg_key): + """Check that the NER update works with a negative annotation that is a different label of the correct one, + or partly overlapping, etc""" + nlp = English() + beam_width = 16 + beam_density = 0.0001 + config = { + "beam_width": beam_width, + "beam_density": beam_density, + "incorrect_spans_key": neg_key, + } + ner = nlp.add_pipe("beam_ner", config=config) + train_text = "Who is Shaka Khan?" + neg_doc = nlp.make_doc(train_text) + ner.add_label("PERSON") + ner.add_label("ORG") + example = Example.from_dict(neg_doc, {"entities": [(7, 17, "PERSON")]}) + example.reference.spans[neg_key] = [ + Span(example.reference, 2, 4, "ORG"), + Span(example.reference, 2, 3, "PERSON"), + Span(example.reference, 1, 4, "PERSON"), + ] + + optimizer = nlp.initialize() + for i in range(2): + losses = {} + nlp.update([example], sgd=optimizer, losses=losses) + + +def test_neg_annotation_conflict(neg_key): + # Check that NER raises for a negative annotation that is THE SAME as a correct one + nlp = English() + beam_width = 16 + beam_density = 0.0001 + config = { + "beam_width": beam_width, + "beam_density": beam_density, + "incorrect_spans_key": neg_key, + } + ner = nlp.add_pipe("beam_ner", config=config) + train_text = "Who is Shaka Khan?" + neg_doc = nlp.make_doc(train_text) + ner.add_label("PERSON") + ner.add_label("LOC") + example = Example.from_dict(neg_doc, {"entities": [(7, 17, "PERSON")]}) + example.reference.spans[neg_key] = [Span(example.reference, 2, 4, "PERSON")] + assert len(example.reference.ents) == 1 + assert example.reference.ents[0].text == "Shaka Khan" + assert example.reference.ents[0].label_ == "PERSON" + assert len(example.reference.spans[neg_key]) == 1 + assert example.reference.spans[neg_key][0].text == "Shaka Khan" + assert example.reference.spans[neg_key][0].label_ == "PERSON" + + optimizer = nlp.initialize() + for i in range(2): + losses = {} + with pytest.raises(ValueError): + nlp.update([example], sgd=optimizer, losses=losses) + + +def test_beam_valid_parse(neg_key): + """Regression test for previously flakey behaviour""" + nlp = English() + beam_width = 16 + beam_density = 0.0001 + config = { + "beam_width": beam_width, + "beam_density": beam_density, + "incorrect_spans_key": neg_key, + } + nlp.add_pipe("beam_ner", config=config) + # fmt: off + tokens = ['FEDERAL', 'NATIONAL', 'MORTGAGE', 'ASSOCIATION', '(', 'Fannie', 'Mae', '):', 'Posted', 'yields', 'on', '30', 'year', 'mortgage', 'commitments', 'for', 'delivery', 'within', '30', 'days', '(', 'priced', 'at', 'par', ')', '9.75', '%', ',', 'standard', 'conventional', 'fixed', '-', 'rate', 'mortgages', ';', '8.70', '%', ',', '6/2', 'rate', 'capped', 'one', '-', 'year', 'adjustable', 'rate', 'mortgages', '.', 'Source', ':', 'Telerate', 'Systems', 'Inc.'] + iob = ['B-ORG', 'I-ORG', 'I-ORG', 'L-ORG', 'O', 'B-ORG', 'L-ORG', 'O', 'O', 'O', 'O', 'B-DATE', 'L-DATE', 'O', 'O', 'O', 'O', 'O', 'B-DATE', 'L-DATE', 'O', 'O', 'O', 'O', 'O', 'B-PERCENT', 'L-PERCENT', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PERCENT', 'L-PERCENT', 'O', 'U-CARDINAL', 'O', 'O', 'B-DATE', 'I-DATE', 'L-DATE', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'] + # fmt: on + + doc = Doc(nlp.vocab, words=tokens) + example = Example.from_dict(doc, {"ner": iob}) + neg_span = Span(example.reference, 50, 53, "ORG") + example.reference.spans[neg_key] = [neg_span] + + optimizer = nlp.initialize() + + for i in range(5): + losses = {} + nlp.update([example], sgd=optimizer, losses=losses) + assert "beam_ner" in losses + + +def test_ner_warns_no_lookups(caplog): + nlp = English() + assert nlp.lang in util.LEXEME_NORM_LANGS + nlp.vocab.lookups = Lookups() + assert not len(nlp.vocab.lookups) + nlp.add_pipe("ner") + with caplog.at_level(logging.DEBUG): + nlp.initialize() + assert "W033" in caplog.text + caplog.clear() + nlp.vocab.lookups.add_table("lexeme_norm") + nlp.vocab.lookups.get_table("lexeme_norm")["a"] = "A" + with caplog.at_level(logging.DEBUG): + nlp.initialize() + assert "W033" not in caplog.text + + +@Language.factory("blocker") +class BlockerComponent1: + def __init__(self, nlp, start, end, name="my_blocker"): + self.start = start + self.end = end + self.name = name + + def __call__(self, doc): + doc.set_ents([], blocked=[doc[self.start : self.end]], default="unmodified") + return doc diff --git a/spacy/tests/parser/test_neural_parser.py b/spacy/tests/parser/test_neural_parser.py new file mode 100644 index 0000000..5bef575 --- /dev/null +++ b/spacy/tests/parser/test_neural_parser.py @@ -0,0 +1,110 @@ +import pytest +from thinc.api import Model + +from spacy import registry +from spacy.pipeline._parser_internals.arc_eager import ArcEager +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.pipeline.transition_parser import Parser +from spacy.tokens.doc import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +@pytest.fixture +def vocab(): + return Vocab() + + +@pytest.fixture +def arc_eager(vocab): + actions = ArcEager.get_actions(left_labels=["L"], right_labels=["R"]) + return ArcEager(vocab.strings, actions) + + +@pytest.fixture +def tok2vec(): + cfg = {"model": DEFAULT_TOK2VEC_MODEL} + tok2vec = registry.resolve(cfg, validate=True)["model"] + tok2vec.initialize() + return tok2vec + + +@pytest.fixture +def parser(vocab, arc_eager): + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + } + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + return Parser(vocab, model, moves=arc_eager, **config) + + +@pytest.fixture +def model(arc_eager, tok2vec, vocab): + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + model.attrs["resize_output"](model, arc_eager.n_moves) + model.initialize() + return model + + +@pytest.fixture +def doc(vocab): + return Doc(vocab, words=["a", "b", "c"]) + + +@pytest.fixture +def gold(doc): + return {"heads": [1, 1, 1], "deps": ["L", "ROOT", "R"]} + + +def test_can_init_nn_parser(parser): + assert isinstance(parser.model, Model) + + +def test_build_model(parser, vocab): + config = { + "learn_tokens": False, + "min_action_freq": 0, + "update_with_oracle_cut_size": 100, + } + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser.model = Parser(vocab, model=model, moves=parser.moves, **config).model + assert parser.model is not None + + +def test_predict_doc(parser, tok2vec, model, doc): + doc.tensor = tok2vec.predict([doc])[0] + parser.model = model + parser(doc) + + +def test_update_doc(parser, model, doc, gold): + parser.model = model + + def optimize(key, weights, gradient): + weights -= 0.001 * gradient + return weights, gradient + + example = Example.from_dict(doc, gold) + parser.update([example], sgd=optimize) + + +@pytest.mark.skip(reason="No longer supported") +def test_predict_doc_beam(parser, model, doc): + parser.model = model + parser(doc, beam_width=32, beam_density=0.001) + + +@pytest.mark.skip(reason="No longer supported") +def test_update_doc_beam(parser, model, doc, gold): + parser.model = model + + def optimize(weights, gradient, key=None): + weights -= 0.001 * gradient + + parser.update_beam((doc, gold), sgd=optimize) diff --git a/spacy/tests/parser/test_nn_beam.py b/spacy/tests/parser/test_nn_beam.py new file mode 100644 index 0000000..f852e5c --- /dev/null +++ b/spacy/tests/parser/test_nn_beam.py @@ -0,0 +1,143 @@ +import hypothesis +import hypothesis.strategies +import numpy +import pytest +from thinc.tests.strategies import ndarrays_of_shape + +from spacy.language import Language +from spacy.pipeline._parser_internals._beam_utils import BeamBatch +from spacy.pipeline._parser_internals.arc_eager import ArcEager +from spacy.pipeline._parser_internals.stateclass import StateClass +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +@pytest.fixture(scope="module") +def vocab(): + return Vocab() + + +@pytest.fixture(scope="module") +def moves(vocab): + aeager = ArcEager(vocab.strings, {}) + aeager.add_action(0, "") + aeager.add_action(1, "") + aeager.add_action(2, "nsubj") + aeager.add_action(2, "punct") + aeager.add_action(2, "aux") + aeager.add_action(2, "nsubjpass") + aeager.add_action(3, "dobj") + aeager.add_action(2, "aux") + aeager.add_action(4, "ROOT") + return aeager + + +@pytest.fixture(scope="module") +def docs(vocab): + return [ + Doc( + vocab, + words=["Rats", "bite", "things"], + heads=[1, 1, 1], + deps=["nsubj", "ROOT", "dobj"], + sent_starts=[True, False, False], + ) + ] + + +@pytest.fixture(scope="module") +def examples(docs): + return [Example(doc, doc.copy()) for doc in docs] + + +@pytest.fixture +def states(docs): + return [StateClass(doc) for doc in docs] + + +@pytest.fixture +def tokvecs(docs, vector_size): + output = [] + for doc in docs: + vec = numpy.random.uniform(-0.1, 0.1, (len(doc), vector_size)) + output.append(numpy.asarray(vec)) + return output + + +@pytest.fixture(scope="module") +def batch_size(docs): + return len(docs) + + +@pytest.fixture(scope="module") +def beam_width(): + return 4 + + +@pytest.fixture(params=[0.0, 0.5, 1.0]) +def beam_density(request): + return request.param + + +@pytest.fixture +def vector_size(): + return 6 + + +@pytest.fixture +def beam(moves, examples, beam_width): + states, golds, _ = moves.init_gold_batch(examples) + return BeamBatch(moves, states, golds, width=beam_width, density=0.0) + + +@pytest.fixture +def scores(moves, batch_size, beam_width): + return numpy.asarray( + numpy.concatenate( + [ + numpy.random.uniform(-0.1, 0.1, (beam_width, moves.n_moves)) + for _ in range(batch_size) + ] + ), + dtype="float32", + ) + + +def test_create_beam(beam): + pass + + +def test_beam_advance(beam, scores): + beam.advance(scores) + + +def test_beam_advance_too_few_scores(beam, scores): + n_state = sum(len(beam) for beam in beam) + scores = scores[:n_state] + with pytest.raises(IndexError): + beam.advance(scores[:-1]) + + +def test_beam_parse(examples, beam_width): + nlp = Language() + parser = nlp.add_pipe("beam_parser") + parser.cfg["beam_width"] = beam_width + parser.add_label("nsubj") + parser.initialize(lambda: examples) + doc = nlp.make_doc("Australia is a country") + parser(doc) + + +@hypothesis.given(hyp=hypothesis.strategies.data()) +def test_beam_density(moves, examples, beam_width, hyp): + beam_density = float(hyp.draw(hypothesis.strategies.floats(0.0, 1.0, width=32))) + states, golds, _ = moves.init_gold_batch(examples) + beam = BeamBatch(moves, states, golds, width=beam_width, density=beam_density) + n_state = sum(len(beam) for beam in beam) + scores = hyp.draw(ndarrays_of_shape((n_state, moves.n_moves))) + beam.advance(scores) + for b in beam: + beam_probs = b.probs + assert b.min_density == beam_density + assert beam_probs[-1] >= beam_probs[0] * beam_density diff --git a/spacy/tests/parser/test_nonproj.py b/spacy/tests/parser/test_nonproj.py new file mode 100644 index 0000000..f4e09fc --- /dev/null +++ b/spacy/tests/parser/test_nonproj.py @@ -0,0 +1,154 @@ +import pytest + +from spacy.pipeline._parser_internals import nonproj +from spacy.pipeline._parser_internals.nonproj import ( + ancestors, + contains_cycle, + is_nonproj_arc, + is_nonproj_tree, +) +from spacy.tokens import Doc + + +@pytest.fixture +def tree(): + return [1, 2, 2, 4, 5, 2, 2] + + +@pytest.fixture +def cyclic_tree(): + return [1, 2, 2, 4, 5, 3, 2] + + +@pytest.fixture +def partial_tree(): + return [1, 2, 2, 4, 5, None, 7, 4, 2] + + +@pytest.fixture +def nonproj_tree(): + return [1, 2, 2, 4, 5, 2, 7, 4, 2] + + +@pytest.fixture +def proj_tree(): + return [1, 2, 2, 4, 5, 2, 7, 5, 2] + + +@pytest.fixture +def multirooted_tree(): + return [3, 2, 0, 3, 3, 7, 7, 3, 7, 10, 7, 10, 11, 12, 18, 16, 18, 17, 12, 3] + + +def test_parser_ancestors(tree, cyclic_tree, partial_tree, multirooted_tree): + assert [a for a in ancestors(3, tree)] == [4, 5, 2] + assert [a for a in ancestors(3, cyclic_tree)] == [4, 5, 3, 4, 5, 3, 4] + assert [a for a in ancestors(3, partial_tree)] == [4, 5, None] + assert [a for a in ancestors(17, multirooted_tree)] == [] + + +def test_parser_contains_cycle(tree, cyclic_tree, partial_tree, multirooted_tree): + assert contains_cycle(tree) is None + assert contains_cycle(cyclic_tree) == {3, 4, 5} + assert contains_cycle(partial_tree) is None + assert contains_cycle(multirooted_tree) is None + + +def test_parser_is_nonproj_arc( + cyclic_tree, nonproj_tree, partial_tree, multirooted_tree +): + assert is_nonproj_arc(0, nonproj_tree) is False + assert is_nonproj_arc(1, nonproj_tree) is False + assert is_nonproj_arc(2, nonproj_tree) is False + assert is_nonproj_arc(3, nonproj_tree) is False + assert is_nonproj_arc(4, nonproj_tree) is False + assert is_nonproj_arc(5, nonproj_tree) is False + assert is_nonproj_arc(6, nonproj_tree) is False + assert is_nonproj_arc(7, nonproj_tree) is True + assert is_nonproj_arc(8, nonproj_tree) is False + assert is_nonproj_arc(7, partial_tree) is False + assert is_nonproj_arc(17, multirooted_tree) is False + assert is_nonproj_arc(16, multirooted_tree) is True + with pytest.raises( + ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]" + ): + is_nonproj_arc(6, cyclic_tree) + + +def test_parser_is_nonproj_tree( + proj_tree, cyclic_tree, nonproj_tree, partial_tree, multirooted_tree +): + assert is_nonproj_tree(proj_tree) is False + assert is_nonproj_tree(nonproj_tree) is True + assert is_nonproj_tree(partial_tree) is False + assert is_nonproj_tree(multirooted_tree) is True + with pytest.raises( + ValueError, match=r"Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]" + ): + is_nonproj_tree(cyclic_tree) + + +def test_parser_pseudoprojectivity(en_vocab): + def deprojectivize(proj_heads, deco_labels): + words = ["whatever "] * len(proj_heads) + doc = Doc(en_vocab, words=words, deps=deco_labels, heads=proj_heads) + nonproj.deprojectivize(doc) + return [t.head.i for t in doc], [token.dep_ for token in doc] + + # fmt: off + tree = [1, 2, 2] + nonproj_tree = [1, 2, 2, 4, 5, 2, 7, 4, 2] + nonproj_tree2 = [9, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1] + cyclic_tree = [1, 2, 2, 4, 5, 3, 2] + labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", "acl", "punct"] + labels2 = ["advmod", "root", "det", "nsubj", "advmod", "det", "dobj", "det", "nmod", "aux", "nmod", "advmod", "det", "amod", "punct"] + cyclic_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "punct"] + # fmt: on + assert nonproj.decompose("X||Y") == ("X", "Y") + assert nonproj.decompose("X") == ("X", "") + assert nonproj.is_decorated("X||Y") is True + assert nonproj.is_decorated("X") is False + nonproj._lift(0, tree) + assert tree == [2, 2, 2] + assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree) == 7 + assert nonproj.get_smallest_nonproj_arc_slow(nonproj_tree2) == 10 + # fmt: off + proj_heads, deco_labels = nonproj.projectivize(nonproj_tree, labels) + with pytest.raises(ValueError, match=r'Found cycle in dependency graph: \[1, 2, 2, 4, 5, 3, 2\]'): + nonproj.projectivize(cyclic_tree, cyclic_labels) + assert proj_heads == [1, 2, 2, 4, 5, 2, 7, 5, 2] + assert deco_labels == ["det", "nsubj", "root", "det", "dobj", "aux", + "nsubj", "acl||dobj", "punct"] + deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels) + assert deproj_heads == nonproj_tree + assert undeco_labels == labels + proj_heads, deco_labels = nonproj.projectivize(nonproj_tree2, labels2) + assert proj_heads == [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1] + assert deco_labels == ["advmod||aux", "root", "det", "nsubj", "advmod", + "det", "dobj", "det", "nmod", "aux", "nmod||dobj", + "advmod", "det", "amod", "punct"] + deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels) + assert deproj_heads == nonproj_tree2 + assert undeco_labels == labels2 + # if decoration is wrong such that there is no head with the desired label + # the structure is kept and the label is undecorated + proj_heads = [1, 2, 2, 4, 5, 2, 7, 5, 2] + deco_labels = ["det", "nsubj", "root", "det", "dobj", "aux", "nsubj", + "acl||iobj", "punct"] + deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels) + assert deproj_heads == proj_heads + assert undeco_labels == ["det", "nsubj", "root", "det", "dobj", "aux", + "nsubj", "acl", "punct"] + # if there are two potential new heads, the first one is chosen even if + # it's wrong + proj_heads = [1, 1, 3, 1, 5, 6, 9, 8, 6, 1, 9, 12, 13, 10, 1] + deco_labels = ["advmod||aux", "root", "det", "aux", "advmod", "det", + "dobj", "det", "nmod", "aux", "nmod||dobj", "advmod", + "det", "amod", "punct"] + + deproj_heads, undeco_labels = deprojectivize(proj_heads, deco_labels) + assert deproj_heads == [3, 1, 3, 1, 5, 6, 9, 8, 6, 1, 6, 12, 13, 10, 1] + assert undeco_labels == ["advmod", "root", "det", "aux", "advmod", "det", + "dobj", "det", "nmod", "aux", "nmod", "advmod", + "det", "amod", "punct"] + # fmt: on diff --git a/spacy/tests/parser/test_parse.py b/spacy/tests/parser/test_parse.py new file mode 100644 index 0000000..3565c62 --- /dev/null +++ b/spacy/tests/parser/test_parse.py @@ -0,0 +1,544 @@ +import pytest +from numpy.testing import assert_equal +from thinc.api import Adam + +from spacy import registry, util +from spacy.attrs import DEP, NORM +from spacy.lang.en import English +from spacy.pipeline import DependencyParser +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + +from ..util import apply_transition_sequence, make_tempdir + +TRAIN_DATA = [ + ( + "They trade mortgage-backed securities.", + { + "heads": [1, 1, 4, 4, 5, 1, 1], + "deps": ["nsubj", "ROOT", "compound", "punct", "nmod", "dobj", "punct"], + }, + ), + ( + "I like London and Berlin.", + { + "heads": [1, 1, 1, 2, 2, 1], + "deps": ["nsubj", "ROOT", "dobj", "cc", "conj", "punct"], + }, + ), +] + + +CONFLICTING_DATA = [ + ( + "I like London and Berlin.", + { + "heads": [1, 1, 1, 2, 2, 1], + "deps": ["nsubj", "ROOT", "dobj", "cc", "conj", "punct"], + }, + ), + ( + "I like London and Berlin.", + { + "heads": [0, 0, 0, 0, 0, 0], + "deps": ["ROOT", "nsubj", "nsubj", "cc", "conj", "punct"], + }, + ), +] + +PARTIAL_DATA = [ + ( + "I like London.", + { + "heads": [1, 1, 1, None], + "deps": ["nsubj", "ROOT", "dobj", None], + }, + ), +] + +eps = 0.1 + + +@pytest.fixture +def vocab(): + return Vocab(lex_attr_getters={NORM: lambda s: s}) + + +@pytest.fixture +def parser(vocab): + vocab.strings.add("ROOT") + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(vocab, model) + parser.cfg["token_vector_width"] = 4 + parser.cfg["hidden_width"] = 32 + # parser.add_label('right') + parser.add_label("left") + parser.initialize(lambda: [_parser_example(parser)]) + sgd = Adam(0.001) + + for i in range(10): + losses = {} + doc = Doc(vocab, words=["a", "b", "c", "d"]) + example = Example.from_dict( + doc, {"heads": [1, 1, 3, 3], "deps": ["left", "ROOT", "left", "ROOT"]} + ) + parser.update([example], sgd=sgd, losses=losses) + return parser + + +def _parser_example(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + gold = {"heads": [1, 1, 3, 3], "deps": ["right", "ROOT", "left", "ROOT"]} + return Example.from_dict(doc, gold) + + +@pytest.mark.issue(2772) +def test_issue2772(en_vocab): + """Test that deprojectivization doesn't mess up sentence boundaries.""" + # fmt: off + words = ["When", "we", "write", "or", "communicate", "virtually", ",", "we", "can", "hide", "our", "true", "feelings", "."] + # fmt: on + # A tree with a non-projective (i.e. crossing) arc + # The arcs (0, 4) and (2, 9) cross. + heads = [4, 2, 9, 2, 2, 4, 9, 9, 9, 9, 12, 12, 9, 9] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert doc[1].is_sent_start is False + + +@pytest.mark.issue(3830) +def test_issue3830_no_subtok(): + """Test that the parser doesn't have subtok label if not learn_tokens""" + config = { + "learn_tokens": False, + } + model = registry.resolve({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(Vocab(), model, **config) + parser.add_label("nsubj") + assert "subtok" not in parser.labels + parser.initialize(lambda: [_parser_example(parser)]) + assert "subtok" not in parser.labels + + +@pytest.mark.issue(3830) +def test_issue3830_with_subtok(): + """Test that the parser does have subtok label if learn_tokens=True.""" + config = { + "learn_tokens": True, + } + model = registry.resolve({"model": DEFAULT_PARSER_MODEL}, validate=True)["model"] + parser = DependencyParser(Vocab(), model, **config) + parser.add_label("nsubj") + assert "subtok" not in parser.labels + parser.initialize(lambda: [_parser_example(parser)]) + assert "subtok" in parser.labels + + +@pytest.mark.issue(7716) +@pytest.mark.xfail(reason="Not fixed yet") +def test_partial_annotation(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[2].is_sent_start = False + # Note that if the following line is used, then doc[2].is_sent_start == False + # doc[3].is_sent_start = False + + doc = parser(doc) + assert doc[2].is_sent_start == False + + +def test_parser_root(en_vocab): + words = ["i", "do", "n't", "have", "other", "assistance"] + heads = [3, 3, 3, 3, 5, 3] + deps = ["nsubj", "aux", "neg", "ROOT", "amod", "dobj"] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + for t in doc: + assert t.dep != 0, t.text + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +@pytest.mark.parametrize("words", [["Hello"]]) +def test_parser_parse_one_word_sentence(en_vocab, en_parser, words): + doc = Doc(en_vocab, words=words, heads=[0], deps=["ROOT"]) + assert len(doc) == 1 + with en_parser.step_through(doc) as _: # noqa: F841 + pass + assert doc[0].dep != 0 + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_parser_initial(en_vocab, en_parser): + words = ["I", "ate", "the", "pizza", "with", "anchovies", "."] + transition = ["L-nsubj", "S", "L-det"] + doc = Doc(en_vocab, words=words) + apply_transition_sequence(en_parser, doc, transition) + assert doc[0].head.i == 1 + assert doc[1].head.i == 1 + assert doc[2].head.i == 3 + assert doc[3].head.i == 3 + + +def test_parser_parse_subtrees(en_vocab, en_parser): + words = ["The", "four", "wheels", "on", "the", "bus", "turned", "quickly"] + heads = [2, 2, 6, 2, 5, 3, 6, 6] + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert len(list(doc[2].lefts)) == 2 + assert len(list(doc[2].rights)) == 1 + assert len(list(doc[2].children)) == 3 + assert len(list(doc[5].lefts)) == 1 + assert len(list(doc[5].rights)) == 0 + assert len(list(doc[5].children)) == 1 + assert len(list(doc[2].subtree)) == 6 + + +def test_parser_merge_pp(en_vocab): + words = ["A", "phrase", "with", "another", "phrase", "occurs"] + heads = [1, 5, 1, 4, 2, 5] + deps = ["det", "nsubj", "prep", "det", "pobj", "ROOT"] + pos = ["DET", "NOUN", "ADP", "DET", "NOUN", "VERB"] + doc = Doc(en_vocab, words=words, deps=deps, heads=heads, pos=pos) + with doc.retokenize() as retokenizer: + for np in doc.noun_chunks: + retokenizer.merge(np, attrs={"lemma": np.lemma_}) + assert doc[0].text == "A phrase" + assert doc[1].text == "with" + assert doc[2].text == "another phrase" + assert doc[3].text == "occurs" + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_parser_arc_eager_finalize_state(en_vocab, en_parser): + words = ["a", "b", "c", "d", "e"] + # right branching + transition = ["R-nsubj", "D", "R-nsubj", "R-nsubj", "D", "R-ROOT"] + tokens = Doc(en_vocab, words=words) + apply_transition_sequence(en_parser, tokens, transition) + + assert tokens[0].n_lefts == 0 + assert tokens[0].n_rights == 2 + assert tokens[0].left_edge.i == 0 + assert tokens[0].right_edge.i == 4 + assert tokens[0].head.i == 0 + + assert tokens[1].n_lefts == 0 + assert tokens[1].n_rights == 0 + assert tokens[1].left_edge.i == 1 + assert tokens[1].right_edge.i == 1 + assert tokens[1].head.i == 0 + + assert tokens[2].n_lefts == 0 + assert tokens[2].n_rights == 2 + assert tokens[2].left_edge.i == 2 + assert tokens[2].right_edge.i == 4 + assert tokens[2].head.i == 0 + + assert tokens[3].n_lefts == 0 + assert tokens[3].n_rights == 0 + assert tokens[3].left_edge.i == 3 + assert tokens[3].right_edge.i == 3 + assert tokens[3].head.i == 2 + + assert tokens[4].n_lefts == 0 + assert tokens[4].n_rights == 0 + assert tokens[4].left_edge.i == 4 + assert tokens[4].right_edge.i == 4 + assert tokens[4].head.i == 2 + + # left branching + transition = ["S", "S", "S", "L-nsubj", "L-nsubj", "L-nsubj", "L-nsubj"] + tokens = Doc(en_vocab, words=words) + apply_transition_sequence(en_parser, tokens, transition) + + assert tokens[0].n_lefts == 0 + assert tokens[0].n_rights == 0 + assert tokens[0].left_edge.i == 0 + assert tokens[0].right_edge.i == 0 + assert tokens[0].head.i == 4 + + assert tokens[1].n_lefts == 0 + assert tokens[1].n_rights == 0 + assert tokens[1].left_edge.i == 1 + assert tokens[1].right_edge.i == 1 + assert tokens[1].head.i == 4 + + assert tokens[2].n_lefts == 0 + assert tokens[2].n_rights == 0 + assert tokens[2].left_edge.i == 2 + assert tokens[2].right_edge.i == 2 + assert tokens[2].head.i == 4 + + assert tokens[3].n_lefts == 0 + assert tokens[3].n_rights == 0 + assert tokens[3].left_edge.i == 3 + assert tokens[3].right_edge.i == 3 + assert tokens[3].head.i == 4 + + assert tokens[4].n_lefts == 4 + assert tokens[4].n_rights == 0 + assert tokens[4].left_edge.i == 0 + assert tokens[4].right_edge.i == 4 + assert tokens[4].head.i == 4 + + +def test_parser_set_sent_starts(en_vocab): + # fmt: off + words = ['Ein', 'Satz', '.', 'Außerdem', 'ist', 'Zimmer', 'davon', 'überzeugt', ',', 'dass', 'auch', 'epige-', '\n', 'netische', 'Mechanismen', 'eine', 'Rolle', 'spielen', ',', 'also', 'Vorgänge', ',', 'die', '\n', 'sich', 'darauf', 'auswirken', ',', 'welche', 'Gene', 'abgelesen', 'werden', 'und', '\n', 'welche', 'nicht', '.', '\n'] + heads = [1, 1, 1, 30, 4, 4, 7, 4, 7, 17, 14, 14, 11, 14, 17, 16, 17, 6, 17, 20, 11, 20, 26, 22, 26, 26, 20, 26, 29, 31, 31, 25, 31, 32, 17, 4, 4, 36] + deps = ['nk', 'ROOT', 'punct', 'mo', 'ROOT', 'sb', 'op', 'pd', 'punct', 'cp', 'mo', 'nk', '', 'nk', 'sb', 'nk', 'oa', 're', 'punct', 'mo', 'app', 'punct', 'sb', '', 'oa', 'op', 'rc', 'punct', 'nk', 'sb', 'oc', 're', 'cd', '', 'oa', 'ng', 'punct', ''] + # fmt: on + doc = Doc(en_vocab, words=words, deps=deps, heads=heads) + for i in range(len(words)): + if i == 0 or i == 3: + assert doc[i].is_sent_start is True + else: + assert doc[i].is_sent_start is False + for sent in doc.sents: + for token in sent: + assert token.head in sent + + +def test_parser_constructor(en_vocab): + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + } + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + DependencyParser(en_vocab, model, **config) + DependencyParser(en_vocab, model) + + +@pytest.mark.parametrize("pipe_name", ["parser", "beam_parser"]) +def test_incomplete_data(pipe_name): + # Test that the parser works with incomplete information + nlp = English() + parser = nlp.add_pipe(pipe_name) + train_examples = [] + for text, annotations in PARTIAL_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + if dep is not None: + parser.add_label(dep) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(150): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses[pipe_name] < 0.0001 + + # test the trained model + test_text = "I like securities." + doc = nlp(test_text) + assert doc[0].dep_ == "nsubj" + assert doc[2].dep_ == "dobj" + assert doc[0].head.i == 1 + assert doc[2].head.i == 1 + + +@pytest.mark.parametrize("pipe_name", ["parser", "beam_parser"]) +def test_overfitting_IO(pipe_name): + # Simple test to try and quickly overfit the dependency parser (normal or beam) + nlp = English() + parser = nlp.add_pipe(pipe_name) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + parser.add_label(dep) + optimizer = nlp.initialize() + # run overfitting + for i in range(200): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses[pipe_name] < 0.0001 + # test the trained model + test_text = "I like securities." + doc = nlp(test_text) + assert doc[0].dep_ == "nsubj" + assert doc[2].dep_ == "dobj" + assert doc[3].dep_ == "punct" + assert doc[0].head.i == 1 + assert doc[2].head.i == 1 + assert doc[3].head.i == 1 + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].dep_ == "nsubj" + assert doc2[2].dep_ == "dobj" + assert doc2[3].dep_ == "punct" + assert doc2[0].head.i == 1 + assert doc2[2].head.i == 1 + assert doc2[3].head.i == 1 + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Just a sentence.", + "Then one more sentence about London.", + "Here is another one.", + "I like London.", + ] + batch_deps_1 = [doc.to_array([DEP]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([DEP]) for doc in nlp.pipe(texts)] + no_batch_deps = [doc.to_array([DEP]) for doc in [nlp(text) for text in texts]] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + +# fmt: off +@pytest.mark.slow +@pytest.mark.parametrize("pipe_name", ["parser", "beam_parser"]) +@pytest.mark.parametrize( + "parser_config", + [ + # TransitionBasedParser V1 + ({"@architectures": "spacy.TransitionBasedParser.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}), + # TransitionBasedParser V2 + ({"@architectures": "spacy.TransitionBasedParser.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "state_type": "parser", "extra_state_tokens": False, "hidden_width": 64, "maxout_pieces": 2, "use_upper": True}), + ], +) +# fmt: on +def test_parser_configs(pipe_name, parser_config): + pipe_config = {"model": parser_config} + nlp = English() + parser = nlp.add_pipe(pipe_name, config=pipe_config) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + parser.add_label(dep) + optimizer = nlp.initialize() + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + +def test_beam_parser_scores(): + # Test that we can get confidence values out of the beam_parser pipe + beam_width = 16 + beam_density = 0.0001 + nlp = English() + config = { + "beam_width": beam_width, + "beam_density": beam_density, + } + parser = nlp.add_pipe("beam_parser", config=config) + train_examples = [] + for text, annotations in CONFLICTING_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + parser.add_label(dep) + optimizer = nlp.initialize() + + # update a bit with conflicting data + for i in range(10): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # test the scores from the beam + test_text = "I like securities." + doc = nlp.make_doc(test_text) + docs = [doc] + beams = parser.predict(docs) + head_scores, label_scores = parser.scored_parses(beams) + + for j in range(len(doc)): + for label in parser.labels: + label_score = label_scores[0][(j, label)] + assert 0 - eps <= label_score <= 1 + eps + for i in range(len(doc)): + head_score = head_scores[0][(j, i)] + assert 0 - eps <= head_score <= 1 + eps + + +def test_beam_overfitting_IO(): + # Simple test to try and quickly overfit the Beam dependency parser + nlp = English() + beam_width = 16 + beam_density = 0.0001 + config = { + "beam_width": beam_width, + "beam_density": beam_density, + } + parser = nlp.add_pipe("beam_parser", config=config) + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for dep in annotations.get("deps", []): + parser.add_label(dep) + optimizer = nlp.initialize() + # run overfitting + for i in range(150): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["beam_parser"] < 0.0001 + # test the scores from the beam + test_text = "I like securities." + docs = [nlp.make_doc(test_text)] + beams = parser.predict(docs) + head_scores, label_scores = parser.scored_parses(beams) + # we only processed one document + head_scores = head_scores[0] + label_scores = label_scores[0] + # test label annotations: 0=nsubj, 2=dobj, 3=punct + assert label_scores[(0, "nsubj")] == pytest.approx(1.0, abs=eps) + assert label_scores[(0, "dobj")] == pytest.approx(0.0, abs=eps) + assert label_scores[(0, "punct")] == pytest.approx(0.0, abs=eps) + assert label_scores[(2, "nsubj")] == pytest.approx(0.0, abs=eps) + assert label_scores[(2, "dobj")] == pytest.approx(1.0, abs=eps) + assert label_scores[(2, "punct")] == pytest.approx(0.0, abs=eps) + assert label_scores[(3, "nsubj")] == pytest.approx(0.0, abs=eps) + assert label_scores[(3, "dobj")] == pytest.approx(0.0, abs=eps) + assert label_scores[(3, "punct")] == pytest.approx(1.0, abs=eps) + # test head annotations: the root is token at index 1 + assert head_scores[(0, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores[(0, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores[(0, 2)] == pytest.approx(0.0, abs=eps) + assert head_scores[(2, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores[(2, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores[(2, 2)] == pytest.approx(0.0, abs=eps) + assert head_scores[(3, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores[(3, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores[(3, 2)] == pytest.approx(0.0, abs=eps) + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + docs2 = [nlp2.make_doc(test_text)] + parser2 = nlp2.get_pipe("beam_parser") + beams2 = parser2.predict(docs2) + head_scores2, label_scores2 = parser2.scored_parses(beams2) + # we only processed one document + head_scores2 = head_scores2[0] + label_scores2 = label_scores2[0] + # check the results again + assert label_scores2[(0, "nsubj")] == pytest.approx(1.0, abs=eps) + assert label_scores2[(0, "dobj")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(0, "punct")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(2, "nsubj")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(2, "dobj")] == pytest.approx(1.0, abs=eps) + assert label_scores2[(2, "punct")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(3, "nsubj")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(3, "dobj")] == pytest.approx(0.0, abs=eps) + assert label_scores2[(3, "punct")] == pytest.approx(1.0, abs=eps) + assert head_scores2[(0, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores2[(0, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores2[(0, 2)] == pytest.approx(0.0, abs=eps) + assert head_scores2[(2, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores2[(2, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores2[(2, 2)] == pytest.approx(0.0, abs=eps) + assert head_scores2[(3, 0)] == pytest.approx(0.0, abs=eps) + assert head_scores2[(3, 1)] == pytest.approx(1.0, abs=eps) + assert head_scores2[(3, 2)] == pytest.approx(0.0, abs=eps) diff --git a/spacy/tests/parser/test_parse_navigate.py b/spacy/tests/parser/test_parse_navigate.py new file mode 100644 index 0000000..d2f684f --- /dev/null +++ b/spacy/tests/parser/test_parse_navigate.py @@ -0,0 +1,126 @@ +import pytest + +from spacy.tokens import Doc + + +@pytest.fixture +def words(): + # fmt: off + return [ + "\n", "It", "was", "a", "bright", "cold", "day", "in", "April", ",", + "and", "the", "clocks", "were", "striking", "thirteen", ".", "\n", + "Winston", "Smith", ",", "his", "chin", "nuzzled", "into", "his", + "breast", "in", "an", "effort", "to", "escape", "the", "\n", "vile", + "wind", ",", "slipped", "quickly", "through", "the", "glass", "doors", + "of", "Victory", "Mansions", ",", "\n", "though", "not", "quickly", + "enough", "to", "prevent", "a", "swirl", "of", "gritty", "dust", + "from", "entering", "\n", "along", "with", "him", ".", "\n\n", "The", + "hallway", "smelt", "of", "boiled", "cabbage", "and", "old", "rag", + "mats", ".", "At", "one", "end", "of", "it", "a", "\n", "coloured", + "poster", ",", "too", "large", "for", "indoor", "display", ",", "had", + "been", "tacked", "to", "the", "wall", ".", "\n", "It", "depicted", + "simply", "an", "enormous", "face", ",", "more", "than", "a", "metre", + "wide", ":", "the", "face", "of", "a", "\n", "man", "of", "about", + "forty", "-", "five", ",", "with", "a", "heavy", "black", "moustache", + "and", "ruggedly", "handsome", "\n", "features", ".", "Winston", "made", + "for", "the", "stairs", ".", "It", "was", "no", "use", "trying", "the", + "lift", ".", "Even", "at", "\n", "the", "best", "of", "times", "it", + "was", "seldom", "working", ",", "and", "at", "present", "the", + "electric", "current", "\n", "was", "cut", "off", "during", "daylight", + "hours", ".", "It", "was", "part", "of", "the", "economy", "drive", + "in", "\n", "preparation", "for", "Hate", "Week", ".", "The", "flat", + "was", "seven", "flights", "up", ",", "and", "Winston", ",", "who", + "\n", "was", "thirty", "-", "nine", "and", "had", "a", "varicose", + "ulcer", "above", "his", "right", "ankle", ",", "went", "slowly", ",", + "\n", "resting", "several", "times", "on", "the", "way", ".", "On", + "each", "landing", ",", "opposite", "the", "lift", "-", "shaft", ",", + "\n", "the", "poster", "with", "the", "enormous", "face", "gazed", + "from", "the", "wall", ".", "It", "was", "one", "of", "those", "\n", + "pictures", "which", "are", "so", "contrived", "that", "the", "eyes", + "follow", "you", "about", "when", "you", "move", ".", "\n", "BIG", + "BROTHER", "IS", "WATCHING", "YOU", ",", "the", "caption", "beneath", + "it", "ran", ".", "\n", ] + # fmt: on + + +@pytest.fixture +def heads(): + # fmt: off + return [ + 1, 2, 2, 6, 6, 6, 2, 6, 7, 2, 2, 12, 14, 14, 2, 14, 14, 16, 19, 23, 23, + 22, 23, 23, 23, 26, 24, 23, 29, 27, 31, 29, 35, 32, 35, 31, 23, 23, 37, + 37, 42, 42, 39, 42, 45, 43, 37, 46, 37, 50, 51, 37, 53, 51, 55, 53, 55, + 58, 56, 53, 59, 60, 60, 62, 63, 23, 65, 68, 69, 69, 69, 72, 70, 72, 76, + 76, 72, 69, 96, 80, 78, 80, 81, 86, 83, 86, 96, 96, 89, 96, 89, 92, 90, + 96, 96, 96, 96, 96, 99, 97, 96, 100, 103, 103, 103, 107, 107, 103, 107, + 111, 111, 112, 113, 107, 103, 116, 136, 116, 120, 118, 117, 120, 125, + 125, 125, 121, 116, 116, 131, 131, 131, 127, 131, 134, 131, 134, 136, + 136, 139, 139, 139, 142, 140, 139, 145, 145, 147, 145, 147, 150, 148, + 145, 153, 162, 153, 156, 162, 156, 157, 162, 162, 162, 162, 162, 162, + 172, 165, 169, 169, 172, 169, 172, 162, 172, 172, 176, 174, 172, 179, + 179, 179, 180, 183, 181, 179, 184, 185, 185, 187, 190, 188, 179, 193, + 194, 194, 196, 194, 196, 194, 194, 218, 200, 204, 202, 200, 207, 207, + 204, 204, 204, 212, 212, 209, 212, 216, 216, 213, 200, 194, 218, 218, + 220, 218, 224, 222, 222, 227, 225, 218, 246, 231, 229, 246, 246, 237, + 237, 237, 233, 246, 238, 241, 246, 241, 245, 245, 242, 246, 246, 249, + 247, 246, 252, 252, 252, 253, 257, 255, 254, 259, 257, 261, 259, 265, + 264, 265, 261, 265, 265, 270, 270, 267, 252, 271, 274, 275, 275, 276, + 283, 283, 280, 283, 280, 281, 283, 283, 284] + # fmt: on + + +def test_parser_parse_navigate_consistency(en_vocab, words, heads): + doc = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads)) + for head in doc: + for child in head.lefts: + assert child.head == head + for child in head.rights: + assert child.head == head + + +def test_parser_parse_navigate_child_consistency(en_vocab, words, heads): + doc = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads)) + lefts = {} + rights = {} + for head in doc: + assert head.i not in lefts + lefts[head.i] = set() + for left in head.lefts: + lefts[head.i].add(left.i) + assert head.i not in rights + rights[head.i] = set() + for right in head.rights: + rights[head.i].add(right.i) + for head in doc: + assert head.n_rights == len(rights[head.i]) + assert head.n_lefts == len(lefts[head.i]) + for child in doc: + if child.i < child.head.i: + assert child.i in lefts[child.head.i] + assert child.i not in rights[child.head.i] + lefts[child.head.i].remove(child.i) + elif child.i > child.head.i: + assert child.i in rights[child.head.i] + assert child.i not in lefts[child.head.i] + rights[child.head.i].remove(child.i) + for head_index, children in lefts.items(): + assert not children + for head_index, children in rights.items(): + assert not children + + +def test_parser_parse_navigate_edges(en_vocab, words, heads): + doc = Doc(en_vocab, words=words, heads=heads, deps=["dep"] * len(heads)) + for token in doc: + subtree = list(token.subtree) + debug = "\t".join((token.text, token.left_edge.text, subtree[0].text)) + assert token.left_edge == subtree[0], debug + debug = "\t".join( + ( + token.text, + token.right_edge.text, + subtree[-1].text, + token.right_edge.head.text, + ) + ) + assert token.right_edge == subtree[-1], debug diff --git a/spacy/tests/parser/test_preset_sbd.py b/spacy/tests/parser/test_preset_sbd.py new file mode 100644 index 0000000..dcbb967 --- /dev/null +++ b/spacy/tests/parser/test_preset_sbd.py @@ -0,0 +1,88 @@ +import pytest +from thinc.api import Adam + +from spacy import registry +from spacy.attrs import NORM +from spacy.pipeline import DependencyParser +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.tokens import Doc +from spacy.training import Example +from spacy.vocab import Vocab + + +@pytest.fixture +def vocab(): + return Vocab(lex_attr_getters={NORM: lambda s: s}) + + +def _parser_example(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + gold = {"heads": [1, 1, 3, 3], "deps": ["right", "ROOT", "left", "ROOT"]} + return Example.from_dict(doc, gold) + + +@pytest.fixture +def parser(vocab): + vocab.strings.add("ROOT") + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(vocab, model) + parser.cfg["token_vector_width"] = 4 + parser.cfg["hidden_width"] = 32 + # parser.add_label('right') + parser.add_label("left") + parser.initialize(lambda: [_parser_example(parser)]) + sgd = Adam(0.001) + + for i in range(10): + losses = {} + doc = Doc(vocab, words=["a", "b", "c", "d"]) + example = Example.from_dict( + doc, {"heads": [1, 1, 3, 3], "deps": ["left", "ROOT", "left", "ROOT"]} + ) + parser.update([example], sgd=sgd, losses=losses) + return parser + + +def test_no_sentences(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc = parser(doc) + assert len(list(doc.sents)) >= 1 + + +def test_sents_1(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[2].sent_start = True + doc = parser(doc) + assert len(list(doc.sents)) >= 2 + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[1].sent_start = False + doc[2].sent_start = True + doc[3].sent_start = False + doc = parser(doc) + assert len(list(doc.sents)) == 2 + + +def test_sents_1_2(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[1].sent_start = True + doc[2].sent_start = True + doc = parser(doc) + assert len(list(doc.sents)) >= 3 + + +def test_sents_1_3(parser): + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[0].is_sent_start = True + doc[1].is_sent_start = True + doc[2].is_sent_start = None + doc[3].is_sent_start = True + doc = parser(doc) + assert len(list(doc.sents)) >= 3 + doc = Doc(parser.vocab, words=["a", "b", "c", "d"]) + doc[0].is_sent_start = True + doc[1].is_sent_start = True + doc[2].is_sent_start = False + doc[3].is_sent_start = True + doc = parser(doc) + assert len(list(doc.sents)) == 3 diff --git a/spacy/tests/parser/test_space_attachment.py b/spacy/tests/parser/test_space_attachment.py new file mode 100644 index 0000000..30e66b3 --- /dev/null +++ b/spacy/tests/parser/test_space_attachment.py @@ -0,0 +1,79 @@ +import pytest + +from spacy.tokens import Doc + +from ..util import apply_transition_sequence + + +def test_parser_space_attachment(en_vocab): + # fmt: off + words = ["This", "is", "a", "test", ".", "\n", "To", "ensure", " ", "spaces", "are", "attached", "well", "."] + heads = [1, 1, 3, 1, 1, 4, 7, 11, 7, 11, 11, 11, 11, 11] + # fmt: on + deps = ["dep"] * len(heads) + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + for sent in doc.sents: + if len(sent) == 1: + assert not sent[-1].is_space + + +def test_parser_sentence_space(en_vocab): + # fmt: off + words = ["I", "look", "forward", "to", "using", "Thingamajig", ".", " ", "I", "'ve", "been", "told", "it", "will", "make", "my", "life", "easier", "..."] + heads = [1, 1, 1, 1, 3, 4, 1, 6, 11, 11, 11, 11, 14, 14, 11, 16, 17, 14, 11] + deps = ["nsubj", "ROOT", "advmod", "prep", "pcomp", "dobj", "punct", "", + "nsubjpass", "aux", "auxpass", "ROOT", "nsubj", "aux", "ccomp", + "poss", "nsubj", "ccomp", "punct"] + # fmt: on + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + assert len(list(doc.sents)) == 2 + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_parser_space_attachment_leading(en_vocab, en_parser): + words = ["\t", "\n", "This", "is", "a", "sentence", "."] + heads = [1, 2, 2, 4, 2, 2] + doc = Doc(en_vocab, words=words, heads=heads) + assert doc[0].is_space + assert doc[1].is_space + assert doc[2].text == "This" + with en_parser.step_through(doc) as stepwise: + pass + assert doc[0].head.i == 2 + assert doc[1].head.i == 2 + assert stepwise.stack == set([2]) + + +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_parser_space_attachment_intermediate_trailing(en_vocab, en_parser): + words = ["This", "is", "\t", "a", "\t\n", "\n", "sentence", ".", "\n\n", "\n"] + heads = [1, 1, 1, 5, 3, 1, 1, 6] + transition = ["L-nsubj", "S", "L-det", "R-attr", "D", "R-punct"] + doc = Doc(en_vocab, words=words, heads=heads) + assert doc[2].is_space + assert doc[4].is_space + assert doc[5].is_space + assert doc[8].is_space + assert doc[9].is_space + apply_transition_sequence(en_parser, doc, transition) + for token in doc: + assert token.dep != 0 or token.is_space + assert [token.head.i for token in doc] == [1, 1, 1, 6, 3, 3, 1, 1, 7, 7] + + +@pytest.mark.parametrize("text,length", [(["\n"], 1), (["\n", "\t", "\n\n", "\t"], 4)]) +@pytest.mark.skip( + reason="The step_through API was removed (but should be brought back)" +) +def test_parser_space_attachment_space(en_parser, text, length): + doc = Doc(en_parser.vocab, words=text) + assert len(doc) == length + with en_parser.step_through(doc) as _: # noqa: F841 + pass + assert doc[0].is_space + for token in doc: + assert token.head.i == length - 1 diff --git a/spacy/tests/parser/test_state.py b/spacy/tests/parser/test_state.py new file mode 100644 index 0000000..0febc3d --- /dev/null +++ b/spacy/tests/parser/test_state.py @@ -0,0 +1,79 @@ +import pytest + +from spacy.pipeline._parser_internals.stateclass import StateClass +from spacy.tokens.doc import Doc +from spacy.vocab import Vocab + + +@pytest.fixture +def vocab(): + return Vocab() + + +@pytest.fixture +def doc(vocab): + return Doc(vocab, words=["a", "b", "c", "d"]) + + +def test_init_state(doc): + state = StateClass(doc) + assert state.stack == [] + assert state.queue == list(range(len(doc))) + assert not state.is_final() + assert state.buffer_length() == 4 + + +def test_push_pop(doc): + state = StateClass(doc) + state.push() + assert state.buffer_length() == 3 + assert state.stack == [0] + assert 0 not in state.queue + state.push() + assert state.stack == [1, 0] + assert 1 not in state.queue + assert state.buffer_length() == 2 + state.pop() + assert state.stack == [0] + assert 1 not in state.queue + + +def test_stack_depth(doc): + state = StateClass(doc) + assert state.stack_depth() == 0 + assert state.buffer_length() == len(doc) + state.push() + assert state.buffer_length() == 3 + assert state.stack_depth() == 1 + + +def test_H(doc): + state = StateClass(doc) + assert state.H(0) == -1 + state.add_arc(1, 0, 0) + assert state.arcs == [{"head": 1, "child": 0, "label": 0}] + assert state.H(0) == 1 + state.add_arc(3, 1, 0) + assert state.H(1) == 3 + + +def test_L(doc): + state = StateClass(doc) + assert state.L(2, 1) == -1 + state.add_arc(2, 1, 0) + assert state.arcs == [{"head": 2, "child": 1, "label": 0}] + assert state.L(2, 1) == 1 + state.add_arc(2, 0, 0) + assert state.L(2, 1) == 0 + assert state.n_L(2) == 2 + + +def test_R(doc): + state = StateClass(doc) + assert state.R(0, 1) == -1 + state.add_arc(0, 1, 0) + assert state.arcs == [{"head": 0, "child": 1, "label": 0}] + assert state.R(0, 1) == 1 + state.add_arc(0, 2, 0) + assert state.R(0, 1) == 2 + assert state.n_R(0) == 2 diff --git a/spacy/tests/pipeline/__init__.py b/spacy/tests/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/pipeline/test_analysis.py b/spacy/tests/pipeline/test_analysis.py new file mode 100644 index 0000000..503b501 --- /dev/null +++ b/spacy/tests/pipeline/test_analysis.py @@ -0,0 +1,115 @@ +import pytest +from mock import Mock + +from spacy.language import Language +from spacy.pipe_analysis import get_attr_info, validate_attrs + + +def test_component_decorator_assigns(): + @Language.component("c1", assigns=["token.tag", "doc.tensor"]) + def test_component1(doc): + return doc + + @Language.component( + "c2", requires=["token.tag", "token.pos"], assigns=["token.lemma", "doc.tensor"] + ) + def test_component2(doc): + return doc + + @Language.component( + "c3", requires=["token.lemma"], assigns=["token._.custom_lemma"] + ) + def test_component3(doc): + return doc + + assert Language.has_factory("c1") + assert Language.has_factory("c2") + assert Language.has_factory("c3") + + nlp = Language() + nlp.add_pipe("c1") + nlp.add_pipe("c2") + problems = nlp.analyze_pipes()["problems"] + assert problems["c2"] == ["token.pos"] + nlp.add_pipe("c3") + assert get_attr_info(nlp, "doc.tensor")["assigns"] == ["c1", "c2"] + nlp.add_pipe("c1", name="c4") + test_component4_meta = nlp.get_pipe_meta("c1") + assert test_component4_meta.factory == "c1" + assert nlp.pipe_names == ["c1", "c2", "c3", "c4"] + assert not Language.has_factory("c4") + assert nlp.pipe_factories["c1"] == "c1" + assert nlp.pipe_factories["c4"] == "c1" + assert get_attr_info(nlp, "doc.tensor")["assigns"] == ["c1", "c2", "c4"] + assert get_attr_info(nlp, "token.pos")["requires"] == ["c2"] + assert nlp("hello world") + + +def test_component_factories_class_func(): + """Test that class components can implement a from_nlp classmethod that + gives them access to the nlp object and config via the factory.""" + + class TestComponent5: + def __call__(self, doc): + return doc + + mock = Mock() + mock.return_value = TestComponent5() + + def test_componen5_factory(nlp, foo: str = "bar", name="c5"): + return mock(nlp, foo=foo) + + Language.factory("c5", func=test_componen5_factory) + assert Language.has_factory("c5") + nlp = Language() + nlp.add_pipe("c5", config={"foo": "bar"}) + assert nlp("hello world") + mock.assert_called_once_with(nlp, foo="bar") + + +def test_analysis_validate_attrs_valid(): + attrs = ["doc.sents", "doc.ents", "token.tag", "token._.xyz", "span._.xyz"] + assert validate_attrs(attrs) + for attr in attrs: + assert validate_attrs([attr]) + with pytest.raises(ValueError): + validate_attrs(["doc.sents", "doc.xyz"]) + + +@pytest.mark.parametrize( + "attr", + [ + "doc", + "doc_ents", + "doc.xyz", + "token.xyz", + "token.tag_", + "token.tag.xyz", + "token._.xyz.abc", + "span.label", + ], +) +def test_analysis_validate_attrs_invalid(attr): + with pytest.raises(ValueError): + validate_attrs([attr]) + + +def test_analysis_validate_attrs_remove_pipe(): + """Test that attributes are validated correctly on remove.""" + + @Language.component("pipe_analysis_c6", assigns=["token.tag"]) + def c1(doc): + return doc + + @Language.component("pipe_analysis_c7", requires=["token.pos"]) + def c2(doc): + return doc + + nlp = Language() + nlp.add_pipe("pipe_analysis_c6") + nlp.add_pipe("pipe_analysis_c7") + problems = nlp.analyze_pipes()["problems"] + assert problems["pipe_analysis_c7"] == ["token.pos"] + nlp.remove_pipe("pipe_analysis_c7") + problems = nlp.analyze_pipes()["problems"] + assert all(p == [] for p in problems.values()) diff --git a/spacy/tests/pipeline/test_annotates_on_update.py b/spacy/tests/pipeline/test_annotates_on_update.py new file mode 100644 index 0000000..d4feebd --- /dev/null +++ b/spacy/tests/pipeline/test_annotates_on_update.py @@ -0,0 +1,113 @@ +from typing import Callable, Iterable, Iterator + +import pytest +from thinc.api import Config + +from spacy.lang.en import English +from spacy.language import Language +from spacy.training import Example +from spacy.training.loop import train +from spacy.util import load_model_from_config, registry + + +@pytest.fixture +def config_str(): + return """ + [nlp] + lang = "en" + pipeline = ["sentencizer","assert_sents"] + disabled = [] + before_creation = null + after_creation = null + after_pipeline_creation = null + batch_size = 1000 + tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"} + + [components] + + [components.assert_sents] + factory = "assert_sents" + + [components.sentencizer] + factory = "sentencizer" + punct_chars = null + + [training] + dev_corpus = "corpora.dev" + train_corpus = "corpora.train" + annotating_components = ["sentencizer"] + max_steps = 2 + + [corpora] + + [corpora.dev] + @readers = "unannotated_corpus" + + [corpora.train] + @readers = "unannotated_corpus" + """ + + +def test_annotates_on_update(): + # The custom component checks for sentence annotation + @Language.factory("assert_sents", default_config={}) + def assert_sents(nlp, name): + return AssertSents(name) + + class AssertSents: + def __init__(self, name, **cfg): + self.name = name + pass + + def __call__(self, doc): + if not doc.has_annotation("SENT_START"): + raise ValueError("No sents") + return doc + + def update(self, examples, *, drop=0.0, sgd=None, losses=None): + for example in examples: + if not example.predicted.has_annotation("SENT_START"): + raise ValueError("No sents") + return {} + + nlp = English() + nlp.add_pipe("sentencizer") + nlp.add_pipe("assert_sents") + + # When the pipeline runs, annotations are set + nlp("This is a sentence.") + + examples = [] + for text in ["a a", "b b", "c c"]: + examples.append(Example(nlp.make_doc(text), nlp(text))) + + for example in examples: + assert not example.predicted.has_annotation("SENT_START") + + # If updating without setting annotations, assert_sents will raise an error + with pytest.raises(ValueError): + nlp.update(examples) + + # Updating while setting annotations for the sentencizer succeeds + nlp.update(examples, annotates=["sentencizer"]) + + +def test_annotating_components_from_config(config_str): + @registry.readers("unannotated_corpus") + def create_unannotated_corpus() -> Callable[[Language], Iterable[Example]]: + return UnannotatedCorpus() + + class UnannotatedCorpus: + def __call__(self, nlp: Language) -> Iterator[Example]: + for text in ["a a", "b b", "c c"]: + doc = nlp.make_doc(text) + yield Example(doc, doc) + + orig_config = Config().from_str(config_str) + nlp = load_model_from_config(orig_config, auto_fill=True, validate=True) + assert nlp.config["training"]["annotating_components"] == ["sentencizer"] + train(nlp) + + nlp.config["training"]["annotating_components"] = [] + with pytest.raises(ValueError): + train(nlp) diff --git a/spacy/tests/pipeline/test_attributeruler.py b/spacy/tests/pipeline/test_attributeruler.py new file mode 100644 index 0000000..06587b4 --- /dev/null +++ b/spacy/tests/pipeline/test_attributeruler.py @@ -0,0 +1,294 @@ +import numpy +import pytest + +from spacy import registry, util +from spacy.lang.en import English +from spacy.pipeline import AttributeRuler +from spacy.tokens import Doc +from spacy.training import Example + +from ..util import make_tempdir + + +@pytest.fixture +def nlp(): + return English() + + +@pytest.fixture +def pattern_dicts(): + return [ + { + "patterns": [[{"ORTH": "a"}], [{"ORTH": "irrelevant"}]], + "attrs": {"LEMMA": "the", "MORPH": "Case=Nom|Number=Plur"}, + }, + # one pattern sets the lemma + {"patterns": [[{"ORTH": "test"}]], "attrs": {"LEMMA": "cat"}}, + # another pattern sets the morphology + { + "patterns": [[{"ORTH": "test"}]], + "attrs": {"MORPH": "Case=Nom|Number=Sing"}, + "index": 0, + }, + ] + + +@pytest.fixture +def tag_map(): + return { + ".": {"POS": "PUNCT", "PunctType": "peri"}, + ",": {"POS": "PUNCT", "PunctType": "comm"}, + } + + +@pytest.fixture +def morph_rules(): + return {"DT": {"the": {"POS": "DET", "LEMMA": "a", "Case": "Nom"}}} + + +def check_tag_map(ruler): + doc = Doc( + ruler.vocab, + words=["This", "is", "a", "test", "."], + tags=["DT", "VBZ", "DT", "NN", "."], + ) + doc = ruler(doc) + for i in range(len(doc)): + if i == 4: + assert doc[i].pos_ == "PUNCT" + assert str(doc[i].morph) == "PunctType=peri" + else: + assert doc[i].pos_ == "" + assert str(doc[i].morph) == "" + + +def check_morph_rules(ruler): + doc = Doc( + ruler.vocab, + words=["This", "is", "the", "test", "."], + tags=["DT", "VBZ", "DT", "NN", "."], + ) + doc = ruler(doc) + for i in range(len(doc)): + if i != 2: + assert doc[i].pos_ == "" + assert str(doc[i].morph) == "" + else: + assert doc[2].pos_ == "DET" + assert doc[2].lemma_ == "a" + assert str(doc[2].morph) == "Case=Nom" + + +def test_attributeruler_init(nlp, pattern_dicts): + a = nlp.add_pipe("attribute_ruler") + for p in pattern_dicts: + a.add(**p) + doc = nlp("This is a test.") + assert doc[2].lemma_ == "the" + assert str(doc[2].morph) == "Case=Nom|Number=Plur" + assert doc[3].lemma_ == "cat" + assert str(doc[3].morph) == "Case=Nom|Number=Sing" + assert doc.has_annotation("LEMMA") + assert doc.has_annotation("MORPH") + + +def test_attributeruler_init_patterns(nlp, pattern_dicts): + # initialize with patterns + ruler = nlp.add_pipe("attribute_ruler") + ruler.initialize(lambda: [], patterns=pattern_dicts) + doc = nlp("This is a test.") + assert doc[2].lemma_ == "the" + assert str(doc[2].morph) == "Case=Nom|Number=Plur" + assert doc[3].lemma_ == "cat" + assert str(doc[3].morph) == "Case=Nom|Number=Sing" + assert doc.has_annotation("LEMMA") + assert doc.has_annotation("MORPH") + nlp.remove_pipe("attribute_ruler") + + # initialize with patterns from misc registry + @registry.misc("attribute_ruler_patterns") + def attribute_ruler_patterns(): + return [ + { + "patterns": [[{"ORTH": "a"}], [{"ORTH": "irrelevant"}]], + "attrs": {"LEMMA": "the", "MORPH": "Case=Nom|Number=Plur"}, + }, + # one pattern sets the lemma + {"patterns": [[{"ORTH": "test"}]], "attrs": {"LEMMA": "cat"}}, + # another pattern sets the morphology + { + "patterns": [[{"ORTH": "test"}]], + "attrs": {"MORPH": "Case=Nom|Number=Sing"}, + "index": 0, + }, + ] + + nlp.config["initialize"]["components"]["attribute_ruler"] = { + "patterns": {"@misc": "attribute_ruler_patterns"} + } + nlp.add_pipe("attribute_ruler") + nlp.initialize() + doc = nlp("This is a test.") + assert doc[2].lemma_ == "the" + assert str(doc[2].morph) == "Case=Nom|Number=Plur" + assert doc[3].lemma_ == "cat" + assert str(doc[3].morph) == "Case=Nom|Number=Sing" + assert doc.has_annotation("LEMMA") + assert doc.has_annotation("MORPH") + + +def test_attributeruler_init_clear(nlp, pattern_dicts): + """Test that initialization clears patterns.""" + ruler = nlp.add_pipe("attribute_ruler") + assert not len(ruler.matcher) + ruler.add_patterns(pattern_dicts) + assert len(ruler.matcher) + ruler.initialize(lambda: []) + assert not len(ruler.matcher) + + +def test_attributeruler_score(nlp, pattern_dicts): + # initialize with patterns + ruler = nlp.add_pipe("attribute_ruler") + ruler.initialize(lambda: [], patterns=pattern_dicts) + doc = nlp("This is a test.") + assert doc[2].lemma_ == "the" + assert str(doc[2].morph) == "Case=Nom|Number=Plur" + assert doc[3].lemma_ == "cat" + assert str(doc[3].morph) == "Case=Nom|Number=Sing" + doc = nlp.make_doc("This is a test.") + dev_examples = [Example.from_dict(doc, {"lemmas": ["this", "is", "a", "cat", "."]})] + scores = nlp.evaluate(dev_examples) + # "cat" is the only correct lemma + assert scores["lemma_acc"] == pytest.approx(0.2) + # no morphs are set + assert scores["morph_acc"] is None + nlp.remove_pipe("attribute_ruler") + + # test with custom scorer + @registry.misc("weird_scorer.v1") + def make_weird_scorer(): + def weird_scorer(examples, weird_score, **kwargs): + return {"weird_score": weird_score} + + return weird_scorer + + ruler = nlp.add_pipe( + "attribute_ruler", config={"scorer": {"@misc": "weird_scorer.v1"}} + ) + ruler.initialize(lambda: [], patterns=pattern_dicts) + scores = nlp.evaluate(dev_examples, scorer_cfg={"weird_score": 0.12345}) + assert scores["weird_score"] == 0.12345 + assert "token_acc" in scores + assert "lemma_acc" not in scores + scores = nlp.evaluate(dev_examples, scorer_cfg={"weird_score": 0.23456}) + assert scores["weird_score"] == 0.23456 + + +def test_attributeruler_rule_order(nlp): + a = AttributeRuler(nlp.vocab) + patterns = [ + {"patterns": [[{"TAG": "VBZ"}]], "attrs": {"POS": "VERB"}}, + {"patterns": [[{"TAG": "VBZ"}]], "attrs": {"POS": "NOUN"}}, + ] + a.add_patterns(patterns) + doc = Doc( + nlp.vocab, + words=["This", "is", "a", "test", "."], + tags=["DT", "VBZ", "DT", "NN", "."], + ) + doc = a(doc) + assert doc[1].pos_ == "NOUN" + + +def test_attributeruler_tag_map(nlp, tag_map): + ruler = AttributeRuler(nlp.vocab) + ruler.load_from_tag_map(tag_map) + check_tag_map(ruler) + + +def test_attributeruler_tag_map_initialize(nlp, tag_map): + ruler = nlp.add_pipe("attribute_ruler") + ruler.initialize(lambda: [], tag_map=tag_map) + check_tag_map(ruler) + + +def test_attributeruler_morph_rules(nlp, morph_rules): + ruler = AttributeRuler(nlp.vocab) + ruler.load_from_morph_rules(morph_rules) + check_morph_rules(ruler) + + +def test_attributeruler_morph_rules_initialize(nlp, morph_rules): + ruler = nlp.add_pipe("attribute_ruler") + ruler.initialize(lambda: [], morph_rules=morph_rules) + check_morph_rules(ruler) + + +def test_attributeruler_indices(nlp): + a = nlp.add_pipe("attribute_ruler") + a.add( + [[{"ORTH": "a"}, {"ORTH": "test"}]], + {"LEMMA": "the", "MORPH": "Case=Nom|Number=Plur"}, + index=0, + ) + a.add( + [[{"ORTH": "This"}, {"ORTH": "is"}]], + {"LEMMA": "was", "MORPH": "Case=Nom|Number=Sing"}, + index=1, + ) + a.add([[{"ORTH": "a"}, {"ORTH": "test"}]], {"LEMMA": "cat"}, index=-1) + + text = "This is a test." + doc = nlp(text) + for i in range(len(doc)): + if i == 1: + assert doc[i].lemma_ == "was" + assert str(doc[i].morph) == "Case=Nom|Number=Sing" + elif i == 2: + assert doc[i].lemma_ == "the" + assert str(doc[i].morph) == "Case=Nom|Number=Plur" + elif i == 3: + assert doc[i].lemma_ == "cat" + else: + assert str(doc[i].morph) == "" + # raises an error when trying to modify a token outside of the match + a.add([[{"ORTH": "a"}, {"ORTH": "test"}]], {"LEMMA": "cat"}, index=2) + with pytest.raises(ValueError): + doc = nlp(text) + # raises an error when trying to modify a token outside of the match + a.add([[{"ORTH": "a"}, {"ORTH": "test"}]], {"LEMMA": "cat"}, index=10) + with pytest.raises(ValueError): + doc = nlp(text) + + +def test_attributeruler_patterns_prop(nlp, pattern_dicts): + a = nlp.add_pipe("attribute_ruler") + a.add_patterns(pattern_dicts) + for p1, p2 in zip(pattern_dicts, a.patterns): + assert p1["patterns"] == p2["patterns"] + assert p1["attrs"] == p2["attrs"] + if p1.get("index"): + assert p1["index"] == p2["index"] + + +def test_attributeruler_serialize(nlp, pattern_dicts): + a = nlp.add_pipe("attribute_ruler") + a.add_patterns(pattern_dicts) + text = "This is a test." + attrs = ["ORTH", "LEMMA", "MORPH"] + doc = nlp(text) + # bytes roundtrip + a_reloaded = AttributeRuler(nlp.vocab).from_bytes(a.to_bytes()) + assert a.to_bytes() == a_reloaded.to_bytes() + doc1 = a_reloaded(nlp.make_doc(text)) + numpy.array_equal(doc.to_array(attrs), doc1.to_array(attrs)) + assert a.patterns == a_reloaded.patterns + # disk roundtrip + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(text) + assert nlp2.get_pipe("attribute_ruler").to_bytes() == a.to_bytes() + assert numpy.array_equal(doc.to_array(attrs), doc2.to_array(attrs)) + assert a.patterns == nlp2.get_pipe("attribute_ruler").patterns diff --git a/spacy/tests/pipeline/test_edit_tree_lemmatizer.py b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py new file mode 100644 index 0000000..5a8f0ae --- /dev/null +++ b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py @@ -0,0 +1,333 @@ +import pickle + +import hypothesis.strategies as st +import pytest +from hypothesis import given + +from spacy import util +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline._edit_tree_internals.edit_trees import EditTrees +from spacy.strings import StringStore +from spacy.training import Example +from spacy.util import make_tempdir + +TRAIN_DATA = [ + ("She likes green eggs", {"lemmas": ["she", "like", "green", "egg"]}), + ("Eat blue ham", {"lemmas": ["eat", "blue", "ham"]}), +] + +PARTIAL_DATA = [ + # partial annotation + ("She likes green eggs", {"lemmas": ["", "like", "green", ""]}), + # misaligned partial annotation + ( + "He hates green eggs", + { + "words": ["He", "hat", "es", "green", "eggs"], + "lemmas": ["", "hat", "e", "green", ""], + }, + ), +] + + +def test_initialize_examples(): + nlp = Language() + lemmatizer = nlp.add_pipe("trainable_lemmatizer") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + # you shouldn't really call this more than once, but for testing it should be fine + nlp.initialize(get_examples=lambda: train_examples) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: None) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: train_examples[0]) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: []) + with pytest.raises(TypeError): + nlp.initialize(get_examples=train_examples) + + +def test_initialize_from_labels(): + nlp = Language() + lemmatizer = nlp.add_pipe("trainable_lemmatizer") + lemmatizer.min_tree_freq = 1 + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + nlp.initialize(get_examples=lambda: train_examples) + + nlp2 = Language() + lemmatizer2 = nlp2.add_pipe("trainable_lemmatizer") + lemmatizer2.initialize( + # We want to check that the strings in replacement nodes are + # added to the string store. Avoid that they get added through + # the examples. + get_examples=lambda: train_examples[:1], + labels=lemmatizer.label_data, + ) + assert lemmatizer2.tree2label == {1: 0, 3: 1, 4: 2, 6: 3} + assert lemmatizer2.label_data == { + "trees": [ + {"orig": "S", "subst": "s"}, + { + "prefix_len": 1, + "suffix_len": 0, + "prefix_tree": 0, + "suffix_tree": 4294967295, + }, + {"orig": "s", "subst": ""}, + { + "prefix_len": 0, + "suffix_len": 1, + "prefix_tree": 4294967295, + "suffix_tree": 2, + }, + { + "prefix_len": 0, + "suffix_len": 0, + "prefix_tree": 4294967295, + "suffix_tree": 4294967295, + }, + {"orig": "E", "subst": "e"}, + { + "prefix_len": 1, + "suffix_len": 0, + "prefix_tree": 5, + "suffix_tree": 4294967295, + }, + ], + "labels": (1, 3, 4, 6), + } + + +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_no_data(top_k): + # Test that the lemmatizer provides a nice error when there's no tagging data / labels + TEXTCAT_DATA = [ + ("I'm so happy.", {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}), + ("I'm so angry", {"cats": {"POSITIVE": 0.0, "NEGATIVE": 1.0}}), + ] + nlp = English() + nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) + nlp.add_pipe("textcat") + + train_examples = [] + for t in TEXTCAT_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + with pytest.raises(ValueError): + nlp.initialize(get_examples=lambda: train_examples) + + +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_incomplete_data(top_k): + # Test that the lemmatizer works with incomplete information + nlp = English() + lemmatizer = nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) + lemmatizer.min_tree_freq = 1 + train_examples = [] + for t in PARTIAL_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["trainable_lemmatizer"] < 0.00001 + + # test the trained model + test_text = "She likes blue eggs" + doc = nlp(test_text) + assert doc[1].lemma_ == "like" + assert doc[2].lemma_ == "blue" + + # Check that incomplete annotations are ignored. + scores, _ = lemmatizer.model([eg.predicted for eg in train_examples], is_train=True) + _, dX = lemmatizer.get_loss(train_examples, scores) + xp = lemmatizer.model.ops.xp + + # Missing annotations. + assert xp.count_nonzero(dX[0][0]) == 0 + assert xp.count_nonzero(dX[0][3]) == 0 + assert xp.count_nonzero(dX[1][0]) == 0 + assert xp.count_nonzero(dX[1][3]) == 0 + + # Misaligned annotations. + assert xp.count_nonzero(dX[1][1]) == 0 + + +@pytest.mark.parametrize("top_k", (1, 5, 30)) +def test_overfitting_IO(top_k): + nlp = English() + lemmatizer = nlp.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) + lemmatizer.min_tree_freq = 1 + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["trainable_lemmatizer"] < 0.00001 + + test_text = "She likes blue eggs" + doc = nlp(test_text) + assert doc[0].lemma_ == "she" + assert doc[1].lemma_ == "like" + assert doc[2].lemma_ == "blue" + assert doc[3].lemma_ == "egg" + + # Check model after a {to,from}_disk roundtrip + with util.make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].lemma_ == "she" + assert doc2[1].lemma_ == "like" + assert doc2[2].lemma_ == "blue" + assert doc2[3].lemma_ == "egg" + + # Check model after a {to,from}_bytes roundtrip + nlp_bytes = nlp.to_bytes() + nlp3 = English() + nlp3.add_pipe("trainable_lemmatizer", config={"top_k": top_k}) + nlp3.from_bytes(nlp_bytes) + doc3 = nlp3(test_text) + assert doc3[0].lemma_ == "she" + assert doc3[1].lemma_ == "like" + assert doc3[2].lemma_ == "blue" + assert doc3[3].lemma_ == "egg" + + # Check model after a pickle roundtrip. + nlp_bytes = pickle.dumps(nlp) + nlp4 = pickle.loads(nlp_bytes) + doc4 = nlp4(test_text) + assert doc4[0].lemma_ == "she" + assert doc4[1].lemma_ == "like" + assert doc4[2].lemma_ == "blue" + assert doc4[3].lemma_ == "egg" + + +def test_lemmatizer_requires_labels(): + nlp = English() + nlp.add_pipe("trainable_lemmatizer") + with pytest.raises(ValueError): + nlp.initialize() + + +def test_lemmatizer_label_data(): + nlp = English() + lemmatizer = nlp.add_pipe("trainable_lemmatizer") + lemmatizer.min_tree_freq = 1 + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + nlp.initialize(get_examples=lambda: train_examples) + + nlp2 = English() + lemmatizer2 = nlp2.add_pipe("trainable_lemmatizer") + lemmatizer2.initialize( + get_examples=lambda: train_examples, labels=lemmatizer.label_data + ) + + # Verify that the labels and trees are the same. + assert lemmatizer.labels == lemmatizer2.labels + assert lemmatizer.trees.to_bytes() == lemmatizer2.trees.to_bytes() + + +def test_dutch(): + strings = StringStore() + trees = EditTrees(strings) + tree = trees.add("deelt", "delen") + assert trees.tree_to_str(tree) == "(m 0 3 () (m 0 2 (s '' 'l') (s 'lt' 'n')))" + + tree = trees.add("gedeeld", "delen") + assert ( + trees.tree_to_str(tree) == "(m 2 3 (s 'ge' '') (m 0 2 (s '' 'l') (s 'ld' 'n')))" + ) + + +def test_from_to_bytes(): + strings = StringStore() + trees = EditTrees(strings) + trees.add("deelt", "delen") + trees.add("gedeeld", "delen") + + b = trees.to_bytes() + + trees2 = EditTrees(strings) + trees2.from_bytes(b) + + # Verify that the nodes did not change. + assert len(trees) == len(trees2) + for i in range(len(trees)): + assert trees.tree_to_str(i) == trees2.tree_to_str(i) + + # Reinserting the same trees should not add new nodes. + trees2.add("deelt", "delen") + trees2.add("gedeeld", "delen") + assert len(trees) == len(trees2) + + +def test_from_to_disk(): + strings = StringStore() + trees = EditTrees(strings) + trees.add("deelt", "delen") + trees.add("gedeeld", "delen") + + trees2 = EditTrees(strings) + with make_tempdir() as temp_dir: + trees_file = temp_dir / "edit_trees.bin" + trees.to_disk(trees_file) + trees2 = trees2.from_disk(trees_file) + + # Verify that the nodes did not change. + assert len(trees) == len(trees2) + for i in range(len(trees)): + assert trees.tree_to_str(i) == trees2.tree_to_str(i) + + # Reinserting the same trees should not add new nodes. + trees2.add("deelt", "delen") + trees2.add("gedeeld", "delen") + assert len(trees) == len(trees2) + + +@given(st.text(), st.text()) +def test_roundtrip(form, lemma): + strings = StringStore() + trees = EditTrees(strings) + tree = trees.add(form, lemma) + assert trees.apply(tree, form) == lemma + + +@given(st.text(alphabet="ab"), st.text(alphabet="ab")) +def test_roundtrip_small_alphabet(form, lemma): + # Test with small alphabets to have more overlap. + strings = StringStore() + trees = EditTrees(strings) + tree = trees.add(form, lemma) + assert trees.apply(tree, form) == lemma + + +def test_unapplicable_trees(): + strings = StringStore() + trees = EditTrees(strings) + tree3 = trees.add("deelt", "delen") + + # Replacement fails. + assert trees.apply(tree3, "deeld") == None + + # Suffix + prefix are too large. + assert trees.apply(tree3, "de") == None + + +def test_empty_strings(): + strings = StringStore() + trees = EditTrees(strings) + no_change = trees.add("xyz", "xyz") + empty = trees.add("", "") + assert no_change == empty diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py new file mode 100644 index 0000000..74dd026 --- /dev/null +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -0,0 +1,1312 @@ +from typing import Any, Callable, Dict, Iterable + +import pytest +from numpy.testing import assert_equal + +from spacy import registry, util +from spacy.attrs import ENT_KB_ID +from spacy.compat import pickle +from spacy.kb import Candidate, InMemoryLookupKB, KnowledgeBase, get_candidates +from spacy.lang.en import English +from spacy.ml import load_kb +from spacy.ml.models.entity_linker import build_span_maker +from spacy.pipeline import EntityLinker +from spacy.pipeline.legacy import EntityLinker_v1 +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.scorer import Scorer +from spacy.tests.util import make_tempdir +from spacy.tokens import Doc, Span +from spacy.training import Example +from spacy.util import ensure_path +from spacy.vocab import Vocab + + +@pytest.fixture +def nlp(): + return English() + + +def assert_almost_equal(a, b): + delta = 0.0001 + assert a - delta <= b <= a + delta + + +@pytest.mark.issue(4674) +def test_issue4674(): + """Test that setting entities with overlapping identifiers does not mess up IO""" + nlp = English() + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + vector1 = [0.9, 1.1, 1.01] + vector2 = [1.8, 2.25, 2.01] + with pytest.warns(UserWarning): + kb.set_entities( + entity_list=["Q1", "Q1"], + freq_list=[32, 111], + vector_list=[vector1, vector2], + ) + assert kb.get_size_entities() == 1 + # dumping to file & loading back in + with make_tempdir() as d: + dir_path = ensure_path(d) + if not dir_path.exists(): + dir_path.mkdir() + file_path = dir_path / "kb" + kb.to_disk(str(file_path)) + kb2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + kb2.from_disk(str(file_path)) + assert kb2.get_size_entities() == 1 + + +@pytest.mark.issue(6730) +def test_issue6730(en_vocab): + """Ensure that the KB does not accept empty strings, but otherwise IO works fine.""" + from spacy.kb.kb_in_memory import InMemoryLookupKB + + kb = InMemoryLookupKB(en_vocab, entity_vector_length=3) + kb.add_entity(entity="1", freq=148, entity_vector=[1, 2, 3]) + + with pytest.raises(ValueError): + kb.add_alias(alias="", entities=["1"], probabilities=[0.4]) + assert kb.contains_alias("") is False + + kb.add_alias(alias="x", entities=["1"], probabilities=[0.2]) + kb.add_alias(alias="y", entities=["1"], probabilities=[0.1]) + + with make_tempdir() as tmp_dir: + kb.to_disk(tmp_dir) + kb.from_disk(tmp_dir) + assert kb.get_size_aliases() == 2 + assert set(kb.get_alias_strings()) == {"x", "y"} + + +@pytest.mark.issue(7065) +def test_issue7065(): + text = "Kathleen Battle sang in Mahler 's Symphony No. 8 at the Cincinnati Symphony Orchestra 's May Festival." + nlp = English() + nlp.add_pipe("sentencizer") + ruler = nlp.add_pipe("entity_ruler") + patterns = [ + { + "label": "THING", + "pattern": [ + {"LOWER": "symphony"}, + {"LOWER": "no"}, + {"LOWER": "."}, + {"LOWER": "8"}, + ], + } + ] + ruler.add_patterns(patterns) + + doc = nlp(text) + sentences = [s for s in doc.sents] + assert len(sentences) == 2 + sent0 = sentences[0] + ent = doc.ents[0] + assert ent.start < sent0.end < ent.end + assert sentences.index(ent.sent) == 0 + + +@pytest.mark.issue(7065) +@pytest.mark.parametrize("entity_in_first_sentence", [True, False]) +def test_sentence_crossing_ents(entity_in_first_sentence: bool): + """Tests if NEL crashes if entities cross sentence boundaries and the first associated sentence doesn't have an + entity. + entity_in_prior_sentence (bool): Whether to include an entity in the first sentence associated with the + sentence-crossing entity. + """ + # Test that the NEL doesn't crash when an entity crosses a sentence boundary + nlp = English() + vector_length = 3 + text = "Mahler 's Symphony No. 8 was beautiful." + entities = [(10, 24, "WORK")] + links = {(10, 24): {"Q7304": 0.0, "Q270853": 1.0}} + if entity_in_first_sentence: + entities.append((0, 6, "PERSON")) + links[(0, 6)] = {"Q7304": 1.0, "Q270853": 0.0} + sent_starts = [1, -1, 0, 0, 0, 1, 0, 0, 0] + doc = nlp(text) + example = Example.from_dict( + doc, {"entities": entities, "links": links, "sent_starts": sent_starts} + ) + train_examples = [example] + + def create_kb(vocab): + # create artificial KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q270853", freq=12, entity_vector=[9, 1, -7]) + mykb.add_alias( + alias="No. 8", + entities=["Q270853"], + probabilities=[1.0], + ) + mykb.add_entity(entity="Q7304", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias( + alias="Mahler", + entities=["Q7304"], + probabilities=[1.0], + ) + return mykb + + # Create the Entity Linker component and add it to the pipeline + entity_linker = nlp.add_pipe("entity_linker", last=True) + entity_linker.set_kb(create_kb) # type: ignore + # train the NEL pipe + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + nlp.update(train_examples, sgd=optimizer) + + # This shouldn't crash. + entity_linker.predict([example.reference]) # type: ignore + + +def test_no_entities(): + # Test that having no entities doesn't crash the model + TRAIN_DATA = [ + ( + "The sky is blue.", + { + "sent_starts": [1, 0, 0, 0, 0], + }, + ) + ] + nlp = English() + vector_length = 3 + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + train_examples.append(Example.from_dict(doc, annotation)) + + def create_kb(vocab): + # create artificial KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) + return mykb + + # Create and train the Entity Linker + entity_linker = nlp.add_pipe("entity_linker", last=True) + entity_linker.set_kb(create_kb) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # adding additional components that are required for the entity_linker + nlp.add_pipe("sentencizer", first=True) + + # this will run the pipeline on the examples and shouldn't crash + nlp.evaluate(train_examples) + + +def test_partial_links(): + # Test that having some entities on the doc without gold links, doesn't crash + TRAIN_DATA = [ + ( + "Russ Cochran his reprints include EC Comics.", + { + "links": {(0, 12): {"Q2146908": 1.0}}, + "entities": [(0, 12, "PERSON")], + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0], + }, + ) + ] + nlp = English() + vector_length = 3 + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + train_examples.append(Example.from_dict(doc, annotation)) + + def create_kb(vocab): + # create artificial KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) + return mykb + + # Create and train the Entity Linker + entity_linker = nlp.add_pipe("entity_linker", last=True) + entity_linker.set_kb(create_kb) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # adding additional components that are required for the entity_linker + nlp.add_pipe("sentencizer", first=True) + patterns = [ + {"label": "PERSON", "pattern": [{"LOWER": "russ"}, {"LOWER": "cochran"}]}, + {"label": "ORG", "pattern": [{"LOWER": "ec"}, {"LOWER": "comics"}]}, + ] + ruler = nlp.add_pipe("entity_ruler", before="entity_linker") + ruler.add_patterns(patterns) + + # this will run the pipeline on the examples and shouldn't crash + results = nlp.evaluate(train_examples) + assert "PERSON" in results["ents_per_type"] + assert "PERSON" in results["nel_f_per_type"] + assert "ORG" in results["ents_per_type"] + assert "ORG" not in results["nel_f_per_type"] + + +def test_kb_valid_entities(nlp): + """Test the valid construction of a KB with 3 entities and two aliases""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[8, 4, 3]) + mykb.add_entity(entity="Q2", freq=5, entity_vector=[2, 1, 0]) + mykb.add_entity(entity="Q3", freq=25, entity_vector=[-1, -6, 5]) + + # adding aliases + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.2]) + mykb.add_alias(alias="adam", entities=["Q2"], probabilities=[0.9]) + + # test the size of the corresponding KB + assert mykb.get_size_entities() == 3 + assert mykb.get_size_aliases() == 2 + + # test retrieval of the entity vectors + assert mykb.get_vector("Q1") == [8, 4, 3] + assert mykb.get_vector("Q2") == [2, 1, 0] + assert mykb.get_vector("Q3") == [-1, -6, 5] + + # test retrieval of prior probabilities + assert_almost_equal(mykb.get_prior_prob(entity="Q2", alias="douglas"), 0.8) + assert_almost_equal(mykb.get_prior_prob(entity="Q3", alias="douglas"), 0.2) + assert_almost_equal(mykb.get_prior_prob(entity="Q342", alias="douglas"), 0.0) + assert_almost_equal(mykb.get_prior_prob(entity="Q3", alias="douglassssss"), 0.0) + + +def test_kb_invalid_entities(nlp): + """Test the invalid construction of a KB with an alias linked to a non-existing entity""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=5, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=25, entity_vector=[3]) + + # adding aliases - should fail because one of the given IDs is not valid + with pytest.raises(ValueError): + mykb.add_alias( + alias="douglas", entities=["Q2", "Q342"], probabilities=[0.8, 0.2] + ) + + +def test_kb_invalid_probabilities(nlp): + """Test the invalid construction of a KB with wrong prior probabilities""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=5, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=25, entity_vector=[3]) + + # adding aliases - should fail because the sum of the probabilities exceeds 1 + with pytest.raises(ValueError): + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.4]) + + +def test_kb_invalid_combination(nlp): + """Test the invalid construction of a KB with non-matching entity and probability lists""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=5, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=25, entity_vector=[3]) + + # adding aliases - should fail because the entities and probabilities vectors are not of equal length + with pytest.raises(ValueError): + mykb.add_alias( + alias="douglas", entities=["Q2", "Q3"], probabilities=[0.3, 0.4, 0.1] + ) + + +def test_kb_invalid_entity_vector(nlp): + """Test the invalid construction of a KB with non-matching entity vector lengths""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1, 2, 3]) + + # this should fail because the kb's expected entity vector length is 3 + with pytest.raises(ValueError): + mykb.add_entity(entity="Q2", freq=5, entity_vector=[2]) + + +def test_kb_default(nlp): + """Test that the default (empty) KB is loaded upon construction""" + entity_linker = nlp.add_pipe("entity_linker", config={}) + assert len(entity_linker.kb) == 0 + with pytest.raises(ValueError, match="E139"): + # this raises an error because the KB is empty + entity_linker.validate_kb() + assert entity_linker.kb.get_size_entities() == 0 + assert entity_linker.kb.get_size_aliases() == 0 + # 64 is the default value from pipeline.entity_linker + assert entity_linker.kb.entity_vector_length == 64 + + +def test_kb_custom_length(nlp): + """Test that the default (empty) KB can be configured with a custom entity length""" + entity_linker = nlp.add_pipe("entity_linker", config={"entity_vector_length": 35}) + assert len(entity_linker.kb) == 0 + assert entity_linker.kb.get_size_entities() == 0 + assert entity_linker.kb.get_size_aliases() == 0 + assert entity_linker.kb.entity_vector_length == 35 + + +def test_kb_initialize_empty(nlp): + """Test that the EL can't initialize without examples""" + entity_linker = nlp.add_pipe("entity_linker") + with pytest.raises(TypeError): + entity_linker.initialize(lambda: []) + + +def test_kb_serialize(nlp): + """Test serialization of the KB""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + with make_tempdir() as d: + # normal read-write behaviour + mykb.to_disk(d / "kb") + mykb.from_disk(d / "kb") + mykb.to_disk(d / "new" / "kb") + mykb.from_disk(d / "new" / "kb") + # allow overwriting an existing file + mykb.to_disk(d / "kb") + with pytest.raises(ValueError): + # can not read from an unknown file + mykb.from_disk(d / "unknown" / "kb") + + +@pytest.mark.issue(9137) +def test_kb_serialize_2(nlp): + v = [5, 6, 7, 8] + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) + kb1.set_entities(["E1"], [1], [v]) + assert kb1.get_vector("E1") == v + with make_tempdir() as d: + kb1.to_disk(d / "kb") + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) + kb2.from_disk(d / "kb") + assert kb2.get_vector("E1") == v + + +def test_kb_set_entities(nlp): + """Test that set_entities entirely overwrites the previous set of entities""" + v = [5, 6, 7, 8] + v1 = [1, 1, 1, 0] + v2 = [2, 2, 2, 3] + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) + kb1.set_entities(["E0"], [1], [v]) + assert kb1.get_entity_strings() == ["E0"] + kb1.set_entities(["E1", "E2"], [1, 9], [v1, v2]) + assert set(kb1.get_entity_strings()) == {"E1", "E2"} + assert kb1.get_vector("E1") == v1 + assert kb1.get_vector("E2") == v2 + with make_tempdir() as d: + kb1.to_disk(d / "kb") + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) + kb2.from_disk(d / "kb") + assert set(kb2.get_entity_strings()) == {"E1", "E2"} + assert kb2.get_vector("E1") == v1 + assert kb2.get_vector("E2") == v2 + + +def test_kb_serialize_vocab(nlp): + """Test serialization of the KB and custom strings""" + entity = "MyFunnyID" + assert entity not in nlp.vocab.strings + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + assert not mykb.contains_entity(entity) + mykb.add_entity(entity, freq=342, entity_vector=[3]) + assert mykb.contains_entity(entity) + assert entity in mykb.vocab.strings + with make_tempdir() as d: + # normal read-write behaviour + mykb.to_disk(d / "kb") + mykb_new = InMemoryLookupKB(Vocab(), entity_vector_length=1) + mykb_new.from_disk(d / "kb") + assert entity in mykb_new.vocab.strings + + +def test_candidate_generation(nlp): + """Test correct candidate generation""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + doc = nlp("douglas adam Adam shrubbery") + + douglas_ent = doc[0:1] + adam_ent = doc[1:2] + Adam_ent = doc[2:3] + shrubbery_ent = doc[3:4] + + # adding entities + mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=12, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=5, entity_vector=[3]) + + # adding aliases + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1]) + mykb.add_alias(alias="adam", entities=["Q2"], probabilities=[0.9]) + + # test the size of the relevant candidates + assert len(get_candidates(mykb, douglas_ent)) == 2 + assert len(get_candidates(mykb, adam_ent)) == 1 + assert len(get_candidates(mykb, Adam_ent)) == 0 # default case sensitive + assert len(get_candidates(mykb, shrubbery_ent)) == 0 + + # test the content of the candidates + assert get_candidates(mykb, adam_ent)[0].entity_ == "Q2" + assert get_candidates(mykb, adam_ent)[0].alias_ == "adam" + assert_almost_equal(get_candidates(mykb, adam_ent)[0].entity_freq, 12) + assert_almost_equal(get_candidates(mykb, adam_ent)[0].prior_prob, 0.9) + + +def test_el_pipe_configuration(nlp): + """Test correct candidate generation as part of the EL pipe""" + nlp.add_pipe("sentencizer") + pattern = {"label": "PERSON", "pattern": [{"LOWER": "douglas"}]} + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns([pattern]) + + def create_kb(vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=1) + kb.add_entity(entity="Q2", freq=12, entity_vector=[2]) + kb.add_entity(entity="Q3", freq=5, entity_vector=[3]) + kb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1]) + return kb + + # run an EL pipe without a trained context encoder, to check the candidate generation step only + entity_linker = nlp.add_pipe("entity_linker", config={"incl_context": False}) + entity_linker.set_kb(create_kb) + # With the default get_candidates function, matching is case-sensitive + text = "Douglas and douglas are not the same." + doc = nlp(text) + assert doc[0].ent_kb_id_ == "NIL" + assert doc[1].ent_kb_id_ == "" + assert doc[2].ent_kb_id_ == "Q2" + + def get_lowercased_candidates(kb, span): + return kb.get_alias_candidates(span.text.lower()) + + def get_lowercased_candidates_batch(kb, spans): + return [get_lowercased_candidates(kb, span) for span in spans] + + @registry.misc("spacy.LowercaseCandidateGenerator.v1") + def create_candidates() -> Callable[ + [InMemoryLookupKB, "Span"], Iterable[Candidate] + ]: + return get_lowercased_candidates + + @registry.misc("spacy.LowercaseCandidateBatchGenerator.v1") + def create_candidates_batch() -> Callable[ + [InMemoryLookupKB, Iterable["Span"]], Iterable[Iterable[Candidate]] + ]: + return get_lowercased_candidates_batch + + # replace the pipe with a new one with with a different candidate generator + entity_linker = nlp.replace_pipe( + "entity_linker", + "entity_linker", + config={ + "incl_context": False, + "get_candidates": {"@misc": "spacy.LowercaseCandidateGenerator.v1"}, + "get_candidates_batch": { + "@misc": "spacy.LowercaseCandidateBatchGenerator.v1" + }, + }, + ) + entity_linker.set_kb(create_kb) + doc = nlp(text) + assert doc[0].ent_kb_id_ == "Q2" + assert doc[1].ent_kb_id_ == "" + assert doc[2].ent_kb_id_ == "Q2" + + +def test_nel_nsents(nlp): + """Test that n_sents can be set through the configuration""" + entity_linker = nlp.add_pipe("entity_linker", config={}) + assert entity_linker.n_sents == 0 + entity_linker = nlp.replace_pipe( + "entity_linker", "entity_linker", config={"n_sents": 2} + ) + assert entity_linker.n_sents == 2 + + +def test_vocab_serialization(nlp): + """Test that string information is retained across storage""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) + q2_hash = mykb.add_entity(entity="Q2", freq=12, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=5, entity_vector=[3]) + + # adding aliases + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.4, 0.1]) + adam_hash = mykb.add_alias(alias="adam", entities=["Q2"], probabilities=[0.9]) + + candidates = mykb.get_alias_candidates("adam") + assert len(candidates) == 1 + assert candidates[0].entity == q2_hash + assert candidates[0].entity_ == "Q2" + assert candidates[0].alias == adam_hash + assert candidates[0].alias_ == "adam" + + with make_tempdir() as d: + mykb.to_disk(d / "kb") + kb_new_vocab = InMemoryLookupKB(Vocab(), entity_vector_length=1) + kb_new_vocab.from_disk(d / "kb") + + candidates = kb_new_vocab.get_alias_candidates("adam") + assert len(candidates) == 1 + assert candidates[0].entity == q2_hash + assert candidates[0].entity_ == "Q2" + assert candidates[0].alias == adam_hash + assert candidates[0].alias_ == "adam" + + assert kb_new_vocab.get_vector("Q2") == [2] + assert_almost_equal(kb_new_vocab.get_prior_prob("Q2", "douglas"), 0.4) + + +def test_append_alias(nlp): + """Test that we can append additional alias-entity pairs""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=12, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=5, entity_vector=[3]) + + # adding aliases + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.4, 0.1]) + mykb.add_alias(alias="adam", entities=["Q2"], probabilities=[0.9]) + + # test the size of the relevant candidates + assert len(mykb.get_alias_candidates("douglas")) == 2 + + # append an alias + mykb.append_alias(alias="douglas", entity="Q1", prior_prob=0.2) + + # test the size of the relevant candidates has been incremented + assert len(mykb.get_alias_candidates("douglas")) == 3 + + # append the same alias-entity pair again should not work (will throw a warning) + with pytest.warns(UserWarning): + mykb.append_alias(alias="douglas", entity="Q1", prior_prob=0.3) + + # test the size of the relevant candidates remained unchanged + assert len(mykb.get_alias_candidates("douglas")) == 3 + + +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_append_invalid_alias(nlp): + """Test that append an alias will throw an error if prior probs are exceeding 1""" + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + + # adding entities + mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=12, entity_vector=[2]) + mykb.add_entity(entity="Q3", freq=5, entity_vector=[3]) + + # adding aliases + mykb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1]) + mykb.add_alias(alias="adam", entities=["Q2"], probabilities=[0.9]) + + # append an alias - should fail because the entities and probabilities vectors are not of equal length + with pytest.raises(ValueError): + mykb.append_alias(alias="douglas", entity="Q1", prior_prob=0.2) + + +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_preserving_links_asdoc(nlp): + """Test that Span.as_doc preserves the existing entity links""" + vector_length = 1 + + def create_kb(vocab): + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + # adding entities + mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) + mykb.add_entity(entity="Q2", freq=8, entity_vector=[1]) + # adding aliases + mykb.add_alias(alias="Boston", entities=["Q1"], probabilities=[0.7]) + mykb.add_alias(alias="Denver", entities=["Q2"], probabilities=[0.6]) + return mykb + + # set up pipeline with NER (Entity Ruler) and NEL (prior probability only, model not trained) + nlp.add_pipe("sentencizer") + patterns = [ + {"label": "GPE", "pattern": "Boston"}, + {"label": "GPE", "pattern": "Denver"}, + ] + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + config = {"incl_prior": False} + entity_linker = nlp.add_pipe("entity_linker", config=config, last=True) + entity_linker.set_kb(create_kb) + nlp.initialize() + assert entity_linker.model.get_dim("nO") == vector_length + + # test whether the entity links are preserved by the `as_doc()` function + text = "She lives in Boston. He lives in Denver." + doc = nlp(text) + for ent in doc.ents: + orig_text = ent.text + orig_kb_id = ent.kb_id_ + sent_doc = ent.sent.as_doc() + for s_ent in sent_doc.ents: + if s_ent.text == orig_text: + assert s_ent.kb_id_ == orig_kb_id + + +def test_preserving_links_ents(nlp): + """Test that doc.ents preserves KB annotations""" + text = "She lives in Boston. He lives in Denver." + doc = nlp(text) + assert len(list(doc.ents)) == 0 + + boston_ent = Span(doc, 3, 4, label="LOC", kb_id="Q1") + doc.ents = [boston_ent] + assert len(list(doc.ents)) == 1 + assert list(doc.ents)[0].label_ == "LOC" + assert list(doc.ents)[0].kb_id_ == "Q1" + + +def test_preserving_links_ents_2(nlp): + """Test that doc.ents preserves KB annotations""" + text = "She lives in Boston. He lives in Denver." + doc = nlp(text) + assert len(list(doc.ents)) == 0 + + loc = doc.vocab.strings.add("LOC") + q1 = doc.vocab.strings.add("Q1") + + doc.ents = [(loc, q1, 3, 4)] + assert len(list(doc.ents)) == 1 + assert list(doc.ents)[0].label_ == "LOC" + assert list(doc.ents)[0].kb_id_ == "Q1" + + +# fmt: off +TRAIN_DATA = [ + ("Russ Cochran captured his first major title with his son as caddie.", + {"links": {(0, 12): {"Q7381115": 0.0, "Q2146908": 1.0}}, + "entities": [(0, 12, "PERSON")], + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}), + ("Russ Cochran his reprints include EC Comics.", + {"links": {(0, 12): {"Q7381115": 1.0, "Q2146908": 0.0}}, + "entities": [(0, 12, "PERSON"), (34, 43, "ART")], + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0]}), + ("Russ Cochran has been publishing comic art.", + {"links": {(0, 12): {"Q7381115": 1.0, "Q2146908": 0.0}}, + "entities": [(0, 12, "PERSON")], + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0]}), + ("Russ Cochran was a member of University of Kentucky's golf team.", + {"links": {(0, 12): {"Q7381115": 0.0, "Q2146908": 1.0}}, + "entities": [(0, 12, "PERSON"), (43, 51, "LOC")], + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}), + # having a blank instance shouldn't break things + ("The weather is nice today.", + {"links": {}, "entities": [], + "sent_starts": [1, -1, 0, 0, 0, 0]}) +] +GOLD_entities = ["Q2146908", "Q7381115", "Q7381115", "Q2146908"] +# fmt: on + + +def test_overfitting_IO_gold_entities(): + # Simple test to try and quickly overfit the NEL component - ensuring the ML models work correctly + nlp = English() + vector_length = 3 + assert "Q2146908" not in nlp.vocab.strings + + # Convert the texts to docs to make sure we have doc.ents set for the training examples + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + train_examples.append(Example.from_dict(doc, annotation)) + + def create_kb(vocab): + # create artificial KB - assign same prior weight to the two russ cochran's + # Q2146908 (Russ Cochran): American golfer + # Q7381115 (Russ Cochran): publisher + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) + mykb.add_alias( + alias="Russ Cochran", + entities=["Q2146908", "Q7381115"], + probabilities=[0.5, 0.5], + ) + return mykb + + # Create the Entity Linker component and add it to the pipeline + entity_linker = nlp.add_pipe( + "entity_linker", last=True, config={"use_gold_ents": True} + ) + assert isinstance(entity_linker, EntityLinker) + entity_linker.set_kb(create_kb) + assert "Q2146908" in entity_linker.vocab.strings + assert "Q2146908" in entity_linker.kb.vocab.strings + + # train the NEL pipe + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert entity_linker.model.get_dim("nO") == vector_length + assert entity_linker.model.get_dim("nO") == entity_linker.kb.entity_vector_length + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["entity_linker"] < 0.001 + + # adding additional components that are required for the entity_linker + nlp.add_pipe("sentencizer", first=True) + + # Add a custom component to recognize "Russ Cochran" as an entity for the example training data + patterns = [ + {"label": "PERSON", "pattern": [{"LOWER": "russ"}, {"LOWER": "cochran"}]} + ] + ruler = nlp.add_pipe("entity_ruler", before="entity_linker") + ruler.add_patterns(patterns) + + # test the trained model + predictions = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + for ent in doc.ents: + predictions.append(ent.kb_id_) + assert predictions == GOLD_entities + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + assert nlp2.pipe_names == nlp.pipe_names + assert "Q2146908" in nlp2.vocab.strings + entity_linker2 = nlp2.get_pipe("entity_linker") + assert "Q2146908" in entity_linker2.vocab.strings + assert "Q2146908" in entity_linker2.kb.vocab.strings + predictions = [] + for text, annotation in TRAIN_DATA: + doc2 = nlp2(text) + for ent in doc2.ents: + predictions.append(ent.kb_id_) + assert predictions == GOLD_entities + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Russ Cochran captured his first major title with his son as caddie.", + "Russ Cochran his reprints include EC Comics.", + "Russ Cochran has been publishing comic art.", + "Russ Cochran was a member of University of Kentucky's golf team.", + ] + batch_deps_1 = [doc.to_array([ENT_KB_ID]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([ENT_KB_ID]) for doc in nlp.pipe(texts)] + no_batch_deps = [doc.to_array([ENT_KB_ID]) for doc in [nlp(text) for text in texts]] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + eval = nlp.evaluate(train_examples) + assert "nel_macro_p" in eval + assert "nel_macro_r" in eval + assert "nel_macro_f" in eval + assert "nel_micro_p" in eval + assert "nel_micro_r" in eval + assert "nel_micro_f" in eval + assert "nel_f_per_type" in eval + assert "PERSON" in eval["nel_f_per_type"] + + assert eval["nel_macro_f"] > 0 + assert eval["nel_micro_f"] > 0 + + +def test_overfitting_IO_with_ner(): + # Simple test to try and overfit the NER and NEL component in combination - ensuring the ML models work correctly + nlp = English() + vector_length = 3 + assert "Q2146908" not in nlp.vocab.strings + + # Convert the texts to docs to make sure we have doc.ents set for the training examples + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + train_examples.append(Example.from_dict(doc, annotation)) + + def create_kb(vocab): + # create artificial KB - assign same prior weight to the two russ cochran's + # Q2146908 (Russ Cochran): American golfer + # Q7381115 (Russ Cochran): publisher + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) + mykb.add_alias( + alias="Russ Cochran", + entities=["Q2146908", "Q7381115"], + probabilities=[0.5, 0.5], + ) + return mykb + + # Create the NER and EL components and add them to the pipeline + ner = nlp.add_pipe("ner", first=True) + entity_linker = nlp.add_pipe( + "entity_linker", last=True, config={"use_gold_ents": False} + ) + entity_linker.set_kb(create_kb) + + train_examples = [] + for text, annotations in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for ent in annotations.get("entities"): + ner.add_label(ent[2]) + optimizer = nlp.initialize() + + # train the NER and NEL pipes + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["ner"] < 0.001 + assert losses["entity_linker"] < 0.001 + + # adding additional components that are required for the entity_linker + nlp.add_pipe("sentencizer", first=True) + + # test the trained model + test_text = "Russ Cochran captured his first major title with his son as caddie." + doc = nlp(test_text) + ents = doc.ents + assert len(ents) == 1 + assert ents[0].text == "Russ Cochran" + assert ents[0].label_ == "PERSON" + assert ents[0].kb_id_ != "NIL" + + # TODO: below assert is still flaky - EL doesn't properly overfit quite yet + # assert ents[0].kb_id_ == "Q2146908" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + assert nlp2.pipe_names == nlp.pipe_names + doc2 = nlp2(test_text) + ents2 = doc2.ents + assert len(ents2) == 1 + assert ents2[0].text == "Russ Cochran" + assert ents2[0].label_ == "PERSON" + assert ents2[0].kb_id_ != "NIL" + + eval = nlp.evaluate(train_examples) + assert "nel_macro_f" in eval + assert "nel_micro_f" in eval + assert "ents_f" in eval + assert "nel_f_per_type" in eval + assert "ents_per_type" in eval + assert "PERSON" in eval["nel_f_per_type"] + assert "PERSON" in eval["ents_per_type"] + + assert eval["nel_macro_f"] > 0 + assert eval["nel_micro_f"] > 0 + assert eval["ents_f"] > 0 + + +def test_kb_serialization(): + # Test that the KB can be used in a pipeline with a different vocab + vector_length = 3 + with make_tempdir() as tmp_dir: + kb_dir = tmp_dir / "kb" + nlp1 = English() + assert "Q2146908" not in nlp1.vocab.strings + mykb = InMemoryLookupKB(nlp1.vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) + assert "Q2146908" in nlp1.vocab.strings + mykb.to_disk(kb_dir) + + nlp2 = English() + assert "RandomWord" not in nlp2.vocab.strings + nlp2.vocab.strings.add("RandomWord") + assert "RandomWord" in nlp2.vocab.strings + assert "Q2146908" not in nlp2.vocab.strings + + # Create the Entity Linker component with the KB from file, and check the final vocab + entity_linker = nlp2.add_pipe("entity_linker", last=True) + entity_linker.set_kb(load_kb(kb_dir)) + assert "Q2146908" in nlp2.vocab.strings + assert "RandomWord" in nlp2.vocab.strings + + +@pytest.mark.xfail(reason="Needs fixing") +def test_kb_pickle(): + # Test that the KB can be pickled + nlp = English() + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + assert not kb_1.contains_alias("Russ Cochran") + kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) + assert kb_1.contains_alias("Russ Cochran") + data = pickle.dumps(kb_1) + kb_2 = pickle.loads(data) + assert kb_2.contains_alias("Russ Cochran") + + +@pytest.mark.xfail(reason="Needs fixing") +def test_nel_pickle(): + # Test that a pipeline with an EL component can be pickled + def create_kb(vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=3) + kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) + return kb + + nlp_1 = English() + nlp_1.add_pipe("ner") + entity_linker_1 = nlp_1.add_pipe("entity_linker", last=True) + entity_linker_1.set_kb(create_kb) + assert nlp_1.pipe_names == ["ner", "entity_linker"] + assert entity_linker_1.kb.contains_alias("Russ Cochran") + + data = pickle.dumps(nlp_1) + nlp_2 = pickle.loads(data) + assert nlp_2.pipe_names == ["ner", "entity_linker"] + entity_linker_2 = nlp_2.get_pipe("entity_linker") + assert entity_linker_2.kb.contains_alias("Russ Cochran") + + +def test_kb_to_bytes(): + # Test that the KB's to_bytes method works correctly + nlp = English() + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + kb_1.add_entity(entity="Q66", freq=9, entity_vector=[1, 2, 3]) + kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) + kb_1.add_alias(alias="Boeing", entities=["Q66"], probabilities=[0.5]) + kb_1.add_alias( + alias="Randomness", entities=["Q66", "Q2146908"], probabilities=[0.1, 0.2] + ) + assert kb_1.contains_alias("Russ Cochran") + kb_bytes = kb_1.to_bytes() + kb_2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) + assert not kb_2.contains_alias("Russ Cochran") + kb_2 = kb_2.from_bytes(kb_bytes) + # check that both KBs are exactly the same + assert kb_1.get_size_entities() == kb_2.get_size_entities() + assert kb_1.entity_vector_length == kb_2.entity_vector_length + assert kb_1.get_entity_strings() == kb_2.get_entity_strings() + assert kb_1.get_vector("Q2146908") == kb_2.get_vector("Q2146908") + assert kb_1.get_vector("Q66") == kb_2.get_vector("Q66") + assert kb_2.contains_alias("Russ Cochran") + assert kb_1.get_size_aliases() == kb_2.get_size_aliases() + assert kb_1.get_alias_strings() == kb_2.get_alias_strings() + assert len(kb_1.get_alias_candidates("Russ Cochran")) == len( + kb_2.get_alias_candidates("Russ Cochran") + ) + assert len(kb_1.get_alias_candidates("Randomness")) == len( + kb_2.get_alias_candidates("Randomness") + ) + + +def test_nel_to_bytes(): + # Test that a pipeline with an EL component can be converted to bytes + def create_kb(vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=3) + kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) + return kb + + nlp_1 = English() + nlp_1.add_pipe("ner") + entity_linker_1 = nlp_1.add_pipe("entity_linker", last=True) + entity_linker_1.set_kb(create_kb) + assert entity_linker_1.kb.contains_alias("Russ Cochran") + assert nlp_1.pipe_names == ["ner", "entity_linker"] + + nlp_bytes = nlp_1.to_bytes() + nlp_2 = English() + nlp_2.add_pipe("ner") + nlp_2.add_pipe("entity_linker", last=True) + assert nlp_2.pipe_names == ["ner", "entity_linker"] + assert not nlp_2.get_pipe("entity_linker").kb.contains_alias("Russ Cochran") + nlp_2 = nlp_2.from_bytes(nlp_bytes) + kb_2 = nlp_2.get_pipe("entity_linker").kb + assert kb_2.contains_alias("Russ Cochran") + assert kb_2.get_vector("Q2146908") == [6, -4, 3] + assert_almost_equal( + kb_2.get_prior_prob(entity="Q2146908", alias="Russ Cochran"), 0.8 + ) + + +def test_scorer_links(): + train_examples = [] + nlp = English() + ref1 = nlp("Julia lives in London happily.") + ref1.ents = [ + Span(ref1, 0, 1, label="PERSON", kb_id="Q2"), + Span(ref1, 3, 4, label="LOC", kb_id="Q3"), + ] + pred1 = nlp("Julia lives in London happily.") + pred1.ents = [ + Span(pred1, 0, 1, label="PERSON", kb_id="Q70"), + Span(pred1, 3, 4, label="LOC", kb_id="Q3"), + ] + train_examples.append(Example(pred1, ref1)) + + ref2 = nlp("She loves London.") + ref2.ents = [ + Span(ref2, 0, 1, label="PERSON", kb_id="Q2"), + Span(ref2, 2, 3, label="LOC", kb_id="Q13"), + ] + pred2 = nlp("She loves London.") + pred2.ents = [ + Span(pred2, 0, 1, label="PERSON", kb_id="Q2"), + Span(pred2, 2, 3, label="LOC", kb_id="NIL"), + ] + train_examples.append(Example(pred2, ref2)) + + ref3 = nlp("London is great.") + ref3.ents = [Span(ref3, 0, 1, label="LOC", kb_id="NIL")] + pred3 = nlp("London is great.") + pred3.ents = [Span(pred3, 0, 1, label="LOC", kb_id="NIL")] + train_examples.append(Example(pred3, ref3)) + + scores = Scorer().score_links(train_examples, negative_labels=["NIL"]) + assert scores["nel_f_per_type"]["PERSON"]["p"] == 1 / 2 + assert scores["nel_f_per_type"]["PERSON"]["r"] == 1 / 2 + assert scores["nel_f_per_type"]["LOC"]["p"] == 1 / 1 + assert scores["nel_f_per_type"]["LOC"]["r"] == 1 / 2 + + assert scores["nel_micro_p"] == 2 / 3 + assert scores["nel_micro_r"] == 2 / 4 + + +# fmt: off +@pytest.mark.parametrize( + "name,config", + [ + ("entity_linker", {"@architectures": "spacy.EntityLinker.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + ("entity_linker", {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + ], +) +# fmt: on +def test_legacy_architectures(name, config): + # Ensure that the legacy architectures still work + vector_length = 3 + nlp = English() + + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp.make_doc(text) + train_examples.append(Example.from_dict(doc, annotation)) + + def create_kb(vocab): + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) + mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) + mykb.add_alias( + alias="Russ Cochran", + entities=["Q2146908", "Q7381115"], + probabilities=[0.5, 0.5], + ) + return mykb + + entity_linker = nlp.add_pipe(name, config={"model": config}) + if config["@architectures"] == "spacy.EntityLinker.v1": + assert isinstance(entity_linker, EntityLinker_v1) + else: + assert isinstance(entity_linker, EntityLinker) + entity_linker.set_kb(create_kb) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + +@pytest.mark.parametrize( + "patterns", + [ + # perfect case + [{"label": "CHARACTER", "pattern": "Kirby"}], + # typo for false negative + [{"label": "PERSON", "pattern": "Korby"}], + # random stuff for false positive + [{"label": "IS", "pattern": "is"}, {"label": "COLOR", "pattern": "pink"}], + ], +) +def test_no_gold_ents(patterns): + # test that annotating components work + TRAIN_DATA = [ + ( + "Kirby is pink", + { + "links": {(0, 5): {"Q613241": 1.0}}, + "entities": [(0, 5, "CHARACTER")], + "sent_starts": [1, 0, 0], + }, + ) + ] + nlp = English() + vector_length = 3 + train_examples = [] + for text, annotation in TRAIN_DATA: + doc = nlp(text) + train_examples.append(Example.from_dict(doc, annotation)) + + # Create a ruler to mark entities + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + + # Apply ruler to examples. In a real pipeline this would be an annotating component. + for eg in train_examples: + eg.predicted = ruler(eg.predicted) + + # Entity ruler is no longer needed (initialization below wipes out the + # patterns and causes warnings) + nlp.remove_pipe("entity_ruler") + + def create_kb(vocab): + # create artificial KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias("Kirby", ["Q613241"], [0.9]) + # Placeholder + mykb.add_entity(entity="pink", freq=12, entity_vector=[7, 2, -5]) + mykb.add_alias("pink", ["pink"], [0.9]) + return mykb + + # Create and train the Entity Linker + entity_linker = nlp.add_pipe( + "entity_linker", config={"use_gold_ents": False}, last=True + ) + entity_linker.set_kb(create_kb) + assert entity_linker.use_gold_ents is False + + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # adding additional components that are required for the entity_linker + nlp.add_pipe("sentencizer", first=True) + + # this will run the pipeline on the examples and shouldn't crash + nlp.evaluate(train_examples) + + +@pytest.mark.issue(9575) +def test_tokenization_mismatch(): + nlp = English() + # include a matching entity so that update isn't skipped + doc1 = Doc( + nlp.vocab, + words=["Kirby", "123456"], + spaces=[True, False], + ents=["B-CHARACTER", "B-CARDINAL"], + ) + doc2 = Doc( + nlp.vocab, + words=["Kirby", "123", "456"], + spaces=[True, False, False], + ents=["B-CHARACTER", "B-CARDINAL", "B-CARDINAL"], + ) + + eg = Example(doc1, doc2) + train_examples = [eg] + vector_length = 3 + + def create_kb(vocab): + # create placeholder KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) + mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias("Kirby", ["Q613241"], [0.9]) + return mykb + + entity_linker = nlp.add_pipe("entity_linker", last=True) + entity_linker.set_kb(create_kb) + + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(2): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + nlp.add_pipe("sentencizer", first=True) + nlp.evaluate(train_examples) + + +def test_abstract_kb_instantiation(): + """Test whether instantiation of abstract KB base class fails.""" + with pytest.raises(TypeError): + KnowledgeBase(None, 3) + + +# fmt: off +@pytest.mark.parametrize( + "meet_threshold,config", + [ + (False, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + (True, {"@architectures": "spacy.EntityLinker.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL}), + ], +) +# fmt: on +def test_threshold(meet_threshold: bool, config: Dict[str, Any]): + """Tests abstention threshold. + meet_threshold (bool): Whether to configure NEL setup so that confidence threshold is met. + config (Dict[str, Any]): NEL architecture config. + """ + nlp = English() + nlp.add_pipe("sentencizer") + text = "Mahler's Symphony No. 8 was beautiful." + entities = [(0, 6, "PERSON")] + links = {(0, 6): {"Q7304": 1.0}} + sent_starts = [1, -1, 0, 0, 0, 0, 0, 0, 0] + entity_id = "Q7304" + doc = nlp(text) + train_examples = [ + Example.from_dict( + doc, {"entities": entities, "links": links, "sent_starts": sent_starts} + ) + ] + + def create_kb(vocab): + # create artificial KB + mykb = InMemoryLookupKB(vocab, entity_vector_length=3) + mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3]) + mykb.add_alias( + alias="Mahler", + entities=[entity_id], + probabilities=[1 if meet_threshold else 0.01], + ) + return mykb + + # Create the Entity Linker component and add it to the pipeline + entity_linker = nlp.add_pipe( + "entity_linker", + last=True, + config={"threshold": 0.99, "model": config}, + ) + entity_linker.set_kb(create_kb) # type: ignore + nlp.initialize(get_examples=lambda: train_examples) + + # Add a custom rule-based component to mimick NER + ruler = nlp.add_pipe("entity_ruler", before="entity_linker") + ruler.add_patterns([{"label": "PERSON", "pattern": [{"LOWER": "mahler"}]}]) # type: ignore + doc = nlp(text) + + assert len(doc.ents) == 1 + assert doc.ents[0].kb_id_ == entity_id if meet_threshold else EntityLinker.NIL + + +def test_span_maker_forward_with_empty(): + """The forward pass of the span maker may have a doc with no entities.""" + nlp = English() + doc1 = nlp("a b c") + ent = doc1[0:1] + ent.label_ = "X" + doc1.ents = [ent] + # no entities + doc2 = nlp("x y z") + + # just to get a model + span_maker = build_span_maker() + span_maker([doc1, doc2], False) diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py new file mode 100644 index 0000000..d0ab003 --- /dev/null +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -0,0 +1,682 @@ +import pytest +from thinc.api import NumpyOps, get_current_ops + +from spacy import registry +from spacy.errors import MatchPatternError +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline import EntityRecognizer, EntityRuler, SpanRuler, merge_entities +from spacy.pipeline.ner import DEFAULT_NER_MODEL +from spacy.tests.util import make_tempdir +from spacy.tokens import Doc, Span + +ENTITY_RULERS = ["entity_ruler", "future_entity_ruler"] + + +@pytest.fixture +def nlp(): + return Language() + + +@pytest.fixture +@registry.misc("entity_ruler_patterns") +def patterns(): + return [ + {"label": "HELLO", "pattern": "hello world"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple", "id": "a1"}, + {"label": "TECH_ORG", "pattern": "Microsoft", "id": "a2"}, + ] + + +@Language.component("add_ent") +def add_ent_component(doc): + doc.ents = [Span(doc, 0, 3, label="ORG")] + return doc + + +@pytest.mark.issue(3345) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue3345(entity_ruler_factory): + """Test case where preset entity crosses sentence boundary.""" + nlp = English() + doc = Doc(nlp.vocab, words=["I", "live", "in", "New", "York"]) + doc[4].is_sent_start = True + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns([{"label": "GPE", "pattern": "New York"}]) + cfg = {"model": DEFAULT_NER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + ner = EntityRecognizer(doc.vocab, model) + # Add the OUT action. I wouldn't have thought this would be necessary... + ner.moves.add_action(5, "") + ner.add_label("GPE") + doc = ruler(doc) + # Get into the state just before "New" + state = ner.moves.init_batch([doc])[0] + ner.moves.apply_transition(state, "O") + ner.moves.apply_transition(state, "O") + ner.moves.apply_transition(state, "O") + # Check that B-GPE is valid. + assert ner.moves.is_valid(state, "B-GPE") + + +@pytest.mark.issue(4849) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue4849(entity_ruler_factory): + nlp = English() + patterns = [ + {"label": "PERSON", "pattern": "joe biden", "id": "joe-biden"}, + {"label": "PERSON", "pattern": "bernie sanders", "id": "bernie-sanders"}, + ] + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"phrase_matcher_attr": "LOWER"}, + ) + ruler.add_patterns(patterns) + text = """ + The left is starting to take aim at Democratic front-runner Joe Biden. + Sen. Bernie Sanders joined in her criticism: "There is no 'middle ground' when it comes to climate policy." + """ + # USING 1 PROCESS + count_ents = 0 + for doc in nlp.pipe([text], n_process=1): + count_ents += len([ent for ent in doc.ents if ent.ent_id > 0]) + assert count_ents == 2 + # USING 2 PROCESSES + if isinstance(get_current_ops, NumpyOps): + count_ents = 0 + for doc in nlp.pipe([text], n_process=2): + count_ents += len([ent for ent in doc.ents if ent.ent_id > 0]) + assert count_ents == 2 + + +@pytest.mark.issue(5918) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue5918(entity_ruler_factory): + # Test edge case when merging entities. + nlp = English() + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "ORG", "pattern": "Digicon Inc"}, + {"label": "ORG", "pattern": "Rotan Mosle Inc's"}, + {"label": "ORG", "pattern": "Rotan Mosle Technology Partners Ltd"}, + ] + ruler.add_patterns(patterns) + + text = """ + Digicon Inc said it has completed the previously-announced disposition + of its computer systems division to an investment group led by + Rotan Mosle Inc's Rotan Mosle Technology Partners Ltd affiliate. + """ + doc = nlp(text) + assert len(doc.ents) == 3 + # make it so that the third span's head is within the entity (ent_iob=I) + # bug #5918 would wrongly transfer that I to the full entity, resulting in 2 instead of 3 final ents. + # TODO: test for logging here + # with pytest.warns(UserWarning): + # doc[29].head = doc[33] + doc = merge_entities(doc) + assert len(doc.ents) == 3 + + +@pytest.mark.issue(8168) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_issue8168(entity_ruler_factory): + nlp = English() + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "ORG", "pattern": "Apple"}, + { + "label": "GPE", + "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}], + "id": "san-francisco", + }, + { + "label": "GPE", + "pattern": [{"LOWER": "san"}, {"LOWER": "fran"}], + "id": "san-francisco", + }, + ] + ruler.add_patterns(patterns) + doc = nlp("San Francisco San Fran") + assert all(t.ent_id_ == "san-francisco" for t in doc) + + +@pytest.mark.issue(8216) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fix8216(nlp, patterns, entity_ruler_factory): + """Test that patterns don't get added excessively.""" + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"validate": True} + ) + ruler.add_patterns(patterns) + pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert pattern_count > 0 + ruler.add_patterns([]) + after_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert after_count == pattern_count + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + assert "HELLO" in ruler + assert "BYE" in ruler + nlp.remove_pipe("entity_ruler") + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + doc = nlp("hello world bye bye") + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "HELLO" + assert doc.ents[1].label_ == "BYE" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_no_patterns_warns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + assert len(ruler) == 0 + assert len(ruler.labels) == 0 + nlp.remove_pipe("entity_ruler") + nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + assert nlp.pipe_names == ["entity_ruler"] + with pytest.warns(UserWarning): + doc = nlp("hello world bye bye") + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_patterns(nlp, patterns, entity_ruler_factory): + # initialize with patterns + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + assert len(ruler.labels) == 0 + ruler.initialize(lambda: [], patterns=patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.ents[0].label_ == "HELLO" + assert doc.ents[1].label_ == "BYE" + nlp.remove_pipe("entity_ruler") + # initialize with patterns from misc registry + nlp.config["initialize"]["components"]["entity_ruler"] = { + "patterns": {"@misc": "entity_ruler_patterns"} + } + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + assert len(ruler.labels) == 0 + nlp.initialize() + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.ents[0].label_ == "HELLO" + assert doc.ents[1].label_ == "BYE" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_init_clear(nlp, patterns, entity_ruler_factory): + """Test that initialization clears patterns.""" + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + ruler.initialize(lambda: []) + assert len(ruler.labels) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_clear(nlp, patterns, entity_ruler_factory): + """Test that initialization clears patterns.""" + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world") + assert len(doc.ents) == 1 + ruler.clear() + assert len(ruler.labels) == 0 + with pytest.warns(UserWarning): + doc = nlp("hello world") + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") + doc = nlp("OH HELLO WORLD bye bye") + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "ORG" + assert doc.ents[1].label_ == "BYE" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_overwrite(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") + doc = nlp("OH HELLO WORLD bye bye") + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "HELLO" + assert doc.ents[0].text == "HELLO" + assert doc.ents[1].label_ == "BYE" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_existing_complex(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) + ruler.add_patterns(patterns) + nlp.add_pipe("add_ent", before="entity_ruler") + doc = nlp("foo foo bye bye") + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "COMPLEX" + assert doc.ents[1].label_ == "BYE" + assert len(doc.ents[0]) == 2 + assert len(doc.ents[1]) == 2 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_entity_id(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe( + entity_ruler_factory, name="entity_ruler", config={"overwrite_ents": True} + ) + ruler.add_patterns(patterns) + doc = nlp("Apple is a technology company") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "TECH_ORG" + assert doc.ents[0].ent_id_ == "a1" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_cfg_ent_id_sep(nlp, patterns, entity_ruler_factory): + config = {"overwrite_ents": True, "ent_id_sep": "**"} + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler", config=config) + ruler.add_patterns(patterns) + doc = nlp("Apple is a technology company") + if isinstance(ruler, EntityRuler): + assert "TECH_ORG**a1" in ruler.phrase_patterns + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "TECH_ORG" + assert doc.ents[0].ent_id_ == "a1" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_bytes(nlp, patterns, entity_ruler_factory): + ruler = EntityRuler(nlp, patterns=patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + ruler_bytes = ruler.to_bytes() + new_ruler = EntityRuler(nlp) + assert len(new_ruler) == 0 + assert len(new_ruler.labels) == 0 + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(patterns) + assert len(new_ruler.labels) == 4 + assert len(new_ruler.patterns) == len(ruler.patterns) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert sorted(new_ruler.labels) == sorted(ruler.labels) + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_phrase_matcher_attr_bytes( + nlp, patterns, entity_ruler_factory +): + ruler = EntityRuler(nlp, phrase_matcher_attr="LOWER", patterns=patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + ruler_bytes = ruler.to_bytes() + new_ruler = EntityRuler(nlp) + assert len(new_ruler) == 0 + assert len(new_ruler.labels) == 0 + assert new_ruler.phrase_matcher_attr is None + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(patterns) + assert len(new_ruler.labels) == 4 + assert new_ruler.phrase_matcher_attr == "LOWER" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_validate(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + validated_ruler = EntityRuler(nlp, validate=True) + + valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} + invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]} + + # invalid pattern raises error without validate + with pytest.raises(ValueError): + ruler.add_patterns([invalid_pattern]) + + # valid pattern is added without errors with validate + validated_ruler.add_patterns([valid_pattern]) + + # invalid pattern raises error with validate + with pytest.raises(MatchPatternError): + validated_ruler.add_patterns([invalid_pattern]) + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_properties(nlp, patterns, entity_ruler_factory): + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) + assert sorted(ruler.labels) == sorted(["HELLO", "BYE", "COMPLEX", "TECH_ORG"]) + assert sorted(ruler.ent_ids) == ["a1", "a2"] + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "FOOBAR", "pattern": "foo bar"}, + {"label": "BARBAZ", "pattern": "bar baz"}, + ] + ruler.add_patterns(patterns) + doc = nlp("foo bar baz") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "FOOBAR" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_pipe(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_disabled(nlp, entity_ruler_factory): + @registry.misc("test_fuzzy_compare_disabled") + def make_test_fuzzy_compare_disabled(): + return lambda x, y, z: False + + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"matcher_fuzzy_compare": {"@misc": "test_fuzzy_compare_disabled"}}, + ) + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("n_process", [1, 2]) +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + texts = ["I enjoy eating Pizza Hut pizza."] + + patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut", "id": "1234"}] + + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + + for doc in nlp.pipe(texts, n_process=2): + for ent in doc.ents: + assert ent.ent_id_ == "1234" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_jsonl(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + with make_tempdir() as d: + ruler.to_disk(d / "test_ruler.jsonl") + ruler.from_disk(d / "test_ruler.jsonl") # read from an existing jsonl file + with pytest.raises(ValueError): + ruler.from_disk(d / "non_existing.jsonl") # read from a bad jsonl file + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_serialize_dir(nlp, patterns, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + ruler.add_patterns(patterns) + with make_tempdir() as d: + ruler.to_disk(d / "test_ruler") + ruler.from_disk(d / "test_ruler") # read from an existing directory + with pytest.raises(ValueError): + ruler.from_disk(d / "non_existing_dir") # read from a bad directory + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_basic(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + {"label": "ORG", "pattern": "ACM"}, + ] + ruler.add_patterns(patterns) + doc = nlp("Dina went to school") + assert len(ruler.patterns) == 3 + assert len(doc.ents) == 1 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher + assert doc.ents[0].label_ == "PERSON" + assert doc.ents[0].text == "Dina" + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina went to school") + assert len(doc.ents) == 0 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher + assert len(ruler.patterns) == 2 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_same_id_multiple_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "DinaCorp", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + ] + ruler.add_patterns(patterns) + doc = nlp("Dina founded DinaCorp and ACME.") + assert len(ruler.patterns) == 3 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" in ruler.phrase_matcher + assert "ORG||dina" in ruler.phrase_matcher + assert len(doc.ents) == 3 + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded DinaCorp and ACME.") + assert len(ruler.patterns) == 1 + if isinstance(ruler, EntityRuler): + assert "PERSON||dina" not in ruler.phrase_matcher + assert "ORG||dina" not in ruler.phrase_matcher + assert len(doc.ents) == 1 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_nonexisting_pattern(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + {"label": "ORG", "pattern": "ACM"}, + ] + ruler.add_patterns(patterns) + assert len(ruler.patterns) == 3 + with pytest.raises(ValueError): + ruler.remove("nepattern") + if isinstance(ruler, SpanRuler): + with pytest.raises(ValueError): + ruler.remove_by_id("nepattern") + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_several_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + {"label": "ORG", "pattern": "ACM"}, + ] + ruler.add_patterns(patterns) + doc = nlp("Dina founded her company ACME.") + assert len(ruler.patterns) == 3 + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "PERSON" + assert doc.ents[0].text == "Dina" + assert doc.ents[1].label_ == "ORG" + assert doc.ents[1].text == "ACME" + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + doc = nlp("Dina founded her company ACME") + assert len(ruler.patterns) == 2 + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "ORG" + assert doc.ents[0].text == "ACME" + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") + doc = nlp("Dina founded her company ACME") + assert len(ruler.patterns) == 1 + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_patterns_in_a_row(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + {"label": "DATE", "pattern": "her birthday", "id": "bday"}, + {"label": "ORG", "pattern": "ACM"}, + ] + ruler.add_patterns(patterns) + doc = nlp("Dina founded her company ACME on her birthday") + assert len(doc.ents) == 3 + assert doc.ents[0].label_ == "PERSON" + assert doc.ents[0].text == "Dina" + assert doc.ents[1].label_ == "ORG" + assert doc.ents[1].text == "ACME" + assert doc.ents[2].label_ == "DATE" + assert doc.ents[2].text == "her birthday" + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + ruler.remove("acme") + ruler.remove("bday") + else: + ruler.remove_by_id("dina") + ruler.remove_by_id("acme") + ruler.remove_by_id("bday") + doc = nlp("Dina went to school") + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_all_patterns(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [ + {"label": "PERSON", "pattern": "Dina", "id": "dina"}, + {"label": "ORG", "pattern": "ACME", "id": "acme"}, + {"label": "DATE", "pattern": "her birthday", "id": "bday"}, + ] + ruler.add_patterns(patterns) + assert len(ruler.patterns) == 3 + if isinstance(ruler, EntityRuler): + ruler.remove("dina") + else: + ruler.remove_by_id("dina") + assert len(ruler.patterns) == 2 + if isinstance(ruler, EntityRuler): + ruler.remove("acme") + else: + ruler.remove_by_id("acme") + assert len(ruler.patterns) == 1 + if isinstance(ruler, EntityRuler): + ruler.remove("bday") + else: + ruler.remove_by_id("bday") + assert len(ruler.patterns) == 0 + with pytest.warns(UserWarning): + doc = nlp("Dina founded her company ACME on her birthday") + assert len(doc.ents) == 0 + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_remove_and_add(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "DATE", "pattern": "last time"}] + ruler.add_patterns(patterns) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "DATE" + assert doc.ents[0].text == "last time" + patterns1 = [{"label": "DATE", "pattern": "this time", "id": "ttime"}] + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "DATE" + assert doc.ents[0].text == "last time" + assert doc.ents[1].label_ == "DATE" + assert doc.ents[1].text == "this time" + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "DATE" + assert doc.ents[0].text == "last time" + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.ents) == 2 + patterns2 = [{"label": "DATE", "pattern": "another time", "id": "ttime"}] + ruler.add_patterns(patterns2) + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 3 + assert len(doc.ents) == 3 + if isinstance(ruler, EntityRuler): + ruler.remove("ttime") + else: + ruler.remove_by_id("ttime") + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 1 + assert len(doc.ents) == 1 diff --git a/spacy/tests/pipeline/test_functions.py b/spacy/tests/pipeline/test_functions.py new file mode 100644 index 0000000..f4db4ee --- /dev/null +++ b/spacy/tests/pipeline/test_functions.py @@ -0,0 +1,102 @@ +import pytest + +from spacy.language import Language +from spacy.pipeline.functions import merge_subtokens +from spacy.tokens import Doc, Span + +from ..doc.test_underscore import clean_underscore # noqa: F401 + + +@pytest.fixture +def doc(en_vocab): + # fmt: off + words = ["This", "is", "a", "sentence", ".", "This", "is", "another", "sentence", ".", "And", "a", "third", "."] + heads = [1, 1, 3, 1, 1, 6, 6, 8, 6, 6, 11, 12, 13, 13] + deps = ["nsubj", "ROOT", "subtok", "attr", "punct", "nsubj", "ROOT", + "subtok", "attr", "punct", "subtok", "subtok", "subtok", "ROOT"] + # fmt: on + return Doc(en_vocab, words=words, heads=heads, deps=deps) + + +@pytest.fixture +def doc2(en_vocab): + words = ["I", "like", "New", "York", "in", "Autumn", "."] + heads = [1, 1, 3, 1, 1, 4, 1] + tags = ["PRP", "IN", "NNP", "NNP", "IN", "NNP", "."] + pos = ["PRON", "VERB", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] + deps = ["ROOT", "prep", "compound", "pobj", "prep", "pobj", "punct"] + doc = Doc(en_vocab, words=words, heads=heads, tags=tags, pos=pos, deps=deps) + doc.ents = [Span(doc, 2, 4, label="GPE")] + return doc + + +def test_merge_subtokens(doc): + doc = merge_subtokens(doc) + # Doc doesn't have spaces, so the result is "And a third ." + # fmt: off + assert [t.text for t in doc] == ["This", "is", "a sentence", ".", "This", "is", "another sentence", ".", "And a third ."] + # fmt: on + + +def test_factories_merge_noun_chunks(doc2): + assert len(doc2) == 7 + nlp = Language() + merge_noun_chunks = nlp.create_pipe("merge_noun_chunks") + merge_noun_chunks(doc2) + assert len(doc2) == 6 + assert doc2[2].text == "New York" + + +def test_factories_merge_ents(doc2): + assert len(doc2) == 7 + assert len(list(doc2.ents)) == 1 + nlp = Language() + merge_entities = nlp.create_pipe("merge_entities") + merge_entities(doc2) + assert len(doc2) == 6 + assert len(list(doc2.ents)) == 1 + assert doc2[2].text == "New York" + + +def test_token_splitter(): + nlp = Language() + config = {"min_length": 20, "split_length": 5} + token_splitter = nlp.add_pipe("token_splitter", config=config) + doc = nlp("aaaaabbbbbcccccdddd e f g") + assert [t.text for t in doc] == ["aaaaabbbbbcccccdddd", "e", "f", "g"] + doc = nlp("aaaaabbbbbcccccdddddeeeeeff g h i") + assert [t.text for t in doc] == [ + "aaaaa", + "bbbbb", + "ccccc", + "ddddd", + "eeeee", + "ff", + "g", + "h", + "i", + ] + assert all(len(t.text) <= token_splitter.split_length for t in doc) + + +@pytest.mark.usefixtures("clean_underscore") +def test_factories_doc_cleaner(): + nlp = Language() + nlp.add_pipe("doc_cleaner") + doc = nlp.make_doc("text") + doc.tensor = [1, 2, 3] + doc = nlp(doc) + assert doc.tensor is None + + nlp = Language() + nlp.add_pipe("doc_cleaner", config={"silent": False}) + with pytest.warns(UserWarning): + doc = nlp("text") + + Doc.set_extension("test_attr", default=-1) + nlp = Language() + nlp.add_pipe("doc_cleaner", config={"attrs": {"_.test_attr": 0}}) + doc = nlp.make_doc("text") + doc._.test_attr = 100 + doc = nlp(doc) + assert doc._.test_attr == 0 diff --git a/spacy/tests/pipeline/test_initialize.py b/spacy/tests/pipeline/test_initialize.py new file mode 100644 index 0000000..71b1222 --- /dev/null +++ b/spacy/tests/pipeline/test_initialize.py @@ -0,0 +1,70 @@ +import pytest +from pydantic import StrictBool +from thinc.api import ConfigValidationError + +from spacy.lang.en import English +from spacy.language import Language +from spacy.training import Example + + +def test_initialize_arguments(): + name = "test_initialize_arguments" + + class CustomTokenizer: + def __init__(self, tokenizer): + self.tokenizer = tokenizer + self.from_initialize = None + + def __call__(self, text): + return self.tokenizer(text) + + def initialize(self, get_examples, nlp, custom: int): + self.from_initialize = custom + + class Component: + def __init__(self): + self.from_initialize = None + + def initialize( + self, get_examples, nlp, custom1: str, custom2: StrictBool = False + ): + self.from_initialize = (custom1, custom2) + + Language.factory(name, func=lambda nlp, name: Component()) + + nlp = English() + nlp.tokenizer = CustomTokenizer(nlp.tokenizer) + example = Example.from_dict(nlp("x"), {}) + get_examples = lambda: [example] + nlp.add_pipe(name) + # The settings here will typically come from the [initialize] block + init_cfg = {"tokenizer": {"custom": 1}, "components": {name: {}}} + nlp.config["initialize"].update(init_cfg) + with pytest.raises(ConfigValidationError) as e: + # Empty config for component, no required custom1 argument + nlp.initialize(get_examples) + errors = e.value.errors + assert len(errors) == 1 + assert errors[0]["loc"] == ("custom1",) + assert errors[0]["type"] == "missing" + init_cfg = { + "tokenizer": {"custom": 1}, + "components": {name: {"custom1": "x", "custom2": 1}}, + } + nlp.config["initialize"].update(init_cfg) + with pytest.raises(ConfigValidationError) as e: + # Wrong type of custom 2 + nlp.initialize(get_examples) + errors = e.value.errors + assert len(errors) == 1 + assert errors[0]["loc"] == ("custom2",) + assert errors[0]["type"] == "bool_type" + init_cfg = { + "tokenizer": {"custom": 1}, + "components": {name: {"custom1": "x"}}, + } + nlp.config["initialize"].update(init_cfg) + nlp.initialize(get_examples) + assert nlp.tokenizer.from_initialize == 1 + pipe = nlp.get_pipe(name) + assert pipe.from_initialize == ("x", False) diff --git a/spacy/tests/pipeline/test_lemmatizer.py b/spacy/tests/pipeline/test_lemmatizer.py new file mode 100644 index 0000000..ccc2e0b --- /dev/null +++ b/spacy/tests/pipeline/test_lemmatizer.py @@ -0,0 +1,115 @@ +import pickle + +import pytest + +from spacy import registry, util +from spacy.lang.en import English +from spacy.lookups import Lookups + +from ..util import make_tempdir + + +@pytest.fixture +def nlp(): + @registry.misc("cope_lookups") + def cope_lookups(): + lookups = Lookups() + lookups.add_table("lemma_lookup", {"cope": "cope", "coped": "cope"}) + lookups.add_table("lemma_index", {"verb": ("cope", "cop")}) + lookups.add_table("lemma_exc", {"verb": {"coping": ("cope",)}}) + lookups.add_table("lemma_rules", {"verb": [["ing", ""]]}) + return lookups + + nlp = English() + nlp.config["initialize"]["components"]["lemmatizer"] = { + "lookups": {"@misc": "cope_lookups"} + } + return nlp + + +def test_lemmatizer_init(nlp): + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"}) + assert isinstance(lemmatizer.lookups, Lookups) + assert not lemmatizer.lookups.tables + assert lemmatizer.mode == "lookup" + with pytest.raises(ValueError): + nlp("test") + nlp.initialize() + assert lemmatizer.lookups.tables + assert nlp("cope")[0].lemma_ == "cope" + assert nlp("coped")[0].lemma_ == "cope" + # replace any tables from spacy-lookups-data + lemmatizer.lookups = Lookups() + # lookup with no tables sets text as lemma + assert nlp("cope")[0].lemma_ == "cope" + assert nlp("coped")[0].lemma_ == "coped" + nlp.remove_pipe("lemmatizer") + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "lookup"}) + with pytest.raises(ValueError): + # Can't initialize without required tables + lemmatizer.initialize(lookups=Lookups()) + lookups = Lookups() + lookups.add_table("lemma_lookup", {}) + lemmatizer.initialize(lookups=lookups) + + +def test_lemmatizer_config(nlp): + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "rule"}) + nlp.initialize() + + # warning if no POS assigned + doc = nlp.make_doc("coping") + with pytest.warns(UserWarning): + doc = lemmatizer(doc) + # warns once by default + doc = lemmatizer(doc) + + # works with POS + doc = nlp.make_doc("coping") + assert doc[0].lemma_ == "" + doc[0].pos_ = "VERB" + doc = lemmatizer(doc) + doc = lemmatizer(doc) + assert doc[0].text == "coping" + assert doc[0].lemma_ == "cope" + + doc = nlp.make_doc("coping") + doc[0].pos_ = "VERB" + assert doc[0].lemma_ == "" + doc = lemmatizer(doc) + assert doc[0].text == "coping" + assert doc[0].lemma_ == "cope" + + +def test_lemmatizer_serialize(nlp): + lemmatizer = nlp.add_pipe("lemmatizer", config={"mode": "rule"}) + nlp.initialize() + + def cope_lookups(): + lookups = Lookups() + lookups.add_table("lemma_lookup", {"cope": "cope", "coped": "cope"}) + lookups.add_table("lemma_index", {"verb": ("cope", "cop")}) + lookups.add_table("lemma_exc", {"verb": {"coping": ("cope",)}}) + lookups.add_table("lemma_rules", {"verb": [["ing", ""]]}) + return lookups + + nlp2 = English() + lemmatizer2 = nlp2.add_pipe("lemmatizer", config={"mode": "rule"}) + lemmatizer2.initialize(lookups=cope_lookups()) + lemmatizer2.from_bytes(lemmatizer.to_bytes()) + assert lemmatizer.to_bytes() == lemmatizer2.to_bytes() + assert lemmatizer.lookups.tables == lemmatizer2.lookups.tables + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2.make_doc("coping") + doc2[0].pos_ = "VERB" + assert doc2[0].lemma_ == "" + doc2 = lemmatizer2(doc2) + assert doc2[0].text == "coping" + assert doc2[0].lemma_ == "cope" + + # Make sure that lemmatizer cache can be pickled + pickle.dumps(lemmatizer2) diff --git a/spacy/tests/pipeline/test_models.py b/spacy/tests/pipeline/test_models.py new file mode 100644 index 0000000..fef0017 --- /dev/null +++ b/spacy/tests/pipeline/test_models.py @@ -0,0 +1,108 @@ +from typing import List + +import numpy +import pytest +from numpy.testing import assert_almost_equal +from thinc.api import Model, data_validation, get_current_ops +from thinc.types import Array2d, Ragged + +from spacy.lang.en import English +from spacy.ml import FeatureExtractor, StaticVectors +from spacy.ml._character_embed import CharacterEmbed +from spacy.tokens import Doc +from spacy.vocab import Vocab + +OPS = get_current_ops() + +texts = ["These are 4 words", "Here just three"] +l0 = [[1, 2], [3, 4], [5, 6], [7, 8]] +l1 = [[9, 8], [7, 6], [5, 4]] +list_floats = [OPS.xp.asarray(l0, dtype="f"), OPS.xp.asarray(l1, dtype="f")] +list_ints = [OPS.xp.asarray(l0, dtype="i"), OPS.xp.asarray(l1, dtype="i")] +array = OPS.xp.asarray(l1, dtype="f") +ragged = Ragged(array, OPS.xp.asarray([2, 1], dtype="i")) + + +def get_docs(): + vocab = Vocab() + for t in texts: + for word in t.split(): + hash_id = vocab.strings.add(word) + vector = numpy.random.uniform(-1, 1, (7,)) + vocab.set_vector(hash_id, vector) + docs = [English(vocab)(t) for t in texts] + return docs + + +# Test components with a model of type Model[List[Doc], List[Floats2d]] +@pytest.mark.parametrize("name", ["tagger", "tok2vec", "morphologizer", "senter"]) +def test_components_batching_list(name): + nlp = English() + proc = nlp.create_pipe(name) + util_batch_unbatch_docs_list(proc.model, get_docs(), list_floats) + + +# Test components with a model of type Model[List[Doc], Floats2d] +@pytest.mark.parametrize("name", ["textcat"]) +def test_components_batching_array(name): + nlp = English() + proc = nlp.create_pipe(name) + util_batch_unbatch_docs_array(proc.model, get_docs(), array) + + +LAYERS = [ + (CharacterEmbed(nM=5, nC=3), get_docs(), list_floats), + (FeatureExtractor([100, 200]), get_docs(), list_ints), + (StaticVectors(), get_docs(), ragged), +] + + +@pytest.mark.parametrize("model,in_data,out_data", LAYERS) +def test_layers_batching_all(model, in_data, out_data): + # In = List[Doc] + if isinstance(in_data, list) and isinstance(in_data[0], Doc): + if isinstance(out_data, OPS.xp.ndarray) and out_data.ndim == 2: + util_batch_unbatch_docs_array(model, in_data, out_data) + elif ( + isinstance(out_data, list) + and isinstance(out_data[0], OPS.xp.ndarray) + and out_data[0].ndim == 2 + ): + util_batch_unbatch_docs_list(model, in_data, out_data) + elif isinstance(out_data, Ragged): + util_batch_unbatch_docs_ragged(model, in_data, out_data) + + +def util_batch_unbatch_docs_list( + model: Model[List[Doc], List[Array2d]], in_data: List[Doc], out_data: List[Array2d] +): + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data) + Y_not_batched = [model.predict([u])[0] for u in in_data] + for i in range(len(Y_batched)): + assert_almost_equal( + OPS.to_numpy(Y_batched[i]), OPS.to_numpy(Y_not_batched[i]), decimal=4 + ) + + +def util_batch_unbatch_docs_array( + model: Model[List[Doc], Array2d], in_data: List[Doc], out_data: Array2d +): + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data).tolist() + Y_not_batched = [model.predict([u])[0].tolist() for u in in_data] + assert_almost_equal(Y_batched, Y_not_batched, decimal=4) + + +def util_batch_unbatch_docs_ragged( + model: Model[List[Doc], Ragged], in_data: List[Doc], out_data: Ragged +): + with data_validation(True): + model.initialize(in_data, out_data) + Y_batched = model.predict(in_data).data.tolist() + Y_not_batched = [] + for u in in_data: + Y_not_batched.extend(model.predict([u]).data.tolist()) + assert_almost_equal(Y_batched, Y_not_batched, decimal=4) diff --git a/spacy/tests/pipeline/test_morphologizer.py b/spacy/tests/pipeline/test_morphologizer.py new file mode 100644 index 0000000..0d895f2 --- /dev/null +++ b/spacy/tests/pipeline/test_morphologizer.py @@ -0,0 +1,226 @@ +import pytest +from numpy.testing import assert_almost_equal, assert_equal +from thinc.api import get_current_ops + +from spacy import util +from spacy.attrs import MORPH +from spacy.lang.en import English +from spacy.language import Language +from spacy.morphology import Morphology +from spacy.tests.util import make_tempdir +from spacy.tokens import Doc +from spacy.training import Example + + +def test_label_types(): + nlp = Language() + morphologizer = nlp.add_pipe("morphologizer") + morphologizer.add_label("Feat=A") + with pytest.raises(ValueError): + morphologizer.add_label(9) + + +TAGS = ["Feat=N", "Feat=V", "Feat=J"] + +TRAIN_DATA = [ + ( + "I like green eggs", + { + "morphs": ["Feat=N", "Feat=V", "Feat=J", "Feat=N"], + "pos": ["NOUN", "VERB", "ADJ", "NOUN"], + }, + ), + # test combinations of morph+POS + ("Eat blue ham", {"morphs": ["Feat=V", "", ""], "pos": ["", "ADJ", ""]}), +] + + +def test_label_smoothing(): + nlp = Language() + morph_no_ls = nlp.add_pipe("morphologizer", "no_label_smoothing") + morph_ls = nlp.add_pipe( + "morphologizer", "label_smoothing", config=dict(label_smoothing=0.05) + ) + train_examples = [] + losses = {} + for tag in TAGS: + morph_no_ls.add_label(tag) + morph_ls.add_label(tag) + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + nlp.initialize(get_examples=lambda: train_examples) + tag_scores, bp_tag_scores = morph_ls.model.begin_update( + [eg.predicted for eg in train_examples] + ) + ops = get_current_ops() + no_ls_grads = ops.to_numpy(morph_no_ls.get_loss(train_examples, tag_scores)[1][0]) + ls_grads = ops.to_numpy(morph_ls.get_loss(train_examples, tag_scores)[1][0]) + assert_almost_equal(ls_grads / no_ls_grads, 0.94285715) + + +def test_no_label(): + nlp = Language() + nlp.add_pipe("morphologizer") + with pytest.raises(ValueError): + nlp.initialize() + + +def test_implicit_label(): + nlp = Language() + nlp.add_pipe("morphologizer") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + nlp.initialize(get_examples=lambda: train_examples) + + +def test_no_resize(): + nlp = Language() + morphologizer = nlp.add_pipe("morphologizer") + morphologizer.add_label("POS" + Morphology.FIELD_SEP + "NOUN") + morphologizer.add_label("POS" + Morphology.FIELD_SEP + "VERB") + nlp.initialize() + # this throws an error because the morphologizer can't be resized after initialization + with pytest.raises(ValueError): + morphologizer.add_label("POS" + Morphology.FIELD_SEP + "ADJ") + + +def test_initialize_examples(): + nlp = Language() + morphologizer = nlp.add_pipe("morphologizer") + morphologizer.add_label("POS" + Morphology.FIELD_SEP + "NOUN") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + # you shouldn't really call this more than once, but for testing it should be fine + nlp.initialize() + nlp.initialize(get_examples=lambda: train_examples) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: None) + with pytest.raises(TypeError): + nlp.initialize(get_examples=train_examples) + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the morphologizer - ensuring the ML models work correctly + nlp = English() + nlp.add_pipe("morphologizer") + train_examples = [] + for inst in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(inst[0]), inst[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["morphologizer"] < 0.00001 + + # test the trained model + test_text = "I like blue ham" + doc = nlp(test_text) + gold_morphs = ["Feat=N", "Feat=V", "", ""] + gold_pos_tags = ["NOUN", "VERB", "ADJ", ""] + assert [str(t.morph) for t in doc] == gold_morphs + assert [t.pos_ for t in doc] == gold_pos_tags + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert [str(t.morph) for t in doc2] == gold_morphs + assert [t.pos_ for t in doc2] == gold_pos_tags + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Just a sentence.", + "Then one more sentence about London.", + "Here is another one.", + "I like London.", + ] + batch_deps_1 = [doc.to_array([MORPH]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([MORPH]) for doc in nlp.pipe(texts)] + no_batch_deps = [doc.to_array([MORPH]) for doc in [nlp(text) for text in texts]] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + # Test without POS + nlp.remove_pipe("morphologizer") + nlp.add_pipe("morphologizer") + for example in train_examples: + for token in example.reference: + token.pos_ = "" + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["morphologizer"] < 0.00001 + + # Test the trained model + test_text = "I like blue ham" + doc = nlp(test_text) + gold_morphs = ["Feat=N", "Feat=V", "", ""] + gold_pos_tags = ["", "", "", ""] + assert [str(t.morph) for t in doc] == gold_morphs + assert [t.pos_ for t in doc] == gold_pos_tags + + # Test overwrite+extend settings + # (note that "" is unset, "_" is set and empty) + morphs = ["Feat=V", "Feat=N", "_"] + doc = Doc(nlp.vocab, words=["blue", "ham", "like"], morphs=morphs) + orig_morphs = [str(t.morph) for t in doc] + orig_pos_tags = [t.pos_ for t in doc] + morphologizer = nlp.get_pipe("morphologizer") + + # don't overwrite or extend + morphologizer.cfg["overwrite"] = False + doc = morphologizer(doc) + assert [str(t.morph) for t in doc] == orig_morphs + assert [t.pos_ for t in doc] == orig_pos_tags + + # overwrite and extend + morphologizer.cfg["overwrite"] = True + morphologizer.cfg["extend"] = True + doc = Doc(nlp.vocab, words=["I", "like"], morphs=["Feat=A|That=A|This=A", ""]) + doc = morphologizer(doc) + assert [str(t.morph) for t in doc] == ["Feat=N|That=A|This=A", "Feat=V"] + + # extend without overwriting + morphologizer.cfg["overwrite"] = False + morphologizer.cfg["extend"] = True + doc = Doc(nlp.vocab, words=["I", "like"], morphs=["Feat=A|That=A|This=A", "That=B"]) + doc = morphologizer(doc) + assert [str(t.morph) for t in doc] == ["Feat=A|That=A|This=A", "Feat=V|That=B"] + + # overwrite without extending + morphologizer.cfg["overwrite"] = True + morphologizer.cfg["extend"] = False + doc = Doc(nlp.vocab, words=["I", "like"], morphs=["Feat=A|That=A|This=A", ""]) + doc = morphologizer(doc) + assert [str(t.morph) for t in doc] == ["Feat=N", "Feat=V"] + + # Test with unset morph and partial POS + nlp.remove_pipe("morphologizer") + nlp.add_pipe("morphologizer") + for example in train_examples: + for token in example.reference: + if token.text == "ham": + token.pos_ = "NOUN" + else: + token.pos_ = "" + token.set_morph(None) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert nlp.get_pipe("morphologizer").labels is not None + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["morphologizer"] < 0.00001 + + # Test the trained model + test_text = "I like blue ham" + doc = nlp(test_text) + gold_morphs = ["", "", "", ""] + gold_pos_tags = ["NOUN", "NOUN", "NOUN", "NOUN"] + assert [str(t.morph) for t in doc] == gold_morphs + assert [t.pos_ for t in doc] == gold_pos_tags diff --git a/spacy/tests/pipeline/test_pipe_factories.py b/spacy/tests/pipeline/test_pipe_factories.py new file mode 100644 index 0000000..a8a6c7d --- /dev/null +++ b/spacy/tests/pipeline/test_pipe_factories.py @@ -0,0 +1,591 @@ +import pytest +from pydantic import StrictInt, StrictStr +from thinc.api import ConfigValidationError, Linear, Model + +import spacy +from spacy.lang.de import German +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.tokens import Doc +from spacy.util import SimpleFrozenDict, combine_score_weights, registry + +from ..util import make_tempdir + + +@pytest.mark.issue(5137) +def test_issue5137(): + factory_name = "test_issue5137" + pipe_name = "my_component" + + @Language.factory(factory_name) + class MyComponent: + def __init__(self, nlp, name=pipe_name, categories="all_categories"): + self.nlp = nlp + self.categories = categories + self.name = name + + def __call__(self, doc): + pass + + def to_disk(self, path, **kwargs): + pass + + def from_disk(self, path, **cfg): + pass + + nlp = English() + my_component = nlp.add_pipe(factory_name, name=pipe_name) + assert my_component.categories == "all_categories" + with make_tempdir() as tmpdir: + nlp.to_disk(tmpdir) + overrides = {"components": {pipe_name: {"categories": "my_categories"}}} + nlp2 = spacy.load(tmpdir, config=overrides) + assert nlp2.get_pipe(pipe_name).categories == "my_categories" + + +def test_pipe_function_component(): + name = "test_component" + + @Language.component(name) + def component(doc: Doc) -> Doc: + return doc + + assert name in registry.factories + nlp = Language() + with pytest.raises(ValueError): + nlp.add_pipe(component) + nlp.add_pipe(name) + assert name in nlp.pipe_names + assert nlp.pipe_factories[name] == name + assert Language.get_factory_meta(name) + assert nlp.get_pipe_meta(name) + pipe = nlp.get_pipe(name) + assert pipe == component + pipe = nlp.create_pipe(name) + assert pipe == component + + +def test_pipe_class_component_init(): + name1 = "test_class_component1" + name2 = "test_class_component2" + + @Language.factory(name1) + class Component1: + def __init__(self, nlp: Language, name: str): + self.nlp = nlp + + def __call__(self, doc: Doc) -> Doc: + return doc + + class Component2: + def __init__(self, nlp: Language, name: str): + self.nlp = nlp + + def __call__(self, doc: Doc) -> Doc: + return doc + + @Language.factory(name2) + def factory(nlp: Language, name=name2): + return Component2(nlp, name) + + nlp = Language() + for name, Component in [(name1, Component1), (name2, Component2)]: + assert name in registry.factories + with pytest.raises(ValueError): + nlp.add_pipe(Component(nlp, name)) + nlp.add_pipe(name) + assert name in nlp.pipe_names + assert nlp.pipe_factories[name] == name + assert Language.get_factory_meta(name) + assert nlp.get_pipe_meta(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe, Component) + assert isinstance(pipe.nlp, Language) + pipe = nlp.create_pipe(name) + assert isinstance(pipe, Component) + assert isinstance(pipe.nlp, Language) + + +def test_pipe_class_component_config(): + name = "test_class_component_config" + + @Language.factory(name) + class Component: + def __init__( + self, nlp: Language, name: str, value1: StrictInt, value2: StrictStr + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + self.is_base = True + self.name = name + + def __call__(self, doc: Doc) -> Doc: + return doc + + @English.factory(name) + class ComponentEN: + def __init__( + self, nlp: Language, name: str, value1: StrictInt, value2: StrictStr + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + self.is_base = False + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + with pytest.raises(ConfigValidationError): # no config provided + nlp.add_pipe(name) + with pytest.raises(ConfigValidationError): # invalid config + nlp.add_pipe(name, config={"value1": "10", "value2": "hello"}) + with pytest.warns(UserWarning): + nlp.add_pipe( + name, config={"value1": 10, "value2": "hello", "name": "wrong_name"} + ) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + assert pipe.is_base is True + assert pipe.name == name + + nlp_en = English() + with pytest.raises(ConfigValidationError): # invalid config + nlp_en.add_pipe(name, config={"value1": "10", "value2": "hello"}) + nlp_en.add_pipe(name, config={"value1": 10, "value2": "hello"}) + pipe = nlp_en.get_pipe(name) + assert isinstance(pipe.nlp, English) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + assert pipe.is_base is False + + +def test_pipe_class_component_defaults(): + name = "test_class_component_defaults" + + @Language.factory(name) + class Component: + def __init__( + self, + nlp: Language, + name: str, + value1: StrictInt = StrictInt(10), + value2: StrictStr = StrictStr("hello"), + ): + self.nlp = nlp + self.value1 = value1 + self.value2 = value2 + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert pipe.value2 == "hello" + + +def test_pipe_class_component_model(): + name = "test_class_component_model" + default_config = { + "model": { + "@architectures": "spacy.TextCatEnsemble.v2", + "tok2vec": DEFAULT_TOK2VEC_MODEL, + "linear_model": { + "@architectures": "spacy.TextCatBOW.v3", + "exclusive_classes": False, + "ngram_size": 1, + "no_output_layer": False, + }, + }, + "value1": 10, + } + + @Language.factory(name, default_config=default_config) + class Component: + def __init__(self, nlp: Language, model: Model, name: str, value1: StrictInt): + self.nlp = nlp + self.model = model + self.value1 = value1 + self.name = name + + def __call__(self, doc: Doc) -> Doc: + return doc + + nlp = Language() + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 10 + assert isinstance(pipe.model, Model) + + +def test_pipe_class_component_model_custom(): + name = "test_class_component_model_custom" + arch = f"{name}.arch" + default_config = {"value1": 1, "model": {"@architectures": arch, "nO": 0, "nI": 0}} + + @Language.factory(name, default_config=default_config) + class Component: + def __init__( + self, + nlp: Language, + model: Model, + name: str, + value1: StrictInt = StrictInt(10), + ): + self.nlp = nlp + self.model = model + self.value1 = value1 + self.name = name + + def __call__(self, doc: Doc) -> Doc: + return doc + + @registry.architectures(arch) + def make_custom_arch(nO: StrictInt, nI: StrictInt): + return Linear(nO, nI) + + nlp = Language() + config = {"value1": 20, "model": {"@architectures": arch, "nO": 1, "nI": 2}} + nlp.add_pipe(name, config=config) + pipe = nlp.get_pipe(name) + assert isinstance(pipe.nlp, Language) + assert pipe.value1 == 20 + assert isinstance(pipe.model, Model) + assert pipe.model.name == "linear" + + nlp = Language() + with pytest.raises(ConfigValidationError): + config = {"value1": "20", "model": {"@architectures": arch, "nO": 1, "nI": 2}} + nlp.add_pipe(name, config=config) + with pytest.raises(ConfigValidationError): + config = {"value1": 20, "model": {"@architectures": arch, "nO": 1.0, "nI": 2.0}} + nlp.add_pipe(name, config=config) + + +def test_pipe_factories_wrong_formats(): + with pytest.raises(ValueError): + # Decorator is not called + @Language.component + def component(foo: int, bar: str): ... + + with pytest.raises(ValueError): + # Decorator is not called + @Language.factory + def factory1(foo: int, bar: str): ... + + with pytest.raises(ValueError): + # Factory function is missing "nlp" and "name" arguments + @Language.factory("test_pipe_factories_missing_args") + def factory2(foo: int, bar: str): ... + + +def test_pipe_factory_meta_config_cleanup(): + """Test that component-specific meta and config entries are represented + correctly and cleaned up when pipes are removed, replaced or renamed.""" + nlp = Language() + nlp.add_pipe("ner", name="ner_component") + nlp.add_pipe("textcat") + assert nlp.get_factory_meta("ner") + assert nlp.get_pipe_meta("ner_component") + assert nlp.get_pipe_config("ner_component") + assert nlp.get_factory_meta("textcat") + assert nlp.get_pipe_meta("textcat") + assert nlp.get_pipe_config("textcat") + nlp.rename_pipe("textcat", "tc") + assert nlp.get_pipe_meta("tc") + assert nlp.get_pipe_config("tc") + with pytest.raises(ValueError): + nlp.remove_pipe("ner") + nlp.remove_pipe("ner_component") + assert "ner_component" not in nlp._pipe_meta + assert "ner_component" not in nlp._pipe_configs + with pytest.raises(ValueError): + nlp.replace_pipe("textcat", "parser") + nlp.replace_pipe("tc", "parser") + assert nlp.get_factory_meta("parser") + assert nlp.get_pipe_meta("tc").factory == "parser" + + +def test_pipe_factories_empty_dict_default(): + """Test that default config values can be empty dicts and that no config + validation error is raised.""" + # TODO: fix this + name = "test_pipe_factories_empty_dict_default" + + @Language.factory(name, default_config={"foo": {}}) + def factory(nlp: Language, name: str, foo: dict): ... + + nlp = Language() + nlp.create_pipe(name) + + +def test_pipe_factories_language_specific(): + """Test that language sub-classes can have their own factories, with + fallbacks to the base factories.""" + name1 = "specific_component1" + name2 = "specific_component2" + Language.component(name1, func=lambda: "base") + English.component(name1, func=lambda: "en") + German.component(name2, func=lambda: "de") + + assert Language.has_factory(name1) + assert not Language.has_factory(name2) + assert English.has_factory(name1) + assert not English.has_factory(name2) + assert German.has_factory(name1) + assert German.has_factory(name2) + + nlp = Language() + assert nlp.create_pipe(name1)() == "base" + with pytest.raises(ValueError): + nlp.create_pipe(name2) + nlp_en = English() + assert nlp_en.create_pipe(name1)() == "en" + with pytest.raises(ValueError): + nlp_en.create_pipe(name2) + nlp_de = German() + assert nlp_de.create_pipe(name1)() == "base" + assert nlp_de.create_pipe(name2)() == "de" + + +def test_language_factories_invalid(): + """Test that assigning directly to Language.factories is now invalid and + raises a custom error.""" + assert isinstance(Language.factories, SimpleFrozenDict) + with pytest.raises(NotImplementedError): + Language.factories["foo"] = "bar" + nlp = Language() + assert isinstance(nlp.factories, SimpleFrozenDict) + assert len(nlp.factories) + with pytest.raises(NotImplementedError): + nlp.factories["foo"] = "bar" + + +@pytest.mark.parametrize( + "weights,override,expected", + [ + ([{"a": 1.0}, {"b": 1.0}, {"c": 1.0}], {}, {"a": 0.33, "b": 0.33, "c": 0.33}), + ([{"a": 1.0}, {"b": 50}, {"c": 100}], {}, {"a": 0.01, "b": 0.33, "c": 0.66}), + ( + [{"a": 0.7, "b": 0.3}, {"c": 1.0}, {"d": 0.5, "e": 0.5}], + {}, + {"a": 0.23, "b": 0.1, "c": 0.33, "d": 0.17, "e": 0.17}, + ), + ( + [{"a": 100, "b": 300}, {"c": 50, "d": 50}], + {}, + {"a": 0.2, "b": 0.6, "c": 0.1, "d": 0.1}, + ), + ([{"a": 0.5, "b": 0.5}, {"b": 1.0}], {}, {"a": 0.33, "b": 0.67}), + ([{"a": 0.5, "b": 0.0}], {}, {"a": 1.0, "b": 0.0}), + ([{"a": 0.5, "b": 0.5}, {"b": 1.0}], {"a": 0.0}, {"a": 0.0, "b": 1.0}), + ([{"a": 0.0, "b": 0.0}, {"c": 0.0}], {}, {"a": 0.0, "b": 0.0, "c": 0.0}), + ([{"a": 0.0, "b": 0.0}, {"c": 1.0}], {}, {"a": 0.0, "b": 0.0, "c": 1.0}), + ( + [{"a": 0.0, "b": 0.0}, {"c": 0.0}], + {"c": 0.2}, + {"a": 0.0, "b": 0.0, "c": 1.0}, + ), + ( + [{"a": 0.5, "b": 0.5, "c": 1.0, "d": 1.0}], + {"a": 0.0, "b": 0.0}, + {"a": 0.0, "b": 0.0, "c": 0.5, "d": 0.5}, + ), + ( + [{"a": 0.5, "b": 0.5, "c": 1.0, "d": 1.0}], + {"a": 0.0, "b": 0.0, "f": 0.0}, + {"a": 0.0, "b": 0.0, "c": 0.5, "d": 0.5, "f": 0.0}, + ), + ], +) +def test_language_factories_combine_score_weights(weights, override, expected): + result = combine_score_weights(weights, override) + assert sum(result.values()) in (0.99, 1.0, 0.0) + assert result == expected + + +def test_language_factories_scores(): + name = "test_language_factories_scores" + func = lambda nlp, name: lambda doc: doc + weights1 = {"a1": 0.5, "a2": 0.5} + weights2 = {"b1": 0.2, "b2": 0.7, "b3": 0.1} + Language.factory(f"{name}1", default_score_weights=weights1, func=func) + Language.factory(f"{name}2", default_score_weights=weights2, func=func) + meta1 = Language.get_factory_meta(f"{name}1") + assert meta1.default_score_weights == weights1 + meta2 = Language.get_factory_meta(f"{name}2") + assert meta2.default_score_weights == weights2 + nlp = Language() + nlp._config["training"]["score_weights"] = {} + nlp.add_pipe(f"{name}1") + nlp.add_pipe(f"{name}2") + cfg = nlp.config["training"] + expected_weights = {"a1": 0.25, "a2": 0.25, "b1": 0.1, "b2": 0.35, "b3": 0.05} + assert cfg["score_weights"] == expected_weights + # Test with custom defaults + config = nlp.config.copy() + config["training"]["score_weights"]["a1"] = 0.0 + config["training"]["score_weights"]["b3"] = 1.3 + nlp = English.from_config(config) + score_weights = nlp.config["training"]["score_weights"] + expected = {"a1": 0.0, "a2": 0.12, "b1": 0.05, "b2": 0.17, "b3": 0.65} + assert score_weights == expected + # Test with null values + config = nlp.config.copy() + config["training"]["score_weights"]["a1"] = None + nlp = English.from_config(config) + score_weights = nlp.config["training"]["score_weights"] + expected = {"a1": None, "a2": 0.12, "b1": 0.05, "b2": 0.17, "b3": 0.66} + assert score_weights == expected + + +def test_pipe_factories_from_source(): + """Test adding components from a source model.""" + source_nlp = English() + source_nlp.add_pipe("tagger", name="my_tagger") + nlp = English() + with pytest.raises(ValueError): + nlp.add_pipe("my_tagger", source="en_core_web_sm") + nlp.add_pipe("my_tagger", source=source_nlp) + assert "my_tagger" in nlp.pipe_names + with pytest.raises(KeyError): + nlp.add_pipe("custom", source=source_nlp) + + +def test_pipe_factories_from_source_language_subclass(): + class CustomEnglishDefaults(English.Defaults): + stop_words = set(["custom", "stop"]) + + @registry.languages("custom_en") + class CustomEnglish(English): + lang = "custom_en" + Defaults = CustomEnglishDefaults + + source_nlp = English() + source_nlp.add_pipe("tagger") + + # custom subclass + nlp = CustomEnglish() + nlp.add_pipe("tagger", source=source_nlp) + assert "tagger" in nlp.pipe_names + + # non-subclass + nlp = German() + nlp.add_pipe("tagger", source=source_nlp) + assert "tagger" in nlp.pipe_names + + # mismatched vectors + nlp = English() + nlp.vocab.vectors.resize((1, 4)) + nlp.vocab.vectors.add("cat", vector=[1, 2, 3, 4]) + with pytest.warns(UserWarning): + nlp.add_pipe("tagger", source=source_nlp) + + +def test_pipe_factories_from_source_custom(): + """Test adding components from a source model with custom components.""" + name = "test_pipe_factories_from_source_custom" + + @Language.factory(name, default_config={"arg": "hello"}) + def test_factory(nlp, name, arg: str): + return lambda doc: doc + + source_nlp = English() + source_nlp.add_pipe("tagger") + source_nlp.add_pipe(name, config={"arg": "world"}) + nlp = English() + nlp.add_pipe(name, source=source_nlp) + assert name in nlp.pipe_names + assert nlp.get_pipe_meta(name).default_config["arg"] == "hello" + config = nlp.config["components"][name] + assert config["factory"] == name + assert config["arg"] == "world" + + +def test_pipe_factories_from_source_config(): + name = "test_pipe_factories_from_source_config" + + @Language.factory(name, default_config={"arg": "hello"}) + def test_factory(nlp, name, arg: str): + return lambda doc: doc + + source_nlp = English() + source_nlp.add_pipe("tagger") + source_nlp.add_pipe(name, name="yolo", config={"arg": "world"}) + dest_nlp_cfg = {"lang": "en", "pipeline": ["parser", "custom"]} + with make_tempdir() as tempdir: + source_nlp.to_disk(tempdir) + dest_components_cfg = { + "parser": {"factory": "parser"}, + "custom": {"source": str(tempdir), "component": "yolo"}, + } + dest_config = {"nlp": dest_nlp_cfg, "components": dest_components_cfg} + nlp = English.from_config(dest_config) + assert nlp.pipe_names == ["parser", "custom"] + assert nlp.pipe_factories == {"parser": "parser", "custom": name} + meta = nlp.get_pipe_meta("custom") + assert meta.factory == name + assert meta.default_config["arg"] == "hello" + config = nlp.config["components"]["custom"] + assert config["factory"] == name + assert config["arg"] == "world" + + +class PipeFactoriesIdempotent: + def __init__(self, nlp, name): ... + + def __call__(self, doc): ... + + +@pytest.mark.parametrize( + "i,func,func2", + [ + (0, lambda nlp, name: lambda doc: doc, lambda doc: doc), + (1, PipeFactoriesIdempotent, PipeFactoriesIdempotent(None, None)), + ], +) +def test_pipe_factories_decorator_idempotent(i, func, func2): + """Check that decorator can be run multiple times if the function is the + same. This is especially relevant for live reloading because we don't + want spaCy to raise an error if a module registering components is reloaded. + """ + name = f"test_pipe_factories_decorator_idempotent_{i}" + for i in range(5): + Language.factory(name, func=func) + nlp = Language() + nlp.add_pipe(name) + Language.factory(name, func=func) + # Make sure it also works for component decorator, which creates the + # factory function + name2 = f"{name}2" + for i in range(5): + Language.component(name2, func=func2) + nlp = Language() + nlp.add_pipe(name) + Language.component(name2, func=func2) + + +def test_pipe_factories_config_excludes_nlp(): + """Test that the extra values we temporarily add to component config + blocks/functions are removed and not copied around. + """ + name = "test_pipe_factories_config_excludes_nlp" + func = lambda nlp, name: lambda doc: doc + Language.factory(name, func=func) + config = { + "nlp": {"lang": "en", "pipeline": [name]}, + "components": {name: {"factory": name}}, + } + nlp = English.from_config(config) + assert nlp.pipe_names == [name] + pipe_cfg = nlp.get_pipe_config(name) + pipe_cfg == {"factory": name} + assert nlp._pipe_configs[name] == {"factory": name} diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py new file mode 100644 index 0000000..9b9786f --- /dev/null +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -0,0 +1,677 @@ +import gc + +import numpy +import pytest +from thinc.api import get_current_ops + +import spacy +from spacy.lang.en import English +from spacy.lang.en.syntax_iterators import noun_chunks +from spacy.language import Language +from spacy.pipeline import TrainablePipe +from spacy.tokens import Doc +from spacy.training import Example +from spacy.util import SimpleFrozenList, get_arg_names, make_tempdir +from spacy.vocab import Vocab + + +@pytest.fixture +def nlp(): + return Language() + + +@Language.component("new_pipe") +def new_pipe(doc): + return doc + + +@Language.component("other_pipe") +def other_pipe(doc): + return doc + + +@pytest.mark.issue(1506) +def test_issue1506(): + def string_generator(): + for _ in range(10001): + yield "It's sentence produced by that bug." + for _ in range(10001): + yield "I erase some hbdsaj lemmas." + for _ in range(10001): + yield "I erase lemmas." + for _ in range(10001): + yield "It's sentence produced by that bug." + for _ in range(10001): + yield "It's sentence produced by that bug." + + nlp = English() + for i, d in enumerate(nlp.pipe(string_generator())): + # We should run cleanup more than one time to actually cleanup data. + # In first run — clean up only mark strings as «not hitted». + if i == 10000 or i == 20000 or i == 30000: + gc.collect() + for t in d: + str(t.lemma_) + + +@pytest.mark.issue(1654) +def test_issue1654(): + nlp = Language(Vocab()) + assert not nlp.pipeline + + @Language.component("component") + def component(doc): + return doc + + nlp.add_pipe("component", name="1") + nlp.add_pipe("component", name="2", after="1") + nlp.add_pipe("component", name="3", after="2") + assert nlp.pipe_names == ["1", "2", "3"] + nlp2 = Language(Vocab()) + assert not nlp2.pipeline + nlp2.add_pipe("component", name="3") + nlp2.add_pipe("component", name="2", before="3") + nlp2.add_pipe("component", name="1", before="2") + assert nlp2.pipe_names == ["1", "2", "3"] + + +@pytest.mark.issue(3880) +def test_issue3880(): + """Test that `nlp.pipe()` works when an empty string ends the batch. + + Fixed in v7.0.5 of Thinc. + """ + texts = ["hello", "world", "", ""] + nlp = English() + nlp.add_pipe("parser").add_label("dep") + nlp.add_pipe("ner").add_label("PERSON") + nlp.add_pipe("tagger").add_label("NN") + nlp.initialize() + for doc in nlp.pipe(texts): + pass + + +@pytest.mark.issue(5082) +def test_issue5082(): + # Ensure the 'merge_entities' pipeline does something sensible for the vectors of the merged tokens + nlp = English() + vocab = nlp.vocab + array1 = numpy.asarray([0.1, 0.5, 0.8], dtype=numpy.float32) + array2 = numpy.asarray([-0.2, -0.6, -0.9], dtype=numpy.float32) + array3 = numpy.asarray([0.3, -0.1, 0.7], dtype=numpy.float32) + array4 = numpy.asarray([0.5, 0, 0.3], dtype=numpy.float32) + array34 = numpy.asarray([0.4, -0.05, 0.5], dtype=numpy.float32) + vocab.set_vector("I", array1) + vocab.set_vector("like", array2) + vocab.set_vector("David", array3) + vocab.set_vector("Bowie", array4) + text = "I like David Bowie" + patterns = [ + {"label": "PERSON", "pattern": [{"LOWER": "david"}, {"LOWER": "bowie"}]} + ] + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + parsed_vectors_1 = [t.vector for t in nlp(text)] + assert len(parsed_vectors_1) == 4 + ops = get_current_ops() + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_1[0]), array1) + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_1[1]), array2) + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_1[2]), array3) + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_1[3]), array4) + nlp.add_pipe("merge_entities") + parsed_vectors_2 = [t.vector for t in nlp(text)] + assert len(parsed_vectors_2) == 3 + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_2[0]), array1) + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_2[1]), array2) + numpy.testing.assert_array_equal(ops.to_numpy(parsed_vectors_2[2]), array34) + + +@pytest.mark.issue(5458) +def test_issue5458(): + # Test that the noun chuncker does not generate overlapping spans + # fmt: off + words = ["In", "an", "era", "where", "markets", "have", "brought", "prosperity", "and", "empowerment", "."] + vocab = Vocab(strings=words) + deps = ["ROOT", "det", "pobj", "advmod", "nsubj", "aux", "relcl", "dobj", "cc", "conj", "punct"] + pos = ["ADP", "DET", "NOUN", "ADV", "NOUN", "AUX", "VERB", "NOUN", "CCONJ", "NOUN", "PUNCT"] + heads = [0, 2, 0, 9, 6, 6, 2, 6, 7, 7, 0] + # fmt: on + en_doc = Doc(vocab, words=words, pos=pos, heads=heads, deps=deps) + en_doc.noun_chunks_iterator = noun_chunks + + # if there are overlapping spans, this will fail with an E102 error "Can't merge non-disjoint spans" + nlp = English() + merge_nps = nlp.create_pipe("merge_noun_chunks") + merge_nps(en_doc) + + +def test_multiple_predictions(): + class DummyPipe(TrainablePipe): + def __init__(self): + self.model = "dummy_model" + + def predict(self, docs): + return ([1, 2, 3], [4, 5, 6]) + + def set_annotations(self, docs, scores): + return docs + + nlp = Language() + doc = nlp.make_doc("foo") + dummy_pipe = DummyPipe() + dummy_pipe(doc) + + +def test_add_pipe_no_name(nlp): + nlp.add_pipe("new_pipe") + assert "new_pipe" in nlp.pipe_names + + +def test_add_pipe_duplicate_name(nlp): + nlp.add_pipe("new_pipe", name="duplicate_name") + with pytest.raises(ValueError): + nlp.add_pipe("new_pipe", name="duplicate_name") + + +@pytest.mark.parametrize("name", ["parser"]) +def test_add_pipe_first(nlp, name): + nlp.add_pipe("new_pipe", name=name, first=True) + assert nlp.pipeline[0][0] == name + + +@pytest.mark.parametrize("name1,name2", [("parser", "lambda_pipe")]) +def test_add_pipe_last(nlp, name1, name2): + Language.component("new_pipe2", func=lambda doc: doc) + nlp.add_pipe("new_pipe2", name=name2) + nlp.add_pipe("new_pipe", name=name1, last=True) + assert nlp.pipeline[0][0] != name1 + assert nlp.pipeline[-1][0] == name1 + + +def test_cant_add_pipe_first_and_last(nlp): + with pytest.raises(ValueError): + nlp.add_pipe("new_pipe", first=True, last=True) + + +@pytest.mark.parametrize("name", ["test_get_pipe"]) +def test_get_pipe(nlp, name): + with pytest.raises(KeyError): + nlp.get_pipe(name) + nlp.add_pipe("new_pipe", name=name) + assert nlp.get_pipe(name) == new_pipe + + +@pytest.mark.parametrize( + "name,replacement,invalid_replacement", + [("test_replace_pipe", "other_pipe", lambda doc: doc)], +) +def test_replace_pipe(nlp, name, replacement, invalid_replacement): + with pytest.raises(ValueError): + nlp.replace_pipe(name, new_pipe) + nlp.add_pipe("new_pipe", name=name) + with pytest.raises(ValueError): + nlp.replace_pipe(name, invalid_replacement) + nlp.replace_pipe(name, replacement) + assert nlp.get_pipe(name) == nlp.create_pipe(replacement) + + +def test_replace_last_pipe(nlp): + nlp.add_pipe("sentencizer") + nlp.add_pipe("ner") + assert nlp.pipe_names == ["sentencizer", "ner"] + nlp.replace_pipe("ner", "ner") + assert nlp.pipe_names == ["sentencizer", "ner"] + + +def test_replace_pipe_config(nlp): + nlp.add_pipe("entity_linker") + nlp.add_pipe("sentencizer") + assert nlp.get_pipe("entity_linker").incl_prior is True + nlp.replace_pipe("entity_linker", "entity_linker", config={"incl_prior": False}) + assert nlp.get_pipe("entity_linker").incl_prior is False + + +@pytest.mark.parametrize("old_name,new_name", [("old_pipe", "new_pipe")]) +def test_rename_pipe(nlp, old_name, new_name): + with pytest.raises(ValueError): + nlp.rename_pipe(old_name, new_name) + nlp.add_pipe("new_pipe", name=old_name) + nlp.rename_pipe(old_name, new_name) + assert nlp.pipeline[0][0] == new_name + + +@pytest.mark.parametrize("name", ["my_component"]) +def test_remove_pipe(nlp, name): + with pytest.raises(ValueError): + nlp.remove_pipe(name) + nlp.add_pipe("new_pipe", name=name) + assert len(nlp.pipeline) == 1 + removed_name, removed_component = nlp.remove_pipe(name) + assert not len(nlp.pipeline) + assert removed_name == name + assert removed_component == new_pipe + + +@pytest.mark.parametrize("name", ["my_component"]) +def test_disable_pipes_method(nlp, name): + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + disabled = nlp.select_pipes(disable=name) + assert not nlp.has_pipe(name) + disabled.restore() + + +@pytest.mark.parametrize("name", ["my_component"]) +def test_enable_pipes_method(nlp, name): + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + disabled = nlp.select_pipes(enable=[]) + assert not nlp.has_pipe(name) + disabled.restore() + + +@pytest.mark.parametrize("name", ["my_component"]) +def test_disable_pipes_context(nlp, name): + """Test that an enabled component stays enabled after running the context manager.""" + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + with nlp.select_pipes(disable=name): + assert not nlp.has_pipe(name) + assert nlp.has_pipe(name) + + +@pytest.mark.parametrize("name", ["my_component"]) +def test_disable_pipes_context_restore(nlp, name): + """Test that a disabled component stays disabled after running the context manager.""" + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + nlp.disable_pipe(name) + assert not nlp.has_pipe(name) + with nlp.select_pipes(disable=name): + assert not nlp.has_pipe(name) + assert not nlp.has_pipe(name) + + +def test_select_pipes_list_arg(nlp): + for name in ["c1", "c2", "c3"]: + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + with nlp.select_pipes(disable=["c1", "c2"]): + assert not nlp.has_pipe("c1") + assert not nlp.has_pipe("c2") + assert nlp.has_pipe("c3") + with nlp.select_pipes(enable="c3"): + assert not nlp.has_pipe("c1") + assert not nlp.has_pipe("c2") + assert nlp.has_pipe("c3") + with nlp.select_pipes(enable=["c1", "c2"], disable="c3"): + assert nlp.has_pipe("c1") + assert nlp.has_pipe("c2") + assert not nlp.has_pipe("c3") + with nlp.select_pipes(enable=[]): + assert not nlp.has_pipe("c1") + assert not nlp.has_pipe("c2") + assert not nlp.has_pipe("c3") + with nlp.select_pipes(enable=["c1", "c2", "c3"], disable=[]): + assert nlp.has_pipe("c1") + assert nlp.has_pipe("c2") + assert nlp.has_pipe("c3") + with nlp.select_pipes(disable=["c1", "c2", "c3"], enable=[]): + assert not nlp.has_pipe("c1") + assert not nlp.has_pipe("c2") + assert not nlp.has_pipe("c3") + + +def test_select_pipes_errors(nlp): + for name in ["c1", "c2", "c3"]: + nlp.add_pipe("new_pipe", name=name) + assert nlp.has_pipe(name) + + with pytest.raises(ValueError): + nlp.select_pipes() + + with pytest.raises(ValueError): + nlp.select_pipes(enable=["c1", "c2"], disable=["c1"]) + + with pytest.raises(ValueError): + nlp.select_pipes(enable=["c1", "c2"], disable=[]) + + with pytest.raises(ValueError): + nlp.select_pipes(enable=[], disable=["c3"]) + + disabled = nlp.select_pipes(disable=["c2"]) + nlp.remove_pipe("c2") + with pytest.raises(ValueError): + disabled.restore() + + +@pytest.mark.parametrize("n_pipes", [100]) +def test_add_lots_of_pipes(nlp, n_pipes): + Language.component("n_pipes", func=lambda doc: doc) + for i in range(n_pipes): + nlp.add_pipe("n_pipes", name=f"pipe_{i}") + assert len(nlp.pipe_names) == n_pipes + + +@pytest.mark.parametrize("component", [lambda doc: doc, {"hello": "world"}]) +def test_raise_for_invalid_components(nlp, component): + with pytest.raises(ValueError): + nlp.add_pipe(component) + + +@pytest.mark.parametrize("component", ["ner", "tagger", "parser", "textcat"]) +def test_pipe_base_class_add_label(nlp, component): + label = "TEST" + pipe = nlp.create_pipe(component) + pipe.add_label(label) + if component == "tagger": + # Tagger always has the default coarse-grained label scheme + assert label in pipe.labels + else: + assert pipe.labels == (label,) + + +def test_pipe_labels(nlp): + input_labels = { + "ner": ["PERSON", "ORG", "GPE"], + "textcat": ["POSITIVE", "NEGATIVE"], + } + for name, labels in input_labels.items(): + nlp.add_pipe(name) + pipe = nlp.get_pipe(name) + for label in labels: + pipe.add_label(label) + assert len(pipe.labels) == len(labels) + + assert len(nlp.pipe_labels) == len(input_labels) + for name, labels in nlp.pipe_labels.items(): + assert sorted(input_labels[name]) == sorted(labels) + + +def test_add_pipe_before_after(): + """Test that before/after works with strings and ints.""" + nlp = Language() + nlp.add_pipe("ner") + with pytest.raises(ValueError): + nlp.add_pipe("textcat", before="parser") + nlp.add_pipe("textcat", before="ner") + assert nlp.pipe_names == ["textcat", "ner"] + with pytest.raises(ValueError): + nlp.add_pipe("parser", before=3) + with pytest.raises(ValueError): + nlp.add_pipe("parser", after=3) + nlp.add_pipe("parser", after=0) + assert nlp.pipe_names == ["textcat", "parser", "ner"] + nlp.add_pipe("tagger", before=2) + assert nlp.pipe_names == ["textcat", "parser", "tagger", "ner"] + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", after=1, first=True) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", before="ner", after=2) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", before=True) + with pytest.raises(ValueError): + nlp.add_pipe("entity_ruler", first=False) + + +def test_disable_enable_pipes(): + name = "test_disable_enable_pipes" + results = {} + + def make_component(name): + results[name] = "" + + def component(doc): + nonlocal results + results[name] = doc.text + return doc + + return component + + c1 = Language.component(f"{name}1", func=make_component(f"{name}1")) + c2 = Language.component(f"{name}2", func=make_component(f"{name}2")) + + nlp = Language() + nlp.add_pipe(f"{name}1") + nlp.add_pipe(f"{name}2") + assert results[f"{name}1"] == "" + assert results[f"{name}2"] == "" + assert nlp.pipeline == [(f"{name}1", c1), (f"{name}2", c2)] + assert nlp.pipe_names == [f"{name}1", f"{name}2"] + nlp.disable_pipe(f"{name}1") + assert nlp.disabled == [f"{name}1"] + assert nlp.component_names == [f"{name}1", f"{name}2"] + assert nlp.pipe_names == [f"{name}2"] + assert nlp.config["nlp"]["disabled"] == [f"{name}1"] + nlp("hello") + assert results[f"{name}1"] == "" # didn't run + assert results[f"{name}2"] == "hello" # ran + nlp.enable_pipe(f"{name}1") + assert nlp.disabled == [] + assert nlp.pipe_names == [f"{name}1", f"{name}2"] + assert nlp.config["nlp"]["disabled"] == [] + nlp("world") + assert results[f"{name}1"] == "world" + assert results[f"{name}2"] == "world" + nlp.disable_pipe(f"{name}2") + nlp.remove_pipe(f"{name}2") + assert nlp.components == [(f"{name}1", c1)] + assert nlp.pipeline == [(f"{name}1", c1)] + assert nlp.component_names == [f"{name}1"] + assert nlp.pipe_names == [f"{name}1"] + assert nlp.disabled == [] + assert nlp.config["nlp"]["disabled"] == [] + nlp.rename_pipe(f"{name}1", name) + assert nlp.components == [(name, c1)] + assert nlp.component_names == [name] + nlp("!") + assert results[f"{name}1"] == "!" + assert results[f"{name}2"] == "world" + with pytest.raises(ValueError): + nlp.disable_pipe(f"{name}2") + nlp.disable_pipe(name) + assert nlp.component_names == [name] + assert nlp.pipe_names == [] + assert nlp.config["nlp"]["disabled"] == [name] + nlp("?") + assert results[f"{name}1"] == "!" + + +def test_pipe_methods_frozen(): + """Test that spaCy raises custom error messages if "frozen" properties are + accessed. We still want to use a list here to not break backwards + compatibility, but users should see an error if they're trying to append + to nlp.pipeline etc.""" + nlp = Language() + ner = nlp.add_pipe("ner") + assert nlp.pipe_names == ["ner"] + for prop in [ + nlp.pipeline, + nlp.pipe_names, + nlp.components, + nlp.component_names, + nlp.disabled, + nlp.factory_names, + ]: + assert isinstance(prop, list) + assert isinstance(prop, SimpleFrozenList) + with pytest.raises(NotImplementedError): + nlp.pipeline.append(("ner2", ner)) + with pytest.raises(NotImplementedError): + nlp.pipe_names.pop() + with pytest.raises(NotImplementedError): + nlp.components.sort() + with pytest.raises(NotImplementedError): + nlp.component_names.clear() + + +@pytest.mark.parametrize( + "pipe", ["tagger", "parser", "ner", "textcat", "morphologizer"] +) +def test_pipe_label_data_exports_labels(pipe): + nlp = Language() + pipe = nlp.add_pipe(pipe) + # Make sure pipe has pipe labels + assert getattr(pipe, "label_data", None) is not None + # Make sure pipe can be initialized with labels + initialize = getattr(pipe, "initialize", None) + assert initialize is not None + assert "labels" in get_arg_names(initialize) + + +@pytest.mark.parametrize("pipe", ["senter", "entity_linker"]) +def test_pipe_label_data_no_labels(pipe): + nlp = Language() + pipe = nlp.add_pipe(pipe) + assert getattr(pipe, "label_data", None) is None + initialize = getattr(pipe, "initialize", None) + if initialize is not None: + assert "labels" not in get_arg_names(initialize) + + +def test_pipe_methods_initialize(): + """Test that the [initialize] config reflects the components correctly.""" + nlp = Language() + nlp.add_pipe("tagger") + assert "tagger" not in nlp.config["initialize"]["components"] + nlp.config["initialize"]["components"]["tagger"] = {"labels": ["hello"]} + assert nlp.config["initialize"]["components"]["tagger"] == {"labels": ["hello"]} + nlp.remove_pipe("tagger") + assert "tagger" not in nlp.config["initialize"]["components"] + nlp.add_pipe("tagger") + assert "tagger" not in nlp.config["initialize"]["components"] + nlp.config["initialize"]["components"]["tagger"] = {"labels": ["hello"]} + nlp.rename_pipe("tagger", "my_tagger") + assert "tagger" not in nlp.config["initialize"]["components"] + assert nlp.config["initialize"]["components"]["my_tagger"] == {"labels": ["hello"]} + nlp.config["initialize"]["components"]["test"] = {"foo": "bar"} + nlp.add_pipe("ner", name="test") + assert "test" in nlp.config["initialize"]["components"] + nlp.remove_pipe("test") + assert "test" not in nlp.config["initialize"]["components"] + + +def test_update_with_annotates(): + name = "test_with_annotates" + results = {} + + def make_component(name): + results[name] = "" + + def component(doc): + nonlocal results + results[name] += doc.text + return doc + + return component + + Language.component(f"{name}1", func=make_component(f"{name}1")) + Language.component(f"{name}2", func=make_component(f"{name}2")) + + components = set([f"{name}1", f"{name}2"]) + + nlp = English() + texts = ["a", "bb", "ccc"] + examples = [] + for text in texts: + examples.append(Example(nlp.make_doc(text), nlp.make_doc(text))) + + for components_to_annotate in [ + [], + [f"{name}1"], + [f"{name}1", f"{name}2"], + [f"{name}2", f"{name}1"], + ]: + for key in results: + results[key] = "" + nlp = English(vocab=nlp.vocab) + nlp.add_pipe(f"{name}1") + nlp.add_pipe(f"{name}2") + nlp.update(examples, annotates=components_to_annotate) + for component in components_to_annotate: + assert results[component] == "".join(eg.predicted.text for eg in examples) + for component in components - set(components_to_annotate): + assert results[component] == "" + + +@pytest.mark.issue(11443) +def test_enable_disable_conflict_with_config(): + """Test conflict between enable/disable w.r.t. `nlp.disabled` set in the config.""" + nlp = English() + nlp.add_pipe("tagger") + nlp.add_pipe("senter") + nlp.add_pipe("sentencizer") + + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + # Expected to succeed, as config and arguments do not conflict. + assert spacy.load( + tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}} + ).disabled == ["senter", "sentencizer"] + # Expected to succeed without warning due to the lack of a conflicting config option. + spacy.load(tmp_dir, enable=["tagger"]) + # Expected to fail due to conflict between enable and disabled. + with pytest.raises(ValueError): + spacy.load( + tmp_dir, + enable=["senter"], + config={"nlp": {"disabled": ["senter", "tagger"]}}, + ) + + +def test_load_disable_enable(): + """Tests spacy.load() with dis-/enabling components.""" + + base_nlp = English() + for pipe in ("sentencizer", "tagger", "parser"): + base_nlp.add_pipe(pipe) + + with make_tempdir() as tmp_dir: + base_nlp.to_disk(tmp_dir) + to_disable = ["parser", "tagger"] + to_enable = ["tagger", "parser"] + single_str = "tagger" + + # Setting only `disable`. + nlp = spacy.load(tmp_dir, disable=to_disable) + assert all([comp_name in nlp.disabled for comp_name in to_disable]) + + # Setting only `enable`. + nlp = spacy.load(tmp_dir, enable=to_enable) + assert all( + [ + (comp_name in nlp.disabled) is (comp_name not in to_enable) + for comp_name in nlp.component_names + ] + ) + + # Loading with a string representing one component + nlp = spacy.load(tmp_dir, exclude=single_str) + assert single_str not in nlp.component_names + + nlp = spacy.load(tmp_dir, disable=single_str) + assert single_str in nlp.component_names + assert single_str not in nlp.pipe_names + assert nlp._disabled == {single_str} + assert nlp.disabled == [single_str] + + # Testing consistent enable/disable combination. + nlp = spacy.load( + tmp_dir, + enable=to_enable, + disable=[ + comp_name + for comp_name in nlp.component_names + if comp_name not in to_enable + ], + ) + assert all( + [ + (comp_name in nlp.disabled) is (comp_name not in to_enable) + for comp_name in nlp.component_names + ] + ) + + # Inconsistent enable/disable combination. + with pytest.raises(ValueError): + spacy.load(tmp_dir, enable=to_enable, disable=["parser"]) diff --git a/spacy/tests/pipeline/test_sentencizer.py b/spacy/tests/pipeline/test_sentencizer.py new file mode 100644 index 0000000..826086f --- /dev/null +++ b/spacy/tests/pipeline/test_sentencizer.py @@ -0,0 +1,177 @@ +import pytest + +import spacy +from spacy.lang.en import English +from spacy.pipeline import Sentencizer +from spacy.tokens import Doc + + +def test_sentencizer(en_vocab): + doc = Doc(en_vocab, words=["Hello", "!", "This", "is", "a", "test", "."]) + sentencizer = Sentencizer(punct_chars=None) + doc = sentencizer(doc) + assert doc.has_annotation("SENT_START") + sent_starts = [t.is_sent_start for t in doc] + sent_ends = [t.is_sent_end for t in doc] + assert sent_starts == [True, False, True, False, False, False, False] + assert sent_ends == [False, True, False, False, False, False, True] + assert len(list(doc.sents)) == 2 + + +def test_sentencizer_pipe(): + texts = ["Hello! This is a test.", "Hi! This is a test."] + nlp = English() + nlp.add_pipe("sentencizer") + for doc in nlp.pipe(texts): + assert doc.has_annotation("SENT_START") + sent_starts = [t.is_sent_start for t in doc] + assert sent_starts == [True, False, True, False, False, False, False] + assert len(list(doc.sents)) == 2 + for ex in nlp.pipe(texts): + doc = ex.doc + assert doc.has_annotation("SENT_START") + sent_starts = [t.is_sent_start for t in doc] + assert sent_starts == [True, False, True, False, False, False, False] + assert len(list(doc.sents)) == 2 + + +def test_sentencizer_empty_docs(): + one_empty_text = [""] + many_empty_texts = ["", "", ""] + some_empty_texts = ["hi", "", "This is a test. Here are two sentences.", ""] + nlp = English() + nlp.add_pipe("sentencizer") + for texts in [one_empty_text, many_empty_texts, some_empty_texts]: + for doc in nlp.pipe(texts): + assert doc.has_annotation("SENT_START") + sent_starts = [t.is_sent_start for t in doc] + if len(doc) == 0: + assert sent_starts == [] + else: + assert len(sent_starts) > 0 + + +@pytest.mark.parametrize( + "words,sent_starts,sent_ends,n_sents", + [ + # The expected result here is that the duplicate punctuation gets merged + # onto the same sentence and no one-token sentence is created for them. + ( + ["Hello", "!", ".", "Test", ".", ".", "ok"], + [True, False, False, True, False, False, True], + [False, False, True, False, False, True, True], + 3, + ), + # We also want to make sure ¡ and ¿ aren't treated as sentence end + # markers, even though they're punctuation + ( + ["¡", "Buen", "día", "!", "Hola", ",", "¿", "qué", "tal", "?"], + [True, False, False, False, True, False, False, False, False, False], + [False, False, False, True, False, False, False, False, False, True], + 2, + ), + # The Token.is_punct check ensures that quotes are handled as well + ( + ['"', "Nice", "!", '"', "I", "am", "happy", "."], + [True, False, False, False, True, False, False, False], + [False, False, False, True, False, False, False, True], + 2, + ), + ], +) +def test_sentencizer_complex(en_vocab, words, sent_starts, sent_ends, n_sents): + doc = Doc(en_vocab, words=words) + sentencizer = Sentencizer(punct_chars=None) + doc = sentencizer(doc) + assert doc.has_annotation("SENT_START") + assert [t.is_sent_start for t in doc] == sent_starts + assert [t.is_sent_end for t in doc] == sent_ends + assert len(list(doc.sents)) == n_sents + + +@pytest.mark.parametrize( + "punct_chars,words,sent_starts,sent_ends,n_sents", + [ + ( + ["~", "?"], + ["Hello", "world", "~", "A", ".", "B", "."], + [True, False, False, True, False, False, False], + [False, False, True, False, False, False, True], + 2, + ), + # Even thought it's not common, the punct_chars should be able to + # handle any tokens + ( + [".", "ö"], + ["Hello", ".", "Test", "ö", "Ok", "."], + [True, False, True, False, True, False], + [False, True, False, True, False, True], + 3, + ), + ], +) +def test_sentencizer_custom_punct( + en_vocab, punct_chars, words, sent_starts, sent_ends, n_sents +): + doc = Doc(en_vocab, words=words) + sentencizer = Sentencizer(punct_chars=punct_chars) + doc = sentencizer(doc) + assert doc.has_annotation("SENT_START") + assert [t.is_sent_start for t in doc] == sent_starts + assert [t.is_sent_end for t in doc] == sent_ends + assert len(list(doc.sents)) == n_sents + + +def test_sentencizer_serialize_bytes(en_vocab): + punct_chars = [".", "~", "+"] + sentencizer = Sentencizer(punct_chars=punct_chars) + assert sentencizer.punct_chars == set(punct_chars) + bytes_data = sentencizer.to_bytes() + new_sentencizer = Sentencizer(punct_chars=None).from_bytes(bytes_data) + assert new_sentencizer.punct_chars == set(punct_chars) + + +@pytest.mark.parametrize( + # fmt: off + "lang,text", + [ + ( + "bn", + "বাংলা ভাষা (বাঙলা, বাঙ্গলা, তথা বাঙ্গালা নামগুলোতেও পরিচিত) একটি ইন্দো-আর্য ভাষা, যা দক্ষিণ এশিয়ার বাঙালি জাতির প্রধান কথ্য ও লেখ্য ভাষা। মাতৃভাষীর সংখ্যায় বাংলা ইন্দো-ইউরোপীয় ভাষা পরিবারের চতুর্থ ও বিশ্বের ষষ্ঠ বৃহত্তম ভাষা।[৫] মোট ব্যবহারকারীর সংখ্যা অনুসারে বাংলা বিশ্বের সপ্তম বৃহত্তম ভাষা। বাংলা সার্বভৌম ভাষাভিত্তিক জাতিরাষ্ট্র বাংলাদেশের একমাত্র রাষ্ট্রভাষা তথা সরকারি ভাষা[৬] এবং ভারতের পশ্চিমবঙ্গ, ত্রিপুরা, আসামের বরাক উপত্যকার সরকারি ভাষা। বঙ্গোপসাগরে অবস্থিত আন্দামান দ্বীপপুঞ্জের প্রধান কথ্য ভাষা বাংলা। এছাড়া ভারতের ঝাড়খণ্ড, বিহার, মেঘালয়, মিজোরাম, উড়িষ্যা রাজ্যগুলোতে উল্লেখযোগ্য পরিমাণে বাংলাভাষী জনগণ রয়েছে। ভারতে হিন্দির পরেই সর্বাধিক প্রচলিত ভাষা বাংলা।[৭][৮] এছাড়াও মধ্য প্রাচ্য, আমেরিকা ও ইউরোপে উল্লেখযোগ্য পরিমাণে বাংলাভাষী অভিবাসী রয়েছে।[৯] সারা বিশ্বে সব মিলিয়ে ২৬ কোটির অধিক লোক দৈনন্দিন জীবনে বাংলা ব্যবহার করে।[২] বাংলাদেশের জাতীয় সঙ্গীত এবং ভারতের জাতীয় সঙ্গীত ও স্তোত্র বাংলাতে রচিত।", + ), + ( + "de", + "Die deutsche Sprache bzw. Deutsch ([dɔʏ̯t͡ʃ]; abgekürzt dt. oder dtsch.) ist eine westgermanische Sprache. Ihr Sprachraum umfasst Deutschland, Österreich, die Deutschschweiz, Liechtenstein, Luxemburg, Ostbelgien, Südtirol, das Elsass und Lothringen sowie Nordschleswig. Außerdem ist sie eine Minderheitensprache in einigen europäischen und außereuropäischen Ländern, z. B. in Rumänien und Südafrika, sowie Nationalsprache im afrikanischen Namibia.", + ), + ( + "hi", + "हिन्दी विश्व की एक प्रमुख भाषा है एवं भारत की राजभाषा है। केन्द्रीय स्तर पर भारत में दूसरी आधिकारिक भाषा अंग्रेजी है। यह हिंदुस्तानी भाषा की एक मानकीकृत रूप है जिसमें संस्कृत के तत्सम तथा तद्भव शब्दों का प्रयोग अधिक है और अरबी-फ़ारसी शब्द कम हैं। हिंदी संवैधानिक रूप से भारत की राजभाषा और भारत की सबसे अधिक बोली और समझी जाने वाली भाषा है। हालाँकि, हिन्दी भारत की राष्ट्रभाषा नहीं है,[3] क्योंकि भारत के संविधान में कोई भी भाषा को ऐसा दर्जा नहीं दिया गया था।[4][5] चीनी के बाद यह विश्व में सबसे अधिक बोली जाने वाली भाषा भी है। विश्व आर्थिक मंच की गणना के अनुसार यह विश्व की दस शक्तिशाली भाषाओं में से एक है।[6]", + ), + ( + "kn", + "ದ್ರಾವಿಡ ಭಾಷೆಗಳಲ್ಲಿ ಪ್ರಾಮುಖ್ಯವುಳ್ಳ ಭಾಷೆಯೂ ಭಾರತದ ಪುರಾತನವಾದ ಭಾಷೆಗಳಲ್ಲಿ ಒಂದೂ ಆಗಿರುವ ಕನ್ನಡ ಭಾಷೆಯನ್ನು ಅದರ ವಿವಿಧ ರೂಪಗಳಲ್ಲಿ ಸುಮಾರು ೪೫ ದಶಲಕ್ಷ ಜನರು ಆಡು ನುಡಿಯಾಗಿ ಬಳಸುತ್ತಲಿದ್ದಾರೆ. ಕನ್ನಡ ಕರ್ನಾಟಕ ರಾಜ್ಯದ ಆಡಳಿತ ಭಾಷೆ.[೧೧] ಜಗತ್ತಿನಲ್ಲಿ ಅತ್ಯಂತ ಹೆಚ್ಚು ಮಂದಿ ಮಾತನಾಡುವ ಭಾಷೆಯೆಂಬ ನೆಲೆಯಲ್ಲಿ ಇಪ್ಪತೊಂಬತ್ತನೆಯ ಸ್ಥಾನ ಕನ್ನಡಕ್ಕಿದೆ. ೨೦೧೧ರ ಜನಗಣತಿಯ ಪ್ರಕಾರ ಜಗತ್ತಿನಲ್ಲಿ ೬.೪ ಕೋಟಿ ಜನಗಳು ಕನ್ನಡ ಮಾತನಾಡುತ್ತಾರೆ ಎಂದು ತಿಳಿದುಬಂದಿದೆ. ಇವರಲ್ಲಿ ೫.೫ ಕೋಟಿ ಜನಗಳ ಮಾತೃಭಾಷೆ ಕನ್ನಡವಾಗಿದೆ. ಬ್ರಾಹ್ಮಿ ಲಿಪಿಯಿಂದ ರೂಪುಗೊಂಡ ಕನ್ನಡ ಲಿಪಿಯನ್ನು ಉಪಯೋಗಿಸಿ ಕನ್ನಡ ಭಾಷೆಯನ್ನು ಬರೆಯಲಾಗುತ್ತದೆ. ಕನ್ನಡ ಬರಹದ ಮಾದರಿಗಳಿಗೆ ಸಾವಿರದ ಐನೂರು ವರುಷಗಳ ಚರಿತ್ರೆಯಿದೆ. ಕ್ರಿ.ಶ. ಆರನೆಯ ಶತಮಾನದ ಪಶ್ಚಿಮ ಗಂಗ ಸಾಮ್ರಾಜ್ಯದ ಕಾಲದಲ್ಲಿ [೧೨] ಮತ್ತು ಒಂಬತ್ತನೆಯ ಶತಮಾನದ ರಾಷ್ಟ್ರಕೂಟ ಸಾಮ್ರಾಜ್ಯದ ಕಾಲದಲ್ಲಿ ಹಳಗನ್ನಡ ಸಾಹಿತ್ಯ ಅತ್ಯಂತ ಹೆಚ್ಚಿನ ರಾಜಾಶ್ರಯ ಪಡೆಯಿತು.[೧೩][೧೪] ಅದಲ್ಲದೆ ಸಾವಿರ ವರುಷಗಳ ಸಾಹಿತ್ಯ ಪರಂಪರೆ ಕನ್ನಡಕ್ಕಿದೆ.[೧೫]ವಿನೋಬಾ ಭಾವೆ ಕನ್ನಡ ಲಿಪಿಯನ್ನು ಲಿಪಿಗಳ ರಾಣಿಯೆಂದು ಹೊಗಳಿದ್ದಾರೆ.[ಸೂಕ್ತ ಉಲ್ಲೇಖನ ಬೇಕು]", + ), + ( + "si", + "ශ්‍රී ලංකාවේ ප්‍රධාන ජාතිය වන සිංහල ජනයාගේ මව් බස සිංහල වෙයි. අද වන විට මිලියන 20 කට අධික සිංහල සහ මිලියන 3කට අධික සිංහල නොවන ජනගහනයක් සිංහල භාෂාව භාවිත කරති. සිංහල‍ ඉන්දු-යුරෝපීය භාෂාවල උප ගණයක් වන ඉන්දු-ආර්ය භාෂා ගණයට අයිති වන අතර මාල දිවයින භාවිත කරන දිවෙහි භාෂාව සිංහලයෙන් පැවත එන්නකි. සිංහල ශ්‍රී ලංකාවේ නිල භාෂාවයි .", + ), + ( + "ta", + "தமிழ் மொழி (Tamil language) தமிழர்களினதும், தமிழ் பேசும் பலரதும் தாய்மொழி ஆகும். தமிழ் திராவிட மொழிக் குடும்பத்தின் முதன்மையான மொழிகளில் ஒன்றும் செம்மொழியும் ஆகும். இந்தியா, இலங்கை, மலேசியா, சிங்கப்பூர் ஆகிய நாடுகளில் அதிக அளவிலும், ஐக்கிய அரபு அமீரகம், தென்னாப்பிரிக்கா, மொரிசியசு, பிஜி, ரீயூனியன், டிரினிடாட் போன்ற நாடுகளில் சிறிய அளவிலும் தமிழ் பேசப்படுகிறது. 1997ஆம் ஆண்டுப் புள்ளி விவரப்படி உலகம் முழுவதிலும் 8 கோடி (80 மில்லியன்) மக்களால் பேசப்படும் தமிழ்[13], ஒரு மொழியைத் தாய்மொழியாகக் கொண்டு பேசும் மக்களின் எண்ணிக்கை அடிப்படையில் பதினெட்டாவது இடத்தில் உள்ளது.[14] இணையத்தில் அதிகம் பயன்படுத்தப்படும் இந்திய மொழிகளில் தமிழ் முதன்மையாக உள்ளதாக 2017 ஆவது ஆண்டில் நடைபெற்ற கூகுள் கணக்கெடுப்பில் தெரிய வந்தது.[15]", + ), + ( + "te", + "ఆంధ్ర ప్రదేశ్, తెలంగాణ రాష్ట్రాల అధికార భాష తెలుగు. భారత దేశంలో తెలుగు మాతృభాషగా మాట్లాడే 8.7 కోట్ల (2001) జనాభాతో [1] ప్రాంతీయ భాషలలో మొదటి స్థానంలో ఉంది. ప్రపంచంలోని ప్రజలు అత్యధికముగా మాట్లాడే భాషలలో 15 స్థానములోనూ, భారత దేశములో హిందీ, తర్వాత స్థానములోనూ నిలుస్తుంది. పాతవైన ప్రపంచ భాష గణాంకాల (ఎథ్నోలాగ్) ప్రకారం ప్రపంచవ్యాప్తంగా 7.4 కోట్లు మందికి మాతృభాషగా ఉంది.[2] మొదటి భాషగా మాట్లాడతారు. అతి ప్రాచీన దేశ భాషలలో సంస్కృతము తమిళముతో బాటు తెలుగు భాషను 2008 అక్టోబరు 31న భారత ప్రభుత్వము గుర్తించింది.", + ), + ( + "ur", + "اُردُو لشکری زبان[8] (یا جدید معیاری اردو) برصغیر کی معیاری زبانوں میں سے ایک ہے۔ یہ پاکستان کی قومی اور رابطہ عامہ کی زبان ہے، جبکہ بھارت کی چھے ریاستوں کی دفتری زبان کا درجہ رکھتی ہے۔ آئین ہند کے مطابق اسے 22 دفتری شناخت زبانوں میں شامل کیا جاچکا ہے۔ 2001ء کی مردم شماری کے مطابق اردو کو بطور مادری زبان بھارت میں 5.01% فیصد لوگ بولتے ہیں اور اس لحاظ سے یہ بھارت کی چھٹی بڑی زبان ہے جبکہ پاکستان میں اسے بطور مادری زبان 7.59% فیصد لوگ استعمال کرتے ہیں، یہ پاکستان کی پانچویں بڑی زبان ہے۔ اردو تاریخی طور پر ہندوستان کی مسلم آبادی سے جڑی ہے۔[حوالہ درکار] بعض ذخیرہ الفاظ کے علاوہ یہ زبان معیاری ہندی سے قابل فہم ہے جو اس خطے کی ہندوؤں سے منسوب ہے۔[حوالہ درکار] زبانِ اردو کو پہچان و ترقی اس وقت ملی جب برطانوی دور میں انگریز حکمرانوں نے اسے فارسی کی بجائے انگریزی کے ساتھ شمالی ہندوستان کے علاقوں اور جموں و کشمیر میں اسے سنہ 1846ء اور پنجاب میں سنہ 1849ء میں بطور دفتری زبان نافذ کیا۔ اس کے علاوہ خلیجی، یورپی، ایشیائی اور امریکی علاقوں میں اردو بولنے والوں کی ایک بڑی تعداد آباد ہے جو بنیادی طور پر جنوبی ایشیاء سے کوچ کرنے والے اہلِ اردو ہیں۔ 1999ء کے اعداد وشمار کے مطابق اردو زبان کے مجموعی متکلمین کی تعداد دس کروڑ ساٹھ لاکھ کے لگ بھگ تھی۔ اس لحاظ سے یہ دنیا کی نویں بڑی زبان ہے۔", + ), + ], + # fmt: on +) +def test_sentencizer_across_scripts(lang, text): + nlp = spacy.blank(lang) + nlp.add_pipe("sentencizer") + doc = nlp(text) + assert len(list(doc.sents)) > 1 diff --git a/spacy/tests/pipeline/test_senter.py b/spacy/tests/pipeline/test_senter.py new file mode 100644 index 0000000..6c76558 --- /dev/null +++ b/spacy/tests/pipeline/test_senter.py @@ -0,0 +1,103 @@ +import pytest +from numpy.testing import assert_equal + +from spacy import util +from spacy.attrs import SENT_START +from spacy.lang.en import English +from spacy.language import Language +from spacy.tests.util import make_tempdir +from spacy.training import Example + + +def test_label_types(): + nlp = Language() + senter = nlp.add_pipe("senter") + with pytest.raises(NotImplementedError): + senter.add_label("A") + + +SENT_STARTS = [0] * 14 +SENT_STARTS[0] = 1 +SENT_STARTS[5] = 1 +SENT_STARTS[9] = 1 + +TRAIN_DATA = [ + ( + "I like green eggs. Eat blue ham. I like purple eggs.", + {"sent_starts": SENT_STARTS}, + ), + ( + "She likes purple eggs. They hate ham. You like yellow eggs.", + {"sent_starts": SENT_STARTS}, + ), +] + + +def test_initialize_examples(): + nlp = Language() + nlp.add_pipe("senter") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + # you shouldn't really call this more than once, but for testing it should be fine + nlp.initialize() + nlp.initialize(get_examples=lambda: train_examples) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: None) + with pytest.raises(TypeError): + nlp.initialize(get_examples=train_examples) + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the senter - ensuring the ML models work correctly + nlp = English() + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + # add some cases where SENT_START == -1 + train_examples[0].reference[10].is_sent_start = False + train_examples[1].reference[1].is_sent_start = False + train_examples[1].reference[11].is_sent_start = False + + nlp.add_pipe("senter") + optimizer = nlp.initialize() + + for i in range(200): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["senter"] < 0.001 + + # test the trained model + test_text = TRAIN_DATA[0][0] + doc = nlp(test_text) + gold_sent_starts = [0] * 14 + gold_sent_starts[0] = 1 + gold_sent_starts[5] = 1 + gold_sent_starts[9] = 1 + assert [int(t.is_sent_start) for t in doc] == gold_sent_starts + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert [int(t.is_sent_start) for t in doc2] == gold_sent_starts + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Just a sentence.", + "Then one more sentence about London.", + "Here is another one.", + "I like London.", + ] + batch_deps_1 = [doc.to_array([SENT_START]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([SENT_START]) for doc in nlp.pipe(texts)] + no_batch_deps = [ + doc.to_array([SENT_START]) for doc in [nlp(text) for text in texts] + ] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + # test internal pipe labels vs. Language.pipe_labels with hidden labels + assert nlp.get_pipe("senter").labels == ("I", "S") + assert "senter" not in nlp.pipe_labels diff --git a/spacy/tests/pipeline/test_span_finder.py b/spacy/tests/pipeline/test_span_finder.py new file mode 100644 index 0000000..47a8a34 --- /dev/null +++ b/spacy/tests/pipeline/test_span_finder.py @@ -0,0 +1,240 @@ +import pytest +from thinc.api import Config + +from spacy import util +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline.span_finder import span_finder_default_config +from spacy.tokens import Doc +from spacy.training import Example +from spacy.util import fix_random_seed, make_tempdir, registry + +SPANS_KEY = "pytest" +TRAIN_DATA = [ + ("Who is Shaka Khan?", {"spans": {SPANS_KEY: [(7, 17)]}}), + ( + "I like London and Berlin.", + {"spans": {SPANS_KEY: [(7, 13), (18, 24)]}}, + ), +] + +TRAIN_DATA_OVERLAPPING = [ + ("Who is Shaka Khan?", {"spans": {SPANS_KEY: [(7, 17)]}}), + ( + "I like London and Berlin", + {"spans": {SPANS_KEY: [(7, 13), (18, 24), (7, 24)]}}, + ), + ("", {"spans": {SPANS_KEY: []}}), +] + + +def make_examples(nlp, data=TRAIN_DATA): + train_examples = [] + for t in data: + eg = Example.from_dict(nlp.make_doc(t[0]), t[1]) + train_examples.append(eg) + return train_examples + + +@pytest.mark.parametrize( + "tokens_predicted, tokens_reference, reference_truths", + [ + ( + ["Mon", ".", "-", "June", "16"], + ["Mon.", "-", "June", "16"], + [(0, 0), (0, 0), (0, 0), (1, 1), (0, 0)], + ), + ( + ["Mon.", "-", "J", "une", "16"], + ["Mon.", "-", "June", "16"], + [(0, 0), (0, 0), (1, 0), (0, 1), (0, 0)], + ), + ( + ["Mon", ".", "-", "June", "16"], + ["Mon.", "-", "June", "1", "6"], + [(0, 0), (0, 0), (0, 0), (1, 1), (0, 0)], + ), + ( + ["Mon.", "-J", "un", "e 16"], + ["Mon.", "-", "June", "16"], + [(0, 0), (0, 0), (0, 0), (0, 0)], + ), + pytest.param( + ["Mon.-June", "16"], + ["Mon.", "-", "June", "16"], + [(0, 1), (0, 0)], + ), + pytest.param( + ["Mon.-", "June", "16"], + ["Mon.", "-", "J", "une", "16"], + [(0, 0), (1, 1), (0, 0)], + ), + pytest.param( + ["Mon.-", "June 16"], + ["Mon.", "-", "June", "16"], + [(0, 0), (1, 0)], + ), + ], +) +def test_loss_alignment_example(tokens_predicted, tokens_reference, reference_truths): + nlp = Language() + predicted = Doc( + nlp.vocab, words=tokens_predicted, spaces=[False] * len(tokens_predicted) + ) + reference = Doc( + nlp.vocab, words=tokens_reference, spaces=[False] * len(tokens_reference) + ) + example = Example(predicted, reference) + example.reference.spans[SPANS_KEY] = [example.reference.char_span(5, 9)] + span_finder = nlp.add_pipe("span_finder", config={"spans_key": SPANS_KEY}) + nlp.initialize() + ops = span_finder.model.ops + if predicted.text != reference.text: + with pytest.raises( + ValueError, match="must match between reference and predicted" + ): + span_finder._get_aligned_truth_scores([example], ops) + return + truth_scores, masks = span_finder._get_aligned_truth_scores([example], ops) + assert len(truth_scores) == len(tokens_predicted) + ops.xp.testing.assert_array_equal(truth_scores, ops.xp.asarray(reference_truths)) + + +def test_span_finder_model(): + nlp = Language() + + docs = [nlp("This is an example."), nlp("This is the second example.")] + docs[0].spans[SPANS_KEY] = [docs[0][3:4]] + docs[1].spans[SPANS_KEY] = [docs[1][3:5]] + + total_tokens = 0 + for doc in docs: + total_tokens += len(doc) + + config = Config().from_str(span_finder_default_config).interpolate() + model = registry.resolve(config)["model"] + + model.initialize(X=docs) + predictions = model.predict(docs) + + assert len(predictions) == total_tokens + assert len(predictions[0]) == 2 + + +def test_span_finder_component(): + nlp = Language() + + docs = [nlp("This is an example."), nlp("This is the second example.")] + docs[0].spans[SPANS_KEY] = [docs[0][3:4]] + docs[1].spans[SPANS_KEY] = [docs[1][3:5]] + + span_finder = nlp.add_pipe("span_finder", config={"spans_key": SPANS_KEY}) + nlp.initialize() + docs = list(span_finder.pipe(docs)) + + assert SPANS_KEY in docs[0].spans + + +@pytest.mark.parametrize( + "min_length, max_length, span_count", + [(0, 0, 0), (None, None, 8), (2, None, 6), (None, 1, 2), (2, 3, 2)], +) +def test_set_annotations_span_lengths(min_length, max_length, span_count): + nlp = Language() + doc = nlp("Me and Jenny goes together like peas and carrots.") + if min_length == 0 and max_length == 0: + with pytest.raises(ValueError, match="Both 'min_length' and 'max_length'"): + span_finder = nlp.add_pipe( + "span_finder", + config={ + "max_length": max_length, + "min_length": min_length, + "spans_key": SPANS_KEY, + }, + ) + return + span_finder = nlp.add_pipe( + "span_finder", + config={ + "max_length": max_length, + "min_length": min_length, + "spans_key": SPANS_KEY, + }, + ) + nlp.initialize() + # Starts [Me, Jenny, peas] + # Ends [Jenny, peas, carrots] + scores = [ + (1, 0), + (0, 0), + (1, 1), + (0, 0), + (0, 0), + (0, 0), + (1, 1), + (0, 0), + (0, 1), + (0, 0), + ] + span_finder.set_annotations([doc], scores) + + assert doc.spans[SPANS_KEY] + assert len(doc.spans[SPANS_KEY]) == span_count + + # Assert below will fail when max_length is set to 0 + if max_length is None: + max_length = float("inf") + if min_length is None: + min_length = 1 + + assert all(min_length <= len(span) <= max_length for span in doc.spans[SPANS_KEY]) + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the span_finder component - ensuring the ML models work correctly + fix_random_seed(0) + nlp = English() + span_finder = nlp.add_pipe("span_finder", config={"spans_key": SPANS_KEY}) + train_examples = make_examples(nlp) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert span_finder.model.get_dim("nO") == 2 + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["span_finder"] < 0.001 + + # test the trained model + test_text = "I like London and Berlin" + doc = nlp(test_text) + spans = doc.spans[SPANS_KEY] + assert len(spans) == 3 + assert set([span.text for span in spans]) == { + "London", + "Berlin", + "London and Berlin", + } + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + spans2 = doc2.spans[SPANS_KEY] + assert len(spans2) == 3 + assert set([span.text for span in spans2]) == { + "London", + "Berlin", + "London and Berlin", + } + + # Test scoring + scores = nlp.evaluate(train_examples) + assert f"spans_{SPANS_KEY}_f" in scores + # It's not perfect 1.0 F1 because it's designed to overgenerate for now. + assert scores[f"spans_{SPANS_KEY}_p"] == 0.75 + assert scores[f"spans_{SPANS_KEY}_r"] == 1.0 + + # also test that the spancat works for just a single entity in a sentence + doc = nlp("London") + assert len(doc.spans[SPANS_KEY]) == 1 diff --git a/spacy/tests/pipeline/test_span_ruler.py b/spacy/tests/pipeline/test_span_ruler.py new file mode 100644 index 0000000..0a8616f --- /dev/null +++ b/spacy/tests/pipeline/test_span_ruler.py @@ -0,0 +1,464 @@ +import pytest +from thinc.api import NumpyOps, get_current_ops + +import spacy +from spacy import registry +from spacy.errors import MatchPatternError +from spacy.tests.util import make_tempdir +from spacy.tokens import Span +from spacy.training import Example + + +@pytest.fixture +@registry.misc("span_ruler_patterns") +def patterns(): + return [ + {"label": "HELLO", "pattern": "hello world", "id": "hello1"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}], "id": "hello2"}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple"}, + {"label": "TECH_ORG", "pattern": "Microsoft"}, + ] + + +@pytest.fixture +def overlapping_patterns(): + return [ + {"label": "FOOBAR", "pattern": "foo bar"}, + {"label": "BARBAZ", "pattern": "bar baz"}, + ] + + +@pytest.fixture +def person_org_patterns(): + return [ + {"label": "PERSON", "pattern": "Dina"}, + {"label": "ORG", "pattern": "ACME"}, + {"label": "ORG", "pattern": "ACM"}, + ] + + +@pytest.fixture +def person_org_date_patterns(person_org_patterns): + return person_org_patterns + [{"label": "DATE", "pattern": "June 14th"}] + + +def test_span_ruler_add_empty(patterns): + """Test that patterns don't get added excessively.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"validate": True}) + ruler.add_patterns(patterns) + pattern_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert pattern_count > 0 + ruler.add_patterns([]) + after_count = sum(len(mm) for mm in ruler.matcher._patterns.values()) + assert after_count == pattern_count + + +def test_span_ruler_init(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + assert "HELLO" in ruler + assert "BYE" in ruler + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].id_ == "hello1" + assert doc.spans["ruler"][1].label_ == "BYE" + assert doc.spans["ruler"][1].id_ == "" + + +def test_span_ruler_no_patterns_warns(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler) == 0 + assert len(ruler.labels) == 0 + assert nlp.pipe_names == ["span_ruler"] + with pytest.warns(UserWarning): + doc = nlp("hello world bye bye") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_init_patterns(patterns): + # initialize with patterns + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + ruler.initialize(lambda: [], patterns=patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + nlp.remove_pipe("span_ruler") + # initialize with patterns from misc registry + nlp.config["initialize"]["components"]["span_ruler"] = { + "patterns": {"@misc": "span_ruler_patterns"} + } + ruler = nlp.add_pipe("span_ruler") + assert len(ruler.labels) == 0 + nlp.initialize() + assert len(ruler.labels) == 4 + doc = nlp("hello world bye bye") + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_init_clear(patterns): + """Test that initialization clears patterns.""" + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + ruler.initialize(lambda: []) + assert len(ruler.labels) == 0 + + +def test_span_ruler_clear(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler.labels) == 4 + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 1 + ruler.clear() + assert len(ruler.labels) == 0 + with pytest.warns(UserWarning): + doc = nlp("hello world") + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_existing(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": False}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0] == doc[0:2] + assert doc.spans["ruler"][1].label_ == "HELLO" + assert doc.spans["ruler"][1].id_ == "hello2" + assert doc.spans["ruler"][2].label_ == "BYE" + assert doc.spans["ruler"][2].id_ == "" + + +def test_span_ruler_existing_overwrite(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + doc = nlp.make_doc("OH HELLO WORLD bye bye") + doc.spans["ruler"] = [doc[0:2]] + doc = nlp(doc) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "HELLO" + assert doc.spans["ruler"][0].text == "HELLO" + assert doc.spans["ruler"][1].label_ == "BYE" + + +def test_span_ruler_serialize_bytes(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + ruler_bytes = ruler.to_bytes() + new_nlp = spacy.blank("xx") + new_ruler = new_nlp.add_pipe("span_ruler") + assert len(new_ruler) == 0 + assert len(new_ruler.labels) == 0 + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(patterns) + assert len(new_ruler.labels) == 4 + assert len(new_ruler.patterns) == len(ruler.patterns) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert sorted(new_ruler.labels) == sorted(ruler.labels) + + +def test_span_ruler_validate(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + validated_ruler = nlp.add_pipe( + "span_ruler", name="validated_span_ruler", config={"validate": True} + ) + + valid_pattern = {"label": "HELLO", "pattern": [{"LOWER": "HELLO"}]} + invalid_pattern = {"label": "HELLO", "pattern": [{"ASDF": "HELLO"}]} + + # invalid pattern raises error without validate + with pytest.raises(ValueError): + ruler.add_patterns([invalid_pattern]) + + # valid pattern is added without errors with validate + validated_ruler.add_patterns([valid_pattern]) + + # invalid pattern raises error with validate + with pytest.raises(MatchPatternError): + validated_ruler.add_patterns([invalid_pattern]) + + +def test_span_ruler_properties(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"overwrite": True}) + ruler.add_patterns(patterns) + assert sorted(ruler.labels) == sorted(set([p["label"] for p in patterns])) + + +def test_span_ruler_overlapping_spans(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + assert doc.spans["ruler"][1].label_ == "BARBAZ" + + +def test_span_ruler_scorer(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(overlapping_patterns) + text = "foo bar baz" + pred_doc = ruler(nlp.make_doc(text)) + assert len(pred_doc.spans["ruler"]) == 2 + assert pred_doc.spans["ruler"][0].label_ == "FOOBAR" + assert pred_doc.spans["ruler"][1].label_ == "BARBAZ" + + ref_doc = nlp.make_doc(text) + ref_doc.spans["ruler"] = [Span(ref_doc, 0, 2, label="FOOBAR")] + scores = nlp.evaluate([Example(pred_doc, ref_doc)]) + assert scores["spans_ruler_p"] == 0.5 + assert scores["spans_ruler_r"] == 1.0 + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_span_ruler_multiprocessing(n_process): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + texts = ["I enjoy eating Pizza Hut pizza."] + + patterns = [{"label": "FASTFOOD", "pattern": "Pizza Hut"}] + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + + for doc in nlp.pipe(texts, n_process=2): + for ent in doc.spans["ruler"]: + assert ent.label_ == "FASTFOOD" + + +def test_span_ruler_serialize_dir(patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(patterns) + with make_tempdir() as d: + ruler.to_disk(d / "test_ruler") + ruler.from_disk(d / "test_ruler") # read from an existing directory + with pytest.raises(ValueError): + ruler.from_disk(d / "non_existing_dir") # read from a bad directory + + +def test_span_ruler_remove_basic(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 0 + assert len(ruler.patterns) == 2 + + +def test_span_ruler_remove_nonexisting_pattern(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + assert len(ruler.patterns) == 3 + with pytest.raises(ValueError): + ruler.remove("NE") + with pytest.raises(ValueError): + ruler.remove_by_id("NE") + + +def test_span_ruler_remove_several_patterns(person_org_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME.")) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + ruler.remove("PERSON") + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "ORG" + assert doc.spans["ruler"][0].text == "ACME" + ruler.remove("ORG") + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME")) + assert len(ruler.patterns) == 0 + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_patterns_in_a_row(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 3 + assert doc.spans["ruler"][0].label_ == "PERSON" + assert doc.spans["ruler"][0].text == "Dina" + assert doc.spans["ruler"][1].label_ == "ORG" + assert doc.spans["ruler"][1].text == "ACME" + assert doc.spans["ruler"][2].label_ == "DATE" + assert doc.spans["ruler"][2].text == "June 14th" + ruler.remove("ORG") + ruler.remove("DATE") + doc = ruler(nlp.make_doc("Dina went to school")) + assert len(doc.spans["ruler"]) == 1 + + +def test_span_ruler_remove_all_patterns(person_org_date_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + ruler.add_patterns(person_org_date_patterns) + assert len(ruler.patterns) == 4 + ruler.remove("PERSON") + assert len(ruler.patterns) == 3 + ruler.remove("ORG") + assert len(ruler.patterns) == 1 + ruler.remove("DATE") + assert len(ruler.patterns) == 0 + with pytest.warns(UserWarning): + doc = ruler(nlp.make_doc("Dina founded the company ACME on June 14th")) + assert len(doc.spans["ruler"]) == 0 + + +def test_span_ruler_remove_and_add(): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler") + patterns1 = [{"label": "DATE1", "pattern": "last time"}] + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + patterns2 = [{"label": "DATE2", "pattern": "this time"}] + ruler.add_patterns(patterns2) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + assert doc.spans["ruler"][0].label_ == "DATE1" + assert doc.spans["ruler"][0].text == "last time" + assert doc.spans["ruler"][1].label_ == "DATE2" + assert doc.spans["ruler"][1].text == "this time" + ruler.remove("DATE1") + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 1 + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "DATE2" + assert doc.spans["ruler"][0].text == "this time" + ruler.add_patterns(patterns1) + doc = ruler( + nlp.make_doc("I saw him last time we met, this time he brought some flowers") + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + patterns3 = [{"label": "DATE3", "pattern": "another time"}] + ruler.add_patterns(patterns3) + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 3 + assert len(doc.spans["ruler"]) == 3 + ruler.remove("DATE3") + doc = ruler( + nlp.make_doc( + "I saw him last time we met, this time he brought some flowers, another time some chocolate." + ) + ) + assert len(ruler.patterns) == 2 + assert len(doc.spans["ruler"]) == 2 + + +def test_span_ruler_spans_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={"spans_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}}, + ) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.spans["ruler"]) == 1 + assert doc.spans["ruler"][0].label_ == "FOOBAR" + + +def test_span_ruler_ents_default_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe("span_ruler", config={"annotate_ents": True}) + ruler.add_patterns(overlapping_patterns) + doc = ruler(nlp.make_doc("foo bar baz")) + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "FOOBAR" + + +def test_span_ruler_ents_overwrite_filter(overlapping_patterns): + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "overwrite": False, + "ents_filter": {"@misc": "spacy.prioritize_new_ents_filter.v1"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + # overlapping ents are clobbered, non-overlapping ents are preserved + doc = nlp.make_doc("foo bar baz a b c") + doc.ents = [Span(doc, 1, 3, label="BARBAZ"), Span(doc, 3, 6, label="ABC")] + doc = ruler(doc) + assert len(doc.ents) == 2 + assert doc.ents[0].label_ == "FOOBAR" + assert doc.ents[1].label_ == "ABC" + + +def test_span_ruler_ents_bad_filter(overlapping_patterns): + @registry.misc("test_pass_through_filter") + def make_pass_through_filter(): + def pass_through_filter(spans1, spans2): + return spans1 + spans2 + + return pass_through_filter + + nlp = spacy.blank("xx") + ruler = nlp.add_pipe( + "span_ruler", + config={ + "annotate_ents": True, + "ents_filter": {"@misc": "test_pass_through_filter"}, + }, + ) + ruler.add_patterns(overlapping_patterns) + with pytest.raises(ValueError): + ruler(nlp.make_doc("foo bar baz")) diff --git a/spacy/tests/pipeline/test_spancat.py b/spacy/tests/pipeline/test_spancat.py new file mode 100644 index 0000000..9405a78 --- /dev/null +++ b/spacy/tests/pipeline/test_spancat.py @@ -0,0 +1,612 @@ +import numpy +import pytest +from numpy.testing import assert_almost_equal, assert_array_equal +from thinc.api import NumpyOps, Ragged, get_current_ops + +from spacy import util +from spacy.lang.en import English +from spacy.language import Language +from spacy.tokens import SpanGroup +from spacy.tokens._dict_proxies import SpanGroups +from spacy.training import Example +from spacy.util import fix_random_seed, make_tempdir, registry + +OPS = get_current_ops() + +SPAN_KEY = "labeled_spans" + +SPANCAT_COMPONENTS = ["spancat", "spancat_singlelabel"] + +TRAIN_DATA = [ + ("Who is Shaka Khan?", {"spans": {SPAN_KEY: [(7, 17, "PERSON")]}}), + ( + "I like London and Berlin.", + {"spans": {SPAN_KEY: [(7, 13, "LOC"), (18, 24, "LOC")]}}, + ), +] + +TRAIN_DATA_OVERLAPPING = [ + ("Who is Shaka Khan?", {"spans": {SPAN_KEY: [(7, 17, "PERSON")]}}), + ( + "I like London and Berlin", + {"spans": {SPAN_KEY: [(7, 13, "LOC"), (18, 24, "LOC"), (7, 24, "DOUBLE_LOC")]}}, + ), + ("", {"spans": {SPAN_KEY: []}}), +] + + +def make_examples(nlp, data=TRAIN_DATA): + train_examples = [] + for t in data: + eg = Example.from_dict(nlp.make_doc(t[0]), t[1]) + train_examples.append(eg) + return train_examples + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_no_label(name): + nlp = Language() + nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + with pytest.raises(ValueError): + nlp.initialize() + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_no_resize(name): + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + spancat.add_label("Thing") + spancat.add_label("Phrase") + assert spancat.labels == ("Thing", "Phrase") + nlp.initialize() + assert spancat.model.get_dim("nO") == spancat._n_labels + # this throws an error because the spancat can't be resized after initialization + with pytest.raises(ValueError): + spancat.add_label("Stuff") + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_implicit_labels(name): + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + assert len(spancat.labels) == 0 + train_examples = make_examples(nlp) + nlp.initialize(get_examples=lambda: train_examples) + assert spancat.labels == ("PERSON", "LOC") + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_explicit_labels(name): + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + assert len(spancat.labels) == 0 + spancat.add_label("PERSON") + spancat.add_label("LOC") + nlp.initialize() + assert spancat.labels == ("PERSON", "LOC") + + +# TODO figure out why this is flaky +@pytest.mark.skip(reason="Test is unreliable for unknown reason") +def test_doc_gc(): + # If the Doc object is garbage collected, the spans won't be functional afterwards + nlp = Language() + spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + spancat.add_label("PERSON") + nlp.initialize() + texts = [ + "Just a sentence.", + "I like London and Berlin", + "I like Berlin", + "I eat ham.", + ] + all_spans = [doc.spans for doc in nlp.pipe(texts)] + for text, spangroups in zip(texts, all_spans): + assert isinstance(spangroups, SpanGroups) + for key, spangroup in spangroups.items(): + assert isinstance(spangroup, SpanGroup) + # XXX This fails with length 0 sometimes + assert len(spangroup) > 0 + with pytest.raises(RuntimeError): + spangroup[0] + + +@pytest.mark.parametrize( + "max_positive,nr_results", [(None, 4), (1, 2), (2, 3), (3, 4), (4, 4)] +) +def test_make_spangroup_multilabel(max_positive, nr_results): + fix_random_seed(0) + nlp = Language() + spancat = nlp.add_pipe( + "spancat", + config={"spans_key": SPAN_KEY, "threshold": 0.5, "max_positive": max_positive}, + ) + doc = nlp.make_doc("Greater London") + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2]) + indices = ngram_suggester([doc])[0].dataXd + assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) + labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat.add_label(label) + scores = numpy.asarray( + [[0.2, 0.4, 0.3, 0.1], [0.1, 0.6, 0.2, 0.4], [0.8, 0.7, 0.3, 0.9]], dtype="f" + ) + spangroup = spancat._make_span_group_multilabel(doc, indices, scores) + assert len(spangroup) == nr_results + + # first span is always the second token "London" + assert spangroup[0].text == "London" + assert spangroup[0].label_ == "City" + assert_almost_equal(0.6, spangroup.attrs["scores"][0], 5) + + # second span depends on the number of positives that were allowed + assert spangroup[1].text == "Greater London" + if max_positive == 1: + assert spangroup[1].label_ == "GreatCity" + assert_almost_equal(0.9, spangroup.attrs["scores"][1], 5) + else: + assert spangroup[1].label_ == "Thing" + assert_almost_equal(0.8, spangroup.attrs["scores"][1], 5) + + if nr_results > 2: + assert spangroup[2].text == "Greater London" + if max_positive == 2: + assert spangroup[2].label_ == "GreatCity" + assert_almost_equal(0.9, spangroup.attrs["scores"][2], 5) + else: + assert spangroup[2].label_ == "City" + assert_almost_equal(0.7, spangroup.attrs["scores"][2], 5) + + assert spangroup[-1].text == "Greater London" + assert spangroup[-1].label_ == "GreatCity" + assert_almost_equal(0.9, spangroup.attrs["scores"][-1], 5) + + +@pytest.mark.parametrize( + "threshold,allow_overlap,nr_results", + [(0.05, True, 3), (0.05, False, 1), (0.5, True, 2), (0.5, False, 1)], +) +def test_make_spangroup_singlelabel(threshold, allow_overlap, nr_results): + fix_random_seed(0) + nlp = Language() + spancat = nlp.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": threshold, + "max_positive": 1, + }, + ) + doc = nlp.make_doc("Greater London") + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2]) + indices = ngram_suggester([doc])[0].dataXd + assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) + labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat.add_label(label) + scores = numpy.asarray( + [[0.2, 0.4, 0.3, 0.1], [0.1, 0.6, 0.2, 0.4], [0.8, 0.7, 0.3, 0.9]], dtype="f" + ) + spangroup = spancat._make_span_group_singlelabel( + doc, indices, scores, allow_overlap + ) + if threshold > 0.4: + if allow_overlap: + assert spangroup[0].text == "London" + assert spangroup[0].label_ == "City" + assert_almost_equal(0.6, spangroup.attrs["scores"][0], 5) + assert spangroup[1].text == "Greater London" + assert spangroup[1].label_ == "GreatCity" + assert spangroup.attrs["scores"][1] == 0.9 + assert_almost_equal(0.9, spangroup.attrs["scores"][1], 5) + else: + assert spangroup[0].text == "Greater London" + assert spangroup[0].label_ == "GreatCity" + assert spangroup.attrs["scores"][0] == 0.9 + else: + if allow_overlap: + assert spangroup[0].text == "Greater" + assert spangroup[0].label_ == "City" + assert spangroup[1].text == "London" + assert spangroup[1].label_ == "City" + assert spangroup[2].text == "Greater London" + assert spangroup[2].label_ == "GreatCity" + else: + assert spangroup[0].text == "Greater London" + + +def test_make_spangroup_negative_label(): + fix_random_seed(0) + nlp_single = Language() + nlp_multi = Language() + spancat_single = nlp_single.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": 0.1, + "max_positive": 1, + }, + ) + spancat_multi = nlp_multi.add_pipe( + "spancat", + config={ + "spans_key": SPAN_KEY, + "threshold": 0.1, + "max_positive": 2, + }, + ) + spancat_single.add_negative_label = True + spancat_multi.add_negative_label = True + doc = nlp_single.make_doc("Greater London") + labels = ["Thing", "City", "Person", "GreatCity"] + for label in labels: + spancat_multi.add_label(label) + spancat_single.add_label(label) + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2]) + indices = ngram_suggester([doc])[0].dataXd + assert_array_equal(OPS.to_numpy(indices), numpy.asarray([[0, 1], [1, 2], [0, 2]])) + scores = numpy.asarray( + [ + [0.2, 0.4, 0.3, 0.1, 0.1], + [0.1, 0.6, 0.2, 0.4, 0.9], + [0.8, 0.7, 0.3, 0.9, 0.1], + ], + dtype="f", + ) + spangroup_multi = spancat_multi._make_span_group_multilabel(doc, indices, scores) + spangroup_single = spancat_single._make_span_group_singlelabel(doc, indices, scores) + assert len(spangroup_single) == 2 + assert spangroup_single[0].text == "Greater" + assert spangroup_single[0].label_ == "City" + assert_almost_equal(0.4, spangroup_single.attrs["scores"][0], 5) + assert spangroup_single[1].text == "Greater London" + assert spangroup_single[1].label_ == "GreatCity" + assert spangroup_single.attrs["scores"][1] == 0.9 + assert_almost_equal(0.9, spangroup_single.attrs["scores"][1], 5) + + assert len(spangroup_multi) == 6 + assert spangroup_multi[0].text == "Greater" + assert spangroup_multi[0].label_ == "City" + assert_almost_equal(0.4, spangroup_multi.attrs["scores"][0], 5) + assert spangroup_multi[1].text == "Greater" + assert spangroup_multi[1].label_ == "Person" + assert_almost_equal(0.3, spangroup_multi.attrs["scores"][1], 5) + assert spangroup_multi[2].text == "London" + assert spangroup_multi[2].label_ == "City" + assert_almost_equal(0.6, spangroup_multi.attrs["scores"][2], 5) + assert spangroup_multi[3].text == "London" + assert spangroup_multi[3].label_ == "GreatCity" + assert_almost_equal(0.4, spangroup_multi.attrs["scores"][3], 5) + assert spangroup_multi[4].text == "Greater London" + assert spangroup_multi[4].label_ == "Thing" + assert spangroup_multi[4].text == "Greater London" + assert_almost_equal(0.8, spangroup_multi.attrs["scores"][4], 5) + assert spangroup_multi[5].text == "Greater London" + assert spangroup_multi[5].label_ == "GreatCity" + assert_almost_equal(0.9, spangroup_multi.attrs["scores"][5], 5) + + +def test_ngram_suggester(en_tokenizer): + # test different n-gram lengths + for size in [1, 2, 3]: + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[size]) + docs = [ + en_tokenizer(text) + for text in [ + "a", + "a b", + "a b c", + "a b c d", + "a b c d e", + "a " * 100, + ] + ] + ngrams = ngram_suggester(docs) + # span sizes are correct + for s in ngrams.data: + assert s[1] - s[0] == size + # spans are within docs + offset = 0 + for i, doc in enumerate(docs): + spans = ngrams.dataXd[offset : offset + ngrams.lengths[i]] + spans_set = set() + for span in spans: + assert 0 <= span[0] < len(doc) + assert 0 < span[1] <= len(doc) + spans_set.add((int(span[0]), int(span[1]))) + # spans are unique + assert spans.shape[0] == len(spans_set) + offset += ngrams.lengths[i] + # the number of spans is correct + assert_array_equal( + OPS.to_numpy(ngrams.lengths), + [max(0, len(doc) - (size - 1)) for doc in docs], + ) + + # test 1-3-gram suggestions + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2, 3]) + docs = [ + en_tokenizer(text) for text in ["a", "a b", "a b c", "a b c d", "a b c d e"] + ] + ngrams = ngram_suggester(docs) + assert_array_equal(OPS.to_numpy(ngrams.lengths), [1, 3, 6, 9, 12]) + assert_array_equal( + OPS.to_numpy(ngrams.data), + [ + # doc 0 + [0, 1], + # doc 1 + [0, 1], + [1, 2], + [0, 2], + # doc 2 + [0, 1], + [1, 2], + [2, 3], + [0, 2], + [1, 3], + [0, 3], + # doc 3 + [0, 1], + [1, 2], + [2, 3], + [3, 4], + [0, 2], + [1, 3], + [2, 4], + [0, 3], + [1, 4], + # doc 4 + [0, 1], + [1, 2], + [2, 3], + [3, 4], + [4, 5], + [0, 2], + [1, 3], + [2, 4], + [3, 5], + [0, 3], + [1, 4], + [2, 5], + ], + ) + + # test some empty docs + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1]) + docs = [en_tokenizer(text) for text in ["", "a", ""]] + ngrams = ngram_suggester(docs) + assert_array_equal(OPS.to_numpy(ngrams.lengths), [len(doc) for doc in docs]) + + # test all empty docs + ngram_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1]) + docs = [en_tokenizer(text) for text in ["", "", ""]] + ngrams = ngram_suggester(docs) + assert_array_equal(OPS.to_numpy(ngrams.lengths), [len(doc) for doc in docs]) + + +def test_ngram_sizes(en_tokenizer): + # test that the range suggester works well + size_suggester = registry.misc.get("spacy.ngram_suggester.v1")(sizes=[1, 2, 3]) + suggester_factory = registry.misc.get("spacy.ngram_range_suggester.v1") + range_suggester = suggester_factory(min_size=1, max_size=3) + docs = [ + en_tokenizer(text) for text in ["a", "a b", "a b c", "a b c d", "a b c d e"] + ] + ngrams_1 = size_suggester(docs) + ngrams_2 = range_suggester(docs) + assert_array_equal(OPS.to_numpy(ngrams_1.lengths), [1, 3, 6, 9, 12]) + assert_array_equal(OPS.to_numpy(ngrams_1.lengths), OPS.to_numpy(ngrams_2.lengths)) + assert_array_equal(OPS.to_numpy(ngrams_1.data), OPS.to_numpy(ngrams_2.data)) + + # one more variation + suggester_factory = registry.misc.get("spacy.ngram_range_suggester.v1") + range_suggester = suggester_factory(min_size=2, max_size=4) + ngrams_3 = range_suggester(docs) + assert_array_equal(OPS.to_numpy(ngrams_3.lengths), [0, 1, 3, 6, 9]) + + +def test_preset_spans_suggester(): + nlp = Language() + docs = [nlp("This is an example."), nlp("This is the second example.")] + docs[0].spans[SPAN_KEY] = [docs[0][3:4]] + docs[1].spans[SPAN_KEY] = [docs[1][0:4], docs[1][3:5]] + suggester = registry.misc.get("spacy.preset_spans_suggester.v1")(spans_key=SPAN_KEY) + candidates = suggester(docs) + assert type(candidates) == Ragged + assert len(candidates) == 2 + assert list(candidates.dataXd[0]) == [3, 4] + assert list(candidates.dataXd[1]) == [0, 4] + assert list(candidates.dataXd[2]) == [3, 5] + assert list(candidates.lengths) == [1, 2] + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the spancat component - ensuring the ML models work correctly + fix_random_seed(0) + nlp = English() + spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + train_examples = make_examples(nlp) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert spancat.model.get_dim("nO") == 2 + assert set(spancat.labels) == {"LOC", "PERSON"} + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["spancat"] < 0.01 + + # test the trained model + test_text = "I like London and Berlin" + doc = nlp(test_text) + assert doc.spans[spancat.key] == doc.spans[SPAN_KEY] + spans = doc.spans[SPAN_KEY] + assert len(spans) == 2 + assert len(spans.attrs["scores"]) == 2 + assert min(spans.attrs["scores"]) > 0.8 + assert set([span.text for span in spans]) == {"London", "Berlin"} + assert set([span.label_ for span in spans]) == {"LOC"} + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + spans2 = doc2.spans[SPAN_KEY] + assert len(spans2) == 2 + assert len(spans2.attrs["scores"]) == 2 + assert min(spans2.attrs["scores"]) > 0.8 + assert set([span.text for span in spans2]) == {"London", "Berlin"} + assert set([span.label_ for span in spans2]) == {"LOC"} + + # Test scoring + scores = nlp.evaluate(train_examples) + assert f"spans_{SPAN_KEY}_f" in scores + assert scores[f"spans_{SPAN_KEY}_p"] == 1.0 + assert scores[f"spans_{SPAN_KEY}_r"] == 1.0 + assert scores[f"spans_{SPAN_KEY}_f"] == 1.0 + + # also test that the spancat works for just a single entity in a sentence + doc = nlp("London") + assert len(doc.spans[spancat.key]) == 1 + + +def test_overfitting_IO_overlapping(): + # Test for overfitting on overlapping entities + fix_random_seed(0) + nlp = English() + spancat = nlp.add_pipe("spancat", config={"spans_key": SPAN_KEY}) + + train_examples = make_examples(nlp, data=TRAIN_DATA_OVERLAPPING) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert spancat.model.get_dim("nO") == 3 + assert set(spancat.labels) == {"PERSON", "LOC", "DOUBLE_LOC"} + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["spancat"] < 0.01 + + # test the trained model + test_text = "I like London and Berlin" + doc = nlp(test_text) + spans = doc.spans[SPAN_KEY] + assert len(spans) == 3 + assert len(spans.attrs["scores"]) == 3 + assert min(spans.attrs["scores"]) > 0.9 + assert set([span.text for span in spans]) == { + "London", + "Berlin", + "London and Berlin", + } + assert set([span.label_ for span in spans]) == {"LOC", "DOUBLE_LOC"} + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + spans2 = doc2.spans[SPAN_KEY] + assert len(spans2) == 3 + assert len(spans2.attrs["scores"]) == 3 + assert min(spans2.attrs["scores"]) > 0.9 + assert set([span.text for span in spans2]) == { + "London", + "Berlin", + "London and Berlin", + } + assert set([span.label_ for span in spans2]) == {"LOC", "DOUBLE_LOC"} + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_zero_suggestions(name): + # Test with a suggester that can return 0 suggestions + @registry.misc("test_mixed_zero_suggester") + def make_mixed_zero_suggester(): + def mixed_zero_suggester(docs, *, ops=None): + if ops is None: + ops = get_current_ops() + spans = [] + lengths = [] + for doc in docs: + if len(doc) > 0 and len(doc) % 2 == 0: + spans.append((0, 1)) + lengths.append(1) + else: + lengths.append(0) + spans = ops.asarray2i(spans) + lengths_array = ops.asarray1i(lengths) + if len(spans) > 0: + output = Ragged(ops.xp.vstack(spans), lengths_array) + else: + output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) + return output + + return mixed_zero_suggester + + fix_random_seed(0) + nlp = English() + spancat = nlp.add_pipe( + name, + config={ + "suggester": {"@misc": "test_mixed_zero_suggester"}, + "spans_key": SPAN_KEY, + }, + ) + train_examples = make_examples(nlp) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert spancat.model.get_dim("nO") == spancat._n_labels + assert set(spancat.labels) == {"LOC", "PERSON"} + + nlp.update(train_examples, sgd=optimizer) + # empty doc + nlp("") + # single doc with zero suggestions + nlp("one") + # single doc with one suggestion + nlp("two two") + # batch with mixed zero/one suggestions + list(nlp.pipe(["one", "two two", "three three three", "", "four four four four"])) + # batch with no suggestions + list(nlp.pipe(["", "one", "three three three"])) + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +def test_set_candidates(name): + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + train_examples = make_examples(nlp) + nlp.initialize(get_examples=lambda: train_examples) + texts = [ + "Just a sentence.", + "I like London and Berlin", + "I like Berlin", + "I eat ham.", + ] + + docs = [nlp(text) for text in texts] + spancat.set_candidates(docs) + + assert len(docs) == len(texts) + assert type(docs[0].spans["candidates"]) == SpanGroup + assert len(docs[0].spans["candidates"]) == 9 + assert docs[0].spans["candidates"][0].text == "Just" + assert docs[0].spans["candidates"][4].text == "Just a" + + +@pytest.mark.parametrize("name", SPANCAT_COMPONENTS) +@pytest.mark.parametrize("n_process", [1, 2]) +def test_spancat_multiprocessing(name, n_process): + if isinstance(get_current_ops, NumpyOps) or n_process < 2: + nlp = Language() + spancat = nlp.add_pipe(name, config={"spans_key": SPAN_KEY}) + train_examples = make_examples(nlp) + nlp.initialize(get_examples=lambda: train_examples) + texts = [ + "Just a sentence.", + "I like London and Berlin", + "I like Berlin", + "I eat ham.", + ] + docs = list(nlp.pipe(texts, n_process=n_process)) + assert len(docs) == len(texts) diff --git a/spacy/tests/pipeline/test_tagger.py b/spacy/tests/pipeline/test_tagger.py new file mode 100644 index 0000000..4b5f1ee --- /dev/null +++ b/spacy/tests/pipeline/test_tagger.py @@ -0,0 +1,242 @@ +import pytest +from numpy.testing import assert_almost_equal, assert_equal +from thinc.api import compounding, get_current_ops + +from spacy import util +from spacy.attrs import TAG +from spacy.lang.en import English +from spacy.language import Language +from spacy.training import Example + +from ..util import make_tempdir + + +@pytest.mark.issue(4348) +def test_issue4348(): + """Test that training the tagger with empty data, doesn't throw errors""" + nlp = English() + example = Example.from_dict(nlp.make_doc(""), {"tags": []}) + TRAIN_DATA = [example, example] + tagger = nlp.add_pipe("tagger") + tagger.add_label("A") + optimizer = nlp.initialize() + for i in range(5): + losses = {} + batches = util.minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001)) + for batch in batches: + nlp.update(batch, sgd=optimizer, losses=losses) + + +def test_label_types(): + nlp = Language() + tagger = nlp.add_pipe("tagger") + tagger.add_label("A") + with pytest.raises(ValueError): + tagger.add_label(9) + + +def test_tagger_initialize_tag_map(): + """Test that Tagger.initialize() without gold tuples does not clobber + the tag map.""" + nlp = Language() + tagger = nlp.add_pipe("tagger") + orig_tag_count = len(tagger.labels) + tagger.add_label("A") + nlp.initialize() + assert orig_tag_count + 1 == len(nlp.get_pipe("tagger").labels) + + +TAGS = ("N", "V", "J") + +TRAIN_DATA = [ + ("I like green eggs", {"tags": ["N", "V", "J", "N"]}), + ("Eat blue ham", {"tags": ["V", "J", "N"]}), +] + +PARTIAL_DATA = [ + # partial annotation + ("I like green eggs", {"tags": ["", "V", "J", ""]}), + # misaligned partial annotation + ( + "He hates green eggs", + { + "words": ["He", "hate", "s", "green", "eggs"], + "tags": ["", "V", "S", "J", ""], + }, + ), +] + + +def test_label_smoothing(): + nlp = Language() + tagger_no_ls = nlp.add_pipe("tagger", "no_label_smoothing") + tagger_ls = nlp.add_pipe( + "tagger", "label_smoothing", config=dict(label_smoothing=0.05) + ) + train_examples = [] + losses = {} + for tag in TAGS: + tagger_no_ls.add_label(tag) + tagger_ls.add_label(tag) + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + nlp.initialize(get_examples=lambda: train_examples) + tag_scores, bp_tag_scores = tagger_ls.model.begin_update( + [eg.predicted for eg in train_examples] + ) + ops = get_current_ops() + no_ls_grads = ops.to_numpy(tagger_no_ls.get_loss(train_examples, tag_scores)[1][0]) + ls_grads = ops.to_numpy(tagger_ls.get_loss(train_examples, tag_scores)[1][0]) + assert_almost_equal(ls_grads / no_ls_grads, 0.925) + + +def test_no_label(): + nlp = Language() + nlp.add_pipe("tagger") + with pytest.raises(ValueError): + nlp.initialize() + + +def test_no_resize(): + nlp = Language() + tagger = nlp.add_pipe("tagger") + tagger.add_label("N") + tagger.add_label("V") + assert tagger.labels == ("N", "V") + nlp.initialize() + assert tagger.model.get_dim("nO") == 2 + # this throws an error because the tagger can't be resized after initialization + with pytest.raises(ValueError): + tagger.add_label("J") + + +def test_implicit_label(): + nlp = Language() + nlp.add_pipe("tagger") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + nlp.initialize(get_examples=lambda: train_examples) + + +def test_initialize_examples(): + nlp = Language() + tagger = nlp.add_pipe("tagger") + train_examples = [] + for tag in TAGS: + tagger.add_label(tag) + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + # you shouldn't really call this more than once, but for testing it should be fine + nlp.initialize() + nlp.initialize(get_examples=lambda: train_examples) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: None) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: train_examples[0]) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: []) + with pytest.raises(TypeError): + nlp.initialize(get_examples=train_examples) + + +def test_no_data(): + # Test that the tagger provides a nice error when there's no tagging data / labels + TEXTCAT_DATA = [ + ("I'm so happy.", {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}), + ("I'm so angry", {"cats": {"POSITIVE": 0.0, "NEGATIVE": 1.0}}), + ] + nlp = English() + nlp.add_pipe("tagger") + nlp.add_pipe("textcat") + train_examples = [] + for t in TEXTCAT_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + with pytest.raises(ValueError): + nlp.initialize(get_examples=lambda: train_examples) + + +def test_incomplete_data(): + # Test that the tagger works with incomplete information + nlp = English() + nlp.add_pipe("tagger") + train_examples = [] + for t in PARTIAL_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["tagger"] < 0.00001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the tagger - ensuring the ML models work correctly + nlp = English() + tagger = nlp.add_pipe("tagger") + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert tagger.model.get_dim("nO") == len(TAGS) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["tagger"] < 0.00001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = [ + "Just a sentence.", + "I like green eggs.", + "Here is another one.", + "I eat ham.", + ] + batch_deps_1 = [doc.to_array([TAG]) for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.to_array([TAG]) for doc in nlp.pipe(texts)] + no_batch_deps = [doc.to_array([TAG]) for doc in [nlp(text) for text in texts]] + assert_equal(batch_deps_1, batch_deps_2) + assert_equal(batch_deps_1, no_batch_deps) + + # Try to unlearn the first 'N' tag with negative annotation + neg_ex = Example.from_dict(nlp.make_doc(test_text), {"tags": ["!N", "V", "J", "N"]}) + + for i in range(20): + losses = {} + nlp.update([neg_ex], sgd=optimizer, losses=losses) + + # test the "untrained" tag + doc3 = nlp(test_text) + assert doc3[0].tag_ != "N" + + +def test_tagger_requires_labels(): + nlp = English() + nlp.add_pipe("tagger") + with pytest.raises(ValueError): + nlp.initialize() diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py new file mode 100644 index 0000000..e749940 --- /dev/null +++ b/spacy/tests/pipeline/test_textcat.py @@ -0,0 +1,973 @@ +import random + +import numpy.random +import pytest +from numpy.testing import assert_almost_equal +from thinc.api import Config, compounding, fix_random_seed, get_current_ops +from wasabi import msg + +import spacy +from spacy import util +from spacy.cli.evaluate import print_prf_per_type, print_textcats_auc_per_cat +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline import TextCategorizer +from spacy.pipeline.textcat import ( + single_label_bow_config, + single_label_cnn_config, + single_label_default_config, +) +from spacy.pipeline.textcat_multilabel import ( + multi_label_bow_config, + multi_label_cnn_config, + multi_label_default_config, +) +from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +from spacy.scorer import Scorer +from spacy.tests.tok2vec import build_lazy_init_tok2vec as _ # noqa: F401 +from spacy.tokens import Doc, DocBin +from spacy.training import Example +from spacy.training.initialize import init_nlp + +# Ensure that the architecture gets added to the registry. +from ..util import make_tempdir + +TRAIN_DATA_SINGLE_LABEL = [ + ("I'm so happy.", {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}), + ("I'm so angry", {"cats": {"POSITIVE": 0.0, "NEGATIVE": 1.0}}), +] + +TRAIN_DATA_MULTI_LABEL = [ + ("I'm angry and confused", {"cats": {"ANGRY": 1.0, "CONFUSED": 1.0, "HAPPY": 0.0}}), + ("I'm confused but happy", {"cats": {"ANGRY": 0.0, "CONFUSED": 1.0, "HAPPY": 1.0}}), +] + +lazy_init_model_config = """ +[model] +@architectures = "test.LazyInitTok2Vec.v1" +width = 96 +""" +LAZY_INIT_TOK2VEC_MODEL = Config().from_str(lazy_init_model_config)["model"] + + +def make_get_examples_single_label(nlp): + train_examples = [] + for t in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + def get_examples(): + return train_examples + + return get_examples + + +def make_get_examples_multi_label(nlp): + train_examples = [] + for t in TRAIN_DATA_MULTI_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + def get_examples(): + return train_examples + + return get_examples + + +@pytest.mark.issue(3611) +def test_issue3611(): + """Test whether adding n-grams in the textcat works even when n > token length of some docs""" + unique_classes = ["offensive", "inoffensive"] + x_train = [ + "This is an offensive text", + "This is the second offensive text", + "inoff", + ] + y_train = ["offensive", "offensive", "inoffensive"] + nlp = spacy.blank("en") + # preparing the data + train_data = [] + for text, train_instance in zip(x_train, y_train): + cat_dict = {label: label == train_instance for label in unique_classes} + train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict})) + # add a text categorizer component + model = { + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 2, + "no_output_layer": False, + } + textcat = nlp.add_pipe("textcat", config={"model": model}, last=True) + for label in unique_classes: + textcat.add_label(label) + # training the network + with nlp.select_pipes(enable="textcat"): + optimizer = nlp.initialize() + for i in range(3): + losses = {} + batches = util.minibatch(train_data, size=compounding(4.0, 32.0, 1.001)) + + for batch in batches: + nlp.update(examples=batch, sgd=optimizer, drop=0.1, losses=losses) + + +@pytest.mark.issue(4030) +def test_issue4030(): + """Test whether textcat works fine with empty doc""" + unique_classes = ["offensive", "inoffensive"] + x_train = [ + "This is an offensive text", + "This is the second offensive text", + "inoff", + ] + y_train = ["offensive", "offensive", "inoffensive"] + nlp = spacy.blank("en") + # preparing the data + train_data = [] + for text, train_instance in zip(x_train, y_train): + cat_dict = {label: label == train_instance for label in unique_classes} + train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict})) + # add a text categorizer component + model = { + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 2, + "no_output_layer": False, + } + textcat = nlp.add_pipe("textcat", config={"model": model}, last=True) + for label in unique_classes: + textcat.add_label(label) + # training the network + with nlp.select_pipes(enable="textcat"): + optimizer = nlp.initialize() + for i in range(3): + losses = {} + batches = util.minibatch(train_data, size=compounding(4.0, 32.0, 1.001)) + + for batch in batches: + nlp.update(examples=batch, sgd=optimizer, drop=0.1, losses=losses) + # processing of an empty doc should result in 0.0 for all categories + doc = nlp("") + assert doc.cats["offensive"] == 0.0 + assert doc.cats["inoffensive"] == 0.0 + + +@pytest.mark.parametrize( + "textcat_config", + [ + single_label_default_config, + single_label_bow_config, + single_label_cnn_config, + multi_label_default_config, + multi_label_bow_config, + multi_label_cnn_config, + ], +) +@pytest.mark.issue(5551) +def test_issue5551(textcat_config): + """Test that after fixing the random seed, the results of the pipeline are truly identical""" + component = "textcat" + + pipe_cfg = Config().from_str(textcat_config) + results = [] + for i in range(3): + fix_random_seed(0) + nlp = English() + text = "Once hot, form ping-pong-ball-sized balls of the mixture, each weighing roughly 25 g." + annots = {"cats": {"Labe1": 1.0, "Label2": 0.0, "Label3": 0.0}} + pipe = nlp.add_pipe(component, config=pipe_cfg, last=True) + for label in set(annots["cats"]): + pipe.add_label(label) + # Train + nlp.initialize() + doc = nlp.make_doc(text) + nlp.update([Example.from_dict(doc, annots)]) + # Store the result of each iteration + result = pipe.model.predict([doc]) + results.append(result[0]) + # All results should be the same because of the fixed seed + assert len(results) == 3 + ops = get_current_ops() + assert_almost_equal(ops.to_numpy(results[0]), ops.to_numpy(results[1]), decimal=5) + assert_almost_equal(ops.to_numpy(results[0]), ops.to_numpy(results[2]), decimal=5) + + +CONFIG_ISSUE_6908 = """ +[paths] +train = "TRAIN_PLACEHOLDER" +raw = null +init_tok2vec = null +vectors = null + +[system] +seed = 0 +gpu_allocator = null + +[nlp] +lang = "en" +pipeline = ["textcat"] +tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"} +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null +batch_size = 1000 + +[components] + +[components.textcat] +factory = "TEXTCAT_PLACEHOLDER" + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths:train} + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths:train} + + +[training] +train_corpus = "corpora.train" +dev_corpus = "corpora.dev" +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +frozen_components = [] +before_to_disk = null + +[pretraining] + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[initialize.components] + +[initialize.components.textcat] +labels = ['label1', 'label2'] + +[initialize.tokenizer] +""" + + +@pytest.mark.parametrize( + "component_name", + ["textcat", "textcat_multilabel"], +) +@pytest.mark.issue(6908) +def test_issue6908(component_name): + """Test initializing textcat with labels in a list""" + + def create_data(out_file): + nlp = spacy.blank("en") + doc = nlp.make_doc("Some text") + doc.cats = {"label1": 0, "label2": 1} + out_data = DocBin(docs=[doc]).to_bytes() + with out_file.open("wb") as file_: + file_.write(out_data) + + with make_tempdir() as tmp_path: + train_path = tmp_path / "train.spacy" + create_data(train_path) + config_str = CONFIG_ISSUE_6908.replace("TEXTCAT_PLACEHOLDER", component_name) + config_str = config_str.replace("TRAIN_PLACEHOLDER", train_path.as_posix()) + config = util.load_config_from_str(config_str) + init_nlp(config) + + +@pytest.mark.issue(7019) +def test_issue7019(): + scores = {"LABEL_A": 0.39829102, "LABEL_B": 0.938298329382, "LABEL_C": None} + print_textcats_auc_per_cat(msg, scores) + scores = { + "LABEL_A": {"p": 0.3420302, "r": 0.3929020, "f": 0.49823928932}, + "LABEL_B": {"p": None, "r": None, "f": None}, + } + print_prf_per_type(msg, scores, name="foo", type="bar") + + +@pytest.mark.issue(9904) +def test_issue9904(): + nlp = Language() + textcat = nlp.add_pipe("textcat") + get_examples = make_get_examples_single_label(nlp) + nlp.initialize(get_examples) + + examples = get_examples() + scores = textcat.predict([eg.predicted for eg in examples]) + + loss = textcat.get_loss(examples, scores)[0] + loss_double_bs = textcat.get_loss(examples * 2, scores.repeat(2, axis=0))[0] + assert loss == pytest.approx(loss_double_bs) + + +@pytest.mark.skip(reason="Test is flakey when run with others") +def test_simple_train(): + nlp = Language() + textcat = nlp.add_pipe("textcat") + textcat.add_label("answer") + nlp.initialize() + for i in range(5): + for text, answer in [ + ("aaaa", 1.0), + ("bbbb", 0), + ("aa", 1.0), + ("bbbbbbbbb", 0.0), + ("aaaaaa", 1), + ]: + nlp.update((text, {"cats": {"answer": answer}})) + doc = nlp("aaa") + assert "answer" in doc.cats + assert doc.cats["answer"] >= 0.5 + + +@pytest.mark.skip(reason="Test is flakey when run with others") +def test_textcat_learns_multilabel(): + random.seed(5) + numpy.random.seed(5) + docs = [] + nlp = Language() + letters = ["a", "b", "c"] + for w1 in letters: + for w2 in letters: + cats = {letter: float(w2 == letter) for letter in letters} + docs.append((Doc(nlp.vocab, words=["d"] * 3 + [w1, w2] + ["d"] * 3), cats)) + random.shuffle(docs) + textcat = TextCategorizer(nlp.vocab, width=8) + for letter in letters: + textcat.add_label(letter) + optimizer = textcat.initialize(lambda: []) + for i in range(30): + losses = {} + examples = [Example.from_dict(doc, {"cats": cats}) for doc, cat in docs] + textcat.update(examples, sgd=optimizer, losses=losses) + random.shuffle(docs) + for w1 in letters: + for w2 in letters: + doc = Doc(nlp.vocab, words=["d"] * 3 + [w1, w2] + ["d"] * 3) + truth = {letter: w2 == letter for letter in letters} + textcat(doc) + for cat, score in doc.cats.items(): + if not truth[cat]: + assert score < 0.5 + else: + assert score > 0.5 + + +@pytest.mark.parametrize("name", ["textcat", "textcat_multilabel"]) +def test_label_types(name): + nlp = Language() + textcat = nlp.add_pipe(name) + textcat.add_label("answer") + with pytest.raises(ValueError): + textcat.add_label(9) + # textcat requires at least two labels + if name == "textcat": + with pytest.raises(ValueError): + nlp.initialize() + else: + nlp.initialize() + + +@pytest.mark.parametrize( + "name,get_examples", + [ + ("textcat", make_get_examples_single_label), + ("textcat_multilabel", make_get_examples_multi_label), + ], +) +def test_invalid_label_value(name, get_examples): + nlp = Language() + textcat = nlp.add_pipe(name) + example_getter = get_examples(nlp) + + def invalid_examples(): + # make one example with an invalid score + examples = example_getter() + ref = examples[0].reference + key = list(ref.cats.keys())[0] + ref.cats[key] = 2.0 + return examples + + with pytest.raises(ValueError): + nlp.initialize(get_examples=invalid_examples) + + +@pytest.mark.parametrize("name", ["textcat", "textcat_multilabel"]) +def test_no_label(name): + nlp = Language() + nlp.add_pipe(name) + with pytest.raises(ValueError): + nlp.initialize() + + +@pytest.mark.parametrize( + "name,get_examples", + [ + ("textcat", make_get_examples_single_label), + ("textcat_multilabel", make_get_examples_multi_label), + ], +) +def test_implicit_label(name, get_examples): + nlp = Language() + nlp.add_pipe(name) + nlp.initialize(get_examples=get_examples(nlp)) + + +# fmt: off +@pytest.mark.slow +@pytest.mark.parametrize( + "name,textcat_config", + [ + # BOW V1 + ("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}), + ("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}), + # ENSEMBLE V1 + ("textcat", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + # ENSEMBLE V2 + ("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}}), + ("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}}), + # CNN + ("textcat", {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), + ], +) +# fmt: on +def test_no_resize(name, textcat_config): + """The old textcat architectures weren't resizable""" + nlp = Language() + pipe_config = {"model": textcat_config} + textcat = nlp.add_pipe(name, config=pipe_config) + textcat.add_label("POSITIVE") + textcat.add_label("NEGATIVE") + nlp.initialize() + assert textcat.model.maybe_get_dim("nO") in [2, None] + # this throws an error because the textcat can't be resized after initialization + with pytest.raises(ValueError): + textcat.add_label("NEUTRAL") + + +# fmt: off +@pytest.mark.parametrize( + "name,textcat_config", + [ + # BOW V3 + ("textcat", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}), + ("textcat", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}), + # CNN + ("textcat", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ], +) +# fmt: on +def test_resize(name, textcat_config): + """The new textcat architectures are resizable""" + nlp = Language() + pipe_config = {"model": textcat_config} + textcat = nlp.add_pipe(name, config=pipe_config) + textcat.add_label("POSITIVE") + textcat.add_label("NEGATIVE") + assert textcat.model.maybe_get_dim("nO") in [2, None] + nlp.initialize() + assert textcat.model.maybe_get_dim("nO") in [2, None] + textcat.add_label("NEUTRAL") + assert textcat.model.maybe_get_dim("nO") in [3, None] + + +# fmt: off +@pytest.mark.parametrize( + "name,textcat_config", + [ + # BOW v3 + ("textcat", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}), + ("textcat", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}), + # REDUCE + ("textcat", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ], +) +# fmt: on +def test_resize_same_results(name, textcat_config): + # Ensure that the resized textcat classifiers still produce the same results for old labels + fix_random_seed(0) + nlp = English() + pipe_config = {"model": textcat_config} + textcat = nlp.add_pipe(name, config=pipe_config) + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert textcat.model.maybe_get_dim("nO") in [2, None] + + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # test the trained model before resizing + test_text = "I am happy." + doc = nlp(test_text) + assert len(doc.cats) == 2 + pos_pred = doc.cats["POSITIVE"] + neg_pred = doc.cats["NEGATIVE"] + + # test the trained model again after resizing + textcat.add_label("NEUTRAL") + doc = nlp(test_text) + assert len(doc.cats) == 3 + assert doc.cats["POSITIVE"] == pos_pred + assert doc.cats["NEGATIVE"] == neg_pred + assert doc.cats["NEUTRAL"] <= 1 + + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + # test the trained model again after training further with new label + doc = nlp(test_text) + assert len(doc.cats) == 3 + assert doc.cats["POSITIVE"] != pos_pred + assert doc.cats["NEGATIVE"] != neg_pred + for cat in doc.cats: + assert doc.cats[cat] <= 1 + + +def test_error_with_multi_labels(): + nlp = Language() + nlp.add_pipe("textcat") + train_examples = [] + for text, annotations in TRAIN_DATA_MULTI_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + with pytest.raises(ValueError): + nlp.initialize(get_examples=lambda: train_examples) + + +# fmt: off +@pytest.mark.parametrize( + "name,textcat_config", + [ + # ENSEMBLE V2 + ("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}), + ("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}), + # PARAMETRIC ATTENTION V1 + ("textcat", {"@architectures": "spacy.TextCatParametricAttention.v1", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatParametricAttention.v1", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "exclusive_classes": False}), + # REDUCE + ("textcat", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "exclusive_classes": True, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ("textcat_multilabel", {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": LAZY_INIT_TOK2VEC_MODEL, "exclusive_classes": False, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ], +) +# fmt: on +def test_tok2vec_lazy_init(name, textcat_config): + # Check that we can properly initialize and use a textcat model using + # a lazily-initialized tok2vec. + nlp = English() + pipe_config = {"model": textcat_config} + textcat = nlp.add_pipe(name, config=pipe_config) + textcat.add_label("POSITIVE") + textcat.add_label("NEGATIVE") + nlp.initialize() + nlp.pipe(["This is a test."]) + + +@pytest.mark.parametrize( + "name,get_examples, train_data", + [ + ("textcat", make_get_examples_single_label, TRAIN_DATA_SINGLE_LABEL), + ("textcat_multilabel", make_get_examples_multi_label, TRAIN_DATA_MULTI_LABEL), + ], +) +def test_initialize_examples(name, get_examples, train_data): + nlp = Language() + textcat = nlp.add_pipe(name) + for text, annotations in train_data: + for label, value in annotations.get("cats").items(): + textcat.add_label(label) + # you shouldn't really call this more than once, but for testing it should be fine + nlp.initialize() + nlp.initialize(get_examples=get_examples(nlp)) + with pytest.raises(TypeError): + nlp.initialize(get_examples=lambda: None) + with pytest.raises(TypeError): + nlp.initialize(get_examples=get_examples()) + + +def test_overfitting_IO(): + # Simple test to try and quickly overfit the single-label textcat component - ensuring the ML models work correctly + fix_random_seed(0) + nlp = English() + textcat = nlp.add_pipe("textcat") + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert textcat.model.get_dim("nO") == 2 + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["textcat"] < 0.01 + + # test the trained model + test_text = "I am happy." + doc = nlp(test_text) + cats = doc.cats + assert cats["POSITIVE"] > 0.9 + assert cats["POSITIVE"] + cats["NEGATIVE"] == pytest.approx(1.0, 0.001) + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + cats2 = doc2.cats + assert cats2["POSITIVE"] > 0.9 + assert cats2["POSITIVE"] + cats2["NEGATIVE"] == pytest.approx(1.0, 0.001) + + # Test scoring + scores = nlp.evaluate(train_examples) + assert scores["cats_micro_f"] == 1.0 + assert scores["cats_macro_f"] == 1.0 + assert scores["cats_macro_auc"] == 1.0 + assert scores["cats_score"] == 1.0 + assert "cats_score_desc" in scores + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = ["Just a sentence.", "I like green eggs.", "I am happy.", "I eat ham."] + batch_cats_1 = [doc.cats for doc in nlp.pipe(texts)] + batch_cats_2 = [doc.cats for doc in nlp.pipe(texts)] + no_batch_cats = [doc.cats for doc in [nlp(text) for text in texts]] + for cats_1, cats_2 in zip(batch_cats_1, batch_cats_2): + for cat in cats_1: + assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5) + for cats_1, cats_2 in zip(batch_cats_1, no_batch_cats): + for cat in cats_1: + assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5) + + +def test_overfitting_IO_multi(): + # Simple test to try and quickly overfit the multi-label textcat component - ensuring the ML models work correctly + fix_random_seed(0) + nlp = English() + textcat = nlp.add_pipe("textcat_multilabel") + + train_examples = [] + for text, annotations in TRAIN_DATA_MULTI_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + assert textcat.model.get_dim("nO") == 3 + + for i in range(100): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + assert losses["textcat_multilabel"] < 0.01 + + # test the trained model + test_text = "I am confused but happy." + doc = nlp(test_text) + cats = doc.cats + assert cats["HAPPY"] > 0.9 + assert cats["CONFUSED"] > 0.9 + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + cats2 = doc2.cats + assert cats2["HAPPY"] > 0.9 + assert cats2["CONFUSED"] > 0.9 + + # Test scoring + scores = nlp.evaluate(train_examples) + assert scores["cats_micro_f"] == 1.0 + assert scores["cats_macro_f"] == 1.0 + assert "cats_score_desc" in scores + + # Make sure that running pipe twice, or comparing to call, always amounts to the same predictions + texts = ["Just a sentence.", "I like green eggs.", "I am happy.", "I eat ham."] + batch_deps_1 = [doc.cats for doc in nlp.pipe(texts)] + batch_deps_2 = [doc.cats for doc in nlp.pipe(texts)] + no_batch_deps = [doc.cats for doc in [nlp(text) for text in texts]] + for cats_1, cats_2 in zip(batch_deps_1, batch_deps_2): + for cat in cats_1: + assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5) + for cats_1, cats_2 in zip(batch_deps_1, no_batch_deps): + for cat in cats_1: + assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5) + + +# fmt: off +@pytest.mark.slow +@pytest.mark.parametrize( + "name,train_data,textcat_config", + [ + # BOW V1 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}), + # ENSEMBLE V1 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v1", "exclusive_classes": False, "pretrained_vectors": None, "width": 64, "embed_size": 2000, "conv_depth": 2, "window_size": 1, "ngram_size": 1, "dropout": None}), + # CNN V1 + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), + # BOW V2 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}), + # BOW V3 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}), + # ENSEMBLE V2 + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}), + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v3", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}), + # CNN V2 (legacy) + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), + # PARAMETRIC ATTENTION V1 + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatParametricAttention.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatParametricAttention.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}), + # REDUCE V1 + ("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatReduce.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False, "use_reduce_first": True, "use_reduce_last": True, "use_reduce_max": True, "use_reduce_mean": True}), + ], +) +# fmt: on +def test_textcat_configs(name, train_data, textcat_config): + pipe_config = {"model": textcat_config} + nlp = English() + textcat = nlp.add_pipe(name, config=pipe_config) + train_examples = [] + for text, annotations in train_data: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + for label, value in annotations.get("cats").items(): + textcat.add_label(label) + optimizer = nlp.initialize() + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + +def test_positive_class(): + nlp = English() + textcat = nlp.add_pipe("textcat") + get_examples = make_get_examples_single_label(nlp) + textcat.initialize(get_examples, labels=["POS", "NEG"], positive_label="POS") + assert textcat.labels == ("POS", "NEG") + assert textcat.cfg["positive_label"] == "POS" + + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + get_examples = make_get_examples_multi_label(nlp) + with pytest.raises(TypeError): + textcat_multilabel.initialize( + get_examples, labels=["POS", "NEG"], positive_label="POS" + ) + textcat_multilabel.initialize(get_examples, labels=["FICTION", "DRAMA"]) + assert textcat_multilabel.labels == ("FICTION", "DRAMA") + assert "positive_label" not in textcat_multilabel.cfg + + +def test_positive_class_not_present(): + nlp = English() + textcat = nlp.add_pipe("textcat") + get_examples = make_get_examples_single_label(nlp) + with pytest.raises(ValueError): + textcat.initialize(get_examples, labels=["SOME", "THING"], positive_label="POS") + + +def test_positive_class_not_binary(): + nlp = English() + textcat = nlp.add_pipe("textcat") + get_examples = make_get_examples_multi_label(nlp) + with pytest.raises(ValueError): + textcat.initialize( + get_examples, labels=["SOME", "THING", "POS"], positive_label="POS" + ) + + +def test_textcat_evaluation(): + train_examples = [] + nlp = English() + ref1 = nlp("one") + ref1.cats = {"winter": 1.0, "summer": 1.0, "spring": 1.0, "autumn": 1.0} + pred1 = nlp("one") + pred1.cats = {"winter": 1.0, "summer": 0.0, "spring": 1.0, "autumn": 1.0} + train_examples.append(Example(pred1, ref1)) + + ref2 = nlp("two") + ref2.cats = {"winter": 0.0, "summer": 0.0, "spring": 1.0, "autumn": 1.0} + pred2 = nlp("two") + pred2.cats = {"winter": 1.0, "summer": 0.0, "spring": 0.0, "autumn": 1.0} + train_examples.append(Example(pred2, ref2)) + + scores = Scorer().score_cats( + train_examples, "cats", labels=["winter", "summer", "spring", "autumn"] + ) + assert scores["cats_f_per_type"]["winter"]["p"] == 1 / 2 + assert scores["cats_f_per_type"]["winter"]["r"] == 1 / 1 + assert scores["cats_f_per_type"]["summer"]["p"] == 0 + assert scores["cats_f_per_type"]["summer"]["r"] == 0 / 1 + assert scores["cats_f_per_type"]["spring"]["p"] == 1 / 1 + assert scores["cats_f_per_type"]["spring"]["r"] == 1 / 2 + assert scores["cats_f_per_type"]["autumn"]["p"] == 2 / 2 + assert scores["cats_f_per_type"]["autumn"]["r"] == 2 / 2 + + assert scores["cats_micro_p"] == 4 / 5 + assert scores["cats_micro_r"] == 4 / 6 + + +@pytest.mark.parametrize( + "multi_label,spring_p", + [(True, 1 / 1), (False, 1 / 2)], +) +def test_textcat_eval_missing(multi_label: bool, spring_p: float): + """ + multi-label: the missing 'spring' in gold_doc_2 doesn't incur a penalty + exclusive labels: the missing 'spring' in gold_doc_2 is interpreted as 0.0""" + train_examples = [] + nlp = English() + + ref1 = nlp("one") + ref1.cats = {"winter": 0.0, "summer": 0.0, "autumn": 0.0, "spring": 1.0} + pred1 = nlp("one") + pred1.cats = {"winter": 0.0, "summer": 0.0, "autumn": 0.0, "spring": 1.0} + train_examples.append(Example(ref1, pred1)) + + ref2 = nlp("two") + # reference 'spring' is missing, pred 'spring' is 1 + ref2.cats = {"winter": 0.0, "summer": 0.0, "autumn": 1.0} + pred2 = nlp("two") + pred2.cats = {"winter": 0.0, "summer": 0.0, "autumn": 0.0, "spring": 1.0} + train_examples.append(Example(pred2, ref2)) + + scores = Scorer().score_cats( + train_examples, + "cats", + labels=["winter", "summer", "spring", "autumn"], + multi_label=multi_label, + ) + assert scores["cats_f_per_type"]["spring"]["p"] == spring_p + assert scores["cats_f_per_type"]["spring"]["r"] == 1 / 1 + + +@pytest.mark.parametrize( + "multi_label,expected_loss", + [(True, 0), (False, 0.125)], +) +def test_textcat_loss(multi_label: bool, expected_loss: float): + """ + multi-label: the missing 'spring' in gold_doc_2 doesn't incur an increase in loss + exclusive labels: the missing 'spring' in gold_doc_2 is interpreted as 0.0 and adds to the loss + """ + train_examples = [] + nlp = English() + + doc1 = nlp("one") + cats1 = {"winter": 0.0, "summer": 0.0, "autumn": 0.0, "spring": 1.0} + train_examples.append(Example.from_dict(doc1, {"cats": cats1})) + + doc2 = nlp("two") + cats2 = {"winter": 0.0, "summer": 0.0, "autumn": 1.0} + train_examples.append(Example.from_dict(doc2, {"cats": cats2})) + + if multi_label: + textcat = nlp.add_pipe("textcat_multilabel") + else: + textcat = nlp.add_pipe("textcat") + assert isinstance(textcat, TextCategorizer) + textcat.initialize(lambda: train_examples) + scores = textcat.model.ops.asarray( + [[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0]], dtype="f" # type: ignore + ) + loss, d_scores = textcat.get_loss(train_examples, scores) + assert loss == expected_loss + + +def test_textcat_multilabel_threshold(): + # Ensure the scorer can be called with a different threshold + nlp = English() + nlp.add_pipe("textcat_multilabel") + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + nlp.initialize(get_examples=lambda: train_examples) + + # score the model (it's not actually trained but that doesn't matter) + scores = nlp.evaluate(train_examples) + assert 0 <= scores["cats_score"] <= 1 + + scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 1.0}) + assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 0 + + scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0}) + macro_f = scores["cats_score"] + assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 + + scores = nlp.evaluate( + train_examples, scorer_cfg={"threshold": 0, "positive_label": "POSITIVE"} + ) + pos_f = scores["cats_score"] + assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 + assert pos_f >= macro_f + + +def test_textcat_multi_threshold(): + # Ensure the scorer can be called with a different threshold + nlp = English() + nlp.add_pipe("textcat_multilabel") + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + nlp.initialize(get_examples=lambda: train_examples) + + # score the model (it's not actually trained but that doesn't matter) + scores = nlp.evaluate(train_examples) + assert 0 <= scores["cats_score"] <= 1 + + scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 1.0}) + assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 0 + + scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0}) + assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 + + +@pytest.mark.parametrize( + "component_name,scorer", + [ + ("textcat", "spacy.textcat_scorer.v1"), + ("textcat_multilabel", "spacy.textcat_multilabel_scorer.v1"), + ], +) +def test_textcat_legacy_scorers(component_name, scorer): + """Check that legacy scorers are registered and produce the expected score + keys.""" + nlp = English() + nlp.add_pipe(component_name, config={"scorer": {"@scorers": scorer}}) + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + nlp.initialize(get_examples=lambda: train_examples) + + # score the model (it's not actually trained but that doesn't matter) + scores = nlp.evaluate(train_examples) + assert 0 <= scores["cats_score"] <= 1 diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py new file mode 100644 index 0000000..ddd9a99 --- /dev/null +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -0,0 +1,651 @@ +import pytest +from numpy.testing import assert_array_equal +from thinc.api import Config, get_current_ops + +from spacy import util +from spacy.lang.en import English +from spacy.ml.models.tok2vec import ( + MaxoutWindowEncoder, + MultiHashEmbed, + build_Tok2Vec_model, +) +from spacy.pipeline.tok2vec import Tok2Vec, Tok2VecListener +from spacy.tokens import Doc +from spacy.training import Example +from spacy.util import registry +from spacy.vocab import Vocab + +from ..util import add_vecs_to_vocab, get_batch, make_tempdir + + +def test_empty_doc(): + width = 128 + embed_size = 2000 + vocab = Vocab() + doc = Doc(vocab, words=[]) + tok2vec = build_Tok2Vec_model( + MultiHashEmbed( + width=width, + rows=[embed_size, embed_size, embed_size, embed_size], + include_static_vectors=False, + attrs=["NORM", "PREFIX", "SUFFIX", "SHAPE"], + ), + MaxoutWindowEncoder(width=width, depth=4, window_size=1, maxout_pieces=3), + ) + tok2vec.initialize() + vectors, backprop = tok2vec.begin_update([doc]) + assert len(vectors) == 1 + assert vectors[0].shape == (0, width) + + +@pytest.mark.parametrize( + "batch_size,width,embed_size", [[1, 128, 2000], [2, 128, 2000], [3, 8, 63]] +) +def test_tok2vec_batch_sizes(batch_size, width, embed_size): + batch = get_batch(batch_size) + tok2vec = build_Tok2Vec_model( + MultiHashEmbed( + width=width, + rows=[embed_size] * 4, + include_static_vectors=False, + attrs=["NORM", "PREFIX", "SUFFIX", "SHAPE"], + ), + MaxoutWindowEncoder(width=width, depth=4, window_size=1, maxout_pieces=3), + ) + tok2vec.initialize() + vectors, backprop = tok2vec.begin_update(batch) + assert len(vectors) == len(batch) + for doc_vec, doc in zip(vectors, batch): + assert doc_vec.shape == (len(doc), width) + + +@pytest.mark.slow +@pytest.mark.parametrize("width", [8]) +@pytest.mark.parametrize( + "embed_arch,embed_config", + # fmt: off + [ + ( + "spacy.MultiHashEmbed.v1", + { + "rows": [100, 100], + "attrs": ["SHAPE", "LOWER"], + "include_static_vectors": False, + }, + ), + ( + "spacy.MultiHashEmbed.v1", + { + "rows": [100, 20], + "attrs": ["ORTH", "PREFIX"], + "include_static_vectors": False, + }, + ), + ( + "spacy.CharacterEmbed.v1", + {"rows": 100, "nM": 64, "nC": 8, "include_static_vectors": False}, + ), + ( + "spacy.CharacterEmbed.v1", + {"rows": 100, "nM": 16, "nC": 2, "include_static_vectors": False}, + ), + ], + # fmt: on +) +@pytest.mark.parametrize( + "tok2vec_arch,encode_arch,encode_config", + # fmt: off + [ + ( + "spacy.Tok2Vec.v1", + "spacy.MaxoutWindowEncoder.v1", + {"window_size": 1, "maxout_pieces": 3, "depth": 2}, + ), + ( + "spacy.Tok2Vec.v2", + "spacy.MaxoutWindowEncoder.v2", + {"window_size": 1, "maxout_pieces": 3, "depth": 2}, + ), + ( + "spacy.Tok2Vec.v1", + "spacy.MishWindowEncoder.v1", + {"window_size": 1, "depth": 6}, + ), + ( + "spacy.Tok2Vec.v2", + "spacy.MishWindowEncoder.v2", + {"window_size": 1, "depth": 6}, + ), + ], + # fmt: on +) +def test_tok2vec_configs( + width, tok2vec_arch, embed_arch, embed_config, encode_arch, encode_config +): + embed = registry.get("architectures", embed_arch) + encode = registry.get("architectures", encode_arch) + tok2vec_model = registry.get("architectures", tok2vec_arch) + + embed_config["width"] = width + encode_config["width"] = width + docs = get_batch(3) + tok2vec = tok2vec_model(embed(**embed_config), encode(**encode_config)) + tok2vec.initialize(docs) + vectors, backprop = tok2vec.begin_update(docs) + assert len(vectors) == len(docs) + assert vectors[0].shape == (len(docs[0]), width) + backprop(vectors) + + +def test_init_tok2vec(): + # Simple test to initialize the default tok2vec + nlp = English() + tok2vec = nlp.add_pipe("tok2vec") + assert tok2vec.listeners == [] + nlp.initialize() + assert tok2vec.model.get_dim("nO") + + +cfg_string = """ + [nlp] + lang = "en" + pipeline = ["tok2vec","tagger"] + + [components] + + [components.tagger] + factory = "tagger" + + [components.tagger.model] + @architectures = "spacy.Tagger.v2" + nO = null + + [components.tagger.model.tok2vec] + @architectures = "spacy.Tok2VecListener.v1" + width = ${components.tok2vec.model.encode.width} + + [components.tok2vec] + factory = "tok2vec" + + [components.tok2vec.model] + @architectures = "spacy.Tok2Vec.v2" + + [components.tok2vec.model.embed] + @architectures = "spacy.MultiHashEmbed.v1" + width = ${components.tok2vec.model.encode.width} + rows = [2000, 1000, 1000, 1000] + attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] + include_static_vectors = false + + [components.tok2vec.model.encode] + @architectures = "spacy.MaxoutWindowEncoder.v2" + width = 96 + depth = 4 + window_size = 1 + maxout_pieces = 3 + """ + +TRAIN_DATA = [ + ( + "I like green eggs", + {"tags": ["N", "V", "J", "N"], "cats": {"preference": 1.0, "imperative": 0.0}}, + ), + ( + "Eat blue ham", + {"tags": ["V", "J", "N"], "cats": {"preference": 0.0, "imperative": 1.0}}, + ), +] + + +@pytest.mark.parametrize("with_vectors", (False, True)) +def test_tok2vec_listener(with_vectors): + orig_config = Config().from_str(cfg_string) + orig_config["components"]["tok2vec"]["model"]["embed"]["include_static_vectors"] = ( + with_vectors + ) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + + if with_vectors: + ops = get_current_ops() + vectors = [ + ("apple", ops.asarray([1, 2, 3])), + ("orange", ops.asarray([-1, -2, -3])), + ("and", ops.asarray([-1, -1, -1])), + ("juice", ops.asarray([5, 5, 10])), + ("pie", ops.asarray([7, 6.3, 8.9])), + ] + add_vecs_to_vocab(nlp.vocab, vectors) + + assert nlp.pipe_names == ["tok2vec", "tagger"] + tagger = nlp.get_pipe("tagger") + tok2vec = nlp.get_pipe("tok2vec") + tagger_tok2vec = tagger.model.get_ref("tok2vec") + assert isinstance(tok2vec, Tok2Vec) + assert isinstance(tagger_tok2vec, Tok2VecListener) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + for tag in t[1]["tags"]: + tagger.add_label(tag) + + # Check that the Tok2Vec component finds its listeners + optimizer = nlp.initialize(lambda: train_examples) + assert tok2vec.listeners == [tagger_tok2vec] + + for i in range(5): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + doc = nlp("Running the pipeline as a whole.") + doc_tensor = tagger_tok2vec.predict([doc])[0] + ops = get_current_ops() + assert_array_equal(ops.to_numpy(doc.tensor), ops.to_numpy(doc_tensor)) + + # test with empty doc + doc = nlp("") + + # TODO: should this warn or error? + nlp.select_pipes(disable="tok2vec") + assert nlp.pipe_names == ["tagger"] + nlp("Running the pipeline with the Tok2Vec component disabled.") + + +def test_tok2vec_listener_callback(): + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + assert nlp.pipe_names == ["tok2vec", "tagger"] + tagger = nlp.get_pipe("tagger") + tok2vec = nlp.get_pipe("tok2vec") + docs = [nlp.make_doc("A random sentence")] + tok2vec.model.initialize(X=docs) + gold_array = [[1.0 for tag in ["V", "Z"]] for word in docs] + label_sample = [tagger.model.ops.asarray(gold_array, dtype="float32")] + tagger.model.initialize(X=docs, Y=label_sample) + docs = [nlp.make_doc("Another entirely random sentence")] + tok2vec.update([Example.from_dict(x, {}) for x in docs]) + Y, get_dX = tagger.model.begin_update(docs) + # assure that the backprop call works (and doesn't hit a 'None' callback) + assert get_dX(Y) is not None + + +def test_tok2vec_listener_overfitting(): + """Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components""" + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses, annotates=["tok2vec"]) + assert losses["tagger"] < 0.00001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + +def test_tok2vec_frozen_not_annotating(): + """Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating""" + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(2): + losses = {} + with pytest.raises( + ValueError, match=r"the tok2vec embedding layer is not updated" + ): + nlp.update( + train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"] + ) + + +def test_tok2vec_frozen_overfitting(): + """Test that a pipeline with a frozen & annotating tok2vec can still overfit""" + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(100): + losses = {} + nlp.update( + train_examples, + sgd=optimizer, + losses=losses, + exclude=["tok2vec"], + annotates=["tok2vec"], + ) + assert losses["tagger"] < 0.0001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + +def test_replace_listeners(): + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + examples = [Example.from_dict(nlp.make_doc("x y"), {"tags": ["V", "Z"]})] + nlp.initialize(lambda: examples) + tok2vec = nlp.get_pipe("tok2vec") + tagger = nlp.get_pipe("tagger") + assert isinstance(tagger.model.layers[0], Tok2VecListener) + assert tok2vec.listener_map["tagger"][0] == tagger.model.layers[0] + assert ( + nlp.config["components"]["tok2vec"]["model"]["@architectures"] + == "spacy.Tok2Vec.v2" + ) + assert ( + nlp.config["components"]["tagger"]["model"]["tok2vec"]["@architectures"] + == "spacy.Tok2VecListener.v1" + ) + nlp.replace_listeners("tok2vec", "tagger", ["model.tok2vec"]) + assert not isinstance(tagger.model.layers[0], Tok2VecListener) + t2v_cfg = nlp.config["components"]["tok2vec"]["model"] + assert t2v_cfg["@architectures"] == "spacy.Tok2Vec.v2" + assert nlp.config["components"]["tagger"]["model"]["tok2vec"] == t2v_cfg + with pytest.raises(ValueError): + nlp.replace_listeners("invalid", "tagger", ["model.tok2vec"]) + with pytest.raises(ValueError): + nlp.replace_listeners("tok2vec", "parser", ["model.tok2vec"]) + with pytest.raises(ValueError): + nlp.replace_listeners("tok2vec", "tagger", ["model.yolo"]) + with pytest.raises(ValueError): + nlp.replace_listeners("tok2vec", "tagger", ["model.tok2vec", "model.yolo"]) + # attempt training with the new pipeline + optimizer = nlp.initialize(lambda: examples) + for i in range(2): + losses = {} + nlp.update(examples, sgd=optimizer, losses=losses) + assert losses["tok2vec"] == 0.0 + assert losses["tagger"] > 0.0 + + +cfg_string_multi = """ + [nlp] + lang = "en" + pipeline = ["tok2vec","tagger", "ner"] + + [components] + + [components.tagger] + factory = "tagger" + + [components.tagger.model] + @architectures = "spacy.Tagger.v2" + nO = null + + [components.tagger.model.tok2vec] + @architectures = "spacy.Tok2VecListener.v1" + width = ${components.tok2vec.model.encode.width} + + [components.ner] + factory = "ner" + + [components.ner.model] + @architectures = "spacy.TransitionBasedParser.v2" + + [components.ner.model.tok2vec] + @architectures = "spacy.Tok2VecListener.v1" + width = ${components.tok2vec.model.encode.width} + + [components.tok2vec] + factory = "tok2vec" + + [components.tok2vec.model] + @architectures = "spacy.Tok2Vec.v2" + + [components.tok2vec.model.embed] + @architectures = "spacy.MultiHashEmbed.v1" + width = ${components.tok2vec.model.encode.width} + rows = [2000, 1000, 1000, 1000] + attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] + include_static_vectors = false + + [components.tok2vec.model.encode] + @architectures = "spacy.MaxoutWindowEncoder.v2" + width = 96 + depth = 4 + window_size = 1 + maxout_pieces = 3 + """ + + +def test_replace_listeners_from_config(): + orig_config = Config().from_str(cfg_string_multi) + nlp = util.load_model_from_config(orig_config, auto_fill=True) + annots = {"tags": ["V", "Z"], "entities": [(0, 1, "A"), (1, 2, "B")]} + examples = [Example.from_dict(nlp.make_doc("x y"), annots)] + nlp.initialize(lambda: examples) + tok2vec = nlp.get_pipe("tok2vec") + tagger = nlp.get_pipe("tagger") + ner = nlp.get_pipe("ner") + assert tok2vec.listening_components == ["tagger", "ner"] + assert any(isinstance(node, Tok2VecListener) for node in ner.model.walk()) + assert any(isinstance(node, Tok2VecListener) for node in tagger.model.walk()) + with make_tempdir() as dir_path: + nlp.to_disk(dir_path) + base_model = str(dir_path) + new_config = { + "nlp": { + "lang": "en", + "pipeline": ["tok2vec", "tagger2", "ner3", "tagger4"], + }, + "components": { + "tok2vec": {"source": base_model}, + "tagger2": { + "source": base_model, + "component": "tagger", + "replace_listeners": ["model.tok2vec"], + }, + "ner3": { + "source": base_model, + "component": "ner", + }, + "tagger4": { + "source": base_model, + "component": "tagger", + }, + }, + } + new_nlp = util.load_model_from_config(new_config, auto_fill=True) + new_nlp.initialize(lambda: examples) + tok2vec = new_nlp.get_pipe("tok2vec") + tagger = new_nlp.get_pipe("tagger2") + ner = new_nlp.get_pipe("ner3") + assert "ner" not in new_nlp.pipe_names + assert "tagger" not in new_nlp.pipe_names + assert tok2vec.listening_components == ["ner3", "tagger4"] + assert any(isinstance(node, Tok2VecListener) for node in ner.model.walk()) + assert not any(isinstance(node, Tok2VecListener) for node in tagger.model.walk()) + t2v_cfg = new_nlp.config["components"]["tok2vec"]["model"] + assert t2v_cfg["@architectures"] == "spacy.Tok2Vec.v2" + assert new_nlp.config["components"]["tagger2"]["model"]["tok2vec"] == t2v_cfg + assert ( + new_nlp.config["components"]["ner3"]["model"]["tok2vec"]["@architectures"] + == "spacy.Tok2VecListener.v1" + ) + assert ( + new_nlp.config["components"]["tagger4"]["model"]["tok2vec"]["@architectures"] + == "spacy.Tok2VecListener.v1" + ) + + +cfg_string_multi_textcat = """ + [nlp] + lang = "en" + pipeline = ["tok2vec","textcat_multilabel","tagger"] + + [components] + + [components.textcat_multilabel] + factory = "textcat_multilabel" + + [components.textcat_multilabel.model] + @architectures = "spacy.TextCatEnsemble.v2" + nO = null + + [components.textcat_multilabel.model.tok2vec] + @architectures = "spacy.Tok2VecListener.v1" + width = ${components.tok2vec.model.encode.width} + + [components.textcat_multilabel.model.linear_model] + @architectures = "spacy.TextCatBOW.v1" + exclusive_classes = false + ngram_size = 1 + no_output_layer = false + + [components.tagger] + factory = "tagger" + + [components.tagger.model] + @architectures = "spacy.Tagger.v2" + nO = null + + [components.tagger.model.tok2vec] + @architectures = "spacy.Tok2VecListener.v1" + width = ${components.tok2vec.model.encode.width} + + [components.tok2vec] + factory = "tok2vec" + + [components.tok2vec.model] + @architectures = "spacy.Tok2Vec.v2" + + [components.tok2vec.model.embed] + @architectures = "spacy.MultiHashEmbed.v1" + width = ${components.tok2vec.model.encode.width} + rows = [2000, 1000, 1000, 1000] + attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] + include_static_vectors = false + + [components.tok2vec.model.encode] + @architectures = "spacy.MaxoutWindowEncoder.v2" + width = 96 + depth = 4 + window_size = 1 + maxout_pieces = 3 + """ + + +def test_tok2vec_listeners_textcat(): + orig_config = Config().from_str(cfg_string_multi_textcat) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + assert nlp.pipe_names == ["tok2vec", "textcat_multilabel", "tagger"] + tagger = nlp.get_pipe("tagger") + textcat = nlp.get_pipe("textcat_multilabel") + tok2vec = nlp.get_pipe("tok2vec") + tagger_tok2vec = tagger.model.get_ref("tok2vec") + textcat_tok2vec = textcat.model.get_ref("tok2vec") + assert isinstance(tok2vec, Tok2Vec) + assert isinstance(tagger_tok2vec, Tok2VecListener) + assert isinstance(textcat_tok2vec, Tok2VecListener) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + optimizer = nlp.initialize(lambda: train_examples) + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + + docs = list(nlp.pipe(["Eat blue ham", "I like green eggs"])) + cats0 = docs[0].cats + assert cats0["preference"] < 0.1 + assert cats0["imperative"] > 0.9 + cats1 = docs[1].cats + assert cats1["preference"] > 0.1 + assert cats1["imperative"] < 0.9 + assert [t.tag_ for t in docs[0]] == ["V", "J", "N"] + assert [t.tag_ for t in docs[1]] == ["N", "V", "J", "N"] + + +def test_tok2vec_listener_source_link_name(): + """The component's internal name and the tok2vec listener map correspond + to the most recently modified pipeline. + """ + orig_config = Config().from_str(cfg_string_multi) + nlp1 = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + assert nlp1.get_pipe("tok2vec").listening_components == ["tagger", "ner"] + + nlp2 = English() + nlp2.add_pipe("tok2vec", source=nlp1) + nlp2.add_pipe("tagger", name="tagger2", source=nlp1) + + # there is no way to have the component have the right name for both + # pipelines, right now the most recently modified pipeline is prioritized + assert nlp1.get_pipe("tagger").name == nlp2.get_pipe("tagger2").name == "tagger2" + + # there is no way to have the tok2vec have the right listener map for both + # pipelines, right now the most recently modified pipeline is prioritized + assert nlp2.get_pipe("tok2vec").listening_components == ["tagger2"] + nlp2.add_pipe("ner", name="ner3", source=nlp1) + assert nlp2.get_pipe("tok2vec").listening_components == ["tagger2", "ner3"] + nlp2.remove_pipe("ner3") + assert nlp2.get_pipe("tok2vec").listening_components == ["tagger2"] + nlp2.remove_pipe("tagger2") + assert nlp2.get_pipe("tok2vec").listening_components == [] + + # at this point the tok2vec component corresponds to nlp2 + assert nlp1.get_pipe("tok2vec").listening_components == [] + + # modifying the nlp1 pipeline syncs the tok2vec listener map back to nlp1 + nlp1.add_pipe("sentencizer") + assert nlp1.get_pipe("tok2vec").listening_components == ["tagger", "ner"] + + # modifying nlp2 syncs it back to nlp2 + nlp2.add_pipe("sentencizer") + assert nlp1.get_pipe("tok2vec").listening_components == [] + + +def test_tok2vec_listener_source_replace_listeners(): + orig_config = Config().from_str(cfg_string_multi) + nlp1 = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + assert nlp1.get_pipe("tok2vec").listening_components == ["tagger", "ner"] + nlp1.replace_listeners("tok2vec", "tagger", ["model.tok2vec"]) + assert nlp1.get_pipe("tok2vec").listening_components == ["ner"] + + nlp2 = English() + nlp2.add_pipe("tok2vec", source=nlp1) + assert nlp2.get_pipe("tok2vec").listening_components == [] + nlp2.add_pipe("tagger", source=nlp1) + assert nlp2.get_pipe("tok2vec").listening_components == [] + nlp2.add_pipe("ner", name="ner2", source=nlp1) + assert nlp2.get_pipe("tok2vec").listening_components == ["ner2"] diff --git a/spacy/tests/registry_contents.json b/spacy/tests/registry_contents.json new file mode 100644 index 0000000..1836d03 --- /dev/null +++ b/spacy/tests/registry_contents.json @@ -0,0 +1,284 @@ +{ + "architectures": [ + "spacy-legacy.CharacterEmbed.v1", + "spacy-legacy.EntityLinker.v1", + "spacy-legacy.HashEmbedCNN.v1", + "spacy-legacy.MaxoutWindowEncoder.v1", + "spacy-legacy.MishWindowEncoder.v1", + "spacy-legacy.MultiHashEmbed.v1", + "spacy-legacy.Tagger.v1", + "spacy-legacy.TextCatBOW.v1", + "spacy-legacy.TextCatCNN.v1", + "spacy-legacy.TextCatEnsemble.v1", + "spacy-legacy.Tok2Vec.v1", + "spacy-legacy.TransitionBasedParser.v1", + "spacy.CharacterEmbed.v2", + "spacy.EntityLinker.v2", + "spacy.HashEmbedCNN.v2", + "spacy.MaxoutWindowEncoder.v2", + "spacy.MishWindowEncoder.v2", + "spacy.MultiHashEmbed.v2", + "spacy.PretrainCharacters.v1", + "spacy.PretrainVectors.v1", + "spacy.SpanCategorizer.v1", + "spacy.SpanFinder.v1", + "spacy.Tagger.v2", + "spacy.TextCatBOW.v2", + "spacy.TextCatBOW.v3", + "spacy.TextCatCNN.v2", + "spacy.TextCatEnsemble.v2", + "spacy.TextCatLowData.v1", + "spacy.TextCatParametricAttention.v1", + "spacy.TextCatReduce.v1", + "spacy.Tok2Vec.v2", + "spacy.Tok2VecListener.v1", + "spacy.TorchBiLSTMEncoder.v1", + "spacy.TransitionBasedParser.v2" + ], + "augmenters": [ + "spacy.combined_augmenter.v1", + "spacy.lower_case.v1", + "spacy.orth_variants.v1" + ], + "batchers": [ + "spacy.batch_by_padded.v1", + "spacy.batch_by_sequence.v1", + "spacy.batch_by_words.v1" + ], + "callbacks": [ + "spacy.copy_from_base_model.v1", + "spacy.models_and_pipes_with_nvtx_range.v1", + "spacy.models_with_nvtx_range.v1" + ], + "cli": [], + "datasets": [], + "displacy_colors": [], + "factories": [ + "attribute_ruler", + "beam_ner", + "beam_parser", + "doc_cleaner", + "entity_linker", + "entity_ruler", + "future_entity_ruler", + "lemmatizer", + "merge_entities", + "merge_noun_chunks", + "merge_subtokens", + "morphologizer", + "ner", + "parser", + "sentencizer", + "senter", + "span_finder", + "span_ruler", + "spancat", + "spancat_singlelabel", + "tagger", + "textcat", + "textcat_multilabel", + "tok2vec", + "token_splitter", + "trainable_lemmatizer" + ], + "initializers": [ + "glorot_normal_init.v1", + "glorot_uniform_init.v1", + "he_normal_init.v1", + "he_uniform_init.v1", + "lecun_normal_init.v1", + "lecun_uniform_init.v1", + "normal_init.v1", + "uniform_init.v1", + "zero_init.v1" + ], + "languages": [], + "layers": [ + "CauchySimilarity.v1", + "ClippedLinear.v1", + "Dish.v1", + "Dropout.v1", + "Embed.v1", + "Gelu.v1", + "HardSigmoid.v1", + "HardSwish.v1", + "HardSwishMobilenet.v1", + "HardTanh.v1", + "HashEmbed.v1", + "LSTM.v1", + "LayerNorm.v1", + "Linear.v1", + "Logistic.v1", + "MXNetWrapper.v1", + "Maxout.v1", + "Mish.v1", + "MultiSoftmax.v1", + "ParametricAttention.v1", + "ParametricAttention.v2", + "PyTorchLSTM.v1", + "PyTorchRNNWrapper.v1", + "PyTorchWrapper.v1", + "PyTorchWrapper.v2", + "PyTorchWrapper.v3", + "Relu.v1", + "ReluK.v1", + "Sigmoid.v1", + "Softmax.v1", + "Softmax.v2", + "SparseLinear.v1", + "SparseLinear.v2", + "Swish.v1", + "add.v1", + "bidirectional.v1", + "chain.v1", + "clone.v1", + "concatenate.v1", + "expand_window.v1", + "list2array.v1", + "list2padded.v1", + "list2ragged.v1", + "noop.v1", + "padded2list.v1", + "premap_ids.v1", + "ragged2list.v1", + "reduce_first.v1", + "reduce_last.v1", + "reduce_max.v1", + "reduce_mean.v1", + "reduce_sum.v1", + "remap_ids.v1", + "remap_ids.v2", + "residual.v1", + "resizable.v1", + "siamese.v1", + "sigmoid_activation.v1", + "softmax_activation.v1", + "spacy-legacy.StaticVectors.v1", + "spacy.CharEmbed.v1", + "spacy.FeatureExtractor.v1", + "spacy.LinearLogistic.v1", + "spacy.PrecomputableAffine.v1", + "spacy.StaticVectors.v2", + "spacy.TransitionModel.v1", + "spacy.extract_ngrams.v1", + "spacy.extract_spans.v1", + "spacy.mean_max_reducer.v1", + "strings2arrays.v1", + "tuplify.v1", + "uniqued.v1", + "with_array.v1", + "with_array2d.v1", + "with_cpu.v1", + "with_flatten.v1", + "with_flatten.v2", + "with_getitem.v1", + "with_list.v1", + "with_padded.v1", + "with_ragged.v1", + "with_reshape.v1" + ], + "lemmatizers": [], + "loggers": [ + "spacy-legacy.ConsoleLogger.v1", + "spacy-legacy.ConsoleLogger.v2", + "spacy-legacy.WandbLogger.v1", + "spacy.ChainLogger.v1", + "spacy.ClearMLLogger.v1", + "spacy.ClearMLLogger.v2", + "spacy.ConsoleLogger.v2", + "spacy.ConsoleLogger.v3", + "spacy.CupyLogger.v1", + "spacy.LookupLogger.v1", + "spacy.MLflowLogger.v1", + "spacy.MLflowLogger.v2", + "spacy.PyTorchLogger.v1", + "spacy.WandbLogger.v1", + "spacy.WandbLogger.v2", + "spacy.WandbLogger.v3", + "spacy.WandbLogger.v4", + "spacy.WandbLogger.v5" + ], + "lookups": [], + "losses": [ + "CategoricalCrossentropy.v1", + "CategoricalCrossentropy.v2", + "CategoricalCrossentropy.v3", + "CosineDistance.v1", + "L2Distance.v1", + "SequenceCategoricalCrossentropy.v1", + "SequenceCategoricalCrossentropy.v2", + "SequenceCategoricalCrossentropy.v3" + ], + "misc": [ + "spacy.CandidateBatchGenerator.v1", + "spacy.CandidateGenerator.v1", + "spacy.EmptyKB.v1", + "spacy.EmptyKB.v2", + "spacy.KBFromFile.v1", + "spacy.LookupsDataLoader.v1", + "spacy.first_longest_spans_filter.v1", + "spacy.levenshtein_compare.v1", + "spacy.ngram_range_suggester.v1", + "spacy.ngram_suggester.v1", + "spacy.preset_spans_suggester.v1", + "spacy.prioritize_existing_ents_filter.v1", + "spacy.prioritize_new_ents_filter.v1" + ], + "models": [], + "ops": [ + "CupyOps", + "MPSOps", + "NumpyOps" + ], + "optimizers": [ + "Adam.v1", + "RAdam.v1", + "SGD.v1" + ], + "readers": [ + "ml_datasets.cmu_movies.v1", + "ml_datasets.dbpedia.v1", + "ml_datasets.imdb_sentiment.v1", + "spacy.Corpus.v1", + "spacy.JsonlCorpus.v1", + "spacy.PlainTextCorpus.v1", + "spacy.read_labels.v1", + "srsly.read_json.v1", + "srsly.read_jsonl.v1", + "srsly.read_msgpack.v1", + "srsly.read_yaml.v1" + ], + "schedules": [ + "compounding.v1", + "constant.v1", + "constant_then.v1", + "cyclic_triangular.v1", + "decaying.v1", + "slanted_triangular.v1", + "warmup_linear.v1" + ], + "scorers": [ + "spacy-legacy.textcat_multilabel_scorer.v1", + "spacy-legacy.textcat_scorer.v1", + "spacy.attribute_ruler_scorer.v1", + "spacy.entity_linker_scorer.v1", + "spacy.entity_ruler_scorer.v1", + "spacy.lemmatizer_scorer.v1", + "spacy.morphologizer_scorer.v1", + "spacy.ner_scorer.v1", + "spacy.overlapping_labeled_spans_scorer.v1", + "spacy.parser_scorer.v1", + "spacy.senter_scorer.v1", + "spacy.span_finder_scorer.v1", + "spacy.spancat_scorer.v1", + "spacy.tagger_scorer.v1", + "spacy.textcat_multilabel_scorer.v2", + "spacy.textcat_scorer.v2" + ], + "tokenizers": [ + "spacy.Tokenizer.v1" + ], + "vectors": [ + "spacy.Vectors.v1" + ] +} diff --git a/spacy/tests/serialize/__init__.py b/spacy/tests/serialize/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/serialize/test_resource_warning.py b/spacy/tests/serialize/test_resource_warning.py new file mode 100644 index 0000000..4cf0ac5 --- /dev/null +++ b/spacy/tests/serialize/test_resource_warning.py @@ -0,0 +1,150 @@ +import warnings +from unittest import TestCase + +import pytest +import srsly +from numpy import zeros + +from spacy.kb.kb_in_memory import InMemoryLookupKB, Writer +from spacy.language import Language +from spacy.pipeline import TrainablePipe +from spacy.vectors import Vectors +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +def nlp(): + return Language() + + +def vectors(): + data = zeros((3, 1), dtype="f") + keys = ["cat", "dog", "rat"] + return Vectors(data=data, keys=keys) + + +def custom_pipe(): + # create dummy pipe partially implementing interface -- only want to test to_disk + class SerializableDummy: + def __init__(self, **cfg): + if cfg: + self.cfg = cfg + else: + self.cfg = None + super(SerializableDummy, self).__init__() + + def to_bytes(self, exclude=tuple(), disable=None, **kwargs): + return srsly.msgpack_dumps({"dummy": srsly.json_dumps(None)}) + + def from_bytes(self, bytes_data, exclude): + return self + + def to_disk(self, path, exclude=tuple(), **kwargs): + pass + + def from_disk(self, path, exclude=tuple(), **kwargs): + return self + + class MyPipe(TrainablePipe): + def __init__(self, vocab, model=True, **cfg): + if cfg: + self.cfg = cfg + else: + self.cfg = None + self.model = SerializableDummy() + self.vocab = vocab + + return MyPipe(Vocab()) + + +def tagger(): + nlp = Language() + tagger = nlp.add_pipe("tagger") + # need to add model for two reasons: + # 1. no model leads to error in serialization, + # 2. the affected line is the one for model serialization + tagger.add_label("A") + nlp.initialize() + return tagger + + +def entity_linker(): + nlp = Language() + + def create_kb(vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=1) + kb.add_entity("test", 0.0, zeros((1,), dtype="f")) + return kb + + entity_linker = nlp.add_pipe("entity_linker") + entity_linker.set_kb(create_kb) + # need to add model for two reasons: + # 1. no model leads to error in serialization, + # 2. the affected line is the one for model serialization + nlp.initialize() + return entity_linker + + +objects_to_test = ( + [nlp, vectors, custom_pipe, tagger, entity_linker], + ["nlp", "vectors", "custom_pipe", "tagger", "entity_linker"], +) + + +def write_obj_and_catch_warnings(obj): + with make_tempdir() as d: + with warnings.catch_warnings(record=True) as warnings_list: + warnings.filterwarnings("always", category=ResourceWarning) + obj.to_disk(d) + # in python3.5 it seems that deprecation warnings are not filtered by filterwarnings + return list(filter(lambda x: isinstance(x, ResourceWarning), warnings_list)) + + +@pytest.mark.parametrize("obj_factory", objects_to_test[0], ids=objects_to_test[1]) +def test_to_disk_resource_warning(obj_factory): + obj = obj_factory() + warnings_list = write_obj_and_catch_warnings(obj) + assert len(warnings_list) == 0 + + +def test_writer_with_path_py35(): + writer = None + with make_tempdir() as d: + path = d / "test" + try: + writer = Writer(path) + except Exception as e: + pytest.fail(str(e)) + finally: + if writer: + writer.close() + + +def test_save_and_load_knowledge_base(): + nlp = Language() + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + with make_tempdir() as d: + path = d / "kb" + try: + kb.to_disk(path) + except Exception as e: + pytest.fail(str(e)) + + try: + kb_loaded = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) + kb_loaded.from_disk(path) + except Exception as e: + pytest.fail(str(e)) + + +class TestToDiskResourceWarningUnittest(TestCase): + def test_resource_warning(self): + items = [x() for x in objects_to_test[0]] + names = objects_to_test[1] + scenarios = zip(items, names) + + for item, name in scenarios: + with self.subTest(msg=name): + warnings_list = write_obj_and_catch_warnings(item) + self.assertEqual(len(warnings_list), 0) diff --git a/spacy/tests/serialize/test_serialize_config.py b/spacy/tests/serialize/test_serialize_config.py new file mode 100644 index 0000000..b36d3ad --- /dev/null +++ b/spacy/tests/serialize/test_serialize_config.py @@ -0,0 +1,574 @@ +import pytest +from catalogue import RegistryError +from thinc.api import Config, ConfigValidationError + +import spacy +from spacy.lang.de import German +from spacy.lang.en import English +from spacy.language import DEFAULT_CONFIG, DEFAULT_CONFIG_PRETRAIN_PATH, Language +from spacy.ml.models import ( + MaxoutWindowEncoder, + MultiHashEmbed, + build_tb_parser_model, + build_Tok2Vec_model, +) +from spacy.schemas import ConfigSchema, ConfigSchemaPretrain +from spacy.training import Example +from spacy.util import ( + load_config, + load_config_from_str, + load_model_from_config, + registry, +) + +from ..util import make_tempdir + +nlp_config_string = """ +[paths] +train = null +dev = null + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} + +[training] + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +size = 666 + +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 342 +depth = 4 +window_size = 1 +embed_size = 2000 +maxout_pieces = 3 +subword_features = true + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.width} +""" + +pretrain_config_string = """ +[paths] +train = null +dev = null + +[corpora] + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} + +[training] + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +size = 666 + +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 342 +depth = 4 +window_size = 1 +embed_size = 2000 +maxout_pieces = 3 +subword_features = true + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.width} + +[pretraining] +""" + + +parser_config_string_upper = """ +[model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 66 +maxout_pieces = 2 +use_upper = true + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 333 +depth = 4 +embed_size = 5555 +window_size = 1 +maxout_pieces = 7 +subword_features = false +""" + + +parser_config_string_no_upper = """ +[model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 66 +maxout_pieces = 2 +use_upper = false + +[model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 333 +depth = 4 +embed_size = 5555 +window_size = 1 +maxout_pieces = 7 +subword_features = false +""" + + +@registry.architectures("my_test_parser") +def my_parser(): + tok2vec = build_Tok2Vec_model( + MultiHashEmbed( + width=321, + attrs=["LOWER", "SHAPE"], + rows=[5432, 5432], + include_static_vectors=False, + ), + MaxoutWindowEncoder(width=321, window_size=3, maxout_pieces=4, depth=2), + ) + parser = build_tb_parser_model( + tok2vec=tok2vec, + state_type="parser", + extra_state_tokens=True, + hidden_width=65, + maxout_pieces=5, + use_upper=True, + ) + return parser + + +@pytest.mark.issue(8190) +def test_issue8190(): + """Test that config overrides are not lost after load is complete.""" + source_cfg = { + "nlp": { + "lang": "en", + }, + "custom": {"key": "value"}, + } + source_nlp = English.from_config(source_cfg) + with make_tempdir() as dir_path: + # We need to create a loadable source pipeline + source_path = dir_path / "test_model" + source_nlp.to_disk(source_path) + nlp = spacy.load(source_path, config={"custom": {"key": "updated_value"}}) + + assert nlp.config["custom"]["key"] == "updated_value" + + +def test_create_nlp_from_config(): + config = Config().from_str(nlp_config_string) + with pytest.raises(ConfigValidationError): + load_model_from_config(config, auto_fill=False) + nlp = load_model_from_config(config, auto_fill=True) + assert nlp.config["training"]["batcher"]["size"] == 666 + assert len(nlp.config["training"]) > 1 + assert nlp.pipe_names == ["tok2vec", "tagger"] + assert len(nlp.config["components"]) == 2 + assert len(nlp.config["nlp"]["pipeline"]) == 2 + nlp.remove_pipe("tagger") + assert len(nlp.config["components"]) == 1 + assert len(nlp.config["nlp"]["pipeline"]) == 1 + with pytest.raises(ValueError): + bad_cfg = {"yolo": {}} + load_model_from_config(Config(bad_cfg), auto_fill=True) + with pytest.raises(ValueError): + bad_cfg = {"pipeline": {"foo": "bar"}} + load_model_from_config(Config(bad_cfg), auto_fill=True) + + +def test_create_nlp_from_pretraining_config(): + """Test that the default pretraining config validates properly""" + config = Config().from_str(pretrain_config_string) + pretrain_config = load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = config.merge(pretrain_config) + registry.resolve(filled["pretraining"], schema=ConfigSchemaPretrain) + + +def test_create_nlp_from_config_multiple_instances(): + """Test that the nlp object is created correctly for a config with multiple + instances of the same component.""" + config = Config().from_str(nlp_config_string) + config["components"] = { + "t2v": config["components"]["tok2vec"], + "tagger1": config["components"]["tagger"], + "tagger2": config["components"]["tagger"], + } + config["nlp"]["pipeline"] = list(config["components"].keys()) + nlp = load_model_from_config(config, auto_fill=True) + assert nlp.pipe_names == ["t2v", "tagger1", "tagger2"] + assert nlp.get_pipe_meta("t2v").factory == "tok2vec" + assert nlp.get_pipe_meta("tagger1").factory == "tagger" + assert nlp.get_pipe_meta("tagger2").factory == "tagger" + pipeline_config = nlp.config["components"] + assert len(pipeline_config) == 3 + assert list(pipeline_config.keys()) == ["t2v", "tagger1", "tagger2"] + assert nlp.config["nlp"]["pipeline"] == ["t2v", "tagger1", "tagger2"] + + +def test_serialize_nlp(): + """Create a custom nlp pipeline from config and ensure it serializes it correctly""" + nlp_config = Config().from_str(nlp_config_string) + nlp = load_model_from_config(nlp_config, auto_fill=True) + nlp.get_pipe("tagger").add_label("A") + nlp.initialize() + assert "tok2vec" in nlp.pipe_names + assert "tagger" in nlp.pipe_names + assert "parser" not in nlp.pipe_names + assert nlp.get_pipe("tagger").model.get_ref("tok2vec").get_dim("nO") == 342 + + with make_tempdir() as d: + nlp.to_disk(d) + nlp2 = spacy.load(d) + assert "tok2vec" in nlp2.pipe_names + assert "tagger" in nlp2.pipe_names + assert "parser" not in nlp2.pipe_names + assert nlp2.get_pipe("tagger").model.get_ref("tok2vec").get_dim("nO") == 342 + + +def test_serialize_custom_nlp(): + """Create a custom nlp pipeline and ensure it serializes it correctly""" + nlp = English() + parser_cfg = dict() + parser_cfg["model"] = {"@architectures": "my_test_parser"} + nlp.add_pipe("parser", config=parser_cfg) + nlp.initialize() + + with make_tempdir() as d: + nlp.to_disk(d) + nlp2 = spacy.load(d) + model = nlp2.get_pipe("parser").model + model.get_ref("tok2vec") + # check that we have the correct settings, not the default ones + assert model.get_ref("upper").get_dim("nI") == 65 + assert model.get_ref("lower").get_dim("nI") == 65 + + +@pytest.mark.parametrize( + "parser_config_string", [parser_config_string_upper, parser_config_string_no_upper] +) +def test_serialize_parser(parser_config_string): + """Create a non-default parser config to check nlp serializes it correctly""" + nlp = English() + model_config = Config().from_str(parser_config_string) + parser = nlp.add_pipe("parser", config=model_config) + parser.add_label("nsubj") + nlp.initialize() + + with make_tempdir() as d: + nlp.to_disk(d) + nlp2 = spacy.load(d) + model = nlp2.get_pipe("parser").model + model.get_ref("tok2vec") + # check that we have the correct settings, not the default ones + if model.attrs["has_upper"]: + assert model.get_ref("upper").get_dim("nI") == 66 + assert model.get_ref("lower").get_dim("nI") == 66 + + +def test_config_nlp_roundtrip(): + """Test that a config produced by the nlp object passes training config + validation.""" + nlp = English() + nlp.add_pipe("entity_ruler") + nlp.add_pipe("ner") + new_nlp = load_model_from_config(nlp.config, auto_fill=False) + assert new_nlp.config == nlp.config + assert new_nlp.pipe_names == nlp.pipe_names + assert new_nlp._pipe_configs == nlp._pipe_configs + assert new_nlp._pipe_meta == nlp._pipe_meta + assert new_nlp._factory_meta == nlp._factory_meta + + +def test_config_nlp_roundtrip_bytes_disk(): + """Test that the config is serialized correctly and not interpolated + by mistake.""" + nlp = English() + nlp_bytes = nlp.to_bytes() + new_nlp = English().from_bytes(nlp_bytes) + assert new_nlp.config == nlp.config + nlp = English() + with make_tempdir() as d: + nlp.to_disk(d) + new_nlp = spacy.load(d) + assert new_nlp.config == nlp.config + + +def test_serialize_config_language_specific(): + """Test that config serialization works as expected with language-specific + factories.""" + name = "test_serialize_config_language_specific" + + @English.factory(name, default_config={"foo": 20}) + def custom_factory(nlp: Language, name: str, foo: int): + return lambda doc: doc + + nlp = Language() + assert not nlp.has_factory(name) + nlp = English() + assert nlp.has_factory(name) + nlp.add_pipe(name, config={"foo": 100}, name="bar") + pipe_config = nlp.config["components"]["bar"] + assert pipe_config["foo"] == 100 + assert pipe_config["factory"] == name + + with make_tempdir() as d: + nlp.to_disk(d) + nlp2 = spacy.load(d) + assert nlp2.has_factory(name) + assert nlp2.pipe_names == ["bar"] + assert nlp2.get_pipe_meta("bar").factory == name + pipe_config = nlp2.config["components"]["bar"] + assert pipe_config["foo"] == 100 + assert pipe_config["factory"] == name + + config = Config().from_str(nlp2.config.to_str()) + config["nlp"]["lang"] = "de" + with pytest.raises(ValueError): + # German doesn't have a factory, only English does + load_model_from_config(config) + + +def test_serialize_config_missing_pipes(): + config = Config().from_str(nlp_config_string) + config["components"].pop("tok2vec") + assert "tok2vec" in config["nlp"]["pipeline"] + assert "tok2vec" not in config["components"] + with pytest.raises(ValueError): + load_model_from_config(config, auto_fill=True) + + +def test_config_overrides(): + overrides_nested = {"nlp": {"lang": "de", "pipeline": ["tagger"]}} + overrides_dot = {"nlp.lang": "de", "nlp.pipeline": ["tagger"]} + # load_model from config with overrides passed directly to Config + config = Config().from_str(nlp_config_string, overrides=overrides_dot) + nlp = load_model_from_config(config, auto_fill=True) + assert isinstance(nlp, German) + assert nlp.pipe_names == ["tagger"] + # Serialized roundtrip with config passed in + base_config = Config().from_str(nlp_config_string) + base_nlp = load_model_from_config(base_config, auto_fill=True) + assert isinstance(base_nlp, English) + assert base_nlp.pipe_names == ["tok2vec", "tagger"] + with make_tempdir() as d: + base_nlp.to_disk(d) + nlp = spacy.load(d, config=overrides_nested) + assert isinstance(nlp, German) + assert nlp.pipe_names == ["tagger"] + with make_tempdir() as d: + base_nlp.to_disk(d) + nlp = spacy.load(d, config=overrides_dot) + assert isinstance(nlp, German) + assert nlp.pipe_names == ["tagger"] + with make_tempdir() as d: + base_nlp.to_disk(d) + nlp = spacy.load(d) + assert isinstance(nlp, English) + assert nlp.pipe_names == ["tok2vec", "tagger"] + + +@pytest.mark.filterwarnings("ignore:\\[W036") +def test_config_overrides_registered_functions(): + nlp = spacy.blank("en") + nlp.add_pipe("attribute_ruler") + with make_tempdir() as d: + nlp.to_disk(d) + nlp_re1 = spacy.load( + d, + config={ + "components": { + "attribute_ruler": { + "scorer": {"@scorers": "spacy.tagger_scorer.v1"} + } + } + }, + ) + assert ( + nlp_re1.config["components"]["attribute_ruler"]["scorer"]["@scorers"] + == "spacy.tagger_scorer.v1" + ) + + @registry.misc("test_some_other_key") + def misc_some_other_key(): + return "some_other_key" + + nlp_re2 = spacy.load( + d, + config={ + "components": { + "attribute_ruler": { + "scorer": { + "@scorers": "spacy.overlapping_labeled_spans_scorer.v1", + "spans_key": {"@misc": "test_some_other_key"}, + } + } + } + }, + ) + assert nlp_re2.config["components"]["attribute_ruler"]["scorer"][ + "spans_key" + ] == {"@misc": "test_some_other_key"} + # run dummy evaluation (will return None scores) in order to test that + # the spans_key value in the nested override is working as intended in + # the config + example = Example.from_dict(nlp_re2.make_doc("a b c"), {}) + scores = nlp_re2.evaluate([example]) + assert "spans_some_other_key_f" in scores + + +def test_config_interpolation(): + config = Config().from_str(nlp_config_string, interpolate=False) + assert config["corpora"]["train"]["path"] == "${paths.train}" + interpolated = config.interpolate() + assert interpolated["corpora"]["train"]["path"] is None + nlp = English.from_config(config) + assert nlp.config["corpora"]["train"]["path"] == "${paths.train}" + # Ensure that variables are preserved in nlp config + width = "${components.tok2vec.model.width}" + assert config["components"]["tagger"]["model"]["tok2vec"]["width"] == width + assert nlp.config["components"]["tagger"]["model"]["tok2vec"]["width"] == width + interpolated2 = nlp.config.interpolate() + assert interpolated2["corpora"]["train"]["path"] is None + assert interpolated2["components"]["tagger"]["model"]["tok2vec"]["width"] == 342 + nlp2 = English.from_config(interpolated) + assert nlp2.config["corpora"]["train"]["path"] is None + assert nlp2.config["components"]["tagger"]["model"]["tok2vec"]["width"] == 342 + + +def test_config_optional_sections(): + config = Config().from_str(nlp_config_string) + config = DEFAULT_CONFIG.merge(config) + assert "pretraining" not in config + filled = registry.fill(config, schema=ConfigSchema, validate=False) + # Make sure that optional "pretraining" block doesn't default to None, + # which would (rightly) cause error because it'd result in a top-level + # key that's not a section (dict). Note that the following roundtrip is + # also how Config.interpolate works under the hood. + new_config = Config().from_str(filled.to_str()) + assert new_config["pretraining"] == {} + + +def test_config_auto_fill_extra_fields(): + config = Config({"nlp": {"lang": "en"}, "training": {}}) + assert load_model_from_config(config, auto_fill=True) + config = Config({"nlp": {"lang": "en"}, "training": {"extra": "hello"}}) + nlp = load_model_from_config(config, auto_fill=True, validate=False) + assert "extra" not in nlp.config["training"] + # Make sure the config generated is valid + load_model_from_config(nlp.config) + + +@pytest.mark.parametrize( + "parser_config_string", [parser_config_string_upper, parser_config_string_no_upper] +) +def test_config_validate_literal(parser_config_string): + nlp = English() + config = Config().from_str(parser_config_string) + config["model"]["state_type"] = "nonsense" + with pytest.raises(ConfigValidationError): + nlp.add_pipe("parser", config=config) + config["model"]["state_type"] = "ner" + nlp.add_pipe("parser", config=config) + + +def test_config_only_resolve_relevant_blocks(): + """Test that only the relevant blocks are resolved in the different methods + and that invalid blocks are ignored if needed. For instance, the [initialize] + shouldn't be resolved at runtime. + """ + nlp = English() + config = nlp.config + config["training"]["before_to_disk"] = {"@misc": "nonexistent"} + config["initialize"]["lookups"] = {"@misc": "nonexistent"} + # This shouldn't resolve [training] or [initialize] + nlp = load_model_from_config(config, auto_fill=True) + # This will raise for nonexistent value + with pytest.raises(RegistryError): + nlp.initialize() + nlp.config["initialize"]["lookups"] = None + nlp.initialize() + + +def test_hyphen_in_config(): + hyphen_config_str = """ + [nlp] + lang = "en" + pipeline = ["my_punctual_component"] + + [components] + + [components.my_punctual_component] + factory = "my_punctual_component" + punctuation = ["?","-"] + """ + + @spacy.Language.factory("my_punctual_component") + class MyPunctualComponent(object): + name = "my_punctual_component" + + def __init__( + self, + nlp, + name, + punctuation, + ): + self.punctuation = punctuation + + nlp = English.from_config(load_config_from_str(hyphen_config_str)) + assert nlp.get_pipe("my_punctual_component").punctuation == ["?", "-"] diff --git a/spacy/tests/serialize/test_serialize_doc.py b/spacy/tests/serialize/test_serialize_doc.py new file mode 100644 index 0000000..eea1344 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_doc.py @@ -0,0 +1,225 @@ +import copy +import pickle + +import numpy +import pytest + +from spacy.attrs import DEP, HEAD +from spacy.lang.en import English +from spacy.language import Language +from spacy.matcher import Matcher, PhraseMatcher +from spacy.tokens import Doc +from spacy.vectors import Vectors +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +@pytest.mark.issue(1727) +def test_issue1727(): + """Test that models with no pretrained vectors can be deserialized + correctly after vectors are added.""" + nlp = Language(Vocab()) + data = numpy.ones((3, 300), dtype="f") + vectors = Vectors(data=data, keys=["I", "am", "Matt"]) + tagger = nlp.create_pipe("tagger") + tagger.add_label("PRP") + assert tagger.cfg.get("pretrained_dims", 0) == 0 + tagger.vocab.vectors = vectors + with make_tempdir() as path: + tagger.to_disk(path) + tagger = nlp.create_pipe("tagger").from_disk(path) + assert tagger.cfg.get("pretrained_dims", 0) == 0 + + +@pytest.mark.issue(1799) +def test_issue1799(): + """Test sentence boundaries are deserialized correctly, even for + non-projective sentences.""" + heads_deps = numpy.asarray( + [ + [1, 397], + [4, 436], + [2, 426], + [1, 402], + [0, 8206900633647566924], + [18446744073709551615, 440], + [18446744073709551614, 442], + ], + dtype="uint64", + ) + doc = Doc(Vocab(), words="Just what I was looking for .".split()) + doc.vocab.strings.add("ROOT") + doc = doc.from_array([HEAD, DEP], heads_deps) + assert len(list(doc.sents)) == 1 + + +@pytest.mark.issue(1834) +def test_issue1834(): + """Test that sentence boundaries & parse/tag flags are not lost + during serialization.""" + words = ["This", "is", "a", "first", "sentence", ".", "And", "another", "one"] + doc = Doc(Vocab(), words=words) + doc[6].is_sent_start = True + new_doc = Doc(doc.vocab).from_bytes(doc.to_bytes()) + assert new_doc[6].sent_start + assert not new_doc.has_annotation("DEP") + assert not new_doc.has_annotation("TAG") + doc = Doc( + Vocab(), + words=words, + tags=["TAG"] * len(words), + heads=[0, 0, 0, 0, 0, 0, 6, 6, 6], + deps=["dep"] * len(words), + ) + new_doc = Doc(doc.vocab).from_bytes(doc.to_bytes()) + assert new_doc[6].sent_start + assert new_doc.has_annotation("DEP") + assert new_doc.has_annotation("TAG") + + +@pytest.mark.issue(1883) +def test_issue1883(): + matcher = Matcher(Vocab()) + matcher.add("pat1", [[{"orth": "hello"}]]) + doc = Doc(matcher.vocab, words=["hello"]) + assert len(matcher(doc)) == 1 + new_matcher = copy.deepcopy(matcher) + new_doc = Doc(new_matcher.vocab, words=["hello"]) + assert len(new_matcher(new_doc)) == 1 + + +@pytest.mark.issue(2564) +def test_issue2564(): + """Test the tagger sets has_annotation("TAG") correctly when used via Language.pipe.""" + nlp = Language() + tagger = nlp.add_pipe("tagger") + tagger.add_label("A") + nlp.initialize() + doc = nlp("hello world") + assert doc.has_annotation("TAG") + docs = nlp.pipe(["hello", "world"]) + piped_doc = next(docs) + assert piped_doc.has_annotation("TAG") + + +@pytest.mark.issue(3248) +def test_issue3248_2(): + """Test that the PhraseMatcher can be pickled correctly.""" + nlp = English() + matcher = PhraseMatcher(nlp.vocab) + matcher.add("TEST1", [nlp("a"), nlp("b"), nlp("c")]) + matcher.add("TEST2", [nlp("d")]) + data = pickle.dumps(matcher) + new_matcher = pickle.loads(data) + assert len(new_matcher) == len(matcher) + + +@pytest.mark.issue(3289) +def test_issue3289(): + """Test that Language.to_bytes handles serializing a pipeline component + with an uninitialized model.""" + nlp = English() + nlp.add_pipe("textcat") + bytes_data = nlp.to_bytes() + new_nlp = English() + new_nlp.add_pipe("textcat") + new_nlp.from_bytes(bytes_data) + + +@pytest.mark.issue(3468) +def test_issue3468(): + """Test that sentence boundaries are set correctly so Doc.has_annotation("SENT_START") can + be restored after serialization.""" + nlp = English() + nlp.add_pipe("sentencizer") + doc = nlp("Hello world") + assert doc[0].is_sent_start + assert doc.has_annotation("SENT_START") + assert len(list(doc.sents)) == 1 + doc_bytes = doc.to_bytes() + new_doc = Doc(nlp.vocab).from_bytes(doc_bytes) + assert new_doc[0].is_sent_start + assert new_doc.has_annotation("SENT_START") + assert len(list(new_doc.sents)) == 1 + + +@pytest.mark.issue(3959) +def test_issue3959(): + """Ensure that a modified pos attribute is serialized correctly.""" + nlp = English() + doc = nlp( + "displaCy uses JavaScript, SVG and CSS to show you how computers understand language" + ) + assert doc[0].pos_ == "" + doc[0].pos_ = "NOUN" + assert doc[0].pos_ == "NOUN" + # usually this is already True when starting from proper models instead of blank English + with make_tempdir() as tmp_dir: + file_path = tmp_dir / "my_doc" + doc.to_disk(file_path) + doc2 = nlp("") + doc2.from_disk(file_path) + assert doc2[0].pos_ == "NOUN" + + +def test_serialize_empty_doc(en_vocab): + doc = Doc(en_vocab) + data = doc.to_bytes() + doc2 = Doc(en_vocab) + doc2.from_bytes(data) + assert len(doc) == len(doc2) + for token1, token2 in zip(doc, doc2): + assert token1.text == token2.text + + +def test_serialize_doc_roundtrip_bytes(en_vocab): + doc = Doc(en_vocab, words=["hello", "world"]) + doc.cats = {"A": 0.5} + doc_b = doc.to_bytes() + new_doc = Doc(en_vocab).from_bytes(doc_b) + assert new_doc.to_bytes() == doc_b + + +def test_serialize_doc_roundtrip_disk(en_vocab): + doc = Doc(en_vocab, words=["hello", "world"]) + with make_tempdir() as d: + file_path = d / "doc" + doc.to_disk(file_path) + doc_d = Doc(en_vocab).from_disk(file_path) + assert doc.to_bytes() == doc_d.to_bytes() + + +def test_serialize_doc_roundtrip_disk_str_path(en_vocab): + doc = Doc(en_vocab, words=["hello", "world"]) + with make_tempdir() as d: + file_path = d / "doc" + file_path = str(file_path) + doc.to_disk(file_path) + doc_d = Doc(en_vocab).from_disk(file_path) + assert doc.to_bytes() == doc_d.to_bytes() + + +def test_serialize_doc_exclude(en_vocab): + doc = Doc(en_vocab, words=["hello", "world"]) + doc.user_data["foo"] = "bar" + new_doc = Doc(en_vocab).from_bytes(doc.to_bytes()) + assert new_doc.user_data["foo"] == "bar" + new_doc = Doc(en_vocab).from_bytes(doc.to_bytes(), exclude=["user_data"]) + assert not new_doc.user_data + new_doc = Doc(en_vocab).from_bytes(doc.to_bytes(exclude=["user_data"])) + assert not new_doc.user_data + + +def test_serialize_doc_span_groups(en_vocab): + doc = Doc(en_vocab, words=["hello", "world", "!"]) + span = doc[0:2] + span.label_ = "test_serialize_doc_span_groups_label" + span.id_ = "test_serialize_doc_span_groups_id" + span.kb_id_ = "test_serialize_doc_span_groups_kb_id" + doc.spans["content"] = [span] + new_doc = Doc(en_vocab).from_bytes(doc.to_bytes()) + assert len(new_doc.spans["content"]) == 1 + assert new_doc.spans["content"][0].label_ == "test_serialize_doc_span_groups_label" + assert new_doc.spans["content"][0].id_ == "test_serialize_doc_span_groups_id" + assert new_doc.spans["content"][0].kb_id_ == "test_serialize_doc_span_groups_kb_id" diff --git a/spacy/tests/serialize/test_serialize_docbin.py b/spacy/tests/serialize/test_serialize_docbin.py new file mode 100644 index 0000000..6f7b100 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_docbin.py @@ -0,0 +1,113 @@ +import pytest + +import spacy +from spacy.lang.en import English +from spacy.tokens import Doc, DocBin +from spacy.tokens.underscore import Underscore + + +@pytest.mark.issue(4367) +def test_issue4367(): + """Test that docbin init goes well""" + DocBin() + DocBin(attrs=["LEMMA"]) + DocBin(attrs=["LEMMA", "ENT_IOB", "ENT_TYPE"]) + + +@pytest.mark.issue(4528) +def test_issue4528(en_vocab): + """Test that user_data is correctly serialized in DocBin.""" + doc = Doc(en_vocab, words=["hello", "world"]) + doc.user_data["foo"] = "bar" + # This is how extension attribute values are stored in the user data + doc.user_data[("._.", "foo", None, None)] = "bar" + doc_bin = DocBin(store_user_data=True) + doc_bin.add(doc) + doc_bin_bytes = doc_bin.to_bytes() + new_doc_bin = DocBin(store_user_data=True).from_bytes(doc_bin_bytes) + new_doc = list(new_doc_bin.get_docs(en_vocab))[0] + assert new_doc.user_data["foo"] == "bar" + assert new_doc.user_data[("._.", "foo", None, None)] == "bar" + + +@pytest.mark.issue(5141) +def test_issue5141(en_vocab): + """Ensure an empty DocBin does not crash on serialization""" + doc_bin = DocBin(attrs=["DEP", "HEAD"]) + assert list(doc_bin.get_docs(en_vocab)) == [] + doc_bin_bytes = doc_bin.to_bytes() + doc_bin_2 = DocBin().from_bytes(doc_bin_bytes) + assert list(doc_bin_2.get_docs(en_vocab)) == [] + + +def test_serialize_doc_bin(): + doc_bin = DocBin( + attrs=["LEMMA", "ENT_IOB", "ENT_TYPE", "NORM", "ENT_ID"], store_user_data=True + ) + texts = ["Some text", "Lots of texts...", "..."] + cats = {"A": 0.5} + nlp = English() + for doc in nlp.pipe(texts): + doc.cats = cats + span = doc[0:2] + span.label_ = "UNUSUAL_SPAN_LABEL" + span.id_ = "UNUSUAL_SPAN_ID" + span.kb_id_ = "UNUSUAL_SPAN_KB_ID" + doc.spans["start"] = [span] + doc[0].norm_ = "UNUSUAL_TOKEN_NORM" + doc[0].ent_id_ = "UNUSUAL_TOKEN_ENT_ID" + doc_bin.add(doc) + bytes_data = doc_bin.to_bytes() + + # Deserialize later, e.g. in a new process + nlp = spacy.blank("en") + doc_bin = DocBin().from_bytes(bytes_data) + reloaded_docs = list(doc_bin.get_docs(nlp.vocab)) + for i, doc in enumerate(reloaded_docs): + assert doc.text == texts[i] + assert doc.cats == cats + assert len(doc.spans) == 1 + assert doc.spans["start"][0].label_ == "UNUSUAL_SPAN_LABEL" + assert doc.spans["start"][0].id_ == "UNUSUAL_SPAN_ID" + assert doc.spans["start"][0].kb_id_ == "UNUSUAL_SPAN_KB_ID" + assert doc[0].norm_ == "UNUSUAL_TOKEN_NORM" + assert doc[0].ent_id_ == "UNUSUAL_TOKEN_ENT_ID" + + +def test_serialize_doc_bin_unknown_spaces(en_vocab): + doc1 = Doc(en_vocab, words=["that", "'s"]) + assert doc1.has_unknown_spaces + assert doc1.text == "that 's " + doc2 = Doc(en_vocab, words=["that", "'s"], spaces=[False, False]) + assert not doc2.has_unknown_spaces + assert doc2.text == "that's" + + doc_bin = DocBin().from_bytes(DocBin(docs=[doc1, doc2]).to_bytes()) + re_doc1, re_doc2 = doc_bin.get_docs(en_vocab) + assert re_doc1.has_unknown_spaces + assert re_doc1.text == "that 's " + assert not re_doc2.has_unknown_spaces + assert re_doc2.text == "that's" + + +@pytest.mark.parametrize( + "writer_flag,reader_flag,reader_value", + [ + (True, True, "bar"), + (True, False, "bar"), + (False, True, "nothing"), + (False, False, "nothing"), + ], +) +def test_serialize_custom_extension(en_vocab, writer_flag, reader_flag, reader_value): + """Test that custom extensions are correctly serialized in DocBin.""" + Doc.set_extension("foo", default="nothing") + doc = Doc(en_vocab, words=["hello", "world"]) + doc._.foo = "bar" + doc_bin_1 = DocBin(store_user_data=writer_flag) + doc_bin_1.add(doc) + doc_bin_bytes = doc_bin_1.to_bytes() + doc_bin_2 = DocBin(store_user_data=reader_flag).from_bytes(doc_bin_bytes) + doc_2 = list(doc_bin_2.get_docs(en_vocab))[0] + assert doc_2._.foo == reader_value + Underscore.doc_extensions = {} diff --git a/spacy/tests/serialize/test_serialize_extension_attrs.py b/spacy/tests/serialize/test_serialize_extension_attrs.py new file mode 100644 index 0000000..2fb56c8 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_extension_attrs.py @@ -0,0 +1,35 @@ +import pytest + +from spacy.tokens import Doc, Token +from spacy.vocab import Vocab + + +@pytest.fixture +def doc_w_attrs(en_tokenizer): + Doc.set_extension("_test_attr", default=False) + Doc.set_extension("_test_prop", getter=lambda doc: len(doc.text)) + Doc.set_extension("_test_method", method=lambda doc, arg: f"{len(doc.text)}{arg}") + doc = en_tokenizer("This is a test.") + doc._._test_attr = "test" + + Token.set_extension("_test_token", default="t0") + doc[1]._._test_token = "t1" + + yield doc + + Doc.remove_extension("_test_attr") + Doc.remove_extension("_test_prop") + Doc.remove_extension("_test_method") + Token.remove_extension("_test_token") + + +def test_serialize_ext_attrs_from_bytes(doc_w_attrs): + doc_b = doc_w_attrs.to_bytes() + doc = Doc(Vocab()).from_bytes(doc_b) + assert doc._.has("_test_attr") + assert doc._._test_attr == "test" + assert doc._._test_prop == len(doc.text) + assert doc._._test_method("test") == f"{len(doc.text)}test" + assert doc[0]._._test_token == "t0" + assert doc[1]._._test_token == "t1" + assert doc[2]._._test_token == "t0" diff --git a/spacy/tests/serialize/test_serialize_kb.py b/spacy/tests/serialize/test_serialize_kb.py new file mode 100644 index 0000000..99eb8cd --- /dev/null +++ b/spacy/tests/serialize/test_serialize_kb.py @@ -0,0 +1,197 @@ +from pathlib import Path +from typing import Any, Callable, Dict, Iterable + +import srsly +from numpy import zeros +from thinc.api import Config + +from spacy import Errors, util +from spacy.kb.kb_in_memory import InMemoryLookupKB +from spacy.util import SimpleFrozenList, ensure_path, load_model_from_config, registry +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +def test_serialize_kb_disk(en_vocab): + # baseline assertions + kb1 = _get_dummy_kb(en_vocab) + _check_kb(kb1) + + # dumping to file & loading back in + with make_tempdir() as d: + dir_path = ensure_path(d) + if not dir_path.exists(): + dir_path.mkdir() + file_path = dir_path / "kb" + kb1.to_disk(str(file_path)) + kb2 = InMemoryLookupKB(vocab=en_vocab, entity_vector_length=3) + kb2.from_disk(str(file_path)) + + # final assertions + _check_kb(kb2) + + +def _get_dummy_kb(vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=3) + kb.add_entity(entity="Q53", freq=33, entity_vector=[0, 5, 3]) + kb.add_entity(entity="Q17", freq=2, entity_vector=[7, 1, 0]) + kb.add_entity(entity="Q007", freq=7, entity_vector=[0, 0, 7]) + kb.add_entity(entity="Q44", freq=342, entity_vector=[4, 4, 4]) + + kb.add_alias(alias="double07", entities=["Q17", "Q007"], probabilities=[0.1, 0.9]) + kb.add_alias( + alias="guy", + entities=["Q53", "Q007", "Q17", "Q44"], + probabilities=[0.3, 0.3, 0.2, 0.1], + ) + kb.add_alias(alias="random", entities=["Q007"], probabilities=[1.0]) + + return kb + + +def _check_kb(kb): + # check entities + assert kb.get_size_entities() == 4 + for entity_string in ["Q53", "Q17", "Q007", "Q44"]: + assert entity_string in kb.get_entity_strings() + for entity_string in ["", "Q0"]: + assert entity_string not in kb.get_entity_strings() + + # check aliases + assert kb.get_size_aliases() == 3 + for alias_string in ["double07", "guy", "random"]: + assert alias_string in kb.get_alias_strings() + for alias_string in ["nothingness", "", "randomnoise"]: + assert alias_string not in kb.get_alias_strings() + + # check candidates & probabilities + candidates = sorted(kb.get_alias_candidates("double07"), key=lambda x: x.entity_) + assert len(candidates) == 2 + + assert candidates[0].entity_ == "Q007" + assert 6.999 < candidates[0].entity_freq < 7.01 + assert candidates[0].entity_vector == [0, 0, 7] + assert candidates[0].alias_ == "double07" + assert 0.899 < candidates[0].prior_prob < 0.901 + + assert candidates[1].entity_ == "Q17" + assert 1.99 < candidates[1].entity_freq < 2.01 + assert candidates[1].entity_vector == [7, 1, 0] + assert candidates[1].alias_ == "double07" + assert 0.099 < candidates[1].prior_prob < 0.101 + + +def test_serialize_subclassed_kb(): + """Check that IO of a custom KB works fine as part of an EL pipe.""" + + config_string = """ + [nlp] + lang = "en" + pipeline = ["entity_linker"] + + [components] + + [components.entity_linker] + factory = "entity_linker" + + [components.entity_linker.generate_empty_kb] + @misc = "kb_test.CustomEmptyKB.v1" + + [initialize] + + [initialize.components] + + [initialize.components.entity_linker] + + [initialize.components.entity_linker.kb_loader] + @misc = "kb_test.CustomKB.v1" + entity_vector_length = 342 + custom_field = 666 + """ + + class SubInMemoryLookupKB(InMemoryLookupKB): + def __init__(self, vocab, entity_vector_length, custom_field): + super().__init__(vocab, entity_vector_length) + self.custom_field = custom_field + + def to_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + """We overwrite InMemoryLookupKB.to_disk() to ensure that self.custom_field is stored as well.""" + path = ensure_path(path) + if not path.exists(): + path.mkdir(parents=True) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + + def serialize_custom_fields(file_path: Path) -> None: + srsly.write_json(file_path, {"custom_field": self.custom_field}) + + serialize = { + "contents": lambda p: self.write_contents(p), + "strings.json": lambda p: self.vocab.strings.to_disk(p), + "custom_fields": lambda p: serialize_custom_fields(p), + } + util.to_disk(path, serialize, exclude) + + def from_disk(self, path, exclude: Iterable[str] = SimpleFrozenList()): + """We overwrite InMemoryLookupKB.from_disk() to ensure that self.custom_field is loaded as well.""" + path = ensure_path(path) + if not path.exists(): + raise ValueError(Errors.E929.format(loc=path)) + if not path.is_dir(): + raise ValueError(Errors.E928.format(loc=path)) + + def deserialize_custom_fields(file_path: Path) -> None: + self.custom_field = srsly.read_json(file_path)["custom_field"] + + deserialize: Dict[str, Callable[[Any], Any]] = { + "contents": lambda p: self.read_contents(p), + "strings.json": lambda p: self.vocab.strings.from_disk(p), + "custom_fields": lambda p: deserialize_custom_fields(p), + } + util.from_disk(path, deserialize, exclude) + + @registry.misc("kb_test.CustomEmptyKB.v1") + def empty_custom_kb() -> Callable[[Vocab, int], SubInMemoryLookupKB]: + def empty_kb_factory(vocab: Vocab, entity_vector_length: int): + return SubInMemoryLookupKB( + vocab=vocab, + entity_vector_length=entity_vector_length, + custom_field=0, + ) + + return empty_kb_factory + + @registry.misc("kb_test.CustomKB.v1") + def custom_kb( + entity_vector_length: int, custom_field: int + ) -> Callable[[Vocab], SubInMemoryLookupKB]: + def custom_kb_factory(vocab): + kb = SubInMemoryLookupKB( + vocab=vocab, + entity_vector_length=entity_vector_length, + custom_field=custom_field, + ) + kb.add_entity("random_entity", 0.0, zeros(entity_vector_length)) + return kb + + return custom_kb_factory + + config = Config().from_str(config_string) + nlp = load_model_from_config(config, auto_fill=True) + nlp.initialize() + + entity_linker = nlp.get_pipe("entity_linker") + assert type(entity_linker.kb) == SubInMemoryLookupKB + assert entity_linker.kb.entity_vector_length == 342 + assert entity_linker.kb.custom_field == 666 + + # Make sure the custom KB is serialized correctly + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + entity_linker2 = nlp2.get_pipe("entity_linker") + # After IO, the KB is the standard one + assert type(entity_linker2.kb) == SubInMemoryLookupKB + assert entity_linker2.kb.entity_vector_length == 342 + assert entity_linker2.kb.custom_field == 666 diff --git a/spacy/tests/serialize/test_serialize_language.py b/spacy/tests/serialize/test_serialize_language.py new file mode 100644 index 0000000..9c36015 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_language.py @@ -0,0 +1,135 @@ +import pickle +import re + +import pytest + +from spacy.lang.en import English +from spacy.lang.it import Italian +from spacy.language import Language +from spacy.tokenizer import Tokenizer +from spacy.training import Example +from spacy.util import load_config_from_str + +from ..util import make_tempdir + + +@pytest.fixture +def meta_data(): + return { + "name": "name-in-fixture", + "version": "version-in-fixture", + "description": "description-in-fixture", + "author": "author-in-fixture", + "email": "email-in-fixture", + "url": "url-in-fixture", + "license": "license-in-fixture", + "vectors": {"width": 0, "vectors": 0, "keys": 0, "name": None}, + } + + +@pytest.mark.issue(2482) +def test_issue2482(): + """Test we can serialize and deserialize a blank NER or parser model.""" + nlp = Italian() + nlp.add_pipe("ner") + b = nlp.to_bytes() + Italian().from_bytes(b) + + +CONFIG_ISSUE_6950 = """ +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v1" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = ${components.tok2vec.model.encode:width} +attrs = ["NORM","PREFIX","SUFFIX","SHAPE"] +rows = [5000,2500,2500,2500] +include_static_vectors = false + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v1" +width = 96 +depth = 4 +window_size = 1 +maxout_pieces = 3 + +[components.ner] +factory = "ner" + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode:width} +upstream = "*" +""" + + +@pytest.mark.issue(6950) +def test_issue6950(): + """Test that the nlp object with initialized tok2vec with listeners pickles + correctly (and doesn't have lambdas). + """ + nlp = English.from_config(load_config_from_str(CONFIG_ISSUE_6950)) + nlp.initialize(lambda: [Example.from_dict(nlp.make_doc("hello"), {"tags": ["V"]})]) + pickle.dumps(nlp) + nlp("hello") + pickle.dumps(nlp) + + +def test_serialize_language_meta_disk(meta_data): + language = Language(meta=meta_data) + with make_tempdir() as d: + language.to_disk(d) + new_language = Language().from_disk(d) + assert new_language.meta == language.meta + + +def test_serialize_with_custom_tokenizer(): + """Test that serialization with custom tokenizer works without token_match. + See: https://support.prodi.gy/t/how-to-save-a-custom-tokenizer/661/2 + """ + prefix_re = re.compile(r"""1/|2/|:[0-9][0-9][A-K]:|:[0-9][0-9]:""") + suffix_re = re.compile(r"""""") + infix_re = re.compile(r"""[~]""") + + def custom_tokenizer(nlp): + return Tokenizer( + nlp.vocab, + {}, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + ) + + nlp = Language() + nlp.tokenizer = custom_tokenizer(nlp) + with make_tempdir() as d: + nlp.to_disk(d) + + +def test_serialize_language_exclude(meta_data): + name = "name-in-fixture" + nlp = Language(meta=meta_data) + assert nlp.meta["name"] == name + new_nlp = Language().from_bytes(nlp.to_bytes()) + assert new_nlp.meta["name"] == name + new_nlp = Language().from_bytes(nlp.to_bytes(), exclude=["meta"]) + assert not new_nlp.meta["name"] == name + new_nlp = Language().from_bytes(nlp.to_bytes(exclude=["meta"])) + assert not new_nlp.meta["name"] == name diff --git a/spacy/tests/serialize/test_serialize_pipeline.py b/spacy/tests/serialize/test_serialize_pipeline.py new file mode 100644 index 0000000..6bbe743 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_pipeline.py @@ -0,0 +1,477 @@ +import pickle + +import pytest +import srsly +from thinc.api import Linear + +import spacy +from spacy import Vocab, load, registry +from spacy.lang.en import English +from spacy.language import Language +from spacy.pipeline import ( + DependencyParser, + EntityRecognizer, + EntityRuler, + SentenceRecognizer, + Tagger, + TextCategorizer, + TrainablePipe, +) +from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +from spacy.pipeline.senter import DEFAULT_SENTER_MODEL +from spacy.pipeline.tagger import DEFAULT_TAGGER_MODEL +from spacy.pipeline.textcat import DEFAULT_SINGLE_TEXTCAT_MODEL +from spacy.tokens import Span +from spacy.util import ensure_path, load_model + +from ..util import make_tempdir + +test_parsers = [DependencyParser, EntityRecognizer] + + +@pytest.fixture +def parser(en_vocab): + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + "beam_width": 1, + "beam_update_prob": 1.0, + "beam_density": 0.0, + } + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(en_vocab, model, **config) + parser.add_label("nsubj") + return parser + + +@pytest.fixture +def blank_parser(en_vocab): + config = { + "learn_tokens": False, + "min_action_freq": 30, + "update_with_oracle_cut_size": 100, + "beam_width": 1, + "beam_update_prob": 1.0, + "beam_density": 0.0, + } + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = DependencyParser(en_vocab, model, **config) + return parser + + +@pytest.fixture +def taggers(en_vocab): + cfg = {"model": DEFAULT_TAGGER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + tagger1 = Tagger(en_vocab, model) + tagger2 = Tagger(en_vocab, model) + return tagger1, tagger2 + + +@pytest.mark.issue(3456) +def test_issue3456(): + # this crashed because of a padding error in layer.ops.unflatten in thinc + nlp = English() + tagger = nlp.add_pipe("tagger") + tagger.add_label("A") + nlp.initialize() + list(nlp.pipe(["hi", ""])) + + +@pytest.mark.issue(3526) +def test_issue_3526_1(en_vocab): + patterns = [ + {"label": "HELLO", "pattern": "hello world"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple", "id": "a1"}, + ] + nlp = Language(vocab=en_vocab) + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) + ruler_bytes = ruler.to_bytes() + assert len(ruler) == len(patterns) + assert len(ruler.labels) == 4 + assert ruler.overwrite + new_ruler = EntityRuler(nlp) + new_ruler = new_ruler.from_bytes(ruler_bytes) + assert len(new_ruler) == len(ruler) + assert len(new_ruler.labels) == 4 + assert new_ruler.overwrite == ruler.overwrite + assert new_ruler.ent_id_sep == ruler.ent_id_sep + + +@pytest.mark.issue(3526) +def test_issue_3526_2(en_vocab): + patterns = [ + {"label": "HELLO", "pattern": "hello world"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple", "id": "a1"}, + ] + nlp = Language(vocab=en_vocab) + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) + bytes_old_style = srsly.msgpack_dumps(ruler.patterns) + new_ruler = EntityRuler(nlp) + new_ruler = new_ruler.from_bytes(bytes_old_style) + assert len(new_ruler) == len(ruler) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert new_ruler.overwrite is not ruler.overwrite + + +@pytest.mark.issue(3526) +def test_issue_3526_3(en_vocab): + patterns = [ + {"label": "HELLO", "pattern": "hello world"}, + {"label": "BYE", "pattern": [{"LOWER": "bye"}, {"LOWER": "bye"}]}, + {"label": "HELLO", "pattern": [{"ORTH": "HELLO"}]}, + {"label": "COMPLEX", "pattern": [{"ORTH": "foo", "OP": "*"}]}, + {"label": "TECH_ORG", "pattern": "Apple", "id": "a1"}, + ] + nlp = Language(vocab=en_vocab) + ruler = EntityRuler(nlp, patterns=patterns, overwrite_ents=True) + with make_tempdir() as tmpdir: + out_file = tmpdir / "entity_ruler" + srsly.write_jsonl(out_file.with_suffix(".jsonl"), ruler.patterns) + new_ruler = EntityRuler(nlp).from_disk(out_file) + for pattern in ruler.patterns: + assert pattern in new_ruler.patterns + assert len(new_ruler) == len(ruler) + assert new_ruler.overwrite is not ruler.overwrite + + +@pytest.mark.issue(3526) +def test_issue_3526_4(en_vocab): + nlp = Language(vocab=en_vocab) + patterns = [{"label": "ORG", "pattern": "Apple"}] + config = {"overwrite_ents": True} + ruler = nlp.add_pipe("entity_ruler", config=config) + ruler.add_patterns(patterns) + with make_tempdir() as tmpdir: + nlp.to_disk(tmpdir) + ruler = nlp.get_pipe("entity_ruler") + assert ruler.patterns == [{"label": "ORG", "pattern": "Apple"}] + assert ruler.overwrite is True + nlp2 = load(tmpdir) + new_ruler = nlp2.get_pipe("entity_ruler") + assert new_ruler.patterns == [{"label": "ORG", "pattern": "Apple"}] + assert new_ruler.overwrite is True + + +@pytest.mark.issue(4042) +def test_issue4042(): + """Test that serialization of an EntityRuler before NER works fine.""" + nlp = English() + # add ner pipe + ner = nlp.add_pipe("ner") + ner.add_label("SOME_LABEL") + nlp.initialize() + # Add entity ruler + patterns = [ + {"label": "MY_ORG", "pattern": "Apple"}, + {"label": "MY_GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]}, + ] + # works fine with "after" + ruler = nlp.add_pipe("entity_ruler", before="ner") + ruler.add_patterns(patterns) + doc1 = nlp("What do you think about Apple ?") + assert doc1.ents[0].label_ == "MY_ORG" + + with make_tempdir() as d: + output_dir = ensure_path(d) + if not output_dir.exists(): + output_dir.mkdir() + nlp.to_disk(output_dir) + nlp2 = load_model(output_dir) + doc2 = nlp2("What do you think about Apple ?") + assert doc2.ents[0].label_ == "MY_ORG" + + +@pytest.mark.issue(4042) +def test_issue4042_bug2(): + """ + Test that serialization of an NER works fine when new labels were added. + This is the second bug of two bugs underlying the issue 4042. + """ + nlp1 = English() + # add ner pipe + ner1 = nlp1.add_pipe("ner") + ner1.add_label("SOME_LABEL") + nlp1.initialize() + # add a new label to the doc + doc1 = nlp1("What do you think about Apple ?") + assert len(ner1.labels) == 1 + assert "SOME_LABEL" in ner1.labels + apple_ent = Span(doc1, 5, 6, label="MY_ORG") + doc1.ents = list(doc1.ents) + [apple_ent] + # Add the label explicitly. Previously we didn't require this. + ner1.add_label("MY_ORG") + ner1(doc1) + assert len(ner1.labels) == 2 + assert "SOME_LABEL" in ner1.labels + assert "MY_ORG" in ner1.labels + with make_tempdir() as d: + # assert IO goes fine + output_dir = ensure_path(d) + if not output_dir.exists(): + output_dir.mkdir() + ner1.to_disk(output_dir) + config = {} + ner2 = nlp1.create_pipe("ner", config=config) + ner2.from_disk(output_dir) + assert len(ner2.labels) == 2 + + +@pytest.mark.issue(4725) +def test_issue4725_1(): + """Ensure the pickling of the NER goes well""" + vocab = Vocab(vectors_name="test_vocab_add_vector") + nlp = English(vocab=vocab) + config = { + "update_with_oracle_cut_size": 111, + } + ner = nlp.create_pipe("ner", config=config) + with make_tempdir() as tmp_path: + with (tmp_path / "ner.pkl").open("wb") as file_: + pickle.dump(ner, file_) + assert ner.cfg["update_with_oracle_cut_size"] == 111 + + with (tmp_path / "ner.pkl").open("rb") as file_: + ner2 = pickle.load(file_) + assert ner2.cfg["update_with_oracle_cut_size"] == 111 + + +@pytest.mark.parametrize("Parser", test_parsers) +def test_serialize_parser_roundtrip_bytes(en_vocab, Parser): + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = Parser(en_vocab, model) + new_parser = Parser(en_vocab, model) + new_parser = new_parser.from_bytes(parser.to_bytes(exclude=["vocab"])) + bytes_2 = new_parser.to_bytes(exclude=["vocab"]) + bytes_3 = parser.to_bytes(exclude=["vocab"]) + assert len(bytes_2) == len(bytes_3) + assert bytes_2 == bytes_3 + + +@pytest.mark.parametrize("Parser", test_parsers) +def test_serialize_parser_strings(Parser): + vocab1 = Vocab() + label = "FunnyLabel" + assert label not in vocab1.strings + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser1 = Parser(vocab1, model) + parser1.add_label(label) + assert label in parser1.vocab.strings + vocab2 = Vocab() + assert label not in vocab2.strings + parser2 = Parser(vocab2, model) + parser2 = parser2.from_bytes(parser1.to_bytes(exclude=["vocab"])) + assert label in parser2.vocab.strings + + +@pytest.mark.parametrize("Parser", test_parsers) +def test_serialize_parser_roundtrip_disk(en_vocab, Parser): + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + parser = Parser(en_vocab, model) + with make_tempdir() as d: + file_path = d / "parser" + parser.to_disk(file_path) + parser_d = Parser(en_vocab, model) + parser_d = parser_d.from_disk(file_path) + parser_bytes = parser.to_bytes(exclude=["model", "vocab"]) + parser_d_bytes = parser_d.to_bytes(exclude=["model", "vocab"]) + assert len(parser_bytes) == len(parser_d_bytes) + assert parser_bytes == parser_d_bytes + + +def test_to_from_bytes(parser, blank_parser): + assert parser.model is not True + assert blank_parser.model is not True + assert blank_parser.moves.n_moves != parser.moves.n_moves + bytes_data = parser.to_bytes(exclude=["vocab"]) + # the blank parser needs to be resized before we can call from_bytes + blank_parser.model.attrs["resize_output"](blank_parser.model, parser.moves.n_moves) + blank_parser.from_bytes(bytes_data) + assert blank_parser.model is not True + assert blank_parser.moves.n_moves == parser.moves.n_moves + + +def test_serialize_tagger_roundtrip_bytes(en_vocab, taggers): + tagger1 = taggers[0] + tagger1_b = tagger1.to_bytes() + tagger1 = tagger1.from_bytes(tagger1_b) + assert tagger1.to_bytes() == tagger1_b + cfg = {"model": DEFAULT_TAGGER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + new_tagger1 = Tagger(en_vocab, model).from_bytes(tagger1_b) + new_tagger1_b = new_tagger1.to_bytes() + assert len(new_tagger1_b) == len(tagger1_b) + assert new_tagger1_b == tagger1_b + + +def test_serialize_tagger_roundtrip_disk(en_vocab, taggers): + tagger1, tagger2 = taggers + with make_tempdir() as d: + file_path1 = d / "tagger1" + file_path2 = d / "tagger2" + tagger1.to_disk(file_path1) + tagger2.to_disk(file_path2) + cfg = {"model": DEFAULT_TAGGER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + tagger1_d = Tagger(en_vocab, model).from_disk(file_path1) + tagger2_d = Tagger(en_vocab, model).from_disk(file_path2) + assert tagger1_d.to_bytes() == tagger2_d.to_bytes() + + +def test_serialize_tagger_strings(en_vocab, de_vocab, taggers): + label = "SomeWeirdLabel" + assert label not in en_vocab.strings + assert label not in de_vocab.strings + tagger = taggers[0] + assert label not in tagger.vocab.strings + with make_tempdir() as d: + # check that custom labels are serialized as part of the component's strings.jsonl + tagger.add_label(label) + assert label in tagger.vocab.strings + file_path = d / "tagger1" + tagger.to_disk(file_path) + # ensure that the custom strings are loaded back in when using the tagger in another pipeline + cfg = {"model": DEFAULT_TAGGER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + tagger2 = Tagger(de_vocab, model).from_disk(file_path) + assert label in tagger2.vocab.strings + + +@pytest.mark.issue(1105) +def test_serialize_textcat_empty(en_vocab): + # See issue #1105 + cfg = {"model": DEFAULT_SINGLE_TEXTCAT_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + textcat = TextCategorizer(en_vocab, model, threshold=0.5) + textcat.to_bytes(exclude=["vocab"]) + + +@pytest.mark.parametrize("Parser", test_parsers) +def test_serialize_pipe_exclude(en_vocab, Parser): + cfg = {"model": DEFAULT_PARSER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + + def get_new_parser(): + new_parser = Parser(en_vocab, model) + return new_parser + + parser = Parser(en_vocab, model) + parser.cfg["foo"] = "bar" + new_parser = get_new_parser().from_bytes(parser.to_bytes(exclude=["vocab"])) + assert "foo" in new_parser.cfg + new_parser = get_new_parser().from_bytes( + parser.to_bytes(exclude=["vocab"]), exclude=["cfg"] + ) + assert "foo" not in new_parser.cfg + new_parser = get_new_parser().from_bytes( + parser.to_bytes(exclude=["cfg"]), exclude=["vocab"] + ) + assert "foo" not in new_parser.cfg + + +def test_serialize_sentencerecognizer(en_vocab): + cfg = {"model": DEFAULT_SENTER_MODEL} + model = registry.resolve(cfg, validate=True)["model"] + sr = SentenceRecognizer(en_vocab, model) + sr_b = sr.to_bytes() + sr_d = SentenceRecognizer(en_vocab, model).from_bytes(sr_b) + assert sr.to_bytes() == sr_d.to_bytes() + + +def test_serialize_pipeline_disable_enable(): + nlp = English() + nlp.add_pipe("ner") + nlp.add_pipe("tagger") + nlp.disable_pipe("tagger") + assert nlp.config["nlp"]["disabled"] == ["tagger"] + config = nlp.config.copy() + nlp2 = English.from_config(config) + assert nlp2.pipe_names == ["ner"] + assert nlp2.component_names == ["ner", "tagger"] + assert nlp2.disabled == ["tagger"] + assert nlp2.config["nlp"]["disabled"] == ["tagger"] + with make_tempdir() as d: + nlp2.to_disk(d) + nlp3 = spacy.load(d) + assert nlp3.pipe_names == ["ner"] + assert nlp3.component_names == ["ner", "tagger"] + with make_tempdir() as d: + nlp3.to_disk(d) + nlp4 = spacy.load(d, disable=["ner"]) + assert nlp4.pipe_names == [] + assert nlp4.component_names == ["ner", "tagger"] + assert nlp4.disabled == ["ner", "tagger"] + with make_tempdir() as d: + nlp.to_disk(d) + nlp5 = spacy.load(d, exclude=["tagger"]) + assert nlp5.pipe_names == ["ner"] + assert nlp5.component_names == ["ner"] + assert nlp5.disabled == [] + + +def test_serialize_custom_trainable_pipe(): + class BadCustomPipe1(TrainablePipe): + def __init__(self, vocab): + pass + + class BadCustomPipe2(TrainablePipe): + def __init__(self, vocab): + self.vocab = vocab + self.model = None + + class CustomPipe(TrainablePipe): + def __init__(self, vocab, model): + self.vocab = vocab + self.model = model + + pipe = BadCustomPipe1(Vocab()) + with pytest.raises(ValueError): + pipe.to_bytes() + with make_tempdir() as d: + with pytest.raises(ValueError): + pipe.to_disk(d) + pipe = BadCustomPipe2(Vocab()) + with pytest.raises(ValueError): + pipe.to_bytes() + with make_tempdir() as d: + with pytest.raises(ValueError): + pipe.to_disk(d) + pipe = CustomPipe(Vocab(), Linear()) + pipe_bytes = pipe.to_bytes() + new_pipe = CustomPipe(Vocab(), Linear()).from_bytes(pipe_bytes) + assert new_pipe.to_bytes() == pipe_bytes + with make_tempdir() as d: + pipe.to_disk(d) + new_pipe = CustomPipe(Vocab(), Linear()).from_disk(d) + assert new_pipe.to_bytes() == pipe_bytes + + +def test_load_without_strings(): + nlp = spacy.blank("en") + orig_strings_length = len(nlp.vocab.strings) + word = "unlikely_word_" * 20 + nlp.vocab.strings.add(word) + assert len(nlp.vocab.strings) == orig_strings_length + 1 + with make_tempdir() as d: + nlp.to_disk(d) + # reload with strings + reloaded_nlp = load(d) + assert len(nlp.vocab.strings) == len(reloaded_nlp.vocab.strings) + assert word in reloaded_nlp.vocab.strings + # reload without strings + reloaded_nlp = load(d, exclude=["strings"]) + assert orig_strings_length == len(reloaded_nlp.vocab.strings) + assert word not in reloaded_nlp.vocab.strings diff --git a/spacy/tests/serialize/test_serialize_span_groups.py b/spacy/tests/serialize/test_serialize_span_groups.py new file mode 100644 index 0000000..85313fc --- /dev/null +++ b/spacy/tests/serialize/test_serialize_span_groups.py @@ -0,0 +1,161 @@ +import pytest + +from spacy.tokens import Span, SpanGroup +from spacy.tokens._dict_proxies import SpanGroups + + +@pytest.mark.issue(10685) +def test_issue10685(en_tokenizer): + """Test `SpanGroups` de/serialization""" + # Start with a Doc with no SpanGroups + doc = en_tokenizer("Will it blend?") + + # Test empty `SpanGroups` de/serialization: + assert len(doc.spans) == 0 + doc.spans.from_bytes(doc.spans.to_bytes()) + assert len(doc.spans) == 0 + + # Test non-empty `SpanGroups` de/serialization: + doc.spans["test"] = SpanGroup(doc, name="test", spans=[doc[0:1]]) + doc.spans["test2"] = SpanGroup(doc, name="test", spans=[doc[1:2]]) + + def assert_spangroups(): + assert len(doc.spans) == 2 + assert doc.spans["test"].name == "test" + assert doc.spans["test2"].name == "test" + assert list(doc.spans["test"]) == [doc[0:1]] + assert list(doc.spans["test2"]) == [doc[1:2]] + + # Sanity check the currently-expected behavior + assert_spangroups() + + # Now test serialization/deserialization: + doc.spans.from_bytes(doc.spans.to_bytes()) + + assert_spangroups() + + +def test_span_groups_serialization_mismatches(en_tokenizer): + """Test the serialization of multiple mismatching `SpanGroups` keys and `SpanGroup.name`s""" + doc = en_tokenizer("How now, brown cow?") + # Some variety: + # 1 SpanGroup where its name matches its key + # 2 SpanGroups that have the same name--which is not a key + # 2 SpanGroups that have the same name--which is a key + # 1 SpanGroup that is a value for 2 different keys (where its name is a key) + # 1 SpanGroup that is a value for 2 different keys (where its name is not a key) + groups = doc.spans + groups["key1"] = SpanGroup(doc, name="key1", spans=[doc[0:1], doc[1:2]]) + groups["key2"] = SpanGroup(doc, name="too", spans=[doc[3:4], doc[4:5]]) + groups["key3"] = SpanGroup(doc, name="too", spans=[doc[1:2], doc[0:1]]) + groups["key4"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + groups["key5"] = SpanGroup(doc, name="key4", spans=[doc[0:1]]) + sg6 = SpanGroup(doc, name="key6", spans=[doc[0:1]]) + groups["key6"] = sg6 + groups["key7"] = sg6 + sg8 = SpanGroup(doc, name="also", spans=[doc[1:2]]) + groups["key8"] = sg8 + groups["key9"] = sg8 + + regroups = SpanGroups(doc).from_bytes(groups.to_bytes()) + + # Assert regroups == groups + assert regroups.keys() == groups.keys() + for key, regroup in regroups.items(): + # Assert regroup == groups[key] + assert regroup.name == groups[key].name + assert list(regroup) == list(groups[key]) + + +@pytest.mark.parametrize( + "spans_bytes,doc_text,expected_spangroups,expected_warning", + # The bytestrings below were generated from an earlier version of spaCy + # that serialized `SpanGroups` as a list of SpanGroup bytes (via SpanGroups.to_bytes). + # Comments preceding the bytestrings indicate from what Doc they were created. + [ + # Empty SpanGroups: + (b"\x90", "", {}, False), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + ( + b"\x91\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04", + "Will it blend?", + {"test": {"name": "test", "spans": [(0, 1)]}}, + False, + ), + # doc = nlp("Will it blend?") + # doc.spans['test'] = SpanGroup(doc, name='test', spans=[doc[0:1]]) + # doc.spans['test2'] = SpanGroup(doc, name='test', spans=[doc[1:2]]) + ( + b"\x92\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x04\xc4C\x83\xa4name\xa4test\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x05\x00\x00\x00\x07", + "Will it blend?", + # We expect only 1 SpanGroup to be in doc.spans in this example + # because there are 2 `SpanGroup`s that have the same .name. See #10685. + {"test": {"name": "test", "spans": [(1, 2)]}}, + True, + ), + # doc = nlp('How now, brown cow?') + # doc.spans['key1'] = SpanGroup(doc, name='key1', spans=[doc[0:1], doc[1:2]]) + # doc.spans['key2'] = SpanGroup(doc, name='too', spans=[doc[3:4], doc[4:5]]) + # doc.spans['key3'] = SpanGroup(doc, name='too', spans=[doc[1:2], doc[0:1]]) + # doc.spans['key4'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + # doc.spans['key5'] = SpanGroup(doc, name='key4', spans=[doc[0:1]]) + ( + b"\x95\xc4m\x83\xa4name\xa4key1\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\t\x00\x00\x00\x0e\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\x0f\x00\x00\x00\x12\xc4l\x83\xa4name\xa3too\xa5attrs\x80\xa5spans\x92\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00\x07\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03\xc4C\x83\xa4name\xa4key4\xa5attrs\x80\xa5spans\x91\xc4(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x03", + "How now, brown cow?", + { + "key1": {"name": "key1", "spans": [(0, 1), (1, 2)]}, + "too": {"name": "too", "spans": [(1, 2), (0, 1)]}, + "key4": {"name": "key4", "spans": [(0, 1)]}, + }, + True, + ), + ], +) +def test_deserialize_span_groups_compat( + en_tokenizer, spans_bytes, doc_text, expected_spangroups, expected_warning +): + """Test backwards-compatibility of `SpanGroups` deserialization. + This uses serializations (bytes) from a prior version of spaCy (before 3.3.1). + + spans_bytes (bytes): Serialized `SpanGroups` object. + doc_text (str): Doc text. + expected_spangroups (dict): + Dict mapping every expected (after deserialization) `SpanGroups` key + to a SpanGroup's "args", where a SpanGroup's args are given as a dict: + {"name": span_group.name, + "spans": [(span0.start, span0.end), ...]} + expected_warning (bool): Whether a warning is to be expected from .from_bytes() + --i.e. if more than 1 SpanGroup has the same .name within the `SpanGroups`. + """ + doc = en_tokenizer(doc_text) + + if expected_warning: + with pytest.warns(UserWarning): + doc.spans.from_bytes(spans_bytes) + else: + # TODO: explicitly check for lack of a warning + doc.spans.from_bytes(spans_bytes) + + assert doc.spans.keys() == expected_spangroups.keys() + for name, spangroup_args in expected_spangroups.items(): + assert doc.spans[name].name == spangroup_args["name"] + spans = [Span(doc, start, end) for start, end in spangroup_args["spans"]] + assert list(doc.spans[name]) == spans + + +def test_span_groups_serialization(en_tokenizer): + doc = en_tokenizer("0 1 2 3 4 5 6") + span_groups = SpanGroups(doc) + spans = [doc[0:2], doc[1:3]] + sg1 = SpanGroup(doc, spans=spans) + span_groups["key1"] = sg1 + span_groups["key2"] = sg1 + span_groups["key3"] = [] + reloaded_span_groups = SpanGroups(doc).from_bytes(span_groups.to_bytes()) + assert span_groups.keys() == reloaded_span_groups.keys() + for key, value in span_groups.items(): + assert all( + span == reloaded_span + for span, reloaded_span in zip(span_groups[key], reloaded_span_groups[key]) + ) diff --git a/spacy/tests/serialize/test_serialize_tokenizer.py b/spacy/tests/serialize/test_serialize_tokenizer.py new file mode 100644 index 0000000..e998a78 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_tokenizer.py @@ -0,0 +1,149 @@ +import pickle +import re + +import pytest + +from spacy.attrs import ENT_IOB, ENT_TYPE +from spacy.lang.en import English +from spacy.tokenizer import Tokenizer +from spacy.tokens import Doc +from spacy.util import ( + compile_infix_regex, + compile_prefix_regex, + compile_suffix_regex, + get_lang_class, + load_model, +) + +from ..util import assert_packed_msg_equal, make_tempdir + + +def load_tokenizer(b): + tok = get_lang_class("en")().tokenizer + tok.from_bytes(b) + return tok + + +@pytest.mark.issue(2833) +def test_issue2833(en_vocab): + """Test that a custom error is raised if a token or span is pickled.""" + doc = Doc(en_vocab, words=["Hello", "world"]) + with pytest.raises(NotImplementedError): + pickle.dumps(doc[0]) + with pytest.raises(NotImplementedError): + pickle.dumps(doc[0:2]) + + +@pytest.mark.issue(3012) +def test_issue3012(en_vocab): + """Test that the is_tagged attribute doesn't get overwritten when we from_array + without tag information.""" + words = ["This", "is", "10", "%", "."] + tags = ["DT", "VBZ", "CD", "NN", "."] + pos = ["DET", "VERB", "NUM", "NOUN", "PUNCT"] + ents = ["O", "O", "B-PERCENT", "I-PERCENT", "O"] + doc = Doc(en_vocab, words=words, tags=tags, pos=pos, ents=ents) + assert doc.has_annotation("TAG") + expected = ("10", "NUM", "CD", "PERCENT") + assert (doc[2].text, doc[2].pos_, doc[2].tag_, doc[2].ent_type_) == expected + header = [ENT_IOB, ENT_TYPE] + ent_array = doc.to_array(header) + doc.from_array(header, ent_array) + assert (doc[2].text, doc[2].pos_, doc[2].tag_, doc[2].ent_type_) == expected + # Serializing then deserializing + doc_bytes = doc.to_bytes() + doc2 = Doc(en_vocab).from_bytes(doc_bytes) + assert (doc2[2].text, doc2[2].pos_, doc2[2].tag_, doc2[2].ent_type_) == expected + + +@pytest.mark.issue(4190) +def test_issue4190(): + def customize_tokenizer(nlp): + prefix_re = compile_prefix_regex(nlp.Defaults.prefixes) + suffix_re = compile_suffix_regex(nlp.Defaults.suffixes) + infix_re = compile_infix_regex(nlp.Defaults.infixes) + # Remove all exceptions where a single letter is followed by a period (e.g. 'h.') + exceptions = { + k: v + for k, v in dict(nlp.Defaults.tokenizer_exceptions).items() + if not (len(k) == 2 and k[1] == ".") + } + new_tokenizer = Tokenizer( + nlp.vocab, + exceptions, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + token_match=nlp.tokenizer.token_match, + faster_heuristics=False, + ) + nlp.tokenizer = new_tokenizer + + test_string = "Test c." + # Load default language + nlp_1 = English() + doc_1a = nlp_1(test_string) + result_1a = [token.text for token in doc_1a] # noqa: F841 + # Modify tokenizer + customize_tokenizer(nlp_1) + doc_1b = nlp_1(test_string) + result_1b = [token.text for token in doc_1b] + # Save and Reload + with make_tempdir() as model_dir: + nlp_1.to_disk(model_dir) + nlp_2 = load_model(model_dir) + # This should be the modified tokenizer + doc_2 = nlp_2(test_string) + result_2 = [token.text for token in doc_2] + assert result_1b == result_2 + assert nlp_2.tokenizer.faster_heuristics is False + + +def test_serialize_custom_tokenizer(en_vocab, en_tokenizer): + """Test that custom tokenizer with not all functions defined or empty + properties can be serialized and deserialized correctly (see #2494, + #4991).""" + tokenizer = Tokenizer(en_vocab, suffix_search=en_tokenizer.suffix_search) + tokenizer_bytes = tokenizer.to_bytes() + Tokenizer(en_vocab).from_bytes(tokenizer_bytes) + + # test that empty/unset values are set correctly on deserialization + tokenizer = get_lang_class("en")().tokenizer + tokenizer.token_match = re.compile("test").match + assert tokenizer.rules != {} + assert tokenizer.token_match is not None + assert tokenizer.url_match is not None + assert tokenizer.prefix_search is not None + assert tokenizer.infix_finditer is not None + tokenizer.from_bytes(tokenizer_bytes) + assert tokenizer.rules == {} + assert tokenizer.token_match is None + assert tokenizer.url_match is None + assert tokenizer.prefix_search is None + assert tokenizer.infix_finditer is None + + tokenizer = Tokenizer(en_vocab, rules={"ABC.": [{"ORTH": "ABC"}, {"ORTH": "."}]}) + tokenizer.rules = {} + tokenizer_bytes = tokenizer.to_bytes() + tokenizer_reloaded = Tokenizer(en_vocab).from_bytes(tokenizer_bytes) + assert tokenizer_reloaded.rules == {} + + +@pytest.mark.parametrize("text", ["I💜you", "they’re", "“hello”"]) +def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text): + tokenizer = en_tokenizer + new_tokenizer = load_tokenizer(tokenizer.to_bytes()) + assert_packed_msg_equal(new_tokenizer.to_bytes(), tokenizer.to_bytes()) + assert new_tokenizer.to_bytes() == tokenizer.to_bytes() + doc1 = tokenizer(text) + doc2 = new_tokenizer(text) + assert [token.text for token in doc1] == [token.text for token in doc2] + + +def test_serialize_tokenizer_roundtrip_disk(en_tokenizer): + tokenizer = en_tokenizer + with make_tempdir() as d: + file_path = d / "tokenizer" + tokenizer.to_disk(file_path) + tokenizer_d = en_tokenizer.from_disk(file_path) + assert tokenizer.to_bytes() == tokenizer_d.to_bytes() diff --git a/spacy/tests/serialize/test_serialize_vocab_strings.py b/spacy/tests/serialize/test_serialize_vocab_strings.py new file mode 100644 index 0000000..fd80c3d --- /dev/null +++ b/spacy/tests/serialize/test_serialize_vocab_strings.py @@ -0,0 +1,195 @@ +import pickle + +import pytest +from thinc.api import get_current_ops + +import spacy +from spacy.lang.en import English +from spacy.strings import StringStore +from spacy.tokens import Doc +from spacy.util import ensure_path, load_model +from spacy.vectors import Vectors +from spacy.vocab import Vocab + +from ..util import make_tempdir + +test_strings = [([], []), (["rats", "are", "cute"], ["i", "like", "rats"])] +test_strings_attrs = [(["rats", "are", "cute"], "Hello")] + + +@pytest.mark.issue(599) +def test_issue599(en_vocab): + doc = Doc(en_vocab) + doc2 = Doc(doc.vocab) + doc2.from_bytes(doc.to_bytes()) + assert doc2.has_annotation("DEP") + + +@pytest.mark.issue(4054) +def test_issue4054(en_vocab): + """Test that a new blank model can be made with a vocab from file, + and that serialization does not drop the language at any point.""" + nlp1 = English() + vocab1 = nlp1.vocab + with make_tempdir() as d: + vocab_dir = ensure_path(d / "vocab") + if not vocab_dir.exists(): + vocab_dir.mkdir() + vocab1.to_disk(vocab_dir) + vocab2 = Vocab().from_disk(vocab_dir) + nlp2 = spacy.blank("en", vocab=vocab2) + nlp_dir = ensure_path(d / "nlp") + if not nlp_dir.exists(): + nlp_dir.mkdir() + nlp2.to_disk(nlp_dir) + nlp3 = load_model(nlp_dir) + assert nlp3.lang == "en" + + +@pytest.mark.issue(4133) +def test_issue4133(en_vocab): + nlp = English() + vocab_bytes = nlp.vocab.to_bytes() + words = ["Apple", "is", "looking", "at", "buying", "a", "startup"] + pos = ["NOUN", "VERB", "ADP", "VERB", "PROPN", "NOUN", "ADP"] + doc = Doc(en_vocab, words=words) + for i, token in enumerate(doc): + token.pos_ = pos[i] + # usually this is already True when starting from proper models instead of blank English + doc_bytes = doc.to_bytes() + vocab = Vocab() + vocab = vocab.from_bytes(vocab_bytes) + doc = Doc(vocab).from_bytes(doc_bytes) + actual = [] + for token in doc: + actual.append(token.pos_) + assert actual == pos + + +@pytest.mark.parametrize("text", ["rat"]) +def test_serialize_vocab(en_vocab, text): + text_hash = en_vocab.strings.add(text) + vocab_bytes = en_vocab.to_bytes(exclude=["lookups"]) + new_vocab = Vocab().from_bytes(vocab_bytes) + assert new_vocab.strings[text_hash] == text + assert new_vocab.to_bytes(exclude=["lookups"]) == vocab_bytes + + +@pytest.mark.parametrize("strings1,strings2", test_strings) +def test_serialize_vocab_roundtrip_bytes(strings1, strings2): + vocab1 = Vocab(strings=strings1) + vocab2 = Vocab(strings=strings2) + vocab1_b = vocab1.to_bytes() + vocab2_b = vocab2.to_bytes() + if strings1 == strings2: + assert vocab1_b == vocab2_b + else: + assert vocab1_b != vocab2_b + vocab1 = vocab1.from_bytes(vocab1_b) + assert vocab1.to_bytes() == vocab1_b + new_vocab1 = Vocab().from_bytes(vocab1_b) + assert new_vocab1.to_bytes() == vocab1_b + assert len(new_vocab1.strings) == len(strings1) + assert sorted([s for s in new_vocab1.strings]) == sorted(strings1) + + +@pytest.mark.parametrize("strings1,strings2", test_strings) +def test_serialize_vocab_roundtrip_disk(strings1, strings2): + vocab1 = Vocab(strings=strings1) + vocab2 = Vocab(strings=strings2) + with make_tempdir() as d: + file_path1 = d / "vocab1" + file_path2 = d / "vocab2" + vocab1.to_disk(file_path1) + vocab2.to_disk(file_path2) + vocab1_d = Vocab().from_disk(file_path1) + vocab2_d = Vocab().from_disk(file_path2) + # check strings rather than lexemes, which are only reloaded on demand + assert set(strings1) == set([s for s in vocab1_d.strings]) + assert set(strings2) == set([s for s in vocab2_d.strings]) + if set(strings1) == set(strings2): + assert [s for s in vocab1_d.strings] == [s for s in vocab2_d.strings] + else: + assert [s for s in vocab1_d.strings] != [s for s in vocab2_d.strings] + + +@pytest.mark.parametrize("strings,lex_attr", test_strings_attrs) +def test_serialize_vocab_lex_attrs_bytes(strings, lex_attr): + vocab1 = Vocab(strings=strings) + vocab2 = Vocab() + vocab1[strings[0]].norm_ = lex_attr + assert vocab1[strings[0]].norm_ == lex_attr + assert vocab2[strings[0]].norm_ != lex_attr + vocab2 = vocab2.from_bytes(vocab1.to_bytes()) + assert vocab2[strings[0]].norm_ == lex_attr + + +@pytest.mark.parametrize("strings,lex_attr", test_strings_attrs) +def test_deserialize_vocab_seen_entries(strings, lex_attr): + # Reported in #2153 + vocab = Vocab(strings=strings) + vocab.from_bytes(vocab.to_bytes()) + assert len(vocab.strings) == len(strings) + + +@pytest.mark.parametrize("strings,lex_attr", test_strings_attrs) +def test_serialize_vocab_lex_attrs_disk(strings, lex_attr): + vocab1 = Vocab(strings=strings) + vocab2 = Vocab() + vocab1[strings[0]].norm_ = lex_attr + assert vocab1[strings[0]].norm_ == lex_attr + assert vocab2[strings[0]].norm_ != lex_attr + with make_tempdir() as d: + file_path = d / "vocab" + vocab1.to_disk(file_path) + vocab2 = vocab2.from_disk(file_path) + assert vocab2[strings[0]].norm_ == lex_attr + + +@pytest.mark.parametrize("strings1,strings2", test_strings) +def test_serialize_stringstore_roundtrip_bytes(strings1, strings2): + sstore1 = StringStore(strings=strings1) + sstore2 = StringStore(strings=strings2) + sstore1_b = sstore1.to_bytes() + sstore2_b = sstore2.to_bytes() + if set(strings1) == set(strings2): + assert sstore1_b == sstore2_b + else: + assert sstore1_b != sstore2_b + sstore1 = sstore1.from_bytes(sstore1_b) + assert sstore1.to_bytes() == sstore1_b + new_sstore1 = StringStore().from_bytes(sstore1_b) + assert new_sstore1.to_bytes() == sstore1_b + assert set(new_sstore1) == set(strings1) + + +@pytest.mark.parametrize("strings1,strings2", test_strings) +def test_serialize_stringstore_roundtrip_disk(strings1, strings2): + sstore1 = StringStore(strings=strings1) + sstore2 = StringStore(strings=strings2) + with make_tempdir() as d: + file_path1 = d / "strings1" + file_path2 = d / "strings2" + sstore1.to_disk(file_path1) + sstore2.to_disk(file_path2) + sstore1_d = StringStore().from_disk(file_path1) + sstore2_d = StringStore().from_disk(file_path2) + assert set(sstore1_d) == set(sstore1) + assert set(sstore2_d) == set(sstore2) + if set(strings1) == set(strings2): + assert set(sstore1_d) == set(sstore2_d) + else: + assert set(sstore1_d) != set(sstore2_d) + + +@pytest.mark.parametrize("strings,lex_attr", test_strings_attrs) +def test_pickle_vocab(strings, lex_attr): + vocab = Vocab(strings=strings) + ops = get_current_ops() + vectors = Vectors(data=ops.xp.zeros((10, 10)), mode="floret", hash_count=1) + vocab.vectors = vectors + vocab[strings[0]].norm_ = lex_attr + vocab_pickled = pickle.dumps(vocab) + vocab_unpickled = pickle.loads(vocab_pickled) + assert vocab.to_bytes() == vocab_unpickled.to_bytes() + assert vocab_unpickled.vectors.mode == "floret" diff --git a/spacy/tests/test_architectures.py b/spacy/tests/test_architectures.py new file mode 100644 index 0000000..3b5804a --- /dev/null +++ b/spacy/tests/test_architectures.py @@ -0,0 +1,16 @@ +import pytest +from catalogue import RegistryError +from thinc.api import Linear + +from spacy import registry + + +def test_get_architecture(): + @registry.architectures("my_test_function") + def create_model(nr_in, nr_out): + return Linear(nr_in, nr_out) + + arch = registry.architectures.get("my_test_function") + assert arch is create_model + with pytest.raises(RegistryError): + registry.architectures.get("not_an_existing_key") diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py new file mode 100644 index 0000000..4bac40f --- /dev/null +++ b/spacy/tests/test_cli.py @@ -0,0 +1,1239 @@ +import math +import os +from collections import Counter +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest +import srsly +from click import NoSuchOption +from packaging.specifiers import SpecifierSet +from thinc.api import Config + +import spacy +from spacy import about +from spacy.cli import download_module, info +from spacy.cli._util import parse_config_overrides, string_to_list, walk_directory +from spacy.cli.apply import apply +from spacy.cli.debug_data import ( + _compile_gold, + _get_distribution, + _get_kl_divergence, + _get_labels_from_model, + _get_labels_from_spancat, + _get_span_characteristics, + _get_spans_length_freq_dist, + _print_span_characteristics, +) +from spacy.cli.download import get_compatibility, get_version +from spacy.cli.evaluate import render_parses +from spacy.cli.find_threshold import find_threshold +from spacy.cli.init_config import RECOMMENDATIONS, fill_config, init_config +from spacy.cli.init_pipeline import _init_labels +from spacy.cli.package import _is_permitted_package_name, get_third_party_dependencies +from spacy.cli.validate import get_model_pkgs +from spacy.lang.en import English +from spacy.lang.nl import Dutch +from spacy.language import Language +from spacy.schemas import RecommendationSchema +from spacy.tokens import Doc, DocBin +from spacy.tokens.span import Span +from spacy.training import Example, docs_to_json, offsets_to_biluo_tags +from spacy.training.converters import conll_ner_to_docs, conllu_to_docs, iob_to_docs +from spacy.util import ENV_VARS, get_minor_version, load_config, load_model_from_config + +from .util import make_tempdir + + +@pytest.mark.issue(4665) +def test_cli_converters_conllu_empty_heads_ner(): + """ + conllu_to_docs should not raise an exception if the HEAD column contains an + underscore + """ + input_data = """ +1 [ _ PUNCT -LRB- _ _ punct _ _ +2 This _ DET DT _ _ det _ _ +3 killing _ NOUN NN _ _ nsubj _ _ +4 of _ ADP IN _ _ case _ _ +5 a _ DET DT _ _ det _ _ +6 respected _ ADJ JJ _ _ amod _ _ +7 cleric _ NOUN NN _ _ nmod _ _ +8 will _ AUX MD _ _ aux _ _ +9 be _ AUX VB _ _ aux _ _ +10 causing _ VERB VBG _ _ root _ _ +11 us _ PRON PRP _ _ iobj _ _ +12 trouble _ NOUN NN _ _ dobj _ _ +13 for _ ADP IN _ _ case _ _ +14 years _ NOUN NNS _ _ nmod _ _ +15 to _ PART TO _ _ mark _ _ +16 come _ VERB VB _ _ acl _ _ +17 . _ PUNCT . _ _ punct _ _ +18 ] _ PUNCT -RRB- _ _ punct _ _ +""" + docs = list(conllu_to_docs(input_data)) + # heads are all 0 + assert not all([t.head.i for t in docs[0]]) + # NER is unset + assert not docs[0].has_annotation("ENT_IOB") + + +@pytest.mark.issue(4924) +def test_issue4924(): + nlp = Language() + example = Example.from_dict(nlp.make_doc(""), {}) + nlp.evaluate([example]) + + +@pytest.mark.issue(7055) +def test_issue7055(): + """Test that fill-config doesn't turn sourced components into factories.""" + source_cfg = { + "nlp": {"lang": "en", "pipeline": ["tok2vec", "tagger"]}, + "components": { + "tok2vec": {"factory": "tok2vec"}, + "tagger": {"factory": "tagger"}, + }, + } + source_nlp = English.from_config(source_cfg) + with make_tempdir() as dir_path: + # We need to create a loadable source pipeline + source_path = dir_path / "test_model" + source_nlp.to_disk(source_path) + base_cfg = { + "nlp": {"lang": "en", "pipeline": ["tok2vec", "tagger", "ner"]}, + "components": { + "tok2vec": {"source": str(source_path)}, + "tagger": {"source": str(source_path)}, + "ner": {"factory": "ner"}, + }, + } + base_cfg = Config(base_cfg) + base_path = dir_path / "base.cfg" + base_cfg.to_disk(base_path) + output_path = dir_path / "config.cfg" + fill_config(output_path, base_path, silent=True) + filled_cfg = load_config(output_path) + assert filled_cfg["components"]["tok2vec"]["source"] == str(source_path) + assert filled_cfg["components"]["tagger"]["source"] == str(source_path) + assert filled_cfg["components"]["ner"]["factory"] == "ner" + assert "model" in filled_cfg["components"]["ner"] + + +@pytest.mark.issue(12566) +@pytest.mark.parametrize( + "factory,output_file", + [("deps", "parses.html"), ("ents", "entities.html"), ("spans", "spans.html")], +) +def test_issue12566(factory: str, output_file: str): + """ + Test if all displaCy types (ents, dep, spans) produce an HTML file + """ + with make_tempdir() as tmp_dir: + # Create sample spaCy file + doc_json = { + "ents": [ + {"end": 54, "label": "nam_adj_country", "start": 44}, + {"end": 83, "label": "nam_liv_person", "start": 69}, + {"end": 100, "label": "nam_pro_title_book", "start": 86}, + ], + "spans": { + "sc": [ + {"end": 54, "kb_id": "", "label": "nam_adj_country", "start": 44}, + {"end": 83, "kb_id": "", "label": "nam_liv_person", "start": 69}, + { + "end": 100, + "kb_id": "", + "label": "nam_pro_title_book", + "start": 86, + }, + ] + }, + "text": "Niedawno czytał em nową książkę znakomitego szkockiego medioznawcy , " + "Briana McNaira - Cultural Chaos .", + "tokens": [ + # fmt: off + { + "id": 0, + "start": 0, + "end": 8, + "tag": "ADV", + "pos": "ADV", + "morph": "Degree=Pos", + "lemma": "niedawno", + "dep": "advmod", + "head": 1, + }, + { + "id": 1, + "start": 9, + "end": 15, + "tag": "PRAET", + "pos": "VERB", + "morph": "Animacy=Hum|Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act", + "lemma": "czytać", + "dep": "ROOT", + "head": 1, + }, + { + "id": 2, + "start": 16, + "end": 18, + "tag": "AGLT", + "pos": "NOUN", + "morph": "Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing", + "lemma": "em", + "dep": "iobj", + "head": 1, + }, + { + "id": 3, + "start": 19, + "end": 23, + "tag": "ADJ", + "pos": "ADJ", + "morph": "Case=Acc|Degree=Pos|Gender=Fem|Number=Sing", + "lemma": "nowy", + "dep": "amod", + "head": 4, + }, + { + "id": 4, + "start": 24, + "end": 31, + "tag": "SUBST", + "pos": "NOUN", + "morph": "Case=Acc|Gender=Fem|Number=Sing", + "lemma": "książka", + "dep": "obj", + "head": 1, + }, + { + "id": 5, + "start": 32, + "end": 43, + "tag": "ADJ", + "pos": "ADJ", + "morph": "Animacy=Nhum|Case=Gen|Degree=Pos|Gender=Masc|Number=Sing", + "lemma": "znakomit", + "dep": "acl", + "head": 4, + }, + { + "id": 6, + "start": 44, + "end": 54, + "tag": "ADJ", + "pos": "ADJ", + "morph": "Animacy=Hum|Case=Gen|Degree=Pos|Gender=Masc|Number=Sing", + "lemma": "szkockiy", + "dep": "amod", + "head": 7, + }, + { + "id": 7, + "start": 55, + "end": 66, + "tag": "SUBST", + "pos": "NOUN", + "morph": "Animacy=Hum|Case=Gen|Gender=Masc|Number=Sing", + "lemma": "medioznawca", + "dep": "iobj", + "head": 5, + }, + { + "id": 8, + "start": 67, + "end": 68, + "tag": "INTERP", + "pos": "PUNCT", + "morph": "PunctType=Comm", + "lemma": ",", + "dep": "punct", + "head": 9, + }, + { + "id": 9, + "start": 69, + "end": 75, + "tag": "SUBST", + "pos": "PROPN", + "morph": "Animacy=Hum|Case=Gen|Gender=Masc|Number=Sing", + "lemma": "Brian", + "dep": "nmod", + "head": 4, + }, + { + "id": 10, + "start": 76, + "end": 83, + "tag": "SUBST", + "pos": "PROPN", + "morph": "Animacy=Hum|Case=Gen|Gender=Masc|Number=Sing", + "lemma": "McNair", + "dep": "flat", + "head": 9, + }, + { + "id": 11, + "start": 84, + "end": 85, + "tag": "INTERP", + "pos": "PUNCT", + "morph": "PunctType=Dash", + "lemma": "-", + "dep": "punct", + "head": 12, + }, + { + "id": 12, + "start": 86, + "end": 94, + "tag": "SUBST", + "pos": "PROPN", + "morph": "Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing", + "lemma": "Cultural", + "dep": "conj", + "head": 4, + }, + { + "id": 13, + "start": 95, + "end": 100, + "tag": "SUBST", + "pos": "NOUN", + "morph": "Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing", + "lemma": "Chaos", + "dep": "flat", + "head": 12, + }, + { + "id": 14, + "start": 101, + "end": 102, + "tag": "INTERP", + "pos": "PUNCT", + "morph": "PunctType=Peri", + "lemma": ".", + "dep": "punct", + "head": 1, + }, + # fmt: on + ], + } + + # Create a .spacy file + nlp = spacy.blank("pl") + doc = Doc(nlp.vocab).from_json(doc_json) + + # Run the evaluate command and check if the html files exist + render_parses( + docs=[doc], output_path=tmp_dir, model_name="", limit=1, **{factory: True} + ) + + assert (tmp_dir / output_file).is_file() + + +def test_cli_info(): + nlp = Dutch() + nlp.add_pipe("textcat") + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + raw_data = info(tmp_dir, exclude=[""]) + assert raw_data["lang"] == "nl" + assert raw_data["components"] == ["textcat"] + + +def test_cli_converters_conllu_to_docs(): + # from NorNE: https://github.com/ltgoslo/norne/blob/3d23274965f513f23aa48455b28b1878dad23c05/ud/nob/no_bokmaal-ud-dev.conllu + lines = [ + "1\tDommer\tdommer\tNOUN\t_\tDefinite=Ind|Gender=Masc|Number=Sing\t2\tappos\t_\tO", + "2\tFinn\tFinn\tPROPN\t_\tGender=Masc\t4\tnsubj\t_\tB-PER", + "3\tEilertsen\tEilertsen\tPROPN\t_\t_\t2\tname\t_\tI-PER", + "4\tavstår\tavstå\tVERB\t_\tMood=Ind|Tense=Pres|VerbForm=Fin\t0\troot\t_\tO", + ] + input_data = "\n".join(lines) + converted_docs = list(conllu_to_docs(input_data, n_sents=1)) + assert len(converted_docs) == 1 + converted = [docs_to_json(converted_docs)] + assert converted[0]["id"] == 0 + assert len(converted[0]["paragraphs"]) == 1 + assert len(converted[0]["paragraphs"][0]["sentences"]) == 1 + sent = converted[0]["paragraphs"][0]["sentences"][0] + assert len(sent["tokens"]) == 4 + tokens = sent["tokens"] + assert [t["orth"] for t in tokens] == ["Dommer", "Finn", "Eilertsen", "avstår"] + assert [t["tag"] for t in tokens] == ["NOUN", "PROPN", "PROPN", "VERB"] + assert [t["head"] for t in tokens] == [1, 2, -1, 0] + assert [t["dep"] for t in tokens] == ["appos", "nsubj", "name", "ROOT"] + ent_offsets = [ + (e[0], e[1], e[2]) for e in converted[0]["paragraphs"][0]["entities"] + ] + biluo_tags = offsets_to_biluo_tags(converted_docs[0], ent_offsets, missing="O") + assert biluo_tags == ["O", "B-PER", "L-PER", "O"] + + +@pytest.mark.parametrize( + "lines", + [ + ( + "1\tDommer\tdommer\tNOUN\t_\tDefinite=Ind|Gender=Masc|Number=Sing\t2\tappos\t_\tname=O", + "2\tFinn\tFinn\tPROPN\t_\tGender=Masc\t4\tnsubj\t_\tSpaceAfter=No|name=B-PER", + "3\tEilertsen\tEilertsen\tPROPN\t_\t_\t2\tname\t_\tname=I-PER", + "4\tavstår\tavstå\tVERB\t_\tMood=Ind|Tense=Pres|VerbForm=Fin\t0\troot\t_\tSpaceAfter=No|name=O", + "5\t.\t$.\tPUNCT\t_\t_\t4\tpunct\t_\tname=B-BAD", + ), + ( + "1\tDommer\tdommer\tNOUN\t_\tDefinite=Ind|Gender=Masc|Number=Sing\t2\tappos\t_\t_", + "2\tFinn\tFinn\tPROPN\t_\tGender=Masc\t4\tnsubj\t_\tSpaceAfter=No|NE=B-PER", + "3\tEilertsen\tEilertsen\tPROPN\t_\t_\t2\tname\t_\tNE=L-PER", + "4\tavstår\tavstå\tVERB\t_\tMood=Ind|Tense=Pres|VerbForm=Fin\t0\troot\t_\tSpaceAfter=No", + "5\t.\t$.\tPUNCT\t_\t_\t4\tpunct\t_\tNE=B-BAD", + ), + ], +) +def test_cli_converters_conllu_to_docs_name_ner_map(lines): + input_data = "\n".join(lines) + converted_docs = list( + conllu_to_docs(input_data, n_sents=1, ner_map={"PER": "PERSON", "BAD": ""}) + ) + assert len(converted_docs) == 1 + converted = [docs_to_json(converted_docs)] + assert converted[0]["id"] == 0 + assert len(converted[0]["paragraphs"]) == 1 + assert converted[0]["paragraphs"][0]["raw"] == "Dommer FinnEilertsen avstår. " + assert len(converted[0]["paragraphs"][0]["sentences"]) == 1 + sent = converted[0]["paragraphs"][0]["sentences"][0] + assert len(sent["tokens"]) == 5 + tokens = sent["tokens"] + assert [t["orth"] for t in tokens] == ["Dommer", "Finn", "Eilertsen", "avstår", "."] + assert [t["tag"] for t in tokens] == ["NOUN", "PROPN", "PROPN", "VERB", "PUNCT"] + assert [t["head"] for t in tokens] == [1, 2, -1, 0, -1] + assert [t["dep"] for t in tokens] == ["appos", "nsubj", "name", "ROOT", "punct"] + ent_offsets = [ + (e[0], e[1], e[2]) for e in converted[0]["paragraphs"][0]["entities"] + ] + biluo_tags = offsets_to_biluo_tags(converted_docs[0], ent_offsets, missing="O") + assert biluo_tags == ["O", "B-PERSON", "L-PERSON", "O", "O"] + + +def test_cli_converters_conllu_to_docs_subtokens(): + # https://raw.githubusercontent.com/ohenrik/nb_news_ud_sm/master/original_data/no-ud-dev-ner.conllu + lines = [ + "1\tDommer\tdommer\tNOUN\t_\tDefinite=Ind|Gender=Masc|Number=Sing\t2\tappos\t_\tname=O", + "2-3\tFE\t_\t_\t_\t_\t_\t_\t_\t_", + "2\tFinn\tFinn\tPROPN\t_\tGender=Masc\t4\tnsubj\t_\tname=B-PER", + "3\tEilertsen\tEilertsen\tX\t_\tGender=Fem|Tense=past\t2\tname\t_\tname=I-PER", + "4\tavstår\tavstå\tVERB\t_\tMood=Ind|Tense=Pres|VerbForm=Fin\t0\troot\t_\tSpaceAfter=No|name=O", + "5\t.\t$.\tPUNCT\t_\t_\t4\tpunct\t_\tname=O", + ] + input_data = "\n".join(lines) + converted_docs = list( + conllu_to_docs( + input_data, n_sents=1, merge_subtokens=True, append_morphology=True + ) + ) + assert len(converted_docs) == 1 + converted = [docs_to_json(converted_docs)] + + assert converted[0]["id"] == 0 + assert len(converted[0]["paragraphs"]) == 1 + assert converted[0]["paragraphs"][0]["raw"] == "Dommer FE avstår. " + assert len(converted[0]["paragraphs"][0]["sentences"]) == 1 + sent = converted[0]["paragraphs"][0]["sentences"][0] + assert len(sent["tokens"]) == 4 + tokens = sent["tokens"] + assert [t["orth"] for t in tokens] == ["Dommer", "FE", "avstår", "."] + assert [t["tag"] for t in tokens] == [ + "NOUN__Definite=Ind|Gender=Masc|Number=Sing", + "PROPN_X__Gender=Fem,Masc|Tense=past", + "VERB__Mood=Ind|Tense=Pres|VerbForm=Fin", + "PUNCT", + ] + assert [t["pos"] for t in tokens] == ["NOUN", "PROPN", "VERB", "PUNCT"] + assert [t["morph"] for t in tokens] == [ + "Definite=Ind|Gender=Masc|Number=Sing", + "Gender=Fem,Masc|Tense=past", + "Mood=Ind|Tense=Pres|VerbForm=Fin", + "", + ] + assert [t["lemma"] for t in tokens] == ["dommer", "Finn Eilertsen", "avstå", "$."] + assert [t["head"] for t in tokens] == [1, 1, 0, -1] + assert [t["dep"] for t in tokens] == ["appos", "nsubj", "ROOT", "punct"] + ent_offsets = [ + (e[0], e[1], e[2]) for e in converted[0]["paragraphs"][0]["entities"] + ] + biluo_tags = offsets_to_biluo_tags(converted_docs[0], ent_offsets, missing="O") + assert biluo_tags == ["O", "U-PER", "O", "O"] + + +def test_cli_converters_iob_to_docs(): + lines = [ + "I|O like|O London|I-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O", + "I|O like|O London|B-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O", + "I|PRP|O like|VBP|O London|NNP|I-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O", + "I|PRP|O like|VBP|O London|NNP|B-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O", + ] + input_data = "\n".join(lines) + converted_docs = list(iob_to_docs(input_data, n_sents=10)) + assert len(converted_docs) == 1 + converted = docs_to_json(converted_docs) + assert converted["id"] == 0 + assert len(converted["paragraphs"]) == 1 + assert len(converted["paragraphs"][0]["sentences"]) == 4 + for i in range(0, 4): + sent = converted["paragraphs"][0]["sentences"][i] + assert len(sent["tokens"]) == 8 + tokens = sent["tokens"] + expected = ["I", "like", "London", "and", "New", "York", "City", "."] + assert [t["orth"] for t in tokens] == expected + assert len(converted_docs[0].ents) == 8 + for ent in converted_docs[0].ents: + assert ent.text in ["New York City", "London"] + + +def test_cli_converters_conll_ner_to_docs(): + lines = [ + "-DOCSTART- -X- O O", + "", + "I\tO", + "like\tO", + "London\tB-GPE", + "and\tO", + "New\tB-GPE", + "York\tI-GPE", + "City\tI-GPE", + ".\tO", + "", + "I O", + "like O", + "London B-GPE", + "and O", + "New B-GPE", + "York I-GPE", + "City I-GPE", + ". O", + "", + "I PRP O", + "like VBP O", + "London NNP B-GPE", + "and CC O", + "New NNP B-GPE", + "York NNP I-GPE", + "City NNP I-GPE", + ". . O", + "", + "I PRP _ O", + "like VBP _ O", + "London NNP _ B-GPE", + "and CC _ O", + "New NNP _ B-GPE", + "York NNP _ I-GPE", + "City NNP _ I-GPE", + ". . _ O", + "", + "I\tPRP\t_\tO", + "like\tVBP\t_\tO", + "London\tNNP\t_\tB-GPE", + "and\tCC\t_\tO", + "New\tNNP\t_\tB-GPE", + "York\tNNP\t_\tI-GPE", + "City\tNNP\t_\tI-GPE", + ".\t.\t_\tO", + ] + input_data = "\n".join(lines) + converted_docs = list(conll_ner_to_docs(input_data, n_sents=10)) + assert len(converted_docs) == 1 + converted = docs_to_json(converted_docs) + assert converted["id"] == 0 + assert len(converted["paragraphs"]) == 1 + assert len(converted["paragraphs"][0]["sentences"]) == 5 + for i in range(0, 5): + sent = converted["paragraphs"][0]["sentences"][i] + assert len(sent["tokens"]) == 8 + tokens = sent["tokens"] + # fmt: off + assert [t["orth"] for t in tokens] == ["I", "like", "London", "and", "New", "York", "City", "."] + # fmt: on + assert len(converted_docs[0].ents) == 10 + for ent in converted_docs[0].ents: + assert ent.text in ["New York City", "London"] + + +@pytest.mark.parametrize( + "args,expected", + [ + # fmt: off + (["--x.foo", "10"], {"x.foo": 10}), + (["--x.foo=10"], {"x.foo": 10}), + (["--x.foo", "bar"], {"x.foo": "bar"}), + (["--x.foo=bar"], {"x.foo": "bar"}), + (["--x.foo", "--x.bar", "baz"], {"x.foo": True, "x.bar": "baz"}), + (["--x.foo", "--x.bar=baz"], {"x.foo": True, "x.bar": "baz"}), + ( + ["--x.foo", "10.1", "--x.bar", "--x.baz", "false"], + {"x.foo": 10.1, "x.bar": True, "x.baz": False}, + ), + ( + ["--x.foo", "10.1", "--x.bar", "--x.baz=false"], + {"x.foo": 10.1, "x.bar": True, "x.baz": False}, + ), + # fmt: on + ], +) +def test_parse_config_overrides(args, expected): + assert parse_config_overrides(args) == expected + + +@pytest.mark.parametrize("args", [["--foo"], ["--x.foo", "bar", "--baz"]]) +def test_parse_config_overrides_invalid(args): + with pytest.raises(NoSuchOption): + parse_config_overrides(args) + + +@pytest.mark.parametrize("args", [["--x.foo", "bar", "baz"], ["x.foo"]]) +def test_parse_config_overrides_invalid_2(args): + with pytest.raises(SystemExit): + parse_config_overrides(args) + + +def test_parse_cli_overrides(): + overrides = "--x.foo bar --x.bar=12 --x.baz false --y.foo=hello" + os.environ[ENV_VARS.CONFIG_OVERRIDES] = overrides + result = parse_config_overrides([]) + assert len(result) == 4 + assert result["x.foo"] == "bar" + assert result["x.bar"] == 12 + assert result["x.baz"] is False + assert result["y.foo"] == "hello" + os.environ[ENV_VARS.CONFIG_OVERRIDES] = "--x" + assert parse_config_overrides([], env_var=None) == {} + with pytest.raises(SystemExit): + parse_config_overrides([]) + os.environ[ENV_VARS.CONFIG_OVERRIDES] = "hello world" + with pytest.raises(SystemExit): + parse_config_overrides([]) + del os.environ[ENV_VARS.CONFIG_OVERRIDES] + + +@pytest.mark.parametrize("lang", ["en", "nl"]) +@pytest.mark.parametrize( + "pipeline", + [ + ["tagger", "parser", "ner"], + [], + ["ner", "textcat", "sentencizer"], + ["morphologizer", "spancat", "entity_linker"], + ["spancat_singlelabel", "textcat_multilabel"], + ], +) +@pytest.mark.parametrize("optimize", ["efficiency", "accuracy"]) +@pytest.mark.parametrize("pretraining", [True, False]) +def test_init_config(lang, pipeline, optimize, pretraining): + # TODO: add more tests and also check for GPU with transformers + config = init_config( + lang=lang, + pipeline=pipeline, + optimize=optimize, + pretraining=pretraining, + gpu=False, + ) + assert isinstance(config, Config) + if pretraining: + config["paths"]["raw_text"] = "my_data.jsonl" + load_model_from_config(config, auto_fill=True) + + +def test_model_recommendations(): + for lang, data in RECOMMENDATIONS.items(): + assert RecommendationSchema(**data) + + +@pytest.mark.parametrize( + "value", + [ + # fmt: off + "parser,textcat,tagger", + " parser, textcat ,tagger ", + "parser,textcat,tagger", + " parser, textcat ,tagger ", + ' "parser"," textcat " ,"tagger "', + " 'parser',' textcat ' ,'tagger '", + "[parser,textcat,tagger]", + '["parser","textcat","tagger"]', + '[" parser" ,"textcat ", " tagger " ]', + "[parser,textcat,tagger]", + "[ parser, textcat , tagger]", + "['parser','textcat','tagger']", + "[' parser' , 'textcat', ' tagger ' ]", + # fmt: on + ], +) +def test_string_to_list(value): + assert string_to_list(value, intify=False) == ["parser", "textcat", "tagger"] + + +@pytest.mark.parametrize( + "value", + [ + # fmt: off + "1,2,3", + "[1,2,3]", + '["1","2","3"]', + '[" 1" ,"2 ", " 3 " ]', + "[' 1' , '2', ' 3 ' ]", + # fmt: on + ], +) +def test_string_to_list_intify(value): + assert string_to_list(value, intify=False) == ["1", "2", "3"] + assert string_to_list(value, intify=True) == [1, 2, 3] + + +def test_download_compatibility(): + spec = SpecifierSet("==" + about.__version__) + spec.prereleases = False + if about.__version__ in spec: + model_name = "en_core_web_sm" + compatibility = get_compatibility() + version = get_version(model_name, compatibility) + assert get_minor_version(about.__version__) == get_minor_version(version) + + +def test_validate_compatibility_table(): + spec = SpecifierSet("==" + about.__version__) + spec.prereleases = False + if about.__version__ in spec: + model_pkgs, compat = get_model_pkgs() + spacy_version = get_minor_version(about.__version__) + current_compat = compat.get(spacy_version, {}) + assert len(current_compat) > 0 + assert "en_core_web_sm" in current_compat + + +@pytest.mark.parametrize("component_name", ["ner", "textcat", "spancat", "tagger"]) +def test_init_labels(component_name): + nlp = Dutch() + component = nlp.add_pipe(component_name) + for label in ["T1", "T2", "T3", "T4"]: + component.add_label(label) + assert len(nlp.get_pipe(component_name).labels) == 4 + + with make_tempdir() as tmp_dir: + _init_labels(nlp, tmp_dir) + + config = init_config( + lang="nl", + pipeline=[component_name], + optimize="efficiency", + gpu=False, + ) + config["initialize"]["components"][component_name] = { + "labels": { + "@readers": "spacy.read_labels.v1", + "path": f"{tmp_dir}/{component_name}.json", + } + } + + nlp2 = load_model_from_config(config, auto_fill=True) + assert len(nlp2.get_pipe(component_name).labels) == 0 + nlp2.initialize() + assert len(nlp2.get_pipe(component_name).labels) == 4 + + +def test_get_third_party_dependencies(): + # We can't easily test the detection of third-party packages here, but we + # can at least make sure that the function and its importlib magic runs. + nlp = Dutch() + # Test with component factory based on Cython module + nlp.add_pipe("tagger") + assert get_third_party_dependencies(nlp.config) == [] + + # Test with legacy function + nlp = Dutch() + nlp.add_pipe( + "textcat", + config={ + "model": { + # Do not update from legacy architecture spacy.TextCatBOW.v1 + "@architectures": "spacy.TextCatBOW.v1", + "exclusive_classes": True, + "ngram_size": 1, + "no_output_layer": False, + } + }, + ) + assert get_third_party_dependencies(nlp.config) == [] + + # Test with lang-specific factory + @Dutch.factory("third_party_test") + def test_factory(nlp, name): + return lambda x: x + + nlp.add_pipe("third_party_test") + # Before #9674 this would throw an exception + get_third_party_dependencies(nlp.config) + + +@pytest.mark.slow +@pytest.mark.parametrize( + "factory_name,pipe_name", + [ + ("ner", "ner"), + ("ner", "my_ner"), + ("spancat", "spancat"), + ("spancat", "my_spancat"), + ], +) +def test_get_labels_from_model(factory_name, pipe_name): + labels = ("A", "B") + + nlp = English() + pipe = nlp.add_pipe(factory_name, name=pipe_name) + for label in labels: + pipe.add_label(label) + nlp.initialize() + assert nlp.get_pipe(pipe_name).labels == labels + if factory_name == "spancat": + assert _get_labels_from_spancat(nlp)[pipe.key] == set(labels) + else: + assert _get_labels_from_model(nlp, factory_name) == set(labels) + + +def test_permitted_package_names(): + # https://www.python.org/dev/peps/pep-0426/#name + assert _is_permitted_package_name("Meine_Bäume") == False + assert _is_permitted_package_name("_package") == False + assert _is_permitted_package_name("package_") == False + assert _is_permitted_package_name(".package") == False + assert _is_permitted_package_name("package.") == False + assert _is_permitted_package_name("-package") == False + assert _is_permitted_package_name("package-") == False + + +def test_debug_data_compile_gold(): + nlp = English() + pred = Doc(nlp.vocab, words=["Token", ".", "New", "York", "City"]) + ref = Doc( + nlp.vocab, + words=["Token", ".", "New York City"], + sent_starts=[True, False, True], + ents=["O", "O", "B-ENT"], + ) + eg = Example(pred, ref) + data = _compile_gold([eg], ["ner"], nlp, True) + assert data["boundary_cross_ents"] == 0 + + pred = Doc(nlp.vocab, words=["Token", ".", "New", "York", "City"]) + ref = Doc( + nlp.vocab, + words=["Token", ".", "New York City"], + sent_starts=[True, False, True], + ents=["O", "B-ENT", "I-ENT"], + ) + eg = Example(pred, ref) + data = _compile_gold([eg], ["ner"], nlp, True) + assert data["boundary_cross_ents"] == 1 + + +@pytest.mark.parametrize("component_name", ["spancat", "spancat_singlelabel"]) +def test_debug_data_compile_gold_for_spans(component_name): + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + data = _compile_gold([eg], [component_name], nlp, True) + + assert data["spancat"][spans_key] == Counter({"ORG": 1, "GPE": 1}) + assert data["spans_length"][spans_key] == {"ORG": [3], "GPE": [1]} + assert data["spans_per_type"][spans_key] == { + "ORG": [Span(ref, 3, 6, "ORG")], + "GPE": [Span(ref, 5, 6, "GPE")], + } + assert data["sb_per_type"][spans_key] == { + "ORG": {"start": [ref[2:3]], "end": [ref[6:7]]}, + "GPE": {"start": [ref[4:5]], "end": [ref[6:7]]}, + } + + +def test_frequency_distribution_is_correct(): + nlp = English() + docs = [ + Doc(nlp.vocab, words=["Bank", "of", "China"]), + Doc(nlp.vocab, words=["China"]), + ] + + expected = Counter({"china": 0.5, "bank": 0.25, "of": 0.25}) + freq_distribution = _get_distribution(docs, normalize=True) + assert freq_distribution == expected + + +def test_kl_divergence_computation_is_correct(): + p = Counter({"a": 0.5, "b": 0.25}) + q = Counter({"a": 0.25, "b": 0.50, "c": 0.15, "d": 0.10}) + result = _get_kl_divergence(p, q) + expected = 0.1733 + assert math.isclose(result, expected, rel_tol=1e-3) + + +def test_get_span_characteristics_return_value(): + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + examples = [eg] + data = _compile_gold(examples, ["spancat"], nlp, True) + span_characteristics = _get_span_characteristics( + examples=examples, compiled_gold=data, spans_key=spans_key + ) + + assert {"sd", "bd", "lengths"}.issubset(span_characteristics.keys()) + assert span_characteristics["min_length"] == 1 + assert span_characteristics["max_length"] == 3 + + +def test_ensure_print_span_characteristics_wont_fail(): + """Test if interface between two methods aren't destroyed if refactored""" + nlp = English() + spans_key = "sc" + + pred = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + pred.spans[spans_key] = [Span(pred, 3, 6, "ORG"), Span(pred, 5, 6, "GPE")] + ref = Doc(nlp.vocab, words=["Welcome", "to", "the", "Bank", "of", "China", "."]) + ref.spans[spans_key] = [Span(ref, 3, 6, "ORG"), Span(ref, 5, 6, "GPE")] + eg = Example(pred, ref) + + examples = [eg] + data = _compile_gold(examples, ["spancat"], nlp, True) + span_characteristics = _get_span_characteristics( + examples=examples, compiled_gold=data, spans_key=spans_key + ) + _print_span_characteristics(span_characteristics) + + +@pytest.mark.parametrize("threshold", [70, 80, 85, 90, 95]) +def test_span_length_freq_dist_threshold_must_be_correct(threshold): + sample_span_lengths = { + "span_type_1": [1, 4, 4, 5], + "span_type_2": [5, 3, 3, 2], + "span_type_3": [3, 1, 3, 3], + } + span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold) + assert sum(span_freqs.values()) >= threshold + + +def test_span_length_freq_dist_output_must_be_correct(): + sample_span_lengths = { + "span_type_1": [1, 4, 4, 5], + "span_type_2": [5, 3, 3, 2], + "span_type_3": [3, 1, 3, 3], + } + threshold = 90 + span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold) + assert sum(span_freqs.values()) >= threshold + assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] + + +def test_applycli_empty_dir(): + with make_tempdir() as data_path: + output = data_path / "test.spacy" + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_docbin(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + # test empty DocBin case + docbin = DocBin() + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_jsonl(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + data = [{"field": "Testing apply cli.", "key": 234}] + data2 = [{"field": "234"}] + srsly.write_jsonl(data_path / "test.jsonl", data) + apply(data_path, output, "blank:en", "field", 1, 1) + srsly.write_jsonl(data_path / "test2.jsonl", data2) + apply(data_path, output, "blank:en", "field", 1, 1) + + +def test_applycli_txt(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + with open(data_path / "test.foo", "w") as ftest: + ftest.write("Testing apply cli.") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_mixed(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + text = "Testing apply cli" + nlp = spacy.blank("en") + doc = nlp(text) + jsonl_data = [{"text": text}] + srsly.write_jsonl(data_path / "test.jsonl", jsonl_data) + docbin = DocBin() + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + with open(data_path / "test.txt", "w") as ftest: + ftest.write(text) + apply(data_path, output, "blank:en", "text", 1, 1) + # Check whether it worked + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert len(result) == 3 + for doc in result: + assert doc.text == text + + +def test_applycli_user_data(): + Doc.set_extension("ext", default=0) + val = ("ext", 0) + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + doc._.ext = val + docbin = DocBin(store_user_data=True) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "", 1, 1) + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert result[0]._.ext == val + + +def test_cli_find_threshold(capsys): + def make_examples(nlp: Language) -> List[Example]: + docs: List[Example] = [] + + for t in [ + ( + "I am angry and confused in the Bank of America.", + { + "cats": {"ANGRY": 1.0, "CONFUSED": 1.0, "HAPPY": 0.0}, + "spans": {"sc": [(31, 46, "ORG")]}, + }, + ), + ( + "I am confused but happy in New York.", + { + "cats": {"ANGRY": 0.0, "CONFUSED": 1.0, "HAPPY": 1.0}, + "spans": {"sc": [(27, 35, "GPE")]}, + }, + ), + ]: + doc = nlp.make_doc(t[0]) + docs.append(Example.from_dict(doc, t[1])) + + return docs + + def init_nlp( + components: Tuple[Tuple[str, Dict[str, Any]], ...] = (), + ) -> Tuple[Language, List[Example]]: + new_nlp = English() + new_nlp.add_pipe( # type: ignore + factory_name="textcat_multilabel", + name="tc_multi", + config={"threshold": 0.9}, + ) + + # Append additional components to pipeline. + for cfn, comp_config in components: + new_nlp.add_pipe(cfn, config=comp_config) + + new_examples = make_examples(new_nlp) + new_nlp.initialize(get_examples=lambda: new_examples) + for i in range(5): + new_nlp.update(new_examples) + + return new_nlp, new_examples + + with make_tempdir() as docs_dir: + # Check whether find_threshold() identifies lowest threshold above 0 as (first) ideal threshold, as this matches + # the current model behavior with the examples above. This can break once the model behavior changes and serves + # mostly as a smoke test. + nlp, examples = init_nlp() + DocBin(docs=[example.reference for example in examples]).to_disk( + docs_dir / "docs.spacy" + ) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + best_threshold, best_score, res = find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="tc_multi", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + assert best_score == max(res.values()) + assert res[1.0] == 0.0 + + # Test with spancat. + nlp, _ = init_nlp((("spancat", {}),)) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + best_threshold, best_score, res = find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="spancat", + threshold_key="threshold", + scores_key="spans_sc_f", + silent=True, + ) + assert best_score == max(res.values()) + assert res[1.0] == 0.0 + + # Having multiple textcat_multilabel components should work, since the name has to be specified. + nlp, _ = init_nlp((("textcat_multilabel", {}),)) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + assert find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="tc_multi", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + + # Specifying the name of an non-existing pipe should fail. + nlp, _ = init_nlp() + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + with pytest.raises(AttributeError): + find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="_", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + + +def test_walk_directory(): + with make_tempdir() as d: + files = [ + "data1.iob", + "data2.iob", + "data3.json", + "data4.conll", + "data5.conll", + "data6.conll", + "data7.txt", + ] + + for f in files: + Path(d / f).touch() + + assert (len(walk_directory(d))) == 7 + assert (len(walk_directory(d, suffix=None))) == 7 + assert (len(walk_directory(d, suffix="json"))) == 1 + assert (len(walk_directory(d, suffix="iob"))) == 2 + assert (len(walk_directory(d, suffix="conll"))) == 3 + assert (len(walk_directory(d, suffix="pdf"))) == 0 + + +def test_debug_data_trainable_lemmatizer_basic(): + examples = [ + ("She likes green eggs", {"lemmas": ["she", "like", "green", "egg"]}), + ("Eat blue ham", {"lemmas": ["eat", "blue", "ham"]}), + ] + nlp = Language() + train_examples = [] + for t in examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + # ref test_edit_tree_lemmatizer::test_initialize_from_labels + # this results in 4 trees + assert len(data["lemmatizer_trees"]) == 4 + + +def test_debug_data_trainable_lemmatizer_partial(): + partial_examples = [ + # partial annotation + ("She likes green eggs", {"lemmas": ["", "like", "green", ""]}), + # misaligned partial annotation + ( + "He hates green eggs", + { + "words": ["He", "hat", "es", "green", "eggs"], + "lemmas": ["", "hat", "e", "green", ""], + }, + ), + ] + nlp = Language() + train_examples = [] + for t in partial_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["partial_lemma_annotations"] == 2 + + +def test_debug_data_trainable_lemmatizer_low_cardinality(): + low_cardinality_examples = [ + ("She likes green eggs", {"lemmas": ["no", "no", "no", "no"]}), + ("Eat blue ham", {"lemmas": ["no", "no", "no"]}), + ] + nlp = Language() + train_examples = [] + for t in low_cardinality_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["n_low_cardinality_lemmas"] == 2 + + +def test_debug_data_trainable_lemmatizer_not_annotated(): + unannotated_examples = [ + ("She likes green eggs", {}), + ("Eat blue ham", {}), + ] + nlp = Language() + train_examples = [] + for t in unannotated_examples: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + + data = _compile_gold(train_examples, ["trainable_lemmatizer"], nlp, True) + assert data["no_lemma_annotations"] == 2 + + +def test_project_api_imports(): + from spacy.cli import project_run + from spacy.cli.project.run import project_run # noqa: F401, F811 + + +def test_download_rejects_relative_urls(monkeypatch): + """Test that we can't tell spacy download to get an arbitrary model by using a + relative path in the filename""" + + monkeypatch.setattr(download_module, "run_command", lambda cmd: None) + monkeypatch.setattr( + download_module, "_get_pip_install_cmd", lambda: ["pip", "install"] + ) + + # Check that normal download works + download_module.download("en_core_web_sm-3.7.1", direct=True) + with pytest.raises(SystemExit): + download_module.download("../en_core_web_sm-3.7.1", direct=True) diff --git a/spacy/tests/test_cli_app.py b/spacy/tests/test_cli_app.py new file mode 100644 index 0000000..5fe9e9b --- /dev/null +++ b/spacy/tests/test_cli_app.py @@ -0,0 +1,428 @@ +import os +from pathlib import Path + +import pytest +import srsly +from typer.testing import CliRunner + +from spacy.cli._util import app, get_git_version +from spacy.tokens import Doc, DocBin, Span + +from .util import make_tempdir, normalize_whitespace + + +def has_git(): + try: + get_git_version() + return True + except RuntimeError: + return False + + +def test_convert_auto(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.iob"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection works + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "Generated output file" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 3 + assert "data1.spacy" in out_files + assert "data2.spacy" in out_files + assert "data3.spacy" in out_files + + +def test_convert_auto_conflict(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.json"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection warns when there are different file types + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "All input files must be same type" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 0 + + +def test_benchmark_accuracy_alias(): + # Verify that the `evaluate` alias works correctly. + result_benchmark = CliRunner().invoke(app, ["benchmark", "accuracy", "--help"]) + result_evaluate = CliRunner().invoke(app, ["evaluate", "--help"]) + assert normalize_whitespace(result_benchmark.stdout) == normalize_whitespace( + result_evaluate.stdout.replace("spacy evaluate", "spacy benchmark accuracy") + ) + + +def test_debug_data_trainable_lemmatizer_cli(en_vocab): + train_docs = [ + Doc(en_vocab, words=["I", "like", "cats"], lemmas=["I", "like", "cat"]), + Doc( + en_vocab, + words=["Dogs", "are", "great", "too"], + lemmas=["dog", "be", "great", "too"], + ), + ] + dev_docs = [ + Doc(en_vocab, words=["Cats", "are", "cute"], lemmas=["cat", "be", "cute"]), + Doc(en_vocab, words=["Pets", "are", "great"], lemmas=["pet", "be", "great"]), + ] + with make_tempdir() as d_in: + train_bin = DocBin(docs=train_docs) + train_bin.to_disk(d_in / "train.spacy") + dev_bin = DocBin(docs=dev_docs) + dev_bin.to_disk(d_in / "dev.spacy") + # `debug data` requires an input pipeline config + CliRunner().invoke( + app, + [ + "init", + "config", + f"{d_in}/config.cfg", + "--lang", + "en", + "--pipeline", + "trainable_lemmatizer", + ], + ) + result_debug_data = CliRunner().invoke( + app, + [ + "debug", + "data", + f"{d_in}/config.cfg", + "--paths.train", + f"{d_in}/train.spacy", + "--paths.dev", + f"{d_in}/dev.spacy", + ], + ) + # Instead of checking specific wording of the output, which may change, + # we'll check that this section of the debug output is present. + assert "= Trainable Lemmatizer =" in result_debug_data.stdout + + +# project tests + +CFG_FILE = "myconfig.cfg" + +SAMPLE_PROJECT = { + "title": "Sample project", + "description": "This is a project for testing", + "assets": [ + { + "dest": "assets/spacy-readme.md", + "url": "https://github.com/explosion/spaCy/raw/dec81508d28b47f09a06203c472b37f00db6c869/README.md", + "checksum": "411b2c89ccf34288fae8ed126bf652f7", + }, + { + "dest": "assets/citation.cff", + "url": "https://github.com/explosion/spaCy/raw/master/CITATION.cff", + "checksum": "c996bfd80202d480eb2e592369714e5e", + "extra": True, + }, + ], + "commands": [ + { + "name": "ok", + "help": "print ok", + "script": ["python -c \"print('okokok')\""], + }, + { + "name": "create", + "help": "make a file", + "script": [f"python -m spacy init config {CFG_FILE}"], + "outputs": [f"{CFG_FILE}"], + }, + ], +} + +SAMPLE_PROJECT_TEXT = srsly.yaml_dumps(SAMPLE_PROJECT) + + +@pytest.fixture +def project_dir(): + with make_tempdir() as pdir: + (pdir / "project.yml").write_text(SAMPLE_PROJECT_TEXT) + yield pdir + + +def test_project_document(project_dir): + readme_path = project_dir / "README.md" + assert not readme_path.exists(), "README already exists" + result = CliRunner().invoke( + app, ["project", "document", str(project_dir), "-o", str(readme_path)] + ) + assert result.exit_code == 0 + assert readme_path.is_file() + text = readme_path.read_text("utf-8") + assert SAMPLE_PROJECT["description"] in text + + +def test_project_assets(project_dir): + asset_dir = project_dir / "assets" + assert not asset_dir.exists(), "Assets dir is already present" + result = CliRunner().invoke(app, ["project", "assets", str(project_dir)]) + assert result.exit_code == 0 + assert (asset_dir / "spacy-readme.md").is_file(), "Assets not downloaded" + # check that extras work + result = CliRunner().invoke(app, ["project", "assets", "--extra", str(project_dir)]) + assert result.exit_code == 0 + assert (asset_dir / "citation.cff").is_file(), "Extras not downloaded" + + +def test_project_run(project_dir): + # make sure dry run works + test_file = project_dir / CFG_FILE + result = CliRunner().invoke( + app, ["project", "run", "--dry", "create", str(project_dir)] + ) + assert result.exit_code == 0 + assert not test_file.is_file() + result = CliRunner().invoke(app, ["project", "run", "create", str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() + result = CliRunner().invoke(app, ["project", "run", "ok", str(project_dir)]) + assert result.exit_code == 0 + assert "okokok" in result.stdout + + +@pytest.mark.skipif(not has_git(), reason="git not installed") +@pytest.mark.parametrize( + "options", + [ + "", + # "--sparse", + "--branch v3", + "--repo https://github.com/explosion/projects --branch v3", + ], +) +def test_project_clone(options): + with make_tempdir() as workspace: + out = workspace / "project" + target = "benchmarks/ner_conll03" + if not options: + options = [] + else: + options = options.split() + result = CliRunner().invoke( + app, ["project", "clone", target, *options, str(out)] + ) + assert result.exit_code == 0 + assert (out / "README.md").is_file() + + +def test_project_push_pull(project_dir): + proj = dict(SAMPLE_PROJECT) + remote = "xyz" + + with make_tempdir() as remote_dir: + proj["remotes"] = {remote: str(remote_dir)} + proj_text = srsly.yaml_dumps(proj) + (project_dir / "project.yml").write_text(proj_text) + + test_file = project_dir / CFG_FILE + result = CliRunner().invoke(app, ["project", "run", "create", str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() + result = CliRunner().invoke(app, ["project", "push", remote, str(project_dir)]) + assert result.exit_code == 0 + test_file.unlink() + assert not test_file.exists() + result = CliRunner().invoke(app, ["project", "pull", remote, str(project_dir)]) + assert result.exit_code == 0 + assert test_file.is_file() + + +def test_find_function_valid(): + # example of architecture in main code base + function = "spacy.TextCatBOW.v3" + result = CliRunner().invoke(app, ["find-function", function, "-r", "architectures"]) + assert f"Found registered function '{function}'" in result.stdout + assert "textcat.py" in result.stdout + + result = CliRunner().invoke(app, ["find-function", function]) + assert f"Found registered function '{function}'" in result.stdout + assert "textcat.py" in result.stdout + + # example of architecture in spacy-legacy + function = "spacy.TextCatBOW.v1" + result = CliRunner().invoke(app, ["find-function", function]) + assert f"Found registered function '{function}'" in result.stdout + assert "spacy_legacy" in result.stdout + assert "textcat.py" in result.stdout + + +def test_find_function_invalid(): + # invalid registry + function = "spacy.TextCatBOW.v3" + registry = "foobar" + result = CliRunner().invoke( + app, ["find-function", function, "--registry", registry] + ) + assert f"Unknown function registry: '{registry}'" in result.stdout + + # invalid function + function = "spacy.TextCatBOW.v666" + result = CliRunner().invoke(app, ["find-function", function]) + assert f"Couldn't find registered function: '{function}'" in result.stdout + + +example_words_1 = ["I", "like", "cats"] +example_words_2 = ["I", "like", "dogs"] +example_lemmas_1 = ["I", "like", "cat"] +example_lemmas_2 = ["I", "like", "dog"] +example_tags = ["PRP", "VBP", "NNS"] +example_morphs = [ + "Case=Nom|Number=Sing|Person=1|PronType=Prs", + "Tense=Pres|VerbForm=Fin", + "Number=Plur", +] +example_deps = ["nsubj", "ROOT", "dobj"] +example_pos = ["PRON", "VERB", "NOUN"] +example_ents = ["O", "O", "I-ANIMAL"] +example_spans = [(2, 3, "ANIMAL")] + +TRAIN_EXAMPLE_1 = dict( + words=example_words_1, + lemmas=example_lemmas_1, + tags=example_tags, + morphs=example_morphs, + deps=example_deps, + heads=[1, 1, 1], + pos=example_pos, + ents=example_ents, + spans=example_spans, + cats={"CAT": 1.0, "DOG": 0.0}, +) +TRAIN_EXAMPLE_2 = dict( + words=example_words_2, + lemmas=example_lemmas_2, + tags=example_tags, + morphs=example_morphs, + deps=example_deps, + heads=[1, 1, 1], + pos=example_pos, + ents=example_ents, + spans=example_spans, + cats={"CAT": 0.0, "DOG": 1.0}, +) + + +@pytest.mark.slow +@pytest.mark.parametrize( + "component,examples", + [ + ("tagger", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ("morphologizer", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ("trainable_lemmatizer", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ("parser", [TRAIN_EXAMPLE_1] * 30), + ("ner", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ("spancat", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ("textcat", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2]), + ], +) +def test_init_config_trainable(component, examples, en_vocab): + if component == "textcat": + train_docs = [] + for example in examples: + doc = Doc(en_vocab, words=example["words"]) + doc.cats = example["cats"] + train_docs.append(doc) + elif component == "spancat": + train_docs = [] + for example in examples: + doc = Doc(en_vocab, words=example["words"]) + doc.spans["sc"] = [ + Span(doc, start, end, label) for start, end, label in example["spans"] + ] + train_docs.append(doc) + else: + train_docs = [] + for example in examples: + # cats, spans are not valid kwargs for instantiating a Doc + example = {k: v for k, v in example.items() if k not in ("cats", "spans")} + doc = Doc(en_vocab, **example) + train_docs.append(doc) + + with make_tempdir() as d_in: + train_bin = DocBin(docs=train_docs) + train_bin.to_disk(d_in / "train.spacy") + dev_bin = DocBin(docs=train_docs) + dev_bin.to_disk(d_in / "dev.spacy") + init_config_result = CliRunner().invoke( + app, + [ + "init", + "config", + f"{d_in}/config.cfg", + "--lang", + "en", + "--pipeline", + component, + ], + ) + assert init_config_result.exit_code == 0 + train_result = CliRunner().invoke( + app, + [ + "train", + f"{d_in}/config.cfg", + "--paths.train", + f"{d_in}/train.spacy", + "--paths.dev", + f"{d_in}/dev.spacy", + "--output", + f"{d_in}/model", + ], + ) + assert train_result.exit_code == 0 + assert Path(d_in / "model" / "model-last").exists() + + +@pytest.mark.slow +@pytest.mark.parametrize( + "component,examples", + [("tagger,parser,morphologizer", [TRAIN_EXAMPLE_1, TRAIN_EXAMPLE_2] * 15)], +) +def test_init_config_trainable_multiple(component, examples, en_vocab): + train_docs = [] + for example in examples: + example = {k: v for k, v in example.items() if k not in ("cats", "spans")} + doc = Doc(en_vocab, **example) + train_docs.append(doc) + + with make_tempdir() as d_in: + train_bin = DocBin(docs=train_docs) + train_bin.to_disk(d_in / "train.spacy") + dev_bin = DocBin(docs=train_docs) + dev_bin.to_disk(d_in / "dev.spacy") + init_config_result = CliRunner().invoke( + app, + [ + "init", + "config", + f"{d_in}/config.cfg", + "--lang", + "en", + "--pipeline", + component, + ], + ) + assert init_config_result.exit_code == 0 + train_result = CliRunner().invoke( + app, + [ + "train", + f"{d_in}/config.cfg", + "--paths.train", + f"{d_in}/train.spacy", + "--paths.dev", + f"{d_in}/dev.spacy", + "--output", + f"{d_in}/model", + ], + ) + assert train_result.exit_code == 0 + assert Path(d_in / "model" / "model-last").exists() diff --git a/spacy/tests/test_displacy.py b/spacy/tests/test_displacy.py new file mode 100644 index 0000000..b83c7db --- /dev/null +++ b/spacy/tests/test_displacy.py @@ -0,0 +1,490 @@ +import numpy +import pytest + +from spacy import displacy +from spacy.displacy.render import DependencyRenderer, EntityRenderer, SpanRenderer +from spacy.lang.en import English +from spacy.lang.fa import Persian +from spacy.tokens import Doc, Span + + +@pytest.mark.issue(2361) +def test_issue2361(de_vocab): + """Test if < is escaped when rendering""" + chars = ("<", ">", "&", """) + words = ["<", ">", "&", '"'] + doc = Doc(de_vocab, words=words, deps=["dep"] * len(words)) + html = displacy.render(doc) + for char in chars: + assert char in html + + +@pytest.mark.issue(2728) +def test_issue2728(en_vocab): + """Test that displaCy ENT visualizer escapes HTML correctly.""" + doc = Doc(en_vocab, words=["test", "", "test"]) + doc.ents = [Span(doc, 0, 1, label="TEST")] + html = displacy.render(doc, style="ent") + assert "<RELEASE>" in html + doc.ents = [Span(doc, 1, 2, label="TEST")] + html = displacy.render(doc, style="ent") + assert "<RELEASE>" in html + + +@pytest.mark.issue(3288) +def test_issue3288(en_vocab): + """Test that retokenization works correctly via displaCy when punctuation + is merged onto the preceeding token and tensor is resized.""" + words = ["Hello", "World", "!", "When", "is", "this", "breaking", "?"] + heads = [1, 1, 1, 4, 4, 6, 4, 4] + deps = ["intj", "ROOT", "punct", "advmod", "ROOT", "det", "nsubj", "punct"] + doc = Doc(en_vocab, words=words, heads=heads, deps=deps) + doc.tensor = numpy.zeros((len(words), 96), dtype="float32") + displacy.render(doc) + + +@pytest.mark.issue(3531) +def test_issue3531(): + """Test that displaCy renderer doesn't require "settings" key.""" + example_dep = { + "words": [ + {"text": "But", "tag": "CCONJ"}, + {"text": "Google", "tag": "PROPN"}, + {"text": "is", "tag": "VERB"}, + {"text": "starting", "tag": "VERB"}, + {"text": "from", "tag": "ADP"}, + {"text": "behind.", "tag": "ADV"}, + ], + "arcs": [ + {"start": 0, "end": 3, "label": "cc", "dir": "left"}, + {"start": 1, "end": 3, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "aux", "dir": "left"}, + {"start": 3, "end": 4, "label": "prep", "dir": "right"}, + {"start": 4, "end": 5, "label": "pcomp", "dir": "right"}, + ], + } + example_ent = { + "text": "But Google is starting from behind.", + "ents": [{"start": 4, "end": 10, "label": "ORG"}], + } + dep_html = displacy.render(example_dep, style="dep", manual=True) + assert dep_html + ent_html = displacy.render(example_ent, style="ent", manual=True) + assert ent_html + + +@pytest.mark.issue(3882) +def test_issue3882(en_vocab): + """Test that displaCy doesn't serialize the doc.user_data when making a + copy of the Doc. + """ + doc = Doc(en_vocab, words=["Hello", "world"], deps=["dep", "dep"]) + doc.user_data["test"] = set() + displacy.parse_deps(doc) + + +@pytest.mark.issue(5447) +def test_issue5447(): + """Test that overlapping arcs get separate levels, unless they're identical.""" + renderer = DependencyRenderer() + words = [ + {"text": "This", "tag": "DT"}, + {"text": "is", "tag": "VBZ"}, + {"text": "a", "tag": "DT"}, + {"text": "sentence.", "tag": "NN"}, + ] + arcs = [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 2, "end": 3, "label": "overlap", "dir": "left"}, + {"end": 3, "label": "overlap", "start": 2, "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "left"}, + ] + renderer.render([{"words": words, "arcs": arcs}]) + assert renderer.highest_level == 3 + + +@pytest.mark.issue(5838) +def test_issue5838(): + # Displacy's EntityRenderer break line + # not working after last entity + sample_text = "First line\nSecond line, with ent\nThird line\nFourth line\n" + nlp = English() + doc = nlp(sample_text) + doc.ents = [Span(doc, 7, 8, label="test")] + html = displacy.render(doc, style="ent") + found = html.count("
") + assert found == 4 + + +def test_displacy_parse_spans(en_vocab): + """Test that spans on a Doc are converted into displaCy's format.""" + doc = Doc(en_vocab, words=["Welcome", "to", "the", "Bank", "of", "China"]) + doc.spans["sc"] = [Span(doc, 3, 6, "ORG"), Span(doc, 5, 6, "GPE")] + spans = displacy.parse_spans(doc) + assert isinstance(spans, dict) + assert spans["text"] == "Welcome to the Bank of China " + assert spans["spans"] == [ + { + "start": 15, + "end": 28, + "start_token": 3, + "end_token": 6, + "label": "ORG", + "kb_id": "", + "kb_url": "#", + }, + { + "start": 23, + "end": 28, + "start_token": 5, + "end_token": 6, + "label": "GPE", + "kb_id": "", + "kb_url": "#", + }, + ] + + +def test_displacy_parse_spans_with_kb_id_options(en_vocab): + """Test that spans with kb_id on a Doc are converted into displaCy's format""" + doc = Doc(en_vocab, words=["Welcome", "to", "the", "Bank", "of", "China"]) + doc.spans["sc"] = [ + Span(doc, 3, 6, "ORG", kb_id="Q790068"), + Span(doc, 5, 6, "GPE", kb_id="Q148"), + ] + + spans = displacy.parse_spans( + doc, {"kb_url_template": "https://wikidata.org/wiki/{}"} + ) + assert isinstance(spans, dict) + assert spans["text"] == "Welcome to the Bank of China " + assert spans["spans"] == [ + { + "start": 15, + "end": 28, + "start_token": 3, + "end_token": 6, + "label": "ORG", + "kb_id": "Q790068", + "kb_url": "https://wikidata.org/wiki/Q790068", + }, + { + "start": 23, + "end": 28, + "start_token": 5, + "end_token": 6, + "label": "GPE", + "kb_id": "Q148", + "kb_url": "https://wikidata.org/wiki/Q148", + }, + ] + + +def test_displacy_parse_spans_different_spans_key(en_vocab): + """Test that spans in a different spans key will be parsed""" + doc = Doc(en_vocab, words=["Welcome", "to", "the", "Bank", "of", "China"]) + doc.spans["sc"] = [Span(doc, 3, 6, "ORG"), Span(doc, 5, 6, "GPE")] + doc.spans["custom"] = [Span(doc, 3, 6, "BANK")] + spans = displacy.parse_spans(doc, options={"spans_key": "custom"}) + + assert isinstance(spans, dict) + assert spans["text"] == "Welcome to the Bank of China " + assert spans["spans"] == [ + { + "start": 15, + "end": 28, + "start_token": 3, + "end_token": 6, + "label": "BANK", + "kb_id": "", + "kb_url": "#", + } + ] + + +def test_displacy_parse_empty_spans_key(en_vocab): + """Test that having an unset spans key doesn't raise an error""" + doc = Doc(en_vocab, words=["Welcome", "to", "the", "Bank", "of", "China"]) + doc.spans["custom"] = [Span(doc, 3, 6, "BANK")] + with pytest.warns(UserWarning, match="W117"): + spans = displacy.parse_spans(doc) + + assert isinstance(spans, dict) + + +def test_displacy_parse_ents(en_vocab): + """Test that named entities on a Doc are converted into displaCy's format.""" + doc = Doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"]) + doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])] + ents = displacy.parse_ents(doc) + assert isinstance(ents, dict) + assert ents["text"] == "But Google is starting from behind " + assert ents["ents"] == [ + {"start": 4, "end": 10, "label": "ORG", "kb_id": "", "kb_url": "#"} + ] + + doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"], kb_id="Q95")] + ents = displacy.parse_ents(doc) + assert isinstance(ents, dict) + assert ents["text"] == "But Google is starting from behind " + assert ents["ents"] == [ + {"start": 4, "end": 10, "label": "ORG", "kb_id": "Q95", "kb_url": "#"} + ] + + +def test_displacy_parse_ents_with_kb_id_options(en_vocab): + """Test that named entities with kb_id on a Doc are converted into displaCy's format.""" + doc = Doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"]) + doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"], kb_id="Q95")] + + ents = displacy.parse_ents( + doc, {"kb_url_template": "https://www.wikidata.org/wiki/{}"} + ) + assert isinstance(ents, dict) + assert ents["text"] == "But Google is starting from behind " + assert ents["ents"] == [ + { + "start": 4, + "end": 10, + "label": "ORG", + "kb_id": "Q95", + "kb_url": "https://www.wikidata.org/wiki/Q95", + } + ] + + +def test_displacy_parse_deps(en_vocab): + """Test that deps and tags on a Doc are converted into displaCy's format.""" + words = ["This", "is", "a", "sentence"] + heads = [1, 1, 3, 1] + pos = ["DET", "VERB", "DET", "NOUN"] + tags = ["DT", "VBZ", "DT", "NN"] + deps = ["nsubj", "ROOT", "det", "attr"] + doc = Doc(en_vocab, words=words, heads=heads, pos=pos, tags=tags, deps=deps) + deps = displacy.parse_deps(doc) + assert isinstance(deps, dict) + assert deps["words"] == [ + {"lemma": None, "text": words[0], "tag": pos[0]}, + {"lemma": None, "text": words[1], "tag": pos[1]}, + {"lemma": None, "text": words[2], "tag": pos[2]}, + {"lemma": None, "text": words[3], "tag": pos[3]}, + ] + assert deps["arcs"] == [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "right"}, + ] + # Test that displacy.parse_deps converts Span to Doc + deps = displacy.parse_deps(doc[:]) + assert isinstance(deps, dict) + assert deps["words"] == [ + {"lemma": None, "text": words[0], "tag": pos[0]}, + {"lemma": None, "text": words[1], "tag": pos[1]}, + {"lemma": None, "text": words[2], "tag": pos[2]}, + {"lemma": None, "text": words[3], "tag": pos[3]}, + ] + assert deps["arcs"] == [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "right"}, + ] + + +def test_displacy_invalid_arcs(): + renderer = DependencyRenderer() + words = [{"text": "This", "tag": "DET"}, {"text": "is", "tag": "VERB"}] + arcs = [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": -1, "end": 2, "label": "det", "dir": "left"}, + ] + with pytest.raises(ValueError): + renderer.render([{"words": words, "arcs": arcs}]) + + +def test_displacy_spans(en_vocab): + """Test that displaCy can render Spans.""" + doc = Doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"]) + doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])] + html = displacy.render(doc[1:4], style="ent") + assert html.startswith("TEST") + # Restore + displacy.set_render_wrapper(lambda html: html) + + +def test_displacy_render_manual_dep(): + """Test displacy.render with manual data for dep style""" + parsed_dep = { + "words": [ + {"text": "This", "tag": "DT"}, + {"text": "is", "tag": "VBZ"}, + {"text": "a", "tag": "DT"}, + {"text": "sentence", "tag": "NN"}, + ], + "arcs": [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "right"}, + ], + "title": "Title", + } + html = displacy.render([parsed_dep], style="dep", manual=True) + for word in parsed_dep["words"]: + assert word["text"] in html + assert word["tag"] in html + + +def test_displacy_render_manual_ent(): + """Test displacy.render with manual data for ent style""" + parsed_ents = [ + { + "text": "But Google is starting from behind.", + "ents": [{"start": 4, "end": 10, "label": "ORG"}], + }, + { + "text": "But Google is starting from behind.", + "ents": [{"start": -100, "end": 100, "label": "COMPANY"}], + "title": "Title", + }, + ] + + html = displacy.render(parsed_ents, style="ent", manual=True) + for parsed_ent in parsed_ents: + assert parsed_ent["ents"][0]["label"] in html + if "title" in parsed_ent: + assert parsed_ent["title"] in html + + +def test_displacy_render_manual_span(): + """Test displacy.render with manual data for span style""" + parsed_spans = [ + { + "text": "Welcome to the Bank of China.", + "spans": [ + {"start_token": 3, "end_token": 6, "label": "ORG"}, + {"start_token": 5, "end_token": 6, "label": "GPE"}, + ], + "tokens": ["Welcome", "to", "the", "Bank", "of", "China", "."], + }, + { + "text": "Welcome to the Bank of China.", + "spans": [ + {"start_token": 3, "end_token": 6, "label": "ORG"}, + {"start_token": 5, "end_token": 6, "label": "GPE"}, + ], + "tokens": ["Welcome", "to", "the", "Bank", "of", "China", "."], + "title": "Title", + }, + ] + + html = displacy.render(parsed_spans, style="span", manual=True) + for parsed_span in parsed_spans: + assert parsed_span["spans"][0]["label"] in html + if "title" in parsed_span: + assert parsed_span["title"] in html + + +def test_displacy_options_case(): + ents = ["foo", "BAR"] + colors = {"FOO": "red", "bar": "green"} + renderer = EntityRenderer({"ents": ents, "colors": colors}) + text = "abcd" + labels = ["foo", "bar", "FOO", "BAR"] + spans = [{"start": i, "end": i + 1, "label": labels[i]} for i in range(len(text))] + result = renderer.render_ents("abcde", spans, None).split("\n\n") + assert "red" in result[0] and "foo" in result[0] + assert "green" in result[1] and "bar" in result[1] + assert "red" in result[2] and "FOO" in result[2] + assert "green" in result[3] and "BAR" in result[3] + + +@pytest.mark.issue(10672) +def test_displacy_manual_sorted_entities(): + doc = { + "text": "But Google is starting from behind.", + "ents": [ + {"start": 14, "end": 22, "label": "SECOND"}, + {"start": 4, "end": 10, "label": "FIRST"}, + ], + "title": None, + } + + html = displacy.render(doc, style="ent", manual=True) + assert html.find("FIRST") < html.find("SECOND") + + +@pytest.mark.issue(12816) +def test_issue12816(en_vocab) -> None: + """Test that displaCy's span visualizer escapes annotated HTML tags correctly.""" + # Create a doc containing an annotated word and an unannotated HTML tag + doc = Doc(en_vocab, words=["test", ""]) + doc.spans["sc"] = [Span(doc, 0, 1, label="test")] + + # Verify that the HTML tag is escaped when unannotated + html = displacy.render(doc, style="span") + assert "<TEST>" in html + + # Annotate the HTML tag + doc.spans["sc"].append(Span(doc, 1, 2, label="test")) + + # Verify that the HTML tag is still escaped + html = displacy.render(doc, style="span") + assert "<TEST>" in html + + +@pytest.mark.issue(13056) +def test_displacy_span_stacking(): + """Test whether span stacking works properly for multiple overlapping spans.""" + spans = [ + {"start_token": 2, "end_token": 5, "label": "SkillNC"}, + {"start_token": 0, "end_token": 2, "label": "Skill"}, + {"start_token": 1, "end_token": 3, "label": "Skill"}, + ] + tokens = ["Welcome", "to", "the", "Bank", "of", "China", "."] + per_token_info = SpanRenderer._assemble_per_token_info(spans=spans, tokens=tokens) + + assert len(per_token_info) == len(tokens) + assert all([len(per_token_info[i]["entities"]) == 1 for i in (0, 3, 4)]) + assert all([len(per_token_info[i]["entities"]) == 2 for i in (1, 2)]) + assert per_token_info[1]["entities"][0]["render_slot"] == 1 + assert per_token_info[1]["entities"][1]["render_slot"] == 2 + assert per_token_info[2]["entities"][0]["render_slot"] == 2 + assert per_token_info[2]["entities"][1]["render_slot"] == 3 diff --git a/spacy/tests/test_errors.py b/spacy/tests/test_errors.py new file mode 100644 index 0000000..a845a52 --- /dev/null +++ b/spacy/tests/test_errors.py @@ -0,0 +1,16 @@ +from inspect import isclass + +import pytest + +from spacy.errors import ErrorsWithCodes + + +class Errors(metaclass=ErrorsWithCodes): + E001 = "error description" + + +def test_add_codes(): + assert Errors.E001 == "[E001] error description" + with pytest.raises(AttributeError): + Errors.E002 + assert isclass(Errors.__class__) diff --git a/spacy/tests/test_factory_imports.py b/spacy/tests/test_factory_imports.py new file mode 100644 index 0000000..7a1b4a7 --- /dev/null +++ b/spacy/tests/test_factory_imports.py @@ -0,0 +1,85 @@ +# coding: utf-8 +"""Test factory import compatibility from original and new locations.""" + +import importlib + +import pytest + + +@pytest.mark.parametrize( + "factory_name,original_module,compat_module", + [ + ("make_tagger", "spacy.pipeline.factories", "spacy.pipeline.tagger"), + ("make_sentencizer", "spacy.pipeline.factories", "spacy.pipeline.sentencizer"), + ("make_ner", "spacy.pipeline.factories", "spacy.pipeline.ner"), + ("make_parser", "spacy.pipeline.factories", "spacy.pipeline.dep_parser"), + ("make_tok2vec", "spacy.pipeline.factories", "spacy.pipeline.tok2vec"), + ("make_spancat", "spacy.pipeline.factories", "spacy.pipeline.spancat"), + ( + "make_spancat_singlelabel", + "spacy.pipeline.factories", + "spacy.pipeline.spancat", + ), + ("make_lemmatizer", "spacy.pipeline.factories", "spacy.pipeline.lemmatizer"), + ("make_entity_ruler", "spacy.pipeline.factories", "spacy.pipeline.entityruler"), + ("make_span_ruler", "spacy.pipeline.factories", "spacy.pipeline.span_ruler"), + ( + "make_edit_tree_lemmatizer", + "spacy.pipeline.factories", + "spacy.pipeline.edit_tree_lemmatizer", + ), + ( + "make_attribute_ruler", + "spacy.pipeline.factories", + "spacy.pipeline.attributeruler", + ), + ( + "make_entity_linker", + "spacy.pipeline.factories", + "spacy.pipeline.entity_linker", + ), + ("make_textcat", "spacy.pipeline.factories", "spacy.pipeline.textcat"), + ("make_token_splitter", "spacy.pipeline.factories", "spacy.pipeline.functions"), + ("make_doc_cleaner", "spacy.pipeline.factories", "spacy.pipeline.functions"), + ( + "make_morphologizer", + "spacy.pipeline.factories", + "spacy.pipeline.morphologizer", + ), + ("make_senter", "spacy.pipeline.factories", "spacy.pipeline.senter"), + ("make_span_finder", "spacy.pipeline.factories", "spacy.pipeline.span_finder"), + ( + "make_multilabel_textcat", + "spacy.pipeline.factories", + "spacy.pipeline.textcat_multilabel", + ), + ("make_beam_ner", "spacy.pipeline.factories", "spacy.pipeline.ner"), + ("make_beam_parser", "spacy.pipeline.factories", "spacy.pipeline.dep_parser"), + ("make_nn_labeller", "spacy.pipeline.factories", "spacy.pipeline.multitask"), + # This one's special because the function was named make_span_ruler, so + # the name in the registrations.py doesn't match the name we make the import hook + # point to. We could make a test just for this but shrug + # ("make_future_entity_ruler", "spacy.pipeline.factories", "spacy.pipeline.span_ruler"), + ], +) +def test_factory_import_compatibility(factory_name, original_module, compat_module): + """Test that factory functions can be imported from both original and compatibility locations.""" + # Import from the original module (registrations.py) + original_module_obj = importlib.import_module(original_module) + original_factory = getattr(original_module_obj, factory_name) + assert original_factory is not None, ( + f"Could not import {factory_name} from {original_module}" + ) + + # Import from the compatibility module (component file) + compat_module_obj = importlib.import_module(compat_module) + compat_factory = getattr(compat_module_obj, factory_name) + assert compat_factory is not None, ( + f"Could not import {factory_name} from {compat_module}" + ) + + # Test that they're the same function (identity) + assert original_factory is compat_factory, ( + f"Factory {factory_name} imported from {original_module} is not the same object " + f"as the one imported from {compat_module}" + ) diff --git a/spacy/tests/test_factory_registrations.py b/spacy/tests/test_factory_registrations.py new file mode 100644 index 0000000..eb69265 --- /dev/null +++ b/spacy/tests/test_factory_registrations.py @@ -0,0 +1,94 @@ +import json +from pathlib import Path + +import pytest + +from spacy.util import registry + +# Path to the reference factory registrations, relative to this file +REFERENCE_FILE = Path(__file__).parent / "factory_registrations.json" + +# Monkey patch the util.is_same_func to handle Cython functions + +from spacy import util + +original_is_same_func = util.is_same_func + + +def patched_is_same_func(func1, func2): + # Handle Cython functions + try: + return original_is_same_func(func1, func2) + except TypeError: + # For Cython functions, just compare the string representation + return str(func1) == str(func2) + + +util.is_same_func = patched_is_same_func + + +@pytest.fixture +def reference_factory_registrations(): + """Load reference factory registrations from JSON file""" + if not REFERENCE_FILE.exists(): + pytest.fail( + f"Reference file {REFERENCE_FILE} not found. Run export_factory_registrations.py first." + ) + + with REFERENCE_FILE.open("r") as f: + return json.load(f) + + +def test_factory_registrations_preserved(reference_factory_registrations): + """Test that all factory registrations from the reference file are still present.""" + # Ensure the registry is populated + registry.ensure_populated() + + # Get all factory registrations + all_factories = registry.factories.get_all() + + # Initialize our data structure to store current factory registrations + current_registrations = {} + + # Process factory registrations + for name, func in all_factories.items(): + # Store information about each factory + try: + module_name = func.__module__ + except (AttributeError, TypeError): + # For Cython functions, just use a placeholder + module_name = str(func).split()[1].split(".")[0] + + try: + func_name = func.__qualname__ + except (AttributeError, TypeError): + # For Cython functions, use the function's name + func_name = ( + func.__name__ + if hasattr(func, "__name__") + else str(func).split()[1].split(".")[-1] + ) + + current_registrations[name] = { + "name": name, + "module": module_name, + "function": func_name, + } + + # Check for missing registrations + missing_registrations = set(reference_factory_registrations.keys()) - set( + current_registrations.keys() + ) + assert not missing_registrations, ( + f"Missing factory registrations: {', '.join(sorted(missing_registrations))}" + ) + + # Check for new registrations (not an error, but informative) + new_registrations = set(current_registrations.keys()) - set( + reference_factory_registrations.keys() + ) + if new_registrations: + # This is not an error, just informative + print( + f"New factory registrations found: {', '.join(sorted(new_registrations))}" + ) diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py new file mode 100644 index 0000000..9818d5d --- /dev/null +++ b/spacy/tests/test_language.py @@ -0,0 +1,800 @@ +import itertools +import logging +import warnings +from unittest import mock + +import pytest +from thinc.api import CupyOps, NumpyOps, get_current_ops + +import spacy +from spacy.lang.de import German +from spacy.lang.en import English +from spacy.language import Language +from spacy.scorer import Scorer +from spacy.tokens import Doc, Span +from spacy.training import Example +from spacy.util import find_matching_language, ignore_error, raise_error, registry +from spacy.vocab import Vocab + +from .util import add_vecs_to_vocab, assert_docs_equal + +try: + import torch + + # Ensure that we don't deadlock in multiprocessing tests. + torch.set_num_threads(1) + torch.set_num_interop_threads(1) +except ImportError: + pass + + +def evil_component(doc): + if "2" in doc.text: + raise ValueError("no dice") + return doc + + +def perhaps_set_sentences(doc): + if not doc.text.startswith("4"): + doc[-1].is_sent_start = True + return doc + + +def assert_sents_error(doc): + if not doc.has_annotation("SENT_START"): + raise ValueError("no sents") + return doc + + +def warn_error(proc_name, proc, docs, e): + logger = logging.getLogger("spacy") + logger.warning("Trouble with component %s.", proc_name) + + +@pytest.fixture +def nlp(): + nlp = Language(Vocab()) + textcat = nlp.add_pipe("textcat") + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + nlp.initialize() + return nlp + + +def test_language_update(nlp): + text = "hello world" + annots = {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}} + wrongkeyannots = {"LABEL": True} + doc = Doc(nlp.vocab, words=text.split(" ")) + example = Example.from_dict(doc, annots) + nlp.update([example]) + + # Not allowed to call with just one Example + with pytest.raises(TypeError): + nlp.update(example) + + # Update with text and dict: not supported anymore since v.3 + with pytest.raises(TypeError): + nlp.update((text, annots)) + # Update with doc object and dict + with pytest.raises(TypeError): + nlp.update((doc, annots)) + + # Create examples badly + with pytest.raises(ValueError): + example = Example.from_dict(doc, None) + with pytest.raises(KeyError): + example = Example.from_dict(doc, wrongkeyannots) + + +def test_language_evaluate(nlp): + text = "hello world" + annots = {"doc_annotation": {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}} + doc = Doc(nlp.vocab, words=text.split(" ")) + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + assert scores["speed"] > 0 + + # test with generator + scores = nlp.evaluate(eg for eg in [example]) + assert scores["speed"] > 0 + + # Not allowed to call with just one Example + with pytest.raises(TypeError): + nlp.evaluate(example) + + # Evaluate with text and dict: not supported anymore since v.3 + with pytest.raises(TypeError): + nlp.evaluate([(text, annots)]) + # Evaluate with doc object and dict + with pytest.raises(TypeError): + nlp.evaluate([(doc, annots)]) + with pytest.raises(TypeError): + nlp.evaluate([text, annots]) + + +def test_evaluate_no_pipe(nlp): + """Test that docs are processed correctly within Language.pipe if the + component doesn't expose a .pipe method.""" + + @Language.component("test_evaluate_no_pipe") + def pipe(doc): + return doc + + text = "hello world" + annots = {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}} + nlp = Language(Vocab()) + doc = nlp(text) + nlp.add_pipe("test_evaluate_no_pipe") + nlp.evaluate([Example.from_dict(doc, annots)]) + + +def test_evaluate_textcat_multilabel(en_vocab): + """Test that evaluate works with a multilabel textcat pipe.""" + nlp = Language(en_vocab) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = {"cats": {"FEATURE": 1.0, "QUESTION": 1.0}} + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + labels = nlp.get_pipe("textcat_multilabel").labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_final(en_vocab): + """Test that evaluate evaluates the final textcat component in a pipeline + with more than one textcat or textcat_multilabel.""" + nlp = Language(en_vocab) + textcat = nlp.add_pipe("textcat") + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # get the labels from the final pipe + labels = nlp.get_pipe(nlp.pipe_names[-1]).labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_separate(en_vocab): + """Test that evaluate can evaluate multiple textcat components separately + with custom scorers.""" + + def custom_textcat_score(examples, **kwargs): + scores = Scorer.score_cats( + examples, + "cats", + multi_label=False, + **kwargs, + ) + return {f"custom_{k}": v for k, v in scores.items()} + + @spacy.registry.scorers("test_custom_textcat_scorer") + def make_custom_textcat_scorer(): + return custom_textcat_score + + nlp = Language(en_vocab) + textcat = nlp.add_pipe( + "textcat", + config={"scorer": {"@scorers": "test_custom_textcat_scorer"}}, + ) + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # check custom scores for the textcat pipe + assert "custom_cats_f_per_type" in scores + labels = nlp.get_pipe("textcat").labels + assert set(scores["custom_cats_f_per_type"].keys()) == set(labels) + # check default scores for the textcat_multilabel pipe + assert "cats_f_per_type" in scores + labels = nlp.get_pipe("textcat_multilabel").labels + assert set(scores["cats_f_per_type"].keys()) == set(labels) + + +def vector_modification_pipe(doc): + doc.vector += 1 + return doc + + +def userdata_pipe(doc): + doc.user_data["foo"] = "bar" + return doc + + +def ner_pipe(doc): + span = Span(doc, 0, 1, label="FIRST") + doc.ents += (span,) + return doc + + +@pytest.fixture +def sample_vectors(): + return [ + ("spacy", [-0.1, -0.2, -0.3]), + ("world", [-0.2, -0.3, -0.4]), + ("pipe", [0.7, 0.8, 0.9]), + ] + + +@pytest.fixture +def nlp2(nlp, sample_vectors): + Language.component( + "test_language_vector_modification_pipe", func=vector_modification_pipe + ) + Language.component("test_language_userdata_pipe", func=userdata_pipe) + Language.component("test_language_ner_pipe", func=ner_pipe) + add_vecs_to_vocab(nlp.vocab, sample_vectors) + nlp.add_pipe("test_language_vector_modification_pipe") + nlp.add_pipe("test_language_ner_pipe") + nlp.add_pipe("test_language_userdata_pipe") + return nlp + + +@pytest.fixture +def texts(): + data = [ + "Hello world.", + "This is spacy.", + "You can use multiprocessing with pipe method.", + "Please try!", + ] + return data + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe(nlp2, n_process, texts): + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + texts = texts * 10 + expecteds = [nlp2(text) for text in texts] + docs = nlp2.pipe(texts, n_process=n_process, batch_size=2) + + for doc, expected_doc in zip(docs, expecteds): + assert_docs_equal(doc, expected_doc) + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_stream(nlp2, n_process, texts): + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + # check if nlp.pipe can handle infinite length iterator properly. + stream_texts = itertools.cycle(texts) + texts0, texts1 = itertools.tee(stream_texts) + expecteds = (nlp2(text) for text in texts0) + docs = nlp2.pipe(texts1, n_process=n_process, batch_size=2) + + n_fetch = 20 + for doc, expected_doc in itertools.islice(zip(docs, expecteds), n_fetch): + assert_docs_equal(doc, expected_doc) + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler(n_process): + """Test that the error handling of nlp.pipe works well""" + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + nlp = English() + nlp.add_pipe("merge_subtokens") + nlp.initialize() + texts = ["Curious to see what will happen to this text.", "And this one."] + # the pipeline fails because there's no parser + with pytest.raises(ValueError): + nlp(texts[0]) + with pytest.raises(ValueError): + list(nlp.pipe(texts, n_process=n_process)) + nlp.set_error_handler(raise_error) + with pytest.raises(ValueError): + list(nlp.pipe(texts, n_process=n_process)) + # set explicitly to ignoring + nlp.set_error_handler(ignore_error) + docs = list(nlp.pipe(texts, n_process=n_process)) + assert len(docs) == 0 + nlp(texts[0]) + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler_custom(en_vocab, n_process): + """Test the error handling of a custom component that has no pipe method""" + Language.component("my_evil_component", func=evil_component) + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + nlp = English() + nlp.add_pipe("my_evil_component") + texts = ["TEXT 111", "TEXT 222", "TEXT 333", "TEXT 342", "TEXT 666"] + with pytest.raises(ValueError): + # the evil custom component throws an error + list(nlp.pipe(texts)) + + nlp.set_error_handler(warn_error) + logger = logging.getLogger("spacy") + with mock.patch.object(logger, "warning") as mock_warning: + # the errors by the evil custom component raise a warning for each + # bad doc + docs = list(nlp.pipe(texts, n_process=n_process)) + # HACK/TODO? the warnings in child processes don't seem to be + # detected by the mock logger + if n_process == 1: + mock_warning.assert_called() + assert mock_warning.call_count == 2 + assert len(docs) + mock_warning.call_count == len(texts) + assert [doc.text for doc in docs] == ["TEXT 111", "TEXT 333", "TEXT 666"] + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler_input_as_tuples(en_vocab, n_process): + """Test the error handling of nlp.pipe with input as tuples""" + Language.component("my_evil_component", func=evil_component) + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + nlp = English() + nlp.add_pipe("my_evil_component") + texts = [ + ("TEXT 111", 111), + ("TEXT 222", 222), + ("TEXT 333", 333), + ("TEXT 342", 342), + ("TEXT 666", 666), + ] + with pytest.raises(ValueError): + list(nlp.pipe(texts, as_tuples=True)) + nlp.set_error_handler(warn_error) + logger = logging.getLogger("spacy") + with mock.patch.object(logger, "warning") as mock_warning: + tuples = list(nlp.pipe(texts, as_tuples=True, n_process=n_process)) + # HACK/TODO? the warnings in child processes don't seem to be + # detected by the mock logger + if n_process == 1: + mock_warning.assert_called() + assert mock_warning.call_count == 2 + assert len(tuples) + mock_warning.call_count == len(texts) + assert (tuples[0][0].text, tuples[0][1]) == ("TEXT 111", 111) + assert (tuples[1][0].text, tuples[1][1]) == ("TEXT 333", 333) + assert (tuples[2][0].text, tuples[2][1]) == ("TEXT 666", 666) + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler_pipe(en_vocab, n_process): + """Test the error handling of a component's pipe method""" + Language.component("my_perhaps_sentences", func=perhaps_set_sentences) + Language.component("assert_sents_error", func=assert_sents_error) + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + texts = [f"{str(i)} is enough. Done" for i in range(100)] + nlp = English() + nlp.add_pipe("my_perhaps_sentences") + nlp.add_pipe("assert_sents_error") + nlp.initialize() + with pytest.raises(ValueError): + # assert_sents_error requires sentence boundaries, will throw an error otherwise + docs = list(nlp.pipe(texts, n_process=n_process, batch_size=10)) + nlp.set_error_handler(ignore_error) + docs = list(nlp.pipe(texts, n_process=n_process, batch_size=10)) + # we lose/ignore the failing 4,40-49 docs + assert len(docs) == 89 + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler_make_doc_actual(n_process): + """Test the error handling for make_doc""" + # TODO: fix so that the following test is the actual behavior + + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + nlp = English() + nlp.max_length = 10 + texts = ["12345678901234567890", "12345"] * 10 + with pytest.raises(ValueError): + list(nlp.pipe(texts, n_process=n_process)) + nlp.default_error_handler = ignore_error + if n_process == 1: + with pytest.raises(ValueError): + list(nlp.pipe(texts, n_process=n_process)) + else: + docs = list(nlp.pipe(texts, n_process=n_process)) + assert len(docs) == 0 + + +@pytest.mark.xfail +@pytest.mark.parametrize("n_process", [1, 2]) +def test_language_pipe_error_handler_make_doc_preferred(n_process): + """Test the error handling for make_doc""" + + ops = get_current_ops() + if isinstance(ops, NumpyOps) or n_process < 2: + nlp = English() + nlp.max_length = 10 + texts = ["12345678901234567890", "12345"] * 10 + with pytest.raises(ValueError): + list(nlp.pipe(texts, n_process=n_process)) + nlp.default_error_handler = ignore_error + docs = list(nlp.pipe(texts, n_process=n_process)) + assert len(docs) == 0 + + +def test_language_from_config_before_after_init(): + name = "test_language_from_config_before_after_init" + ran_before = False + ran_after = False + ran_after_pipeline = False + ran_before_init = False + ran_after_init = False + + @registry.callbacks(f"{name}_before") + def make_before_creation(): + def before_creation(lang_cls): + nonlocal ran_before + ran_before = True + assert lang_cls is English + lang_cls.Defaults.foo = "bar" + return lang_cls + + return before_creation + + @registry.callbacks(f"{name}_after") + def make_after_creation(): + def after_creation(nlp): + nonlocal ran_after + ran_after = True + assert isinstance(nlp, English) + assert nlp.pipe_names == [] + assert nlp.Defaults.foo == "bar" + nlp.meta["foo"] = "bar" + return nlp + + return after_creation + + @registry.callbacks(f"{name}_after_pipeline") + def make_after_pipeline_creation(): + def after_pipeline_creation(nlp): + nonlocal ran_after_pipeline + ran_after_pipeline = True + assert isinstance(nlp, English) + assert nlp.pipe_names == ["sentencizer"] + assert nlp.Defaults.foo == "bar" + assert nlp.meta["foo"] == "bar" + nlp.meta["bar"] = "baz" + return nlp + + return after_pipeline_creation + + @registry.callbacks(f"{name}_before_init") + def make_before_init(): + def before_init(nlp): + nonlocal ran_before_init + ran_before_init = True + nlp.meta["before_init"] = "before" + return nlp + + return before_init + + @registry.callbacks(f"{name}_after_init") + def make_after_init(): + def after_init(nlp): + nonlocal ran_after_init + ran_after_init = True + nlp.meta["after_init"] = "after" + return nlp + + return after_init + + config = { + "nlp": { + "pipeline": ["sentencizer"], + "before_creation": {"@callbacks": f"{name}_before"}, + "after_creation": {"@callbacks": f"{name}_after"}, + "after_pipeline_creation": {"@callbacks": f"{name}_after_pipeline"}, + }, + "components": {"sentencizer": {"factory": "sentencizer"}}, + "initialize": { + "before_init": {"@callbacks": f"{name}_before_init"}, + "after_init": {"@callbacks": f"{name}_after_init"}, + }, + } + nlp = English.from_config(config) + assert nlp.Defaults.foo == "bar" + assert nlp.meta["foo"] == "bar" + assert nlp.meta["bar"] == "baz" + assert "before_init" not in nlp.meta + assert "after_init" not in nlp.meta + assert nlp.pipe_names == ["sentencizer"] + assert nlp("text") + nlp.initialize() + assert nlp.meta["before_init"] == "before" + assert nlp.meta["after_init"] == "after" + assert all( + [ran_before, ran_after, ran_after_pipeline, ran_before_init, ran_after_init] + ) + + +def test_language_from_config_before_after_init_invalid(): + """Check that an error is raised if function doesn't return nlp.""" + name = "test_language_from_config_before_after_init_invalid" + registry.callbacks(f"{name}_before1", func=lambda: lambda nlp: None) + registry.callbacks(f"{name}_before2", func=lambda: lambda nlp: nlp()) + registry.callbacks(f"{name}_after1", func=lambda: lambda nlp: None) + registry.callbacks(f"{name}_after1", func=lambda: lambda nlp: English) + + for callback_name in [f"{name}_before1", f"{name}_before2"]: + config = {"nlp": {"before_creation": {"@callbacks": callback_name}}} + with pytest.raises(ValueError): + English.from_config(config) + for callback_name in [f"{name}_after1", f"{name}_after2"]: + config = {"nlp": {"after_creation": {"@callbacks": callback_name}}} + with pytest.raises(ValueError): + English.from_config(config) + for callback_name in [f"{name}_after1", f"{name}_after2"]: + config = {"nlp": {"after_pipeline_creation": {"@callbacks": callback_name}}} + with pytest.raises(ValueError): + English.from_config(config) + + +def test_language_whitespace_tokenizer(): + """Test the custom whitespace tokenizer from the docs.""" + + class WhitespaceTokenizer: + def __init__(self, vocab): + self.vocab = vocab + + def __call__(self, text): + words = text.split(" ") + spaces = [True] * len(words) + # Avoid zero-length tokens + for i, word in enumerate(words): + if word == "": + words[i] = " " + spaces[i] = False + # Remove the final trailing space + if words[-1] == " ": + words = words[0:-1] + spaces = spaces[0:-1] + else: + spaces[-1] = False + + return Doc(self.vocab, words=words, spaces=spaces) + + nlp = spacy.blank("en") + nlp.tokenizer = WhitespaceTokenizer(nlp.vocab) + text = " What's happened to me? he thought. It wasn't a dream. " + doc = nlp(text) + assert doc.text == text + + +def test_language_custom_tokenizer(): + """Test that a fully custom tokenizer can be plugged in via the registry.""" + name = "test_language_custom_tokenizer" + + class CustomTokenizer: + """Dummy "tokenizer" that splits on spaces and adds prefix to each word.""" + + def __init__(self, nlp, prefix): + self.vocab = nlp.vocab + self.prefix = prefix + + def __call__(self, text): + words = [f"{self.prefix}{word}" for word in text.split(" ")] + return Doc(self.vocab, words=words) + + @registry.tokenizers(name) + def custom_create_tokenizer(prefix: str = "_"): + def create_tokenizer(nlp): + return CustomTokenizer(nlp, prefix=prefix) + + return create_tokenizer + + config = {"nlp": {"tokenizer": {"@tokenizers": name}}} + nlp = English.from_config(config) + doc = nlp("hello world") + assert [t.text for t in doc] == ["_hello", "_world"] + doc = list(nlp.pipe(["hello world"]))[0] + assert [t.text for t in doc] == ["_hello", "_world"] + + +def test_language_from_config_invalid_lang(): + """Test that calling Language.from_config raises an error and lang defined + in config needs to match language-specific subclasses.""" + config = {"nlp": {"lang": "en"}} + with pytest.raises(ValueError): + Language.from_config(config) + with pytest.raises(ValueError): + German.from_config(config) + + +def test_spacy_blank(): + nlp = spacy.blank("en") + assert nlp.config["training"]["dropout"] == 0.1 + config = {"training": {"dropout": 0.2}} + meta = {"name": "my_custom_model"} + nlp = spacy.blank("en", config=config, meta=meta) + assert nlp.config["training"]["dropout"] == 0.2 + assert nlp.meta["name"] == "my_custom_model" + + +@pytest.mark.parametrize( + "lang,target", + [ + ("fra", "fr"), + ("fre", "fr"), + ("iw", "he"), + ("mo", "ro"), + ("scc", "sr"), + ("mul", "xx"), + ("zxx", None), + ], +) +def test_language_matching(lang, target): + """ + Test that we can look up languages by equivalent or nearly-equivalent + language codes. + """ + assert find_matching_language(lang) == target + + +@pytest.mark.parametrize( + "lang,target", + [ + ("en", "en"), + ("fra", "fr"), + ("fre", "fr"), + ("iw", "he"), + ("mo", "ro"), + ("scc", "sr"), + ("mul", "xx"), + ("xx", "xx"), + ], +) +def test_blank_languages(lang, target): + """ + Test that we can get spacy.blank in various languages, including codes + that are defined to be equivalent or that match by CLDR language matching. + """ + nlp = spacy.blank(lang) + assert nlp.lang == target + + +@pytest.mark.parametrize("value", [False, None, ["x", "y"], Language, Vocab]) +def test_language_init_invalid_vocab(value): + err_fragment = "invalid value" + with pytest.raises(ValueError) as e: + Language(value) + assert err_fragment in str(e.value) + + +def test_language_source_and_vectors(nlp2): + nlp = Language(Vocab()) + textcat = nlp.add_pipe("textcat") + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + nlp.initialize() + long_string = "thisisalongstring" + assert long_string not in nlp.vocab.strings + assert long_string not in nlp2.vocab.strings + nlp.vocab.strings.add(long_string) + assert nlp.vocab.vectors.to_bytes() != nlp2.vocab.vectors.to_bytes() + vectors_bytes = nlp.vocab.vectors.to_bytes() + with pytest.warns(UserWarning): + nlp2.add_pipe("textcat", name="textcat2", source=nlp) + # strings should be added + assert long_string in nlp2.vocab.strings + # vectors should remain unmodified + assert nlp.vocab.vectors.to_bytes() == vectors_bytes + + +@pytest.mark.parametrize("n_process", [1, 2]) +def test_pass_doc_to_pipeline(nlp, n_process): + texts = ["cats", "dogs", "guinea pigs"] + docs = [nlp.make_doc(text) for text in texts] + assert not any(len(doc.cats) for doc in docs) + doc = nlp(docs[0]) + assert doc.text == texts[0] + assert len(doc.cats) > 0 + if isinstance(get_current_ops(), NumpyOps) or n_process < 2: + # Catch warnings to ensure that all worker processes exited + # successfully. + with warnings.catch_warnings(): + warnings.simplefilter("error") + docs = nlp.pipe(docs, n_process=n_process) + assert [doc.text for doc in docs] == texts + assert all(len(doc.cats) for doc in docs) + + +def test_invalid_arg_to_pipeline(nlp): + str_list = ["This is a text.", "This is another."] + with pytest.raises(ValueError): + nlp(str_list) # type: ignore + assert len(list(nlp.pipe(str_list))) == 2 + int_list = [1, 2, 3] + with pytest.raises(ValueError): + list(nlp.pipe(int_list)) # type: ignore + with pytest.raises(ValueError): + nlp(int_list) # type: ignore + + +@pytest.mark.skipif( + not isinstance(get_current_ops(), CupyOps), reason="test requires GPU" +) +def test_multiprocessing_gpu_warning(nlp2, texts): + texts = texts * 10 + docs = nlp2.pipe(texts, n_process=2, batch_size=2) + + with pytest.warns(UserWarning, match="multiprocessing with GPU models"): + with pytest.raises(ValueError): + # Trigger multi-processing. + for _ in docs: + pass + + +def test_dot_in_factory_names(nlp): + Language.component("my_evil_component", func=evil_component) + nlp.add_pipe("my_evil_component") + + with pytest.raises(ValueError, match="not permitted"): + Language.component("my.evil.component.v1", func=evil_component) + + with pytest.raises(ValueError, match="not permitted"): + Language.factory("my.evil.component.v1", func=evil_component) + + +def test_component_return(): + """Test that an error is raised if components return a type other than a + doc.""" + nlp = English() + + @Language.component("test_component_good_pipe") + def good_pipe(doc): + return doc + + nlp.add_pipe("test_component_good_pipe") + nlp("text") + nlp.remove_pipe("test_component_good_pipe") + + @Language.component("test_component_bad_pipe") + def bad_pipe(doc): + return doc.text + + nlp.add_pipe("test_component_bad_pipe") + with pytest.raises(ValueError, match="instead of a Doc"): + nlp("text") diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py new file mode 100644 index 0000000..309c57b --- /dev/null +++ b/spacy/tests/test_misc.py @@ -0,0 +1,492 @@ +import ctypes +import os +from pathlib import Path + +import pytest +from pydantic import ValidationError +from thinc.api import ( + Config, + ConfigValidationError, + CupyOps, + MPSOps, + NumpyOps, + Optimizer, + get_current_ops, + set_current_ops, +) +from thinc.compat import has_cupy_gpu, has_torch_mps_gpu + +from spacy import prefer_gpu, require_cpu, require_gpu, util +from spacy.about import __version__ as spacy_version +from spacy.lang.en import English +from spacy.lang.nl import Dutch +from spacy.language import DEFAULT_CONFIG_PATH +from spacy.ml._precomputable_affine import ( + PrecomputableAffine, + _backprop_precomputable_affine_padding, +) +from spacy.schemas import ConfigSchemaTraining, TokenPattern, TokenPatternSchema +from spacy.training.batchers import minibatch_by_words +from spacy.util import ( + SimpleFrozenList, + dot_to_object, + find_available_port, + import_file, + to_ternary_int, +) + +from .util import get_random_doc, make_tempdir + + +@pytest.fixture +def is_admin(): + """Determine if the tests are run as admin or not.""" + try: + admin = os.getuid() == 0 + except AttributeError: + admin = ctypes.windll.shell32.IsUserAnAdmin() != 0 + + return admin + + +@pytest.mark.issue(6207) +def test_issue6207(en_tokenizer): + doc = en_tokenizer("zero one two three four five six") + + # Make spans + s1 = doc[:4] + s2 = doc[3:6] # overlaps with s1 + s3 = doc[5:7] # overlaps with s2, not s1 + + result = util.filter_spans((s1, s2, s3)) + assert s1 in result + assert s2 not in result + assert s3 in result + + +@pytest.mark.issue(6258) +def test_issue6258(): + """Test that the non-empty constraint pattern field is respected""" + # These one is valid + TokenPatternSchema(pattern=[TokenPattern()]) + # But an empty pattern list should fail to validate + # based on the schema's constraint + with pytest.raises(ValidationError): + TokenPatternSchema(pattern=[]) + + +@pytest.mark.parametrize("text", ["hello/world", "hello world"]) +def test_util_ensure_path_succeeds(text): + path = util.ensure_path(text) + assert isinstance(path, Path) + + +@pytest.mark.parametrize( + "package,result", [("numpy", True), ("sfkodskfosdkfpsdpofkspdof", False)] +) +def test_util_is_package(package, result): + """Test that an installed package via pip is recognised by util.is_package.""" + assert util.is_package(package) is result + + +@pytest.mark.parametrize("package", ["thinc"]) +def test_util_get_package_path(package): + """Test that a Path object is returned for a package name.""" + path = util.get_package_path(package) + assert isinstance(path, Path) + + +def test_PrecomputableAffine(nO=4, nI=5, nF=3, nP=2): + model = PrecomputableAffine(nO=nO, nI=nI, nF=nF, nP=nP).initialize() + assert model.get_param("W").shape == (nF, nO, nP, nI) + tensor = model.ops.alloc((10, nI)) + Y, get_dX = model.begin_update(tensor) + assert Y.shape == (tensor.shape[0] + 1, nF, nO, nP) + dY = model.ops.alloc((15, nO, nP)) + ids = model.ops.alloc((15, nF)) + ids[1, 2] = -1 + dY[1] = 1 + assert not model.has_grad("pad") + d_pad = _backprop_precomputable_affine_padding(model, dY, ids) + assert d_pad[0, 2, 0, 0] == 1.0 + ids.fill(0.0) + dY.fill(0.0) + dY[0] = 0 + ids[1, 2] = 0 + ids[1, 1] = -1 + ids[1, 0] = -1 + dY[1] = 1 + ids[2, 0] = -1 + dY[2] = 5 + d_pad = _backprop_precomputable_affine_padding(model, dY, ids) + assert d_pad[0, 0, 0, 0] == 6 + assert d_pad[0, 1, 0, 0] == 1 + assert d_pad[0, 2, 0, 0] == 0 + + +def test_prefer_gpu(): + current_ops = get_current_ops() + if has_cupy_gpu: + assert prefer_gpu() + assert isinstance(get_current_ops(), CupyOps) + elif has_torch_mps_gpu: + assert prefer_gpu() + assert isinstance(get_current_ops(), MPSOps) + else: + assert not prefer_gpu() + set_current_ops(current_ops) + + +def test_require_gpu(): + current_ops = get_current_ops() + if has_cupy_gpu: + require_gpu() + assert isinstance(get_current_ops(), CupyOps) + elif has_torch_mps_gpu: + require_gpu() + assert isinstance(get_current_ops(), MPSOps) + set_current_ops(current_ops) + + +def test_require_cpu(): + current_ops = get_current_ops() + require_cpu() + assert isinstance(get_current_ops(), NumpyOps) + try: + import cupy # noqa: F401 + + require_gpu() + assert isinstance(get_current_ops(), CupyOps) + except ImportError: + pass + require_cpu() + assert isinstance(get_current_ops(), NumpyOps) + set_current_ops(current_ops) + + +def test_ascii_filenames(): + """Test that all filenames in the project are ASCII. + See: https://twitter.com/_inesmontani/status/1177941471632211968 + """ + root = Path(__file__).parent.parent + for path in root.glob("**/*"): + assert all(ord(c) < 128 for c in path.name), path.name + + +def test_load_model_blank_shortcut(): + """Test that using a model name like "blank:en" works as a shortcut for + spacy.blank("en"). + """ + nlp = util.load_model("blank:en") + assert nlp.lang == "en" + assert nlp.pipeline == [] + + # ImportError for loading an unsupported language + with pytest.raises(ImportError): + util.load_model("blank:zxx") + + # ImportError for requesting an invalid language code that isn't registered + with pytest.raises(ImportError): + util.load_model("blank:fjsfijsdof") + + +@pytest.mark.parametrize( + "version,constraint,compatible", + [ + (spacy_version, spacy_version, True), + (spacy_version, f">={spacy_version}", True), + ("3.0.0", "2.0.0", False), + ("3.2.1", ">=2.0.0", True), + ("2.2.10a1", ">=1.0.0,<2.1.1", False), + ("3.0.0.dev3", ">=1.2.3,<4.5.6", True), + ("n/a", ">=1.2.3,<4.5.6", None), + ("1.2.3", "n/a", None), + ("n/a", "n/a", None), + ], +) +def test_is_compatible_version(version, constraint, compatible): + assert util.is_compatible_version(version, constraint) is compatible + + +@pytest.mark.parametrize( + "constraint,expected", + [ + ("3.0.0", False), + ("==3.0.0", False), + (">=2.3.0", True), + (">2.0.0", True), + ("<=2.0.0", True), + (">2.0.0,<3.0.0", False), + (">=2.0.0,<3.0.0", False), + ("!=1.1,>=1.0,~=1.0", True), + ("n/a", None), + ], +) +def test_is_unconstrained_version(constraint, expected): + assert util.is_unconstrained_version(constraint) is expected + + +@pytest.mark.parametrize( + "a1,a2,b1,b2,is_match", + [ + ("3.0.0", "3.0", "3.0.1", "3.0", True), + ("3.1.0", "3.1", "3.2.1", "3.2", False), + ("xxx", None, "1.2.3.dev0", "1.2", False), + ], +) +def test_minor_version(a1, a2, b1, b2, is_match): + assert util.get_minor_version(a1) == a2 + assert util.get_minor_version(b1) == b2 + assert util.is_minor_version_match(a1, b1) is is_match + assert util.is_minor_version_match(a2, b2) is is_match + + +@pytest.mark.parametrize( + "dot_notation,expected", + [ + ( + {"token.pos": True, "token._.xyz": True}, + {"token": {"pos": True, "_": {"xyz": True}}}, + ), + ( + {"training.batch_size": 128, "training.optimizer.learn_rate": 0.01}, + {"training": {"batch_size": 128, "optimizer": {"learn_rate": 0.01}}}, + ), + ( + {"attribute_ruler.scorer.@scorers": "spacy.tagger_scorer.v1"}, + {"attribute_ruler": {"scorer": {"@scorers": "spacy.tagger_scorer.v1"}}}, + ), + ], +) +def test_dot_to_dict(dot_notation, expected): + result = util.dot_to_dict(dot_notation) + assert result == expected + assert util.dict_to_dot(result) == dot_notation + + +@pytest.mark.parametrize( + "dot_notation,expected", + [ + ( + {"token.pos": True, "token._.xyz": True}, + {"token": {"pos": True, "_": {"xyz": True}}}, + ), + ( + {"training.batch_size": 128, "training.optimizer.learn_rate": 0.01}, + {"training": {"batch_size": 128, "optimizer": {"learn_rate": 0.01}}}, + ), + ( + {"attribute_ruler.scorer": {"@scorers": "spacy.tagger_scorer.v1"}}, + {"attribute_ruler": {"scorer": {"@scorers": "spacy.tagger_scorer.v1"}}}, + ), + ], +) +def test_dot_to_dict_overrides(dot_notation, expected): + result = util.dot_to_dict(dot_notation) + assert result == expected + assert util.dict_to_dot(result, for_overrides=True) == dot_notation + + +def test_set_dot_to_object(): + config = {"foo": {"bar": 1, "baz": {"x": "y"}}, "test": {"a": {"b": "c"}}} + with pytest.raises(KeyError): + util.set_dot_to_object(config, "foo.bar.baz", 100) + with pytest.raises(KeyError): + util.set_dot_to_object(config, "hello.world", 100) + with pytest.raises(KeyError): + util.set_dot_to_object(config, "test.a.b.c", 100) + util.set_dot_to_object(config, "foo.bar", 100) + assert config["foo"]["bar"] == 100 + util.set_dot_to_object(config, "foo.baz.x", {"hello": "world"}) + assert config["foo"]["baz"]["x"]["hello"] == "world" + assert config["test"]["a"]["b"] == "c" + util.set_dot_to_object(config, "foo", 123) + assert config["foo"] == 123 + util.set_dot_to_object(config, "test", "hello") + assert dict(config) == {"foo": 123, "test": "hello"} + + +@pytest.mark.parametrize( + "doc_sizes, expected_batches", + [ + ([400, 400, 199], [3]), + ([400, 400, 199, 3], [4]), + ([400, 400, 199, 3, 200], [3, 2]), + ([400, 400, 199, 3, 1], [5]), + ([400, 400, 199, 3, 1, 1500], [5]), # 1500 will be discarded + ([400, 400, 199, 3, 1, 200], [3, 3]), + ([400, 400, 199, 3, 1, 999], [3, 3]), + ([400, 400, 199, 3, 1, 999, 999], [3, 2, 1, 1]), + ([1, 2, 999], [3]), + ([1, 2, 999, 1], [4]), + ([1, 200, 999, 1], [2, 2]), + ([1, 999, 200, 1], [2, 2]), + ], +) +def test_util_minibatch(doc_sizes, expected_batches): + docs = [get_random_doc(doc_size) for doc_size in doc_sizes] + tol = 0.2 + batch_size = 1000 + batches = list( + minibatch_by_words(docs, size=batch_size, tolerance=tol, discard_oversize=True) + ) + assert [len(batch) for batch in batches] == expected_batches + + max_size = batch_size + batch_size * tol + for batch in batches: + assert sum([len(doc) for doc in batch]) < max_size + + +@pytest.mark.parametrize( + "doc_sizes, expected_batches", + [ + ([400, 4000, 199], [1, 2]), + ([400, 400, 199, 3000, 200], [1, 4]), + ([400, 400, 199, 3, 1, 1500], [1, 5]), + ([400, 400, 199, 3000, 2000, 200, 200], [1, 1, 3, 2]), + ([1, 2, 9999], [1, 2]), + ([2000, 1, 2000, 1, 1, 1, 2000], [1, 1, 1, 4]), + ], +) +def test_util_minibatch_oversize(doc_sizes, expected_batches): + """Test that oversized documents are returned in their own batch""" + docs = [get_random_doc(doc_size) for doc_size in doc_sizes] + tol = 0.2 + batch_size = 1000 + batches = list( + minibatch_by_words(docs, size=batch_size, tolerance=tol, discard_oversize=False) + ) + assert [len(batch) for batch in batches] == expected_batches + + +def test_util_dot_section(): + cfg_string = """ + [nlp] + lang = "en" + pipeline = ["textcat"] + + [components] + + [components.textcat] + factory = "textcat" + + [components.textcat.model] + @architectures = "spacy.TextCatBOW.v3" + exclusive_classes = true + length = 262144 + ngram_size = 1 + no_output_layer = false + """ + nlp_config = Config().from_str(cfg_string) + en_nlp = util.load_model_from_config(nlp_config, auto_fill=True) + default_config = Config().from_disk(DEFAULT_CONFIG_PATH) + default_config["nlp"]["lang"] = "nl" + nl_nlp = util.load_model_from_config(default_config, auto_fill=True) + # Test that creation went OK + assert isinstance(en_nlp, English) + assert isinstance(nl_nlp, Dutch) + assert nl_nlp.pipe_names == [] + assert en_nlp.pipe_names == ["textcat"] + # not exclusive_classes + assert en_nlp.get_pipe("textcat").model.attrs["multi_label"] is False + # Test that default values got overwritten + assert en_nlp.config["nlp"]["pipeline"] == ["textcat"] + assert nl_nlp.config["nlp"]["pipeline"] == [] # default value [] + # Test proper functioning of 'dot_to_object' + with pytest.raises(KeyError): + dot_to_object(en_nlp.config, "nlp.pipeline.tagger") + with pytest.raises(KeyError): + dot_to_object(en_nlp.config, "nlp.unknownattribute") + T = util.registry.resolve(nl_nlp.config["training"], schema=ConfigSchemaTraining) + assert isinstance(dot_to_object({"training": T}, "training.optimizer"), Optimizer) + + +def test_simple_frozen_list(): + t = SimpleFrozenList(["foo", "bar"]) + assert t == ["foo", "bar"] + assert t.index("bar") == 1 # okay method + with pytest.raises(NotImplementedError): + t.append("baz") + with pytest.raises(NotImplementedError): + t.sort() + with pytest.raises(NotImplementedError): + t.extend(["baz"]) + with pytest.raises(NotImplementedError): + t.pop() + t = SimpleFrozenList(["foo", "bar"], error="Error!") + with pytest.raises(NotImplementedError): + t.append("baz") + + +def test_resolve_dot_names(): + config = { + "training": {"optimizer": {"@optimizers": "Adam.v1"}}, + "foo": {"bar": "training.optimizer", "baz": "training.xyz"}, + } + result = util.resolve_dot_names(config, ["training.optimizer"]) + assert isinstance(result[0], Optimizer) + with pytest.raises(ConfigValidationError) as e: + util.resolve_dot_names(config, ["training.xyz", "training.optimizer"]) + errors = e.value.errors + assert len(errors) == 1 + assert errors[0]["loc"] == ["training", "xyz"] + + +def test_import_code(): + code_str = """ +from spacy import Language + +class DummyComponent: + def __init__(self, vocab, name): + pass + + def initialize(self, get_examples, *, nlp, dummy_param: int): + pass + +@Language.factory( + "dummy_component", +) +def make_dummy_component( + nlp: Language, name: str +): + return DummyComponent(nlp.vocab, name) +""" + + with make_tempdir() as temp_dir: + code_path = os.path.join(temp_dir, "code.py") + with open(code_path, "w") as fileh: + fileh.write(code_str) + + import_file("python_code", code_path) + config = {"initialize": {"components": {"dummy_component": {"dummy_param": 1}}}} + nlp = English.from_config(config) + nlp.add_pipe("dummy_component") + nlp.initialize() + + +def test_to_ternary_int(): + assert to_ternary_int(True) == 1 + assert to_ternary_int(None) == 0 + assert to_ternary_int(False) == -1 + assert to_ternary_int(1) == 1 + assert to_ternary_int(1.0) == 1 + assert to_ternary_int(0) == 0 + assert to_ternary_int(0.0) == 0 + assert to_ternary_int(-1) == -1 + assert to_ternary_int(5) == -1 + assert to_ternary_int(-10) == -1 + assert to_ternary_int("string") == -1 + assert to_ternary_int([0, "string"]) == -1 + + +def test_find_available_port(): + host = "0.0.0.0" + port = 5001 + assert find_available_port(port, host) == port, "Port 5001 isn't free" + + from wsgiref.simple_server import demo_app, make_server + + with make_server(host, port, demo_app) as httpd: + with pytest.warns(UserWarning, match="already in use"): + found_port = find_available_port(port, host, auto_select=True) + assert found_port == port + 1, "Didn't find next port" diff --git a/spacy/tests/test_models.py b/spacy/tests/test_models.py new file mode 100644 index 0000000..706203f --- /dev/null +++ b/spacy/tests/test_models.py @@ -0,0 +1,301 @@ +from typing import List + +import numpy +import pytest +from numpy.testing import assert_array_almost_equal, assert_array_equal +from thinc.api import ( + Adam, + Logistic, + Ragged, + Relu, + chain, + fix_random_seed, + reduce_mean, + set_dropout_rate, +) + +from spacy.lang.en import English +from spacy.lang.en.examples import sentences as EN_SENTENCES +from spacy.ml.extract_spans import _get_span_indices, extract_spans +from spacy.ml.models import ( + MaxoutWindowEncoder, + MultiHashEmbed, + build_bow_text_classifier, + build_simple_cnn_text_classifier, + build_spancat_model, + build_Tok2Vec_model, +) +from spacy.ml.staticvectors import StaticVectors +from spacy.util import registry + + +def get_textcat_bow_kwargs(): + return { + "exclusive_classes": True, + "ngram_size": 1, + "no_output_layer": False, + "nO": 34, + } + + +def get_textcat_cnn_kwargs(): + return {"tok2vec": make_test_tok2vec(), "exclusive_classes": False, "nO": 13} + + +def get_all_params(model): + params = [] + for node in model.walk(): + for name in node.param_names: + params.append(node.get_param(name).ravel()) + return node.ops.xp.concatenate(params) + + +def get_docs(): + nlp = English() + return list(nlp.pipe(EN_SENTENCES + [" ".join(EN_SENTENCES)])) + + +def get_gradient(model, Y): + if isinstance(Y, model.ops.xp.ndarray): + dY = model.ops.alloc(Y.shape, dtype=Y.dtype) + dY += model.ops.xp.random.uniform(-1.0, 1.0, Y.shape) + return dY + elif isinstance(Y, List): + return [get_gradient(model, y) for y in Y] + else: + raise ValueError(f"Could not get gradient for type {type(Y)}") + + +def get_tok2vec_kwargs(): + # This actually creates models, so seems best to put it in a function. + return { + "embed": MultiHashEmbed( + width=32, + rows=[500, 500, 500], + attrs=["NORM", "PREFIX", "SHAPE"], + include_static_vectors=False, + ), + "encode": MaxoutWindowEncoder( + width=32, depth=2, maxout_pieces=2, window_size=1 + ), + } + + +def make_test_tok2vec(): + return build_Tok2Vec_model(**get_tok2vec_kwargs()) + + +def test_multi_hash_embed(): + embed = MultiHashEmbed( + width=32, + rows=[500, 500, 500], + attrs=["NORM", "PREFIX", "SHAPE"], + include_static_vectors=False, + ) + hash_embeds = [node for node in embed.walk() if node.name == "hashembed"] + assert len(hash_embeds) == 3 + # Check they look at different columns. + assert sorted(he.attrs["column"] for he in hash_embeds) == [0, 1, 2] + # Check they use different seeds + assert len(set(he.attrs["seed"] for he in hash_embeds)) == 3 + # Check they all have the same number of rows + assert [he.get_dim("nV") for he in hash_embeds] == [500, 500, 500] + # Now try with different row factors + embed = MultiHashEmbed( + width=32, + rows=[1000, 50, 250], + attrs=["NORM", "PREFIX", "SHAPE"], + include_static_vectors=False, + ) + hash_embeds = [node for node in embed.walk() if node.name == "hashembed"] + assert [he.get_dim("nV") for he in hash_embeds] == [1000, 50, 250] + + +@pytest.mark.parametrize( + "seed,model_func,kwargs", + [ + (0, build_Tok2Vec_model, get_tok2vec_kwargs()), + (0, build_bow_text_classifier, get_textcat_bow_kwargs()), + (0, build_simple_cnn_text_classifier, get_textcat_cnn_kwargs()), + ], +) +def test_models_initialize_consistently(seed, model_func, kwargs): + fix_random_seed(seed) + model1 = model_func(**kwargs) + model1.initialize() + fix_random_seed(seed) + model2 = model_func(**kwargs) + model2.initialize() + params1 = get_all_params(model1) + params2 = get_all_params(model2) + assert_array_equal(model1.ops.to_numpy(params1), model2.ops.to_numpy(params2)) + + +@pytest.mark.parametrize( + "seed,model_func,kwargs,get_X", + [ + (0, build_Tok2Vec_model, get_tok2vec_kwargs(), get_docs), + (0, build_bow_text_classifier, get_textcat_bow_kwargs(), get_docs), + (0, build_simple_cnn_text_classifier, get_textcat_cnn_kwargs(), get_docs), + ], +) +def test_models_predict_consistently(seed, model_func, kwargs, get_X): + fix_random_seed(seed) + model1 = model_func(**kwargs).initialize() + Y1 = model1.predict(get_X()) + fix_random_seed(seed) + model2 = model_func(**kwargs).initialize() + Y2 = model2.predict(get_X()) + + if model1.has_ref("tok2vec"): + tok2vec1 = model1.get_ref("tok2vec").predict(get_X()) + tok2vec2 = model2.get_ref("tok2vec").predict(get_X()) + for i in range(len(tok2vec1)): + for j in range(len(tok2vec1[i])): + assert_array_equal( + numpy.asarray(model1.ops.to_numpy(tok2vec1[i][j])), + numpy.asarray(model2.ops.to_numpy(tok2vec2[i][j])), + ) + + try: + Y1 = model1.ops.to_numpy(Y1) + Y2 = model2.ops.to_numpy(Y2) + except Exception: + pass + if isinstance(Y1, numpy.ndarray): + assert_array_equal(Y1, Y2) + elif isinstance(Y1, List): + assert len(Y1) == len(Y2) + for y1, y2 in zip(Y1, Y2): + try: + y1 = model1.ops.to_numpy(y1) + y2 = model2.ops.to_numpy(y2) + except Exception: + pass + assert_array_equal(y1, y2) + else: + raise ValueError(f"Could not compare type {type(Y1)}") + + +@pytest.mark.parametrize( + "seed,dropout,model_func,kwargs,get_X", + [ + (0, 0.2, build_Tok2Vec_model, get_tok2vec_kwargs(), get_docs), + (0, 0.2, build_bow_text_classifier, get_textcat_bow_kwargs(), get_docs), + (0, 0.2, build_simple_cnn_text_classifier, get_textcat_cnn_kwargs(), get_docs), + ], +) +def test_models_update_consistently(seed, dropout, model_func, kwargs, get_X): + def get_updated_model(): + fix_random_seed(seed) + optimizer = Adam(0.001) + model = model_func(**kwargs).initialize() + initial_params = get_all_params(model) + set_dropout_rate(model, dropout) + for _ in range(5): + Y, get_dX = model.begin_update(get_X()) + dY = get_gradient(model, Y) + get_dX(dY) + model.finish_update(optimizer) + updated_params = get_all_params(model) + with pytest.raises(AssertionError): + assert_array_equal( + model.ops.to_numpy(initial_params), model.ops.to_numpy(updated_params) + ) + return model + + model1 = get_updated_model() + model2 = get_updated_model() + assert_array_almost_equal( + model1.ops.to_numpy(get_all_params(model1)), + model2.ops.to_numpy(get_all_params(model2)), + decimal=5, + ) + + +@pytest.mark.parametrize("model_func,kwargs", [(StaticVectors, {"nO": 128, "nM": 300})]) +def test_empty_docs(model_func, kwargs): + nlp = English() + model = model_func(**kwargs).initialize() + # Test the layer can be called successfully with 0, 1 and 2 empty docs. + for n_docs in range(3): + docs = [nlp("") for _ in range(n_docs)] + # Test predict + model.predict(docs) + # Test backprop + output, backprop = model.begin_update(docs) + backprop(output) + + +def test_init_extract_spans(): + extract_spans().initialize() + + +def test_extract_spans_span_indices(): + model = extract_spans().initialize() + spans = Ragged( + model.ops.asarray([[0, 3], [2, 3], [5, 7]], dtype="i"), + model.ops.asarray([2, 1], dtype="i"), + ) + x_lengths = model.ops.asarray([5, 10], dtype="i") + indices = _get_span_indices(model.ops, spans, x_lengths) + assert list(indices) == [0, 1, 2, 2, 10, 11] + + +def test_extract_spans_forward_backward(): + model = extract_spans().initialize() + X = Ragged(model.ops.alloc2f(15, 4), model.ops.asarray([5, 10], dtype="i")) + spans = Ragged( + model.ops.asarray([[0, 3], [2, 3], [5, 7]], dtype="i"), + model.ops.asarray([2, 1], dtype="i"), + ) + Y, backprop = model.begin_update((X, spans)) + assert list(Y.lengths) == [3, 1, 2] + assert Y.dataXd.shape == (6, 4) + dX, spans2 = backprop(Y) + assert spans2 is spans + assert dX.dataXd.shape == X.dataXd.shape + assert list(dX.lengths) == list(X.lengths) + + +def test_spancat_model_init(): + model = build_spancat_model( + build_Tok2Vec_model(**get_tok2vec_kwargs()), reduce_mean(), Logistic() + ) + model.initialize() + + +def test_spancat_model_forward_backward(nO=5): + tok2vec = build_Tok2Vec_model(**get_tok2vec_kwargs()) + docs = get_docs() + spans_list = [] + lengths = [] + for doc in docs: + spans_list.append(doc[:2]) + spans_list.append(doc[1:4]) + lengths.append(2) + spans = Ragged( + tok2vec.ops.asarray([[s.start, s.end] for s in spans_list], dtype="i"), + tok2vec.ops.asarray(lengths, dtype="i"), + ) + model = build_spancat_model( + tok2vec, reduce_mean(), chain(Relu(nO=nO), Logistic()) + ).initialize(X=(docs, spans)) + + Y, backprop = model((docs, spans), is_train=True) + assert Y.shape == (spans.dataXd.shape[0], nO) + backprop(Y) + + +def test_textcat_reduce_invalid_args(): + textcat_reduce = registry.architectures.get("spacy.TextCatReduce.v1") + tok2vec = make_test_tok2vec() + with pytest.raises(ValueError, match=r"must be used with at least one reduction"): + textcat_reduce( + tok2vec=tok2vec, + exclusive_classes=False, + use_reduce_first=False, + use_reduce_last=False, + use_reduce_max=False, + use_reduce_mean=False, + ) diff --git a/spacy/tests/test_pickles.py b/spacy/tests/test_pickles.py new file mode 100644 index 0000000..e3acd27 --- /dev/null +++ b/spacy/tests/test_pickles.py @@ -0,0 +1,62 @@ +import numpy +import pytest +import srsly + +from spacy.attrs import NORM +from spacy.lang.en import English +from spacy.strings import StringStore +from spacy.tokens import Doc +from spacy.vocab import Vocab + + +@pytest.mark.parametrize("text1,text2", [("hello", "bye")]) +def test_pickle_string_store(text1, text2): + stringstore = StringStore() + store1 = stringstore[text1] + store2 = stringstore[text2] + data = srsly.pickle_dumps(stringstore, protocol=-1) + unpickled = srsly.pickle_loads(data) + assert unpickled[text1] == store1 + assert unpickled[text2] == store2 + assert len(stringstore) == len(unpickled) + + +@pytest.mark.parametrize("text1,text2", [("dog", "cat")]) +def test_pickle_vocab(text1, text2): + vocab = Vocab( + lex_attr_getters={int(NORM): lambda string: string[:-1]}, + get_noun_chunks=English.Defaults.syntax_iterators.get("noun_chunks"), + ) + vocab.set_vector("dog", numpy.ones((5,), dtype="f")) + lex1 = vocab[text1] + lex2 = vocab[text2] + assert lex1.norm_ == text1[:-1] + assert lex2.norm_ == text2[:-1] + data = srsly.pickle_dumps(vocab) + unpickled = srsly.pickle_loads(data) + assert unpickled[text1].orth == lex1.orth + assert unpickled[text2].orth == lex2.orth + assert unpickled[text1].norm == lex1.norm + assert unpickled[text2].norm == lex2.norm + assert unpickled[text1].norm != unpickled[text2].norm + assert unpickled.vectors is not None + assert unpickled.get_noun_chunks is not None + assert list(vocab["dog"].vector) == [1.0, 1.0, 1.0, 1.0, 1.0] + + +def test_pickle_doc(en_vocab): + words = ["a", "b", "c"] + deps = ["dep"] * len(words) + heads = [0] * len(words) + doc = Doc( + en_vocab, + words=words, + deps=deps, + heads=heads, + ) + data = srsly.pickle_dumps(doc) + unpickled = srsly.pickle_loads(data) + assert [t.text for t in unpickled] == words + assert [t.dep_ for t in unpickled] == deps + assert [t.head.i for t in unpickled] == heads + assert list(doc.noun_chunks) == [] diff --git a/spacy/tests/test_registry_population.py b/spacy/tests/test_registry_population.py new file mode 100644 index 0000000..e72f3d9 --- /dev/null +++ b/spacy/tests/test_registry_population.py @@ -0,0 +1,54 @@ +import json +from pathlib import Path + +import pytest + +from spacy.util import registry + +# Path to the reference registry contents, relative to this file +REFERENCE_FILE = Path(__file__).parent / "registry_contents.json" + + +@pytest.fixture +def reference_registry(): + """Load reference registry contents from JSON file""" + if not REFERENCE_FILE.exists(): + pytest.fail(f"Reference file {REFERENCE_FILE} not found.") + + with REFERENCE_FILE.open("r") as f: + return json.load(f) + + +def test_registry_types(reference_registry): + """Test that all registry types match the reference""" + # Get current registry types + current_registry_types = set(registry.get_registry_names()) + expected_registry_types = set(reference_registry.keys()) + + # Check for missing registry types + missing_types = expected_registry_types - current_registry_types + assert not missing_types, f"Missing registry types: {', '.join(missing_types)}" + + +def test_registry_entries(reference_registry): + """Test that all registry entries are present""" + # Check each registry's entries + for registry_name, expected_entries in reference_registry.items(): + # Skip if this registry type doesn't exist + if not hasattr(registry, registry_name): + pytest.fail(f"Registry '{registry_name}' does not exist.") + + # Get current entries + reg = getattr(registry, registry_name) + current_entries = sorted(list(reg.get_all().keys())) + + # Compare entries + expected_set = set(expected_entries) + current_set = set(current_entries) + + # Check for missing entries - these would indicate our new registry population + # mechanism is missing something + missing_entries = expected_set - current_set + assert not missing_entries, ( + f"Registry '{registry_name}' missing entries: {', '.join(missing_entries)}" + ) diff --git a/spacy/tests/test_scorer.py b/spacy/tests/test_scorer.py new file mode 100644 index 0000000..95daf04 --- /dev/null +++ b/spacy/tests/test_scorer.py @@ -0,0 +1,537 @@ +import pytest +from numpy.testing import assert_almost_equal, assert_array_almost_equal +from pytest import approx + +from spacy.lang.en import English +from spacy.scorer import PRFScore, ROCAUCScore, Scorer, _roc_auc_score, _roc_curve +from spacy.tokens import Doc, Span +from spacy.training import Example +from spacy.training.iob_utils import offsets_to_biluo_tags + +test_las_apple = [ + [ + "Apple is looking at buying U.K. startup for $ 1 billion", + { + "heads": [2, 2, 2, 2, 3, 6, 4, 4, 10, 10, 7], + "deps": [ + "nsubj", + "aux", + "ROOT", + "prep", + "pcomp", + "compound", + "dobj", + "prep", + "quantmod", + "compound", + "pobj", + ], + }, + ] +] + +test_ner_cardinal = [ + ["100 - 200", {"entities": [[0, 3, "CARDINAL"], [6, 9, "CARDINAL"]]}] +] + +test_ner_apple = [ + [ + "Apple is looking at buying U.K. startup for $1 billion", + {"entities": [(0, 5, "ORG"), (27, 31, "GPE"), (44, 54, "MONEY")]}, + ] +] + + +@pytest.fixture +def tagged_doc(): + text = "Sarah's sister flew to Silicon Valley via London." + tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."] + pos = [ + "PROPN", + "PART", + "NOUN", + "VERB", + "ADP", + "PROPN", + "PROPN", + "ADP", + "PROPN", + "PUNCT", + ] + morphs = [ + "NounType=prop|Number=sing", + "Poss=yes", + "Number=sing", + "Tense=past|VerbForm=fin", + "", + "NounType=prop|Number=sing", + "NounType=prop|Number=sing", + "", + "NounType=prop|Number=sing", + "PunctType=peri", + ] + nlp = English() + doc = nlp(text) + for i in range(len(tags)): + doc[i].tag_ = tags[i] + doc[i].pos_ = pos[i] + doc[i].set_morph(morphs[i]) + if i > 0: + doc[i].is_sent_start = False + return doc + + +@pytest.fixture +def sented_doc(): + text = "One sentence. Two sentences. Three sentences." + nlp = English() + doc = nlp(text) + for i in range(len(doc)): + if i % 3 == 0: + doc[i].is_sent_start = True + else: + doc[i].is_sent_start = False + return doc + + +def test_tokenization(sented_doc): + scorer = Scorer() + gold = {"sent_starts": [t.sent_start for t in sented_doc]} + example = Example.from_dict(sented_doc, gold) + scores = scorer.score([example]) + assert scores["token_acc"] == 1.0 + + nlp = English() + example.predicted = Doc( + nlp.vocab, + words=["One", "sentence.", "Two", "sentences.", "Three", "sentences."], + spaces=[True, True, True, True, True, False], + ) + example.predicted[1].is_sent_start = False + scores = scorer.score([example]) + assert scores["token_acc"] == 0.5 + assert scores["token_p"] == 0.5 + assert scores["token_r"] == approx(0.33333333) + assert scores["token_f"] == 0.4 + + # per-component scoring + scorer = Scorer() + scores = scorer.score([example], per_component=True) + assert scores["tokenizer"]["token_acc"] == 0.5 + assert scores["tokenizer"]["token_p"] == 0.5 + assert scores["tokenizer"]["token_r"] == approx(0.33333333) + assert scores["tokenizer"]["token_f"] == 0.4 + + +def test_sents(sented_doc): + scorer = Scorer() + gold = {"sent_starts": [t.sent_start for t in sented_doc]} + example = Example.from_dict(sented_doc, gold) + scores = scorer.score([example]) + assert scores["sents_f"] == 1.0 + + # One sentence start is moved + gold["sent_starts"][3] = 0 + gold["sent_starts"][4] = 1 + example = Example.from_dict(sented_doc, gold) + scores = scorer.score([example]) + assert scores["sents_f"] == approx(0.3333333) + + +def test_las_per_type(en_vocab): + # Gold and Doc are identical + scorer = Scorer() + examples = [] + for input_, annot in test_las_apple: + doc = Doc( + en_vocab, words=input_.split(" "), heads=annot["heads"], deps=annot["deps"] + ) + gold = {"heads": annot["heads"], "deps": annot["deps"]} + example = Example.from_dict(doc, gold) + examples.append(example) + results = scorer.score(examples) + + assert results["dep_uas"] == 1.0 + assert results["dep_las"] == 1.0 + assert results["dep_las_per_type"]["nsubj"]["p"] == 1.0 + assert results["dep_las_per_type"]["nsubj"]["r"] == 1.0 + assert results["dep_las_per_type"]["nsubj"]["f"] == 1.0 + assert results["dep_las_per_type"]["compound"]["p"] == 1.0 + assert results["dep_las_per_type"]["compound"]["r"] == 1.0 + assert results["dep_las_per_type"]["compound"]["f"] == 1.0 + + # One dep is incorrect in Doc + scorer = Scorer() + examples = [] + for input_, annot in test_las_apple: + doc = Doc( + en_vocab, words=input_.split(" "), heads=annot["heads"], deps=annot["deps"] + ) + gold = {"heads": annot["heads"], "deps": annot["deps"]} + doc[0].dep_ = "compound" + example = Example.from_dict(doc, gold) + examples.append(example) + results = scorer.score(examples) + + assert results["dep_uas"] == 1.0 + assert_almost_equal(results["dep_las"], 0.9090909) + assert results["dep_las_per_type"]["nsubj"]["p"] == 0 + assert results["dep_las_per_type"]["nsubj"]["r"] == 0 + assert results["dep_las_per_type"]["nsubj"]["f"] == 0 + assert_almost_equal(results["dep_las_per_type"]["compound"]["p"], 0.666666666) + assert results["dep_las_per_type"]["compound"]["r"] == 1.0 + assert results["dep_las_per_type"]["compound"]["f"] == 0.8 + + +def test_ner_per_type(en_vocab): + # Gold and Doc are identical + scorer = Scorer() + examples = [] + for input_, annot in test_ner_cardinal: + doc = Doc( + en_vocab, words=input_.split(" "), ents=["B-CARDINAL", "O", "B-CARDINAL"] + ) + entities = offsets_to_biluo_tags(doc, annot["entities"]) + example = Example.from_dict(doc, {"entities": entities}) + # a hack for sentence boundaries + example.predicted[1].is_sent_start = False + example.reference[1].is_sent_start = False + examples.append(example) + results = scorer.score(examples) + + assert results["ents_p"] == 1.0 + assert results["ents_r"] == 1.0 + assert results["ents_f"] == 1.0 + assert results["ents_per_type"]["CARDINAL"]["p"] == 1.0 + assert results["ents_per_type"]["CARDINAL"]["r"] == 1.0 + assert results["ents_per_type"]["CARDINAL"]["f"] == 1.0 + + # Doc has one missing and one extra entity + # Entity type MONEY is not present in Doc + scorer = Scorer() + examples = [] + for input_, annot in test_ner_apple: + doc = Doc( + en_vocab, + words=input_.split(" "), + ents=["B-ORG", "O", "O", "O", "O", "B-GPE", "B-ORG", "O", "O", "O"], + ) + entities = offsets_to_biluo_tags(doc, annot["entities"]) + example = Example.from_dict(doc, {"entities": entities}) + # a hack for sentence boundaries + example.predicted[1].is_sent_start = False + example.reference[1].is_sent_start = False + examples.append(example) + results = scorer.score(examples) + + assert results["ents_p"] == approx(0.6666666) + assert results["ents_r"] == approx(0.6666666) + assert results["ents_f"] == approx(0.6666666) + assert "GPE" in results["ents_per_type"] + assert "MONEY" in results["ents_per_type"] + assert "ORG" in results["ents_per_type"] + assert results["ents_per_type"]["GPE"]["p"] == 1.0 + assert results["ents_per_type"]["GPE"]["r"] == 1.0 + assert results["ents_per_type"]["GPE"]["f"] == 1.0 + assert results["ents_per_type"]["MONEY"]["p"] == 0 + assert results["ents_per_type"]["MONEY"]["r"] == 0 + assert results["ents_per_type"]["MONEY"]["f"] == 0 + assert results["ents_per_type"]["ORG"]["p"] == 0.5 + assert results["ents_per_type"]["ORG"]["r"] == 1.0 + assert results["ents_per_type"]["ORG"]["f"] == approx(0.6666666) + + +def test_tag_score(tagged_doc): + # Gold and Doc are identical + scorer = Scorer() + gold = { + "tags": [t.tag_ for t in tagged_doc], + "pos": [t.pos_ for t in tagged_doc], + "morphs": [str(t.morph) for t in tagged_doc], + "sent_starts": [1 if t.is_sent_start else -1 for t in tagged_doc], + } + example = Example.from_dict(tagged_doc, gold) + results = scorer.score([example]) + + assert results["tag_acc"] == 1.0 + assert results["pos_acc"] == 1.0 + assert results["morph_acc"] == 1.0 + assert results["morph_micro_f"] == 1.0 + assert results["morph_per_feat"]["NounType"]["f"] == 1.0 + + # Gold annotation is modified + scorer = Scorer() + tags = [t.tag_ for t in tagged_doc] + tags[0] = "NN" + pos = [t.pos_ for t in tagged_doc] + pos[1] = "X" + morphs = [str(t.morph) for t in tagged_doc] + morphs[1] = "Number=sing" + morphs[2] = "Number=plur" + gold = { + "tags": tags, + "pos": pos, + "morphs": morphs, + "sent_starts": gold["sent_starts"], + } + example = Example.from_dict(tagged_doc, gold) + results = scorer.score([example]) + + assert results["tag_acc"] == 0.9 + assert results["pos_acc"] == 0.9 + assert results["morph_acc"] == approx(0.8) + assert results["morph_micro_f"] == approx(0.8461538) + assert results["morph_per_feat"]["NounType"]["f"] == 1.0 + assert results["morph_per_feat"]["Poss"]["f"] == 0.0 + assert results["morph_per_feat"]["Number"]["f"] == approx(0.72727272) + + # per-component scoring + scorer = Scorer() + results = scorer.score([example], per_component=True) + assert results["tagger"]["tag_acc"] == 0.9 + assert results["morphologizer"]["pos_acc"] == 0.9 + assert results["morphologizer"]["morph_acc"] == approx(0.8) + + +def test_partial_annotation(en_tokenizer): + pred_doc = en_tokenizer("a b c d e") + pred_doc[0].tag_ = "A" + pred_doc[0].pos_ = "X" + pred_doc[0].set_morph("Feat=Val") + pred_doc[0].dep_ = "dep" + + # unannotated reference + ref_doc = en_tokenizer("a b c d e") + ref_doc.has_unknown_spaces = True + example = Example(pred_doc, ref_doc) + scorer = Scorer() + scores = scorer.score([example]) + for key in scores: + # cats doesn't have an unset state + if key.startswith("cats"): + continue + assert scores[key] is None + + # partially annotated reference, not overlapping with predicted annotation + ref_doc = en_tokenizer("a b c d e") + ref_doc.has_unknown_spaces = True + ref_doc[1].tag_ = "A" + ref_doc[1].pos_ = "X" + ref_doc[1].set_morph("Feat=Val") + ref_doc[1].dep_ = "dep" + example = Example(pred_doc, ref_doc) + scorer = Scorer() + scores = scorer.score([example]) + assert scores["token_acc"] is None + assert scores["tag_acc"] == 0.0 + assert scores["pos_acc"] == 0.0 + assert scores["morph_acc"] == 0.0 + assert scores["dep_uas"] == 1.0 + assert scores["dep_las"] == 0.0 + assert scores["sents_f"] is None + + # partially annotated reference, overlapping with predicted annotation + ref_doc = en_tokenizer("a b c d e") + ref_doc.has_unknown_spaces = True + ref_doc[0].tag_ = "A" + ref_doc[0].pos_ = "X" + ref_doc[1].set_morph("Feat=Val") + ref_doc[1].dep_ = "dep" + example = Example(pred_doc, ref_doc) + scorer = Scorer() + scores = scorer.score([example]) + assert scores["token_acc"] is None + assert scores["tag_acc"] == 1.0 + assert scores["pos_acc"] == 1.0 + assert scores["morph_acc"] == 0.0 + assert scores["dep_uas"] == 1.0 + assert scores["dep_las"] == 0.0 + assert scores["sents_f"] is None + + +def test_roc_auc_score(): + # Binary classification, toy tests from scikit-learn test suite + y_true = [0, 1] + y_score = [0, 1] + tpr, fpr, _ = _roc_curve(y_true, y_score) + roc_auc = _roc_auc_score(y_true, y_score) + assert_array_almost_equal(tpr, [0, 0, 1]) + assert_array_almost_equal(fpr, [0, 1, 1]) + assert_almost_equal(roc_auc, 1.0) + + y_true = [0, 1] + y_score = [1, 0] + tpr, fpr, _ = _roc_curve(y_true, y_score) + roc_auc = _roc_auc_score(y_true, y_score) + assert_array_almost_equal(tpr, [0, 1, 1]) + assert_array_almost_equal(fpr, [0, 0, 1]) + assert_almost_equal(roc_auc, 0.0) + + y_true = [1, 0] + y_score = [1, 1] + tpr, fpr, _ = _roc_curve(y_true, y_score) + roc_auc = _roc_auc_score(y_true, y_score) + assert_array_almost_equal(tpr, [0, 1]) + assert_array_almost_equal(fpr, [0, 1]) + assert_almost_equal(roc_auc, 0.5) + + y_true = [1, 0] + y_score = [1, 0] + tpr, fpr, _ = _roc_curve(y_true, y_score) + roc_auc = _roc_auc_score(y_true, y_score) + assert_array_almost_equal(tpr, [0, 0, 1]) + assert_array_almost_equal(fpr, [0, 1, 1]) + assert_almost_equal(roc_auc, 1.0) + + y_true = [1, 0] + y_score = [0.5, 0.5] + tpr, fpr, _ = _roc_curve(y_true, y_score) + roc_auc = _roc_auc_score(y_true, y_score) + assert_array_almost_equal(tpr, [0, 1]) + assert_array_almost_equal(fpr, [0, 1]) + assert_almost_equal(roc_auc, 0.5) + + # same result as above with ROCAUCScore wrapper + score = ROCAUCScore() + score.score_set(0.5, 1) + score.score_set(0.5, 0) + assert_almost_equal(score.score, 0.5) + + # check that errors are raised in undefined cases and score is -inf + y_true = [0, 0] + y_score = [0.25, 0.75] + with pytest.raises(ValueError): + _roc_auc_score(y_true, y_score) + + score = ROCAUCScore() + score.score_set(0.25, 0) + score.score_set(0.75, 0) + with pytest.raises(ValueError): + _ = score.score # noqa: F841 + + y_true = [1, 1] + y_score = [0.25, 0.75] + with pytest.raises(ValueError): + _roc_auc_score(y_true, y_score) + + score = ROCAUCScore() + score.score_set(0.25, 1) + score.score_set(0.75, 1) + with pytest.raises(ValueError): + _ = score.score # noqa: F841 + + +def test_score_spans(): + nlp = English() + text = "This is just a random sentence." + key = "my_spans" + gold = nlp.make_doc(text) + pred = nlp.make_doc(text) + spans = [] + spans.append(gold.char_span(0, 4, label="PERSON")) + spans.append(gold.char_span(0, 7, label="ORG")) + spans.append(gold.char_span(8, 12, label="ORG")) + gold.spans[key] = spans + + def span_getter(doc, span_key): + return doc.spans[span_key] + + # Predict exactly the same, but overlapping spans will be discarded + pred.spans[key] = gold.spans[key].copy(doc=pred) + eg = Example(pred, gold) + scores = Scorer.score_spans([eg], attr=key, getter=span_getter) + assert scores[f"{key}_p"] == 1.0 + assert scores[f"{key}_r"] < 1.0 + + # Allow overlapping, now both precision and recall should be 100% + pred.spans[key] = gold.spans[key].copy(doc=pred) + eg = Example(pred, gold) + scores = Scorer.score_spans([eg], attr=key, getter=span_getter, allow_overlap=True) + assert scores[f"{key}_p"] == 1.0 + assert scores[f"{key}_r"] == 1.0 + + # Change the predicted labels + new_spans = [Span(pred, span.start, span.end, label="WRONG") for span in spans] + pred.spans[key] = new_spans + eg = Example(pred, gold) + scores = Scorer.score_spans([eg], attr=key, getter=span_getter, allow_overlap=True) + assert scores[f"{key}_p"] == 0.0 + assert scores[f"{key}_r"] == 0.0 + assert f"{key}_per_type" in scores + + # Discard labels from the evaluation + scores = Scorer.score_spans( + [eg], attr=key, getter=span_getter, allow_overlap=True, labeled=False + ) + assert scores[f"{key}_p"] == 1.0 + assert scores[f"{key}_r"] == 1.0 + assert f"{key}_per_type" not in scores + + +def test_prf_score(): + cand = {"hi", "ho"} + gold1 = {"yo", "hi"} + gold2 = set() + + a = PRFScore() + a.score_set(cand=cand, gold=gold1) + assert (a.precision, a.recall, a.fscore) == approx((0.5, 0.5, 0.5)) + + b = PRFScore() + b.score_set(cand=cand, gold=gold2) + assert (b.precision, b.recall, b.fscore) == approx((0.0, 0.0, 0.0)) + + c = a + b + assert (c.precision, c.recall, c.fscore) == approx((0.25, 0.5, 0.33333333)) + + a += b + assert (a.precision, a.recall, a.fscore) == approx( + (c.precision, c.recall, c.fscore) + ) + + +def test_score_cats(en_tokenizer): + text = "some text" + gold_doc = en_tokenizer(text) + gold_doc.cats = {"POSITIVE": 1.0, "NEGATIVE": 0.0} + pred_doc = en_tokenizer(text) + pred_doc.cats = {"POSITIVE": 0.75, "NEGATIVE": 0.25} + example = Example(pred_doc, gold_doc) + # threshold is ignored for multi_label=False + scores1 = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=False, + positive_label="POSITIVE", + threshold=0.1, + ) + scores2 = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=False, + positive_label="POSITIVE", + threshold=0.9, + ) + assert scores1["cats_score"] == 1.0 + assert scores2["cats_score"] == 1.0 + assert scores1 == scores2 + # threshold is relevant for multi_label=True + scores = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=True, + threshold=0.9, + ) + assert scores["cats_macro_f"] == 0.0 + # threshold is relevant for multi_label=True + scores = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=True, + threshold=0.1, + ) + assert scores["cats_macro_f"] == 0.5 diff --git a/spacy/tests/test_ty.py b/spacy/tests/test_ty.py new file mode 100644 index 0000000..2037520 --- /dev/null +++ b/spacy/tests/test_ty.py @@ -0,0 +1,18 @@ +import spacy +from spacy import ty + + +def test_component_types(): + nlp = spacy.blank("en") + tok2vec = nlp.create_pipe("tok2vec") + tagger = nlp.create_pipe("tagger") + entity_ruler = nlp.create_pipe("entity_ruler") + assert isinstance(tok2vec, ty.TrainableComponent) + assert isinstance(tagger, ty.TrainableComponent) + assert not isinstance(entity_ruler, ty.TrainableComponent) + assert isinstance(tok2vec, ty.InitializableComponent) + assert isinstance(tagger, ty.InitializableComponent) + assert isinstance(entity_ruler, ty.InitializableComponent) + assert isinstance(tok2vec, ty.ListenedToComponent) + assert not isinstance(tagger, ty.ListenedToComponent) + assert not isinstance(entity_ruler, ty.ListenedToComponent) diff --git a/spacy/tests/tok2vec.py b/spacy/tests/tok2vec.py new file mode 100644 index 0000000..7e7b689 --- /dev/null +++ b/spacy/tests/tok2vec.py @@ -0,0 +1,36 @@ +from typing import List + +from thinc.api import Model +from thinc.types import Floats2d + +from spacy.tokens import Doc +from spacy.util import registry + + +@registry.architectures("test.LazyInitTok2Vec.v1") +def build_lazy_init_tok2vec(*, width: int) -> Model[List[Doc], List[Floats2d]]: + """tok2vec model of which the output size is only known after + initialization. This implementation does not output meaningful + embeddings, it is strictly for testing.""" + return Model( + "lazy_init_tok2vec", + lazy_init_tok2vec_forward, + init=lazy_init_tok2vec_init, + dims={"nO": None}, + attrs={"width": width}, + ) + + +def lazy_init_tok2vec_init(model: Model, X=None, Y=None): + width = model.attrs["width"] + model.set_dim("nO", width) + + +def lazy_init_tok2vec_forward(model: Model, X: List[Doc], is_train: bool): + width = model.get_dim("nO") + Y = [model.ops.alloc2f(len(doc), width) for doc in X] + + def backprop(dY): + return [] + + return Y, backprop diff --git a/spacy/tests/tokenizer/__init__.py b/spacy/tests/tokenizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/tokenizer/sun.txt b/spacy/tests/tokenizer/sun.txt new file mode 100644 index 0000000..1b04f8b --- /dev/null +++ b/spacy/tests/tokenizer/sun.txt @@ -0,0 +1,4 @@ +The Sun is the star at the center of the Solar System. It is almost perfectly spherical and consists of hot plasma interwoven with magnetic fields.[12][13] It has a diameter of about 1,392,684 km (865,374 mi),[5] around 109 times that of Earth, and its mass (1.989×1030 kilograms, approximately 330,000 times the mass of Earth) accounts for about 99.86% of the total mass of the Solar System.[14] Chemically, about three quarters of the Sun's mass consists of hydrogen, while the rest is mostly helium. The remaining 1.69% (equal to 5,600 times the mass of Earth) consists of heavier elements, including oxygen, carbon, neon and iron, among others.[15] + +The Sun formed about 4.567 billion[a][16] years ago from the gravitational collapse of a region within a large molecular cloud. Most of the matter gathered in the center, while the rest flattened into an orbiting disk that would become the Solar System. The central mass became increasingly hot and dense, eventually initiating thermonuclear fusion in its core. It is thought that almost all stars form by this process. The Sun is a G-type main-sequence star (G2V) based on spectral class and it is informally designated as a yellow dwarf because its visible radiation is most intense in the yellow-green portion of the spectrum, and although it is actually white in color, from the surface of the Earth it may appear yellow because of atmospheric scattering of blue light.[17] In the spectral class label, G2 indicates its surface temperature, of approximately 5778 K (5505 °C), and V indicates that the Sun, like most stars, is a main-sequence star, and thus generates its energy by nuclear fusion of hydrogen nuclei into helium. In its core, the Sun fuses about 620 million metric tons of hydrogen each second.[18][19] +Once regarded by astronomers as a small and relatively insignificant star, the Sun is now thought to be brighter than about 85% of the stars in the Milky Way, most of which are red dwarfs.[20][21] The absolute magnitude of the Sun is +4.83; however, as the star closest to Earth, the Sun is by far the brightest object in the sky with an apparent magnitude of −26.74.[22][23] This is about 13 billion times brighter than the next brightest star, Sirius, with an apparent magnitude of −1.46. The Sun's hot corona continuously expands in space creating the solar wind, a stream of charged particles that extends to the heliopause at roughly 100 astronomical units. The bubble in the interstellar medium formed by the solar wind, the heliosphere, is the largest continuous structure in the Solar System.[24][25] diff --git a/spacy/tests/tokenizer/test_exceptions.py b/spacy/tests/tokenizer/test_exceptions.py new file mode 100644 index 0000000..1f8f52c --- /dev/null +++ b/spacy/tests/tokenizer/test_exceptions.py @@ -0,0 +1,54 @@ +import sys + +import pytest + + +def test_tokenizer_handles_emoticons(tokenizer): + # Tweebo challenge (CMU) + text = ( + """:o :/ :'( >:o (: :) >.< XD -__- o.O ;D :-) @_@ :P 8D :1 >:( :D =| :> ....""" + ) + tokens = tokenizer(text) + assert tokens[0].text == ":o" + assert tokens[1].text == ":/" + assert tokens[2].text == ":'(" + assert tokens[3].text == ">:o" + assert tokens[4].text == "(:" + assert tokens[5].text == ":)" + assert tokens[6].text == ">.<" + assert tokens[7].text == "XD" + assert tokens[8].text == "-__-" + assert tokens[9].text == "o.O" + assert tokens[10].text == ";D" + assert tokens[11].text == ":-)" + assert tokens[12].text == "@_@" + assert tokens[13].text == ":P" + assert tokens[14].text == "8D" + assert tokens[15].text == ":1" + assert tokens[16].text == ">:(" + assert tokens[17].text == ":D" + assert tokens[18].text == "=|" + assert tokens[19].text == ":>" + assert tokens[20].text == "...." + + +@pytest.mark.parametrize("text,length", [("108)", 2), ("XDN", 1)]) +def test_tokenizer_excludes_false_pos_emoticons(tokenizer, text, length): + tokens = tokenizer(text) + assert len(tokens) == length + + +@pytest.mark.parametrize( + "text,length", [("can you still dunk?🍕🍔😵LOL", 8), ("i💙you", 3), ("🤘🤘yay!", 4)] +) +def test_tokenizer_handles_emoji(tokenizer, text, length): + # These break on narrow unicode builds, e.g. Windows + if sys.maxunicode >= 1114111: + tokens = tokenizer(text) + assert len(tokens) == length + + +def test_tokenizer_degree(tokenizer): + for u in "cfkCFK": + assert [t.text for t in tokenizer(f"°{u}.")] == ["°", f"{u}", "."] + assert [t[1] for t in tokenizer.explain(f"°{u}.")] == ["°", f"{u}", "."] diff --git a/spacy/tests/tokenizer/test_explain.py b/spacy/tests/tokenizer/test_explain.py new file mode 100644 index 0000000..f475284 --- /dev/null +++ b/spacy/tests/tokenizer/test_explain.py @@ -0,0 +1,145 @@ +import re +import string + +import hypothesis +import hypothesis.strategies +import pytest + +import spacy +from spacy.tokenizer import Tokenizer +from spacy.util import get_lang_class + +# Only include languages with no external dependencies +# "is" seems to confuse importlib, so we're also excluding it for now +# excluded: ja, ru, th, uk, vi, zh, is +LANGUAGES = [ + pytest.param("fr", marks=pytest.mark.slow()), + pytest.param("af", marks=pytest.mark.slow()), + pytest.param("ar", marks=pytest.mark.slow()), + pytest.param("bg", marks=pytest.mark.slow()), + "bn", + pytest.param("bo", marks=pytest.mark.slow()), + pytest.param("ca", marks=pytest.mark.slow()), + pytest.param("cs", marks=pytest.mark.slow()), + pytest.param("da", marks=pytest.mark.slow()), + pytest.param("de", marks=pytest.mark.slow()), + "el", + "en", + pytest.param("es", marks=pytest.mark.slow()), + pytest.param("et", marks=pytest.mark.slow()), + pytest.param("fa", marks=pytest.mark.slow()), + pytest.param("fi", marks=pytest.mark.slow()), + "fr", + pytest.param("ga", marks=pytest.mark.slow()), + pytest.param("he", marks=pytest.mark.slow()), + pytest.param("hi", marks=pytest.mark.slow()), + pytest.param("hr", marks=pytest.mark.slow()), + "hu", + pytest.param("id", marks=pytest.mark.slow()), + pytest.param("it", marks=pytest.mark.slow()), + pytest.param("kn", marks=pytest.mark.slow()), + pytest.param("lb", marks=pytest.mark.slow()), + pytest.param("lt", marks=pytest.mark.slow()), + pytest.param("lv", marks=pytest.mark.slow()), + pytest.param("nb", marks=pytest.mark.slow()), + pytest.param("nl", marks=pytest.mark.slow()), + "pl", + pytest.param("pt", marks=pytest.mark.slow()), + pytest.param("ro", marks=pytest.mark.slow()), + pytest.param("si", marks=pytest.mark.slow()), + pytest.param("sk", marks=pytest.mark.slow()), + pytest.param("sl", marks=pytest.mark.slow()), + pytest.param("sq", marks=pytest.mark.slow()), + pytest.param("sr", marks=pytest.mark.slow()), + pytest.param("sv", marks=pytest.mark.slow()), + pytest.param("ta", marks=pytest.mark.slow()), + pytest.param("te", marks=pytest.mark.slow()), + pytest.param("tl", marks=pytest.mark.slow()), + pytest.param("tr", marks=pytest.mark.slow()), + pytest.param("tt", marks=pytest.mark.slow()), + pytest.param("ur", marks=pytest.mark.slow()), + pytest.param("kmr", marks=pytest.mark.slow()), +] + + +@pytest.mark.parametrize("lang", LANGUAGES) +def test_tokenizer_explain(lang): + tokenizer = get_lang_class(lang)().tokenizer + examples = pytest.importorskip(f"spacy.lang.{lang}.examples") + for sentence in examples.sentences: + tokens = [t.text for t in tokenizer(sentence) if not t.is_space] + debug_tokens = [t[1] for t in tokenizer.explain(sentence)] + assert tokens == debug_tokens + + +def test_tokenizer_explain_special_matcher(en_vocab): + suffix_re = re.compile(r"[\.]$") + infix_re = re.compile(r"[/]") + rules = {"a.": [{"ORTH": "a."}]} + tokenizer = Tokenizer( + en_vocab, + rules=rules, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + ) + tokens = [t.text for t in tokenizer("a/a.")] + explain_tokens = [t[1] for t in tokenizer.explain("a/a.")] + assert tokens == explain_tokens + + +def test_tokenizer_explain_special_matcher_whitespace(en_vocab): + rules = {":]": [{"ORTH": ":]"}]} + tokenizer = Tokenizer( + en_vocab, + rules=rules, + ) + text = ": ]" + tokens = [t.text for t in tokenizer(text)] + explain_tokens = [t[1] for t in tokenizer.explain(text)] + assert tokens == explain_tokens + + +@hypothesis.strategies.composite +def sentence_strategy(draw: hypothesis.strategies.DrawFn, max_n_words: int = 4) -> str: + """ + Composite strategy for fuzzily generating sentence with varying interpunctation. + + draw (hypothesis.strategies.DrawFn): Protocol for drawing function allowing to fuzzily pick from hypothesis' + strategies. + max_n_words (int): Max. number of words in generated sentence. + RETURNS (str): Fuzzily generated sentence. + """ + + punctuation_and_space_regex = "|".join( + [*[re.escape(p) for p in string.punctuation], r"\s"] + ) + sentence = [ + [ + draw(hypothesis.strategies.text(min_size=1)), + draw(hypothesis.strategies.from_regex(punctuation_and_space_regex)), + ] + for _ in range( + draw(hypothesis.strategies.integers(min_value=2, max_value=max_n_words)) + ) + ] + + return " ".join([token for token_pair in sentence for token in token_pair]) + + +@pytest.mark.xfail +@pytest.mark.parametrize("lang", LANGUAGES) +@hypothesis.given(sentence=sentence_strategy()) +def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None: + """ + Tests whether output of tokenizer.explain() matches tokenizer output. Input generated by hypothesis. + lang (str): Language to test. + text (str): Fuzzily generated sentence to tokenize. + """ + + tokenizer: Tokenizer = spacy.blank(lang).tokenizer + # Tokenizer.explain is not intended to handle whitespace or control + # characters in the same way as Tokenizer + sentence = re.sub(r"\s+", " ", sentence).strip() + tokens = [t.text for t in tokenizer(sentence)] + debug_tokens = [t[1] for t in tokenizer.explain(sentence)] + assert tokens == debug_tokens, f"{tokens}, {debug_tokens}, {sentence}" diff --git a/spacy/tests/tokenizer/test_naughty_strings.py b/spacy/tests/tokenizer/test_naughty_strings.py new file mode 100644 index 0000000..b22dabb --- /dev/null +++ b/spacy/tests/tokenizer/test_naughty_strings.py @@ -0,0 +1,115 @@ +import pytest + +# Examples taken from the "Big List of Naughty Strings" +# https://github.com/minimaxir/big-list-of-naughty-strings + + +NAUGHTY_STRINGS = [ + # ASCII punctuation + r",./;'[]\-=", + r'<>?:"{}|_+', + r'!@#$%^&*()`~"', + # Unicode additional control characters, byte order marks + r"­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪", + r"￾", + # Unicode Symbols + r"Ω≈ç√∫˜µ≤≥÷", + r"åß∂ƒ©˙∆˚¬…æ", + "œ∑´®†¥¨ˆøπ“‘", + r"¡™£¢∞§¶•ªº–≠", + r"¸˛Ç◊ı˜Â¯˘¿", + r"ÅÍÎÏ˝ÓÔÒÚÆ☃", + r"Œ„´‰ˇÁ¨ˆØ∏”’", + r"`⁄€‹›fifl‡°·‚—±", + r"⅛⅜⅝⅞", + r"ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя", + r"٠١٢٣٤٥٦٧٨٩", + # Unicode Subscript/Superscript/Accents + r"⁰⁴⁵", + r"₀₁₂", + r"⁰⁴⁵₀₁₂", + r"ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็", + r" ̄ ̄", + # Two-Byte Characters + r"田中さんにあげて下さい", + r"パーティーへ行かないか", + r"和製漢語", + r"部落格", + r"사회과학원 어학연구소", + r"찦차를 타고 온 펲시맨과 쑛다리 똠방각하", + r"社會科學院語學研究所", + r"울란바토르", + r"𠜎𠜱𠝹𠱓𠱸𠲖𠳏", + # Japanese Emoticons + r"ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ", + r"(｡◕ ∀ ◕｡)", + r"｀ｨ(´∀｀∩", + r"__ﾛ(,_,*)", + r"・(￣∀￣)・:*:", + r"ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ", + r",。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’", + r"(╯°□°）╯︵ ┻━┻)" "(ﾉಥ益ಥ）ﾉ ┻━┻", + r"┬─┬ノ( º _ ºノ)", + r"( ͡° ͜ʖ ͡°)", + # Emoji + r"😍", + r"👩🏽", + r"👾 🙇 💁 🙅 🙆 🙋 🙎 🙍", + r"🐵 🙈 🙉 🙊", + r"❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙", + r"✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿", + r"🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧", + r"0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟", + # Regional Indicator Symbols + r"🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸", + r"🇺🇸🇷🇺🇸🇦🇫🇦🇲", + r"🇺🇸🇷🇺🇸🇦", + # Unicode Numbers + r"１２３", + r"١٢٣", + # Right-To-Left Strings + r"ثم نفس سقطت وبالتحديد،, جزيرتي باستخدام أن دنو. إذ هنا؟ الستار وتنصيب كان. أهّل ايطاليا، بريطانيا-فرنسا قد أخذ. سليمان، إتفاقية بين ما, يذكر الحدود أي بعد, معاملة بولندا، الإطلاق عل إيو.", + r"إيو.", + r"בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ", + r"הָיְתָהtestالصفحات التّحول", + r"﷽", + r"ﷺ", + r"مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ،", + # Trick Unicode + r"‪‪test‪", + r"‫test", + r"
test
", + r"test⁠test", + r"⁦test⁧", + # Zalgo Text + r"Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣", + r"̡͓̞ͅI̗̘̦͝n͇͇͙v̮̫ok̲̫̙͈i̖͙̭̹̠̞n̡̻̮̣̺g̲͈͙̭͙̬͎ ̰t͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨o͚̪͡f̘̣̬ ̖̘͖̟͙̮c҉͔̫͖͓͇͖ͅh̵̤̣͚͔á̗̼͕ͅo̼̣̥s̱͈̺̖̦̻͢.̛̖̞̠̫̰", + r"̗̺͖̹̯͓Ṯ̤͍̥͇͈h̲́e͏͓̼̗̙̼̣͔ ͇̜̱̠͓͍ͅN͕͠e̗̱z̘̝̜̺͙p̤̺̹͍̯͚e̠̻̠͜r̨̤͍̺̖͔̖̖d̠̟̭̬̝͟i̦͖̩͓͔̤a̠̗̬͉̙n͚͜ ̻̞̰͚ͅh̵͉i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟", + r"̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕", + r"Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮", + # Unicode Upsidedown + r"˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥", + r"00˙Ɩ$-", + # Unicode font + r"Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ", + r"𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠", + r"𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌", + r"𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈", + r"𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰", + r"𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘", + r"𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐", + r"⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢", + # File paths + r"../../../../../../../../../../../etc/passwd%00", + r"../../../../../../../../../../../etc/hosts", + # iOS Vulnerabilities + r"Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗", + r"🏳0🌈️", +] + + +@pytest.mark.slow +@pytest.mark.parametrize("text", NAUGHTY_STRINGS) +def test_tokenizer_naughty_strings(tokenizer, text): + tokens = tokenizer(text) + assert tokens.text_with_ws == text diff --git a/spacy/tests/tokenizer/test_tokenizer.py b/spacy/tests/tokenizer/test_tokenizer.py new file mode 100644 index 0000000..1ea5f78 --- /dev/null +++ b/spacy/tests/tokenizer/test_tokenizer.py @@ -0,0 +1,557 @@ +import re + +import numpy +import pytest + +from spacy.lang.de import German +from spacy.lang.en import English +from spacy.symbols import ORTH +from spacy.tokenizer import Tokenizer +from spacy.tokens import Doc +from spacy.training import Example +from spacy.util import ( + compile_infix_regex, + compile_prefix_regex, + compile_suffix_regex, + ensure_path, +) +from spacy.vocab import Vocab + + +@pytest.mark.issue(743) +def test_issue743(): + doc = Doc(Vocab(), ["hello", "world"]) + token = doc[0] + s = set([token]) + items = list(s) + assert items[0] is token + + +@pytest.mark.issue(801) +@pytest.mark.skip( + reason="Can not be fixed unless with variable-width lookbehinds, cf. PR #3218" +) +@pytest.mark.parametrize( + "text,tokens", + [ + ('"deserve,"--and', ['"', "deserve", ',"--', "and"]), + ("exception;--exclusive", ["exception", ";--", "exclusive"]), + ("day.--Is", ["day", ".--", "Is"]), + ("refinement:--just", ["refinement", ":--", "just"]), + ("memories?--To", ["memories", "?--", "To"]), + ("Useful.=--Therefore", ["Useful", ".=--", "Therefore"]), + ("=Hope.=--Pandora", ["=", "Hope", ".=--", "Pandora"]), + ], +) +def test_issue801(en_tokenizer, text, tokens): + """Test that special characters + hyphens are split correctly.""" + doc = en_tokenizer(text) + assert len(doc) == len(tokens) + assert [t.text for t in doc] == tokens + + +@pytest.mark.issue(1061) +def test_issue1061(): + """Test special-case works after tokenizing. Was caching problem.""" + text = "I like _MATH_ even _MATH_ when _MATH_, except when _MATH_ is _MATH_! but not _MATH_." + tokenizer = English().tokenizer + doc = tokenizer(text) + assert "MATH" in [w.text for w in doc] + assert "_MATH_" not in [w.text for w in doc] + + tokenizer.add_special_case("_MATH_", [{ORTH: "_MATH_"}]) + doc = tokenizer(text) + assert "_MATH_" in [w.text for w in doc] + assert "MATH" not in [w.text for w in doc] + + # For sanity, check it works when pipeline is clean. + tokenizer = English().tokenizer + tokenizer.add_special_case("_MATH_", [{ORTH: "_MATH_"}]) + doc = tokenizer(text) + assert "_MATH_" in [w.text for w in doc] + assert "MATH" not in [w.text for w in doc] + + +@pytest.mark.issue(1963) +def test_issue1963(en_tokenizer): + """Test that doc.merge() resizes doc.tensor""" + doc = en_tokenizer("a b c d") + doc.tensor = numpy.ones((len(doc), 128), dtype="f") + with doc.retokenize() as retokenizer: + retokenizer.merge(doc[0:2]) + assert len(doc) == 3 + assert doc.tensor.shape == (3, 128) + + +@pytest.mark.skip( + reason="Can not be fixed without variable-width look-behind (which we don't want)" +) +@pytest.mark.issue(1235) +def test_issue1235(): + """Test that g is not split of if preceded by a number and a letter""" + nlp = English() + testwords = "e2g 2g 52g" + doc = nlp(testwords) + assert len(doc) == 5 + assert doc[0].text == "e2g" + assert doc[1].text == "2" + assert doc[2].text == "g" + assert doc[3].text == "52" + assert doc[4].text == "g" + + +@pytest.mark.issue(1242) +def test_issue1242(): + nlp = English() + doc = nlp("") + assert len(doc) == 0 + docs = list(nlp.pipe(["", "hello"])) + assert len(docs[0]) == 0 + assert len(docs[1]) == 1 + + +@pytest.mark.issue(1257) +def test_issue1257(): + """Test that tokens compare correctly.""" + doc1 = Doc(Vocab(), words=["a", "b", "c"]) + doc2 = Doc(Vocab(), words=["a", "c", "e"]) + assert doc1[0] != doc2[0] + assert not doc1[0] == doc2[0] + + +@pytest.mark.issue(1375) +def test_issue1375(): + """Test that token.nbor() raises IndexError for out-of-bounds access.""" + doc = Doc(Vocab(), words=["0", "1", "2"]) + with pytest.raises(IndexError): + assert doc[0].nbor(-1) + assert doc[1].nbor(-1).text == "0" + with pytest.raises(IndexError): + assert doc[2].nbor(1) + assert doc[1].nbor(1).text == "2" + + +@pytest.mark.issue(1488) +def test_issue1488(): + """Test that tokenizer can parse DOT inside non-whitespace separators""" + prefix_re = re.compile(r"""[\[\("']""") + suffix_re = re.compile(r"""[\]\)"']""") + infix_re = re.compile(r"""[-~\.]""") + simple_url_re = re.compile(r"""^https?://""") + + def my_tokenizer(nlp): + return Tokenizer( + nlp.vocab, + {}, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + token_match=simple_url_re.match, + ) + + nlp = English() + nlp.tokenizer = my_tokenizer(nlp) + doc = nlp("This is a test.") + for token in doc: + assert token.text + + +@pytest.mark.issue(1494) +def test_issue1494(): + """Test if infix_finditer works correctly""" + infix_re = re.compile(r"""[^a-z]""") + test_cases = [ + ("token 123test", ["token", "1", "2", "3", "test"]), + ("token 1test", ["token", "1test"]), + ("hello...test", ["hello", ".", ".", ".", "test"]), + ] + + def new_tokenizer(nlp): + return Tokenizer(nlp.vocab, {}, infix_finditer=infix_re.finditer) + + nlp = English() + nlp.tokenizer = new_tokenizer(nlp) + for text, expected in test_cases: + assert [token.text for token in nlp(text)] == expected + + +@pytest.mark.skip( + reason="Can not be fixed without iterative looping between prefix/suffix and infix" +) +@pytest.mark.issue(2070) +def test_issue2070(): + """Test that checks that a dot followed by a quote is handled + appropriately. + """ + # Problem: The dot is now properly split off, but the prefix/suffix rules + # are not applied again afterwards. This means that the quote will still be + # attached to the remaining token. + nlp = English() + doc = nlp('First sentence."A quoted sentence" he said ...') + assert len(doc) == 11 + + +@pytest.mark.issue(2926) +def test_issue2926(fr_tokenizer): + """Test that the tokenizer correctly splits tokens separated by a slash (/) + ending in a digit. + """ + doc = fr_tokenizer("Learn html5/css3/javascript/jquery") + assert len(doc) == 8 + assert doc[0].text == "Learn" + assert doc[1].text == "html5" + assert doc[2].text == "/" + assert doc[3].text == "css3" + assert doc[4].text == "/" + assert doc[5].text == "javascript" + assert doc[6].text == "/" + assert doc[7].text == "jquery" + + +@pytest.mark.parametrize( + "text", + [ + "ABLEItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume TABLE ItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume ItemColumn IAcceptance Limits of ErrorIn-Service Limits of ErrorColumn IIColumn IIIColumn IVColumn VComputed VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeUnder Registration of\xa0VolumeOver Registration of\xa0VolumeCubic FeetCubic FeetCubic FeetCubic FeetCubic Feet1Up to 10.0100.0050.0100.005220.0200.0100.0200.010350.0360.0180.0360.0184100.0500.0250.0500.0255Over 100.5% of computed volume0.25% of computed volume0.5% of computed volume0.25% of computed volume", + "oow.jspsearch.eventoracleopenworldsearch.technologyoraclesolarissearch.technologystoragesearch.technologylinuxsearch.technologyserverssearch.technologyvirtualizationsearch.technologyengineeredsystemspcodewwmkmppscem:", + ], +) +@pytest.mark.issue(2626) +def test_issue2626_2835(en_tokenizer, text): + """Check that sentence doesn't cause an infinite loop in the tokenizer.""" + doc = en_tokenizer(text) + assert doc + + +@pytest.mark.issue(2656) +def test_issue2656(en_tokenizer): + """Test that tokenizer correctly splits off punctuation after numbers with + decimal points. + """ + doc = en_tokenizer("I went for 40.3, and got home by 10.0.") + assert len(doc) == 11 + assert doc[0].text == "I" + assert doc[1].text == "went" + assert doc[2].text == "for" + assert doc[3].text == "40.3" + assert doc[4].text == "," + assert doc[5].text == "and" + assert doc[6].text == "got" + assert doc[7].text == "home" + assert doc[8].text == "by" + assert doc[9].text == "10.0" + assert doc[10].text == "." + + +@pytest.mark.issue(2754) +def test_issue2754(en_tokenizer): + """Test that words like 'a' and 'a.m.' don't get exceptional norm values.""" + a = en_tokenizer("a") + assert a[0].norm_ == "a" + am = en_tokenizer("am") + assert am[0].norm_ == "am" + + +@pytest.mark.issue(3002) +def test_issue3002(): + """Test that the tokenizer doesn't hang on a long list of dots""" + nlp = German() + doc = nlp( + "880.794.982.218.444.893.023.439.794.626.120.190.780.624.990.275.671 ist eine lange Zahl" + ) + assert len(doc) == 5 + + +@pytest.mark.skip(reason="default suffix rules avoid one upper-case letter before dot") +@pytest.mark.issue(3449) +def test_issue3449(): + nlp = English() + nlp.add_pipe("sentencizer") + text1 = "He gave the ball to I. Do you want to go to the movies with I?" + text2 = "He gave the ball to I. Do you want to go to the movies with I?" + text3 = "He gave the ball to I.\nDo you want to go to the movies with I?" + t1 = nlp(text1) + t2 = nlp(text2) + t3 = nlp(text3) + assert t1[5].text == "I" + assert t2[5].text == "I" + assert t3[5].text == "I" + + +@pytest.mark.parametrize( + "text,words", [("A'B C", ["A", "'", "B", "C"]), ("A-B", ["A-B"])] +) +def test_gold_misaligned(en_tokenizer, text, words): + doc = en_tokenizer(text) + Example.from_dict(doc, {"words": words}) + + +def test_tokenizer_handles_no_word(tokenizer): + tokens = tokenizer("") + assert len(tokens) == 0 + + +@pytest.mark.parametrize("text", ["lorem"]) +def test_tokenizer_handles_single_word(tokenizer, text): + tokens = tokenizer(text) + assert tokens[0].text == text + + +def test_tokenizer_handles_punct(tokenizer): + text = "Lorem, ipsum." + tokens = tokenizer(text) + assert len(tokens) == 4 + assert tokens[0].text == "Lorem" + assert tokens[1].text == "," + assert tokens[2].text == "ipsum" + assert tokens[1].text != "Lorem" + + +def test_tokenizer_handles_punct_braces(tokenizer): + text = "Lorem, (ipsum)." + tokens = tokenizer(text) + assert len(tokens) == 6 + + +def test_tokenizer_handles_digits(tokenizer): + exceptions = ["hu", "bn"] + text = "Lorem ipsum: 1984." + tokens = tokenizer(text) + + if tokens[0].lang_ not in exceptions: + assert len(tokens) == 5 + assert tokens[0].text == "Lorem" + assert tokens[3].text == "1984" + + +@pytest.mark.parametrize( + "text", + ["google.com", "python.org", "spacy.io", "explosion.ai", "http://www.google.com"], +) +def test_tokenizer_keep_urls(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 1 + + +@pytest.mark.parametrize("text", ["NASDAQ:GOOG"]) +def test_tokenizer_colons(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize( + "text", ["hello123@example.com", "hi+there@gmail.it", "matt@explosion.ai"] +) +def test_tokenizer_keeps_email(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 1 + + +def test_tokenizer_handles_long_text(tokenizer): + text = """Lorem ipsum dolor sit amet, consectetur adipiscing elit + +Cras egestas orci non porttitor maximus. +Maecenas quis odio id dolor rhoncus dignissim. Curabitur sed velit at orci ultrices sagittis. Nulla commodo euismod arcu eget vulputate. + +Phasellus tincidunt, augue quis porta finibus, massa sapien consectetur augue, non lacinia enim nibh eget ipsum. Vestibulum in bibendum mauris. + +"Nullam porta fringilla enim, a dictum orci consequat in." Mauris nec malesuada justo.""" + + tokens = tokenizer(text) + assert len(tokens) > 5 + + +@pytest.mark.parametrize("file_name", ["sun.txt"]) +def test_tokenizer_handle_text_from_file(tokenizer, file_name): + loc = ensure_path(__file__).parent / file_name + with loc.open("r", encoding="utf8") as infile: + text = infile.read() + assert len(text) != 0 + tokens = tokenizer(text) + assert len(tokens) > 100 + + +def test_tokenizer_suspected_freeing_strings(tokenizer): + text1 = "Lorem dolor sit amet, consectetur adipiscing elit." + text2 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit." + tokens1 = tokenizer(text1) + tokens2 = tokenizer(text2) + assert tokens1[0].text == "Lorem" + assert tokens2[0].text == "Lorem" + + +@pytest.mark.parametrize("text,tokens", [("lorem", [{"orth": "lo"}, {"orth": "rem"}])]) +def test_tokenizer_add_special_case(tokenizer, text, tokens): + tokenizer.add_special_case(text, tokens) + doc = tokenizer(text) + assert doc[0].text == tokens[0]["orth"] + assert doc[1].text == tokens[1]["orth"] + + +@pytest.mark.parametrize( + "text,tokens", + [ + ("lorem", [{"orth": "lo"}, {"orth": "re"}]), + ("lorem", [{"orth": "lo", "tag": "A"}, {"orth": "rem"}]), + ], +) +def test_tokenizer_validate_special_case(tokenizer, text, tokens): + with pytest.raises(ValueError): + tokenizer.add_special_case(text, tokens) + + +@pytest.mark.parametrize( + "text,tokens", [("lorem", [{"orth": "lo", "norm": "LO"}, {"orth": "rem"}])] +) +def test_tokenizer_add_special_case_tag(text, tokens): + vocab = Vocab() + tokenizer = Tokenizer(vocab, {}, None, None, None) + tokenizer.add_special_case(text, tokens) + doc = tokenizer(text) + assert doc[0].text == tokens[0]["orth"] + assert doc[0].norm_ == tokens[0]["norm"] + assert doc[1].text == tokens[1]["orth"] + + +def test_tokenizer_special_cases_with_affixes(tokenizer): + text = '(((_SPECIAL_ A/B, A/B-A/B")' + tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}]) + tokenizer.add_special_case("A/B", [{"orth": "A/B"}]) + doc = tokenizer(text) + assert [token.text for token in doc] == [ + "(", + "(", + "(", + "_SPECIAL_", + "A/B", + ",", + "A/B", + "-", + "A/B", + '"', + ")", + ] + + +def test_tokenizer_special_cases_with_affixes_preserve_spacy(): + tokenizer = English().tokenizer + # reset all special cases + tokenizer.rules = {} + + # in-place modification (only merges) + text = "''a'' " + tokenizer.add_special_case("''", [{"ORTH": "''"}]) + assert tokenizer(text).text == text + + # not in-place (splits and merges) + tokenizer.add_special_case("ab", [{"ORTH": "a"}, {"ORTH": "b"}]) + text = "ab ab ab ''ab ab'' ab'' ''ab" + assert tokenizer(text).text == text + + +def test_tokenizer_special_cases_with_period(tokenizer): + text = "_SPECIAL_." + tokenizer.add_special_case("_SPECIAL_", [{"orth": "_SPECIAL_"}]) + doc = tokenizer(text) + assert [token.text for token in doc] == ["_SPECIAL_", "."] + + +def test_tokenizer_special_cases_idx(tokenizer): + text = "the _ID'X_" + tokenizer.add_special_case("_ID'X_", [{"orth": "_ID"}, {"orth": "'X_"}]) + doc = tokenizer(text) + assert doc[1].idx == 4 + assert doc[2].idx == 7 + + +def test_tokenizer_special_cases_spaces(tokenizer): + assert [t.text for t in tokenizer("a b c")] == ["a", "b", "c"] + tokenizer.add_special_case("a b c", [{"ORTH": "a b c"}]) + assert [t.text for t in tokenizer("a b c")] == ["a b c"] + + +def test_tokenizer_flush_cache(en_vocab): + suffix_re = re.compile(r"[\.]$") + tokenizer = Tokenizer( + en_vocab, + suffix_search=suffix_re.search, + ) + assert [t.text for t in tokenizer("a.")] == ["a", "."] + tokenizer.suffix_search = None + assert [t.text for t in tokenizer("a.")] == ["a."] + + +def test_tokenizer_flush_specials(en_vocab): + suffix_re = re.compile(r"[\.]$") + rules = {"a a": [{"ORTH": "a a"}]} + tokenizer1 = Tokenizer( + en_vocab, + suffix_search=suffix_re.search, + rules=rules, + ) + assert [t.text for t in tokenizer1("a a.")] == ["a a", "."] + tokenizer1.rules = {} + assert [t.text for t in tokenizer1("a a.")] == ["a", "a", "."] + + +def test_tokenizer_prefix_suffix_overlap_lookbehind(en_vocab): + # the prefix and suffix matches overlap in the suffix lookbehind + prefixes = ["a(?=.)"] + suffixes = [r"(?<=\w)\.", r"(?<=a)\d+\."] + prefix_re = compile_prefix_regex(prefixes) + suffix_re = compile_suffix_regex(suffixes) + tokenizer = Tokenizer( + en_vocab, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + ) + tokens = [t.text for t in tokenizer("a10.")] + assert tokens == ["a", "10", "."] + explain_tokens = [t[1] for t in tokenizer.explain("a10.")] + assert tokens == explain_tokens + + +def test_tokenizer_infix_prefix(en_vocab): + # the prefix and suffix matches overlap in the suffix lookbehind + infixes = ["±"] + suffixes = ["%"] + infix_re = compile_infix_regex(infixes) + suffix_re = compile_suffix_regex(suffixes) + tokenizer = Tokenizer( + en_vocab, + infix_finditer=infix_re.finditer, + suffix_search=suffix_re.search, + ) + tokens = [t.text for t in tokenizer("±10%")] + assert tokens == ["±10", "%"] + explain_tokens = [t[1] for t in tokenizer.explain("±10%")] + assert tokens == explain_tokens + + +@pytest.mark.issue(10086) +def test_issue10086(en_tokenizer): + """Test special case works when part of infix substring.""" + text = "No--don't see" + + # without heuristics: do n't + en_tokenizer.faster_heuristics = False + doc = en_tokenizer(text) + assert "n't" in [w.text for w in doc] + assert "do" in [w.text for w in doc] + + # with (default) heuristics: don't + en_tokenizer.faster_heuristics = True + doc = en_tokenizer(text) + assert "don't" in [w.text for w in doc] + + +def test_tokenizer_initial_special_case_explain(en_vocab): + tokenizer = Tokenizer( + en_vocab, + token_match=re.compile("^id$").match, + rules={ + "id": [{"ORTH": "i"}, {"ORTH": "d"}], + }, + ) + tokens = [t.text for t in tokenizer("id")] + explain_tokens = [t[1] for t in tokenizer.explain("id")] + assert tokens == explain_tokens diff --git a/spacy/tests/tokenizer/test_urls.py b/spacy/tests/tokenizer/test_urls.py new file mode 100644 index 0000000..ff8812b --- /dev/null +++ b/spacy/tests/tokenizer/test_urls.py @@ -0,0 +1,205 @@ +import pytest + +from spacy.lang.tokenizer_exceptions import BASE_EXCEPTIONS + +URLS_BASIC = [ + "http://www.nytimes.com/2016/04/20/us/politics/new-york-primary-preview.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=a-lede-package-region®ion=top-news&WT.nav=top-news&_r=0", + "www.red-stars.com", + "mailto:foo.bar@baz.com", +] + +URLS_FULL = URLS_BASIC + [ + "mailto:foo-bar@baz-co.com", + "mailto:foo-bar@baz-co.com?subject=hi", + "www.google.com?q=google", + "http://foo.com/blah_(wikipedia)#cite-1", +] + +# URL SHOULD_MATCH and SHOULD_NOT_MATCH patterns courtesy of https://mathiasbynens.be/demo/url-regex +URLS_SHOULD_MATCH = [ + "http://foo.com/blah_blah", + "http://BlahBlah.com/Blah_Blah", + "http://foo.com/blah_blah/", + "http://www.example.com/wpstyle/?p=364", + "https://www.example.com/foo/?bar=baz&inga=42&quux", + "http://userid:password@example.com:8080", + "http://userid:password@example.com:8080/", + "http://userid@example.com", + "http://userid@example.com/", + "http://userid@example.com:8080", + "http://userid@example.com:8080/", + "http://userid:password@example.com", + "http://userid:password@example.com/", + "http://142.42.1.1/", + "http://142.42.1.1:8080/", + "http://foo.com/blah_(wikipedia)#cite-1", + "http://foo.com/blah_(wikipedia)_blah#cite-1", + "http://foo.com/unicode_(✪)_in_parens", + "http://foo.com/(something)?after=parens", + "http://code.google.com/events/#&product=browser", + "http://j.mp", + "ftp://foo.bar/baz", + "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "http://1337.net", + "http://a.b-c.de", + "http://223.255.255.254", + "http://a.b--c.de/", # this is a legit domain name see: https://gist.github.com/dperini/729294 comment on 9/9/2014 + "ssh://login@server.com:12345/repository.git", + "svn+ssh://user@ssh.yourdomain.com/path", + pytest.param( + "chrome://extensions/?id=mhjfbmdgcfjbbpaeojofohoefgiehjai", + marks=pytest.mark.xfail(), + ), + pytest.param( + "chrome-extension://mhjfbmdgcfjbbpaeojofohoefgiehjai", marks=pytest.mark.xfail() + ), + "http://foo.com/blah_blah_(wikipedia)", + "http://foo.com/blah_blah_(wikipedia)_(again)", + "http://www.foo.co.uk", + "http://www.foo.co.uk/", + "http://www.foo.co.uk/blah/blah", + "http://⌘.ws", + "http://⌘.ws/", + "http://☺.damowmow.com/", + "http://✪df.ws/123", + "http://➡.ws/䨹", + "http://مثال.إختبار", + "http://例子.测试", + "http://उदाहरण.परीक्षा", +] + +URLS_SHOULD_NOT_MATCH = [ + "http://", + "http://.", + "http://..", + "http://../", + "http://?", + "http://??", + "http://??/", + "http://#", + "http://##", + "http://##/", + "http://foo.bar?q=Spaces should be encoded", + "//", + "//a", + "///a", + "///", + "http:///a", + "rdar://1234", + "h://test", + "http:// shouldfail.com", + ":// should fail", + "http://foo.bar/foo(bar)baz quux", + "http://-error-.invalid/", + "http://a.b-.co", + "http://0.0.0.0", + "http://10.1.1.0", + "http://10.1.1.255", + "http://224.1.1.1", + "http://123.123.123", + "http://3628126748", + "http://.www.foo.bar/", + "http://.www.foo.bar./", + "http://10.1.1.1", + "NASDAQ:GOOG", + "http://-a.b.co", + pytest.param("foo.com", marks=pytest.mark.xfail()), + "http://1.1.1.1.1", + "http://www.foo.bar./", +] + + +# Punctuation we want to check is split away before the URL +PREFIXES = ["(", '"', ">"] + + +# Punctuation we want to check is split away after the URL +SUFFIXES = ['"', ":", ">"] + + +@pytest.mark.parametrize("url", URLS_SHOULD_MATCH) +def test_should_match(en_tokenizer, url): + assert en_tokenizer.url_match(url) is not None + + +@pytest.mark.parametrize("url", URLS_SHOULD_NOT_MATCH) +def test_should_not_match(en_tokenizer, url): + assert en_tokenizer.url_match(url) is None + + +@pytest.mark.parametrize("url", URLS_BASIC) +def test_tokenizer_handles_simple_url(tokenizer, url): + tokens = tokenizer(url) + assert len(tokens) == 1 + assert tokens[0].text == url + + +@pytest.mark.parametrize("url", URLS_BASIC) +def test_tokenizer_handles_simple_surround_url(tokenizer, url): + tokens = tokenizer("(" + url + ")") + assert len(tokens) == 3 + assert tokens[0].text == "(" + assert tokens[1].text == url + assert tokens[2].text == ")" + + +@pytest.mark.slow +@pytest.mark.parametrize("prefix", PREFIXES) +@pytest.mark.parametrize("url", URLS_FULL) +def test_tokenizer_handles_prefixed_url(tokenizer, prefix, url): + tokens = tokenizer(prefix + url) + assert len(tokens) == 2 + assert tokens[0].text == prefix + assert tokens[1].text == url + + +@pytest.mark.slow +@pytest.mark.parametrize("suffix", SUFFIXES) +@pytest.mark.parametrize("url", URLS_FULL) +def test_tokenizer_handles_suffixed_url(tokenizer, url, suffix): + tokens = tokenizer(url + suffix) + assert len(tokens) == 2 + assert tokens[0].text == url + assert tokens[1].text == suffix + + +@pytest.mark.slow +@pytest.mark.parametrize("prefix", PREFIXES) +@pytest.mark.parametrize("suffix", SUFFIXES) +@pytest.mark.parametrize("url", URLS_FULL) +def test_tokenizer_handles_surround_url(tokenizer, prefix, suffix, url): + tokens = tokenizer(prefix + url + suffix) + assert len(tokens) == 3 + assert tokens[0].text == prefix + assert tokens[1].text == url + assert tokens[2].text == suffix + + +@pytest.mark.slow +@pytest.mark.parametrize("prefix1", PREFIXES) +@pytest.mark.parametrize("prefix2", PREFIXES) +@pytest.mark.parametrize("url", URLS_FULL) +def test_tokenizer_handles_two_prefix_url(tokenizer, prefix1, prefix2, url): + tokens = tokenizer(prefix1 + prefix2 + url) + assert len(tokens) == 3 + assert tokens[0].text == prefix1 + assert tokens[1].text == prefix2 + assert tokens[2].text == url + + +@pytest.mark.slow +@pytest.mark.parametrize("suffix1", SUFFIXES) +@pytest.mark.parametrize("suffix2", SUFFIXES) +@pytest.mark.parametrize("url", URLS_FULL) +def test_tokenizer_handles_two_suffix_url(tokenizer, suffix1, suffix2, url): + tokens = tokenizer(url + suffix1 + suffix2) + if suffix1 + suffix2 in BASE_EXCEPTIONS: + assert len(tokens) == 2 + assert tokens[0].text == url + assert tokens[1].text == suffix1 + suffix2 + else: + assert len(tokens) == 3 + assert tokens[0].text == url + assert tokens[1].text == suffix1 + assert tokens[2].text == suffix2 diff --git a/spacy/tests/tokenizer/test_whitespace.py b/spacy/tests/tokenizer/test_whitespace.py new file mode 100644 index 0000000..d68bb9e --- /dev/null +++ b/spacy/tests/tokenizer/test_whitespace.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.mark.parametrize("text", ["lorem ipsum"]) +def test_tokenizer_splits_single_space(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 2 + + +@pytest.mark.parametrize("text", ["lorem ipsum"]) +def test_tokenizer_splits_double_space(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 + assert tokens[1].text == " " + + +@pytest.mark.parametrize("text", ["lorem ipsum "]) +def test_tokenizer_handles_double_trailing_ws(tokenizer, text): + tokens = tokenizer(text) + assert repr(tokens.text_with_ws) == repr(text) + + +@pytest.mark.parametrize("text", ["lorem\nipsum"]) +def test_tokenizer_splits_newline(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 + assert tokens[1].text == "\n" + + +@pytest.mark.parametrize("text", ["lorem \nipsum"]) +def test_tokenizer_splits_newline_space(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["lorem \nipsum"]) +def test_tokenizer_splits_newline_double_space(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 + + +@pytest.mark.parametrize("text", ["lorem \n ipsum"]) +def test_tokenizer_splits_newline_space_wrap(tokenizer, text): + tokens = tokenizer(text) + assert len(tokens) == 3 diff --git a/spacy/tests/training/__init__.py b/spacy/tests/training/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/training/test_augmenters.py b/spacy/tests/training/test_augmenters.py new file mode 100644 index 0000000..49a8301 --- /dev/null +++ b/spacy/tests/training/test_augmenters.py @@ -0,0 +1,245 @@ +import random +from contextlib import contextmanager + +import pytest + +from spacy.lang.en import English +from spacy.pipeline._parser_internals.nonproj import contains_cycle +from spacy.tokens import Doc, DocBin, Span +from spacy.training import Corpus, Example +from spacy.training.augment import ( + create_lower_casing_augmenter, + create_orth_variants_augmenter, + make_whitespace_variant, +) + +from ..util import make_tempdir + + +@contextmanager +def make_docbin(docs, name="roundtrip.spacy"): + with make_tempdir() as tmpdir: + output_file = tmpdir / name + DocBin(docs=docs).to_disk(output_file) + yield output_file + + +@pytest.fixture +def nlp(): + return English() + + +@pytest.fixture +def doc(nlp): + # fmt: off + words = ["Sarah", "'s", "sister", "flew", "to", "Silicon", "Valley", "via", "London", "."] + tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."] + pos = ["PROPN", "PART", "NOUN", "VERB", "ADP", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] + ents = ["B-PERSON", "I-PERSON", "O", "", "O", "B-LOC", "I-LOC", "O", "B-GPE", "O"] + cats = {"TRAVEL": 1.0, "BAKING": 0.0} + # fmt: on + doc = Doc(nlp.vocab, words=words, tags=tags, pos=pos, ents=ents) + doc.cats = cats + return doc + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_make_orth_variants(nlp): + single = [ + {"tags": ["NFP"], "variants": ["…", "..."]}, + {"tags": [":"], "variants": ["-", "—", "–", "--", "---", "——"]}, + ] + # fmt: off + words = ["\n\n", "A", "\t", "B", "a", "b", "…", "...", "-", "—", "–", "--", "---", "——"] + tags = ["_SP", "NN", "\t", "NN", "NN", "NN", "NFP", "NFP", ":", ":", ":", ":", ":", ":"] + # fmt: on + spaces = [True] * len(words) + spaces[0] = False + spaces[2] = False + doc = Doc(nlp.vocab, words=words, spaces=spaces, tags=tags) + augmenter = create_orth_variants_augmenter( + level=0.2, lower=0.5, orth_variants={"single": single} + ) + with make_docbin([doc] * 10) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + # Due to randomness, only test that it works without errors + list(reader(nlp)) + + # check that the following settings lowercase everything + augmenter = create_orth_variants_augmenter( + level=1.0, lower=1.0, orth_variants={"single": single} + ) + with make_docbin([doc] * 10) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + for example in reader(nlp): + for token in example.reference: + assert token.text == token.text.lower() + + # check that lowercasing is applied without tags + doc = Doc(nlp.vocab, words=words, spaces=[True] * len(words)) + augmenter = create_orth_variants_augmenter( + level=1.0, lower=1.0, orth_variants={"single": single} + ) + with make_docbin([doc] * 10) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + for example in reader(nlp): + for ex_token, doc_token in zip(example.reference, doc): + assert ex_token.text == doc_token.text.lower() + + # check that no lowercasing is applied with lower=0.0 + doc = Doc(nlp.vocab, words=words, spaces=[True] * len(words)) + augmenter = create_orth_variants_augmenter( + level=1.0, lower=0.0, orth_variants={"single": single} + ) + with make_docbin([doc] * 10) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + for example in reader(nlp): + for ex_token, doc_token in zip(example.reference, doc): + assert ex_token.text == doc_token.text + + +def test_lowercase_augmenter(nlp, doc): + augmenter = create_lower_casing_augmenter(level=1.0) + with make_docbin([doc]) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + corpus = list(reader(nlp)) + eg = corpus[0] + assert eg.reference.text == doc.text.lower() + assert eg.predicted.text == doc.text.lower() + ents = [(e.start, e.end, e.label) for e in doc.ents] + assert [(e.start, e.end, e.label) for e in eg.reference.ents] == ents + for ref_ent, orig_ent in zip(eg.reference.ents, doc.ents): + assert ref_ent.text == orig_ent.text.lower() + assert [t.ent_iob for t in doc] == [t.ent_iob for t in eg.reference] + assert [t.pos_ for t in eg.reference] == [t.pos_ for t in doc] + + # check that augmentation works when lowercasing leads to different + # predicted tokenization + words = ["A", "B", "CCC."] + doc = Doc(nlp.vocab, words=words) + with make_docbin([doc]) as output_file: + reader = Corpus(output_file, augmenter=augmenter) + corpus = list(reader(nlp)) + eg = corpus[0] + assert eg.reference.text == doc.text.lower() + assert eg.predicted.text == doc.text.lower() + assert [t.text for t in eg.reference] == [t.lower() for t in words] + assert [t.text for t in eg.predicted] == [ + t.text for t in nlp.make_doc(doc.text.lower()) + ] + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_custom_data_augmentation(nlp, doc): + def create_spongebob_augmenter(randomize: bool = False): + def augment(nlp, example): + text = example.text + if randomize: + ch = [c.lower() if random.random() < 0.5 else c.upper() for c in text] + else: + ch = [c.lower() if i % 2 else c.upper() for i, c in enumerate(text)] + example_dict = example.to_dict() + doc = nlp.make_doc("".join(ch)) + example_dict["token_annotation"]["ORTH"] = [t.text for t in doc] + yield example + yield example.from_dict(doc, example_dict) + + return augment + + with make_docbin([doc]) as output_file: + reader = Corpus(output_file, augmenter=create_spongebob_augmenter()) + corpus = list(reader(nlp)) + orig_text = "Sarah 's sister flew to Silicon Valley via London . " + augmented = "SaRaH 's sIsTeR FlEw tO SiLiCoN VaLlEy vIa lOnDoN . " + assert corpus[0].text == orig_text + assert corpus[0].reference.text == orig_text + assert corpus[0].predicted.text == orig_text + assert corpus[1].text == augmented + assert corpus[1].reference.text == augmented + assert corpus[1].predicted.text == augmented + ents = [(e.start, e.end, e.label) for e in doc.ents] + assert [(e.start, e.end, e.label) for e in corpus[0].reference.ents] == ents + assert [(e.start, e.end, e.label) for e in corpus[1].reference.ents] == ents + + +def test_make_whitespace_variant(nlp): + # fmt: off + text = "They flew to New York City.\nThen they drove to Washington, D.C." + words = ["They", "flew", "to", "New", "York", "City", ".", "\n", "Then", "they", "drove", "to", "Washington", ",", "D.C."] + spaces = [True, True, True, True, True, False, False, False, True, True, True, True, False, True, False] + tags = ["PRP", "VBD", "IN", "NNP", "NNP", "NNP", ".", "_SP", "RB", "PRP", "VBD", "IN", "NNP", ",", "NNP"] + lemmas = ["they", "fly", "to", "New", "York", "City", ".", "\n", "then", "they", "drive", "to", "Washington", ",", "D.C."] + heads = [1, 1, 1, 4, 5, 2, 1, 10, 10, 10, 10, 10, 11, 12, 12] + deps = ["nsubj", "ROOT", "prep", "compound", "compound", "pobj", "punct", "dep", "advmod", "nsubj", "ROOT", "prep", "pobj", "punct", "appos"] + ents = ["O", "", "O", "B-GPE", "I-GPE", "I-GPE", "O", "O", "O", "O", "O", "O", "B-GPE", "O", "B-GPE"] + # fmt: on + doc = Doc( + nlp.vocab, + words=words, + spaces=spaces, + tags=tags, + lemmas=lemmas, + heads=heads, + deps=deps, + ents=ents, + ) + assert doc.text == text + example = Example(nlp.make_doc(text), doc) + # whitespace is only added internally in entity spans + mod_ex = make_whitespace_variant(nlp, example, " ", 3) + assert mod_ex.reference.ents[0].text == "New York City" + mod_ex = make_whitespace_variant(nlp, example, " ", 4) + assert mod_ex.reference.ents[0].text == "New York City" + mod_ex = make_whitespace_variant(nlp, example, " ", 5) + assert mod_ex.reference.ents[0].text == "New York City" + mod_ex = make_whitespace_variant(nlp, example, " ", 6) + assert mod_ex.reference.ents[0].text == "New York City" + # add a space at every possible position + for i in range(len(doc) + 1): + mod_ex = make_whitespace_variant(nlp, example, " ", i) + assert mod_ex.reference[i].is_space + # adds annotation when the doc contains at least partial annotation + assert [t.tag_ for t in mod_ex.reference] == tags[:i] + ["_SP"] + tags[i:] + assert [t.lemma_ for t in mod_ex.reference] == lemmas[:i] + [" "] + lemmas[i:] + assert [t.dep_ for t in mod_ex.reference] == deps[:i] + ["dep"] + deps[i:] + # does not add partial annotation if doc does not contain this feature + assert not mod_ex.reference.has_annotation("POS") + assert not mod_ex.reference.has_annotation("MORPH") + # produces well-formed trees + assert not contains_cycle([t.head.i for t in mod_ex.reference]) + assert len(list(doc.sents)) == 2 + if i == 0: + assert mod_ex.reference[i].head.i == 1 + else: + assert mod_ex.reference[i].head.i == i - 1 + # adding another space also produces well-formed trees + for j in (3, 8, 10): + mod_ex2 = make_whitespace_variant(nlp, mod_ex, "\t\t\n", j) + assert not contains_cycle([t.head.i for t in mod_ex2.reference]) + assert len(list(doc.sents)) == 2 + assert mod_ex2.reference[j].head.i == j - 1 + # entities are well-formed + assert len(doc.ents) == len(mod_ex.reference.ents) + # there is one token with missing entity information + assert any(t.ent_iob == 0 for t in mod_ex.reference) + for ent in mod_ex.reference.ents: + assert not ent[0].is_space + assert not ent[-1].is_space + + # no modifications if: + # partial dependencies + example.reference[0].dep_ = "" + mod_ex = make_whitespace_variant(nlp, example, " ", 5) + assert mod_ex.text == example.reference.text + example.reference[0].dep_ = "nsubj" # reset + + # spans + example.reference.spans["spans"] = [example.reference[0:5]] + mod_ex = make_whitespace_variant(nlp, example, " ", 5) + assert mod_ex.text == example.reference.text + del example.reference.spans["spans"] # reset + + # links + example.reference.ents = [Span(doc, 0, 2, label="ENT", kb_id="Q123")] + mod_ex = make_whitespace_variant(nlp, example, " ", 5) + assert mod_ex.text == example.reference.text diff --git a/spacy/tests/training/test_corpus.py b/spacy/tests/training/test_corpus.py new file mode 100644 index 0000000..ded6a53 --- /dev/null +++ b/spacy/tests/training/test_corpus.py @@ -0,0 +1,78 @@ +from contextlib import contextmanager +from pathlib import Path +from typing import Generator, Iterable, List, Tuple + +import pytest + +from spacy.lang.en import English +from spacy.training import Example, PlainTextCorpus +from spacy.util import make_tempdir + +# Intentional newlines to check that they are skipped. +PLAIN_TEXT_DOC = """ + +This is a doc. It contains two sentences. +This is another doc. + +A third doc. + +""" + +PLAIN_TEXT_DOC_TOKENIZED = [ + [ + "This", + "is", + "a", + "doc", + ".", + "It", + "contains", + "two", + "sentences", + ".", + ], + ["This", "is", "another", "doc", "."], + ["A", "third", "doc", "."], +] + + +@pytest.mark.parametrize("min_length", [0, 5]) +@pytest.mark.parametrize("max_length", [0, 5]) +def test_plain_text_reader(min_length, max_length): + nlp = English() + with _string_to_tmp_file(PLAIN_TEXT_DOC) as file_path: + corpus = PlainTextCorpus( + file_path, min_length=min_length, max_length=max_length + ) + + check = [ + doc + for doc in PLAIN_TEXT_DOC_TOKENIZED + if len(doc) >= min_length and (max_length == 0 or len(doc) <= max_length) + ] + reference, predicted = _examples_to_tokens(corpus(nlp)) + + assert reference == check + assert predicted == check + + +@contextmanager +def _string_to_tmp_file(s: str) -> Generator[Path, None, None]: + with make_tempdir() as d: + file_path = Path(d) / "string.txt" + with open(file_path, "w", encoding="utf-8") as f: + f.write(s) + yield file_path + + +def _examples_to_tokens( + examples: Iterable[Example], +) -> Tuple[List[List[str]], List[List[str]]]: + reference = [] + predicted = [] + + for eg in examples: + reference.append([t.text for t in eg.reference]) + predicted.append([t.text for t in eg.predicted]) + + return reference, predicted diff --git a/spacy/tests/training/test_logger.py b/spacy/tests/training/test_logger.py new file mode 100644 index 0000000..4875002 --- /dev/null +++ b/spacy/tests/training/test_logger.py @@ -0,0 +1,30 @@ +import pytest + +import spacy +from spacy.training import loggers + + +@pytest.fixture() +def nlp(): + nlp = spacy.blank("en") + nlp.add_pipe("ner") + return nlp + + +@pytest.fixture() +def info(): + return { + "losses": {"ner": 100}, + "other_scores": {"ENTS_F": 0.85, "ENTS_P": 0.90, "ENTS_R": 0.80}, + "epoch": 100, + "step": 125, + "score": 85, + } + + +def test_console_logger(nlp, info): + console_logger = loggers.console_logger( + progress_bar=True, console_output=True, output_file=None + ) + log_step, finalize = console_logger(nlp) + log_step(info) diff --git a/spacy/tests/training/test_new_example.py b/spacy/tests/training/test_new_example.py new file mode 100644 index 0000000..88f8199 --- /dev/null +++ b/spacy/tests/training/test_new_example.py @@ -0,0 +1,472 @@ +import pytest + +from spacy.tokens import Doc +from spacy.training.example import Example +from spacy.util import to_ternary_int +from spacy.vocab import Vocab + + +def test_Example_init_requires_doc_objects(): + vocab = Vocab() + with pytest.raises(TypeError): + Example(None, None) + with pytest.raises(TypeError): + Example(Doc(vocab, words=["hi"]), None) + with pytest.raises(TypeError): + Example(None, Doc(vocab, words=["hi"])) + + +def test_Example_from_dict_basic(): + example = Example.from_dict( + Doc(Vocab(), words=["hello", "world"]), {"words": ["hello", "world"]} + ) + assert isinstance(example.x, Doc) + assert isinstance(example.y, Doc) + + +@pytest.mark.parametrize( + "annots", [{"words": ["ice", "cream"], "weirdannots": ["something", "such"]}] +) +def test_Example_from_dict_invalid(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + with pytest.raises(KeyError): + Example.from_dict(predicted, annots) + + +@pytest.mark.parametrize( + "pred_words", [["ice", "cream"], ["icecream"], ["i", "ce", "cream"]] +) +@pytest.mark.parametrize("annots", [{"words": ["icecream"], "tags": ["NN"]}]) +def test_Example_from_dict_with_tags(pred_words, annots): + vocab = Vocab() + predicted = Doc(vocab, words=pred_words) + example = Example.from_dict(predicted, annots) + for i, token in enumerate(example.reference): + assert token.tag_ == annots["tags"][i] + aligned_tags = example.get_aligned("TAG", as_string=True) + assert aligned_tags == ["NN" for _ in predicted] + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_aligned_tags(): + pred_words = ["Apply", "some", "sunscreen", "unless", "you", "can", "not"] + gold_words = ["Apply", "some", "sun", "screen", "unless", "you", "cannot"] + gold_tags = ["VERB", "DET", "NOUN", "NOUN", "SCONJ", "PRON", "VERB"] + annots = {"words": gold_words, "tags": gold_tags} + vocab = Vocab() + predicted = Doc(vocab, words=pred_words) + example1 = Example.from_dict(predicted, annots) + aligned_tags1 = example1.get_aligned("TAG", as_string=True) + assert aligned_tags1 == ["VERB", "DET", "NOUN", "SCONJ", "PRON", "VERB", "VERB"] + # ensure that to_dict works correctly + example2 = Example.from_dict(predicted, example1.to_dict()) + aligned_tags2 = example2.get_aligned("TAG", as_string=True) + assert aligned_tags2 == ["VERB", "DET", "NOUN", "SCONJ", "PRON", "VERB", "VERB"] + + +def test_aligned_tags_multi(): + pred_words = ["Applysome", "sunscreen", "unless", "you", "can", "not"] + gold_words = ["Apply", "somesun", "screen", "unless", "you", "cannot"] + gold_tags = ["VERB", "DET", "NOUN", "SCONJ", "PRON", "VERB"] + annots = {"words": gold_words, "tags": gold_tags} + vocab = Vocab() + predicted = Doc(vocab, words=pred_words) + example = Example.from_dict(predicted, annots) + aligned_tags = example.get_aligned("TAG", as_string=True) + assert aligned_tags == [None, None, "SCONJ", "PRON", "VERB", "VERB"] + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "London", "and", "Berlin", "."], + "deps": ["nsubj", "ROOT", "dobj", "cc", "conj", "punct"], + "heads": [1, 1, 1, 2, 2, 1], + } + ], +) +def test_Example_from_dict_with_parse(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + for i, token in enumerate(example.reference): + assert token.dep_ == annots["deps"][i] + assert token.head.i == annots["heads"][i] + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["Sarah", "'s", "sister", "flew"], + "morphs": [ + "NounType=prop|Number=sing", + "Poss=yes", + "Number=sing", + "Tense=past|VerbForm=fin", + ], + } + ], +) +def test_Example_from_dict_with_morphology(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + for i, token in enumerate(example.reference): + assert str(token.morph) == annots["morphs"][i] + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["This", "is", "one", "sentence", "this", "is", "another"], + "sent_starts": [1, False, 0, None, True, -1, -5.7], + } + ], +) +def test_Example_from_dict_with_sent_start(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.sents)) == 2 + for i, token in enumerate(example.reference): + if to_ternary_int(annots["sent_starts"][i]) == 1: + assert token.is_sent_start is True + elif to_ternary_int(annots["sent_starts"][i]) == 0: + assert token.is_sent_start is None + else: + assert token.is_sent_start is False + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["This", "is", "a", "sentence"], + "cats": {"cat1": 1.0, "cat2": 0.0, "cat3": 0.5}, + } + ], +) +def test_Example_from_dict_with_cats(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.cats)) == 3 + assert example.reference.cats["cat1"] == 1.0 + assert example.reference.cats["cat2"] == 0.0 + assert example.reference.cats["cat3"] == 0.5 + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "entities": [(7, 15, "LOC"), (20, 26, "LOC")], + } + ], +) +def test_Example_from_dict_with_entities(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.ents)) == 2 + # fmt: off + assert [example.reference[i].ent_iob_ for i in range(7)] == ["O", "O", "B", "I", "O", "B", "O"] + assert example.get_aligned("ENT_IOB") == [2, 2, 3, 1, 2, 3, 2] + # fmt: on + assert example.reference[2].ent_type_ == "LOC" + assert example.reference[3].ent_type_ == "LOC" + assert example.reference[5].ent_type_ == "LOC" + + +def test_Example_from_dict_with_empty_entities(): + annots = { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "entities": [], + } + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + # entities as empty list sets everything to O + assert example.reference.has_annotation("ENT_IOB") + assert len(list(example.reference.ents)) == 0 + assert all(token.ent_iob_ == "O" for token in example.reference) + # various unset/missing entities leaves entities unset + annots["entities"] = None + example = Example.from_dict(predicted, annots) + assert not example.reference.has_annotation("ENT_IOB") + annots.pop("entities", None) + example = Example.from_dict(predicted, annots) + assert not example.reference.has_annotation("ENT_IOB") + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "entities": [ + (0, 4, "LOC"), + (21, 27, "LOC"), + ], # not aligned to token boundaries + } + ], +) +def test_Example_from_dict_with_entities_invalid(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + with pytest.warns(UserWarning): + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.ents)) == 0 + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "entities": [ + (7, 15, "LOC"), + (11, 15, "LOC"), + (20, 26, "LOC"), + ], # overlapping + } + ], +) +def test_Example_from_dict_with_entities_overlapping(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + with pytest.raises(ValueError): + Example.from_dict(predicted, annots) + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": { + "cities": [(7, 15, "LOC"), (20, 26, "LOC")], + "people": [(0, 1, "PERSON")], + }, + } + ], +) +def test_Example_from_dict_with_spans(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.ents)) == 0 + assert len(list(example.reference.spans["cities"])) == 2 + assert len(list(example.reference.spans["people"])) == 1 + for span in example.reference.spans["cities"]: + assert span.label_ == "LOC" + for span in example.reference.spans["people"]: + assert span.label_ == "PERSON" + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": { + "cities": [(7, 15, "LOC"), (11, 15, "LOC"), (20, 26, "LOC")], + "people": [(0, 1, "PERSON")], + }, + } + ], +) +def test_Example_from_dict_with_spans_overlapping(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(list(example.reference.ents)) == 0 + assert len(list(example.reference.spans["cities"])) == 3 + assert len(list(example.reference.spans["people"])) == 1 + for span in example.reference.spans["cities"]: + assert span.label_ == "LOC" + for span in example.reference.spans["people"]: + assert span.label_ == "PERSON" + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": [(0, 1, "PERSON")], + }, + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": {"cities": (7, 15, "LOC")}, + }, + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": {"cities": [7, 11]}, + }, + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "spans": {"cities": [[7]]}, + }, + ], +) +def test_Example_from_dict_with_spans_invalid(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + with pytest.raises(ValueError): + Example.from_dict(predicted, annots) + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "entities": [(7, 15, "LOC"), (20, 26, "LOC")], + "links": { + (7, 15): {"Q60": 1.0, "Q64": 0.0}, + (20, 26): {"Q60": 0.0, "Q64": 1.0}, + }, + } + ], +) +def test_Example_from_dict_with_links(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert example.reference[0].ent_kb_id_ == "" + assert example.reference[1].ent_kb_id_ == "" + assert example.reference[2].ent_kb_id_ == "Q60" + assert example.reference[3].ent_kb_id_ == "Q60" + assert example.reference[4].ent_kb_id_ == "" + assert example.reference[5].ent_kb_id_ == "Q64" + assert example.reference[6].ent_kb_id_ == "" + + +@pytest.mark.parametrize( + "annots", + [ + { + "words": ["I", "like", "New", "York", "and", "Berlin", "."], + "links": {(7, 14): {"Q7381115": 1.0, "Q2146908": 0.0}}, + } + ], +) +def test_Example_from_dict_with_links_invalid(annots): + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + with pytest.raises(ValueError): + Example.from_dict(predicted, annots) + + +def test_Example_from_dict_sentences(): + vocab = Vocab() + predicted = Doc(vocab, words=["One", "sentence", ".", "one", "more"]) + annots = {"sent_starts": [1, 0, 0, 1, 0]} + ex = Example.from_dict(predicted, annots) + assert len(list(ex.reference.sents)) == 2 + + # this currently throws an error - bug or feature? + # predicted = Doc(vocab, words=["One", "sentence", "not", "one", "more"]) + # annots = {"sent_starts": [1, 0, 0, 0, 0]} + # ex = Example.from_dict(predicted, annots) + # assert len(list(ex.reference.sents)) == 1 + + predicted = Doc(vocab, words=["One", "sentence", "not", "one", "more"]) + annots = {"sent_starts": [1, -1, 0, 0, 0]} + ex = Example.from_dict(predicted, annots) + assert len(list(ex.reference.sents)) == 1 + + +def test_Example_missing_deps(): + vocab = Vocab() + words = ["I", "like", "London", "and", "Berlin", "."] + deps = ["nsubj", "ROOT", "dobj", "cc", "conj", "punct"] + heads = [1, 1, 1, 2, 2, 1] + annots_head_only = {"words": words, "heads": heads} + annots_head_dep = {"words": words, "heads": heads, "deps": deps} + predicted = Doc(vocab, words=words) + + # when not providing deps, the head information is considered to be missing + # in this case, the token's heads refer to themselves + example_1 = Example.from_dict(predicted, annots_head_only) + assert [t.head.i for t in example_1.reference] == [0, 1, 2, 3, 4, 5] + + # when providing deps, the head information is actually used + example_2 = Example.from_dict(predicted, annots_head_dep) + assert [t.head.i for t in example_2.reference] == heads + + +def test_Example_missing_heads(): + vocab = Vocab() + words = ["I", "like", "London", "and", "Berlin", "."] + deps = ["nsubj", "ROOT", "dobj", None, "conj", "punct"] + heads = [1, 1, 1, None, 2, 1] + annots = {"words": words, "heads": heads, "deps": deps} + predicted = Doc(vocab, words=words) + + example = Example.from_dict(predicted, annots) + parsed_heads = [t.head.i for t in example.reference] + assert parsed_heads[0] == heads[0] + assert parsed_heads[1] == heads[1] + assert parsed_heads[2] == heads[2] + assert parsed_heads[4] == heads[4] + assert parsed_heads[5] == heads[5] + expected = [True, True, True, False, True, True] + assert [t.has_head() for t in example.reference] == expected + # Ensure that the missing head doesn't create an artificial new sentence start + expected = [True, False, False, False, False, False] + assert example.get_aligned_sent_starts() == expected + + +def test_Example_aligned_whitespace(en_vocab): + words = ["a", " ", "b"] + tags = ["A", "SPACE", "B"] + predicted = Doc(en_vocab, words=words) + reference = Doc(en_vocab, words=words, tags=tags) + + example = Example(predicted, reference) + assert example.get_aligned("TAG", as_string=True) == tags + + +@pytest.mark.issue("11260") +def test_issue11260(): + annots = { + "words": ["I", "like", "New", "York", "."], + "spans": { + "cities": [(7, 15, "LOC", "")], + "people": [(0, 1, "PERSON", "")], + }, + } + vocab = Vocab() + predicted = Doc(vocab, words=annots["words"]) + example = Example.from_dict(predicted, annots) + assert len(example.reference.spans["cities"]) == 1 + assert len(example.reference.spans["people"]) == 1 + + output_dict = example.to_dict() + assert "spans" in output_dict["doc_annotation"] + assert output_dict["doc_annotation"]["spans"]["cities"] == annots["spans"]["cities"] + assert output_dict["doc_annotation"]["spans"]["people"] == annots["spans"]["people"] + + output_example = Example.from_dict(predicted, output_dict) + + assert len(output_example.reference.spans["cities"]) == len( + example.reference.spans["cities"] + ) + assert len(output_example.reference.spans["people"]) == len( + example.reference.spans["people"] + ) + for span in example.reference.spans["cities"]: + assert span.label_ == "LOC" + assert span.text == "New York" + assert span.start_char == 7 + for span in example.reference.spans["people"]: + assert span.label_ == "PERSON" + assert span.text == "I" + assert span.start_char == 0 diff --git a/spacy/tests/training/test_pretraining.py b/spacy/tests/training/test_pretraining.py new file mode 100644 index 0000000..f33089f --- /dev/null +++ b/spacy/tests/training/test_pretraining.py @@ -0,0 +1,382 @@ +from pathlib import Path + +import numpy as np +import pytest +import srsly +from thinc.api import Config, get_current_ops + +from spacy import util +from spacy.lang.en import English +from spacy.language import DEFAULT_CONFIG_PATH, DEFAULT_CONFIG_PRETRAIN_PATH +from spacy.ml.models.multi_task import create_pretrain_vectors +from spacy.tokens import Doc, DocBin +from spacy.training.initialize import init_nlp +from spacy.training.loop import train +from spacy.training.pretrain import pretrain +from spacy.vectors import Vectors +from spacy.vocab import Vocab + +from ..util import make_tempdir + +pretrain_string_listener = """ +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 342 +depth = 4 +window_size = 1 +embed_size = 2000 +maxout_pieces = 3 +subword_features = true + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.width} + +[pretraining] +max_epochs = 5 + +[training] +max_epochs = 5 +""" + +pretrain_string_internal = """ +[nlp] +lang = "en" +pipeline = ["tagger"] + +[components] + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" + +[components.tagger.model.tok2vec] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 342 +depth = 4 +window_size = 1 +embed_size = 2000 +maxout_pieces = 3 +subword_features = true + +[pretraining] +max_epochs = 5 + +[training] +max_epochs = 5 +""" + + +pretrain_string_vectors = """ +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.HashEmbedCNN.v1" +pretrained_vectors = null +width = 342 +depth = 4 +window_size = 1 +embed_size = 2000 +maxout_pieces = 3 +subword_features = true + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.width} + +[pretraining] +max_epochs = 5 + +[pretraining.objective] +@architectures = spacy.PretrainVectors.v1 +maxout_pieces = 3 +hidden_size = 300 +loss = cosine + +[training] +max_epochs = 5 +""" + +CHAR_OBJECTIVES = [ + {}, + {"@architectures": "spacy.PretrainCharacters.v1"}, + { + "@architectures": "spacy.PretrainCharacters.v1", + "maxout_pieces": 5, + "hidden_size": 42, + "n_characters": 2, + }, +] + +VECTOR_OBJECTIVES = [ + { + "@architectures": "spacy.PretrainVectors.v1", + "maxout_pieces": 3, + "hidden_size": 300, + "loss": "cosine", + }, + { + "@architectures": "spacy.PretrainVectors.v1", + "maxout_pieces": 2, + "hidden_size": 200, + "loss": "L2", + }, +] + + +def test_pretraining_default(): + """Test that pretraining defaults to a character objective""" + config = Config().from_str(pretrain_string_internal) + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + assert "PretrainCharacters" in filled["pretraining"]["objective"]["@architectures"] + + +@pytest.mark.parametrize("objective", CHAR_OBJECTIVES) +@pytest.mark.parametrize("skip_last", (True, False)) +def test_pretraining_tok2vec_characters(objective, skip_last): + """Test that pretraining works with the character objective""" + config = Config().from_str(pretrain_string_listener) + config["pretraining"]["objective"] = objective + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + with make_tempdir() as tmp_dir: + file_path = write_sample_jsonl(tmp_dir) + filled["paths"]["raw_text"] = file_path + filled = filled.interpolate() + assert filled["pretraining"]["component"] == "tok2vec" + pretrain(filled, tmp_dir, skip_last=skip_last) + assert Path(tmp_dir / "model0.bin").exists() + assert Path(tmp_dir / "model4.bin").exists() + assert not Path(tmp_dir / "model5.bin").exists() + if skip_last: + assert not Path(tmp_dir / "model-last.bin").exists() + else: + assert Path(tmp_dir / "model-last.bin").exists() + + +@pytest.mark.parametrize("objective", VECTOR_OBJECTIVES) +def test_pretraining_tok2vec_vectors_fail(objective): + """Test that pretraining doesn't works with the vectors objective if there are no static vectors""" + config = Config().from_str(pretrain_string_listener) + config["pretraining"]["objective"] = objective + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + with make_tempdir() as tmp_dir: + file_path = write_sample_jsonl(tmp_dir) + filled["paths"]["raw_text"] = file_path + filled = filled.interpolate() + assert filled["initialize"]["vectors"] is None + with pytest.raises(ValueError): + pretrain(filled, tmp_dir) + + +@pytest.mark.parametrize("objective", VECTOR_OBJECTIVES) +def test_pretraining_tok2vec_vectors(objective): + """Test that pretraining works with the vectors objective and static vectors defined""" + config = Config().from_str(pretrain_string_listener) + config["pretraining"]["objective"] = objective + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + with make_tempdir() as tmp_dir: + file_path = write_sample_jsonl(tmp_dir) + filled["paths"]["raw_text"] = file_path + nlp_path = write_vectors_model(tmp_dir) + filled["initialize"]["vectors"] = nlp_path + filled = filled.interpolate() + pretrain(filled, tmp_dir) + + +@pytest.mark.parametrize("config", [pretrain_string_internal, pretrain_string_listener]) +def test_pretraining_tagger_tok2vec(config): + """Test pretraining of the tagger's tok2vec layer (via a listener)""" + config = Config().from_str(pretrain_string_listener) + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + with make_tempdir() as tmp_dir: + file_path = write_sample_jsonl(tmp_dir) + filled["paths"]["raw_text"] = file_path + filled["pretraining"]["component"] = "tagger" + filled["pretraining"]["layer"] = "tok2vec" + filled = filled.interpolate() + pretrain(filled, tmp_dir) + assert Path(tmp_dir / "model0.bin").exists() + assert Path(tmp_dir / "model4.bin").exists() + assert Path(tmp_dir / "model-last.bin").exists() + assert not Path(tmp_dir / "model5.bin").exists() + + +def test_pretraining_tagger(): + """Test pretraining of the tagger itself will throw an error (not an appropriate tok2vec layer)""" + config = Config().from_str(pretrain_string_internal) + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + with make_tempdir() as tmp_dir: + file_path = write_sample_jsonl(tmp_dir) + filled["paths"]["raw_text"] = file_path + filled["pretraining"]["component"] = "tagger" + filled = filled.interpolate() + with pytest.raises(ValueError): + pretrain(filled, tmp_dir) + + +# Try to debug segfault on windows +def test_pretraining_training(): + """Test that training can use a pretrained Tok2Vec model""" + config = Config().from_str(pretrain_string_internal) + nlp = util.load_model_from_config(config, auto_fill=True, validate=False) + filled = nlp.config + pretrain_config = util.load_config(DEFAULT_CONFIG_PRETRAIN_PATH) + filled = pretrain_config.merge(filled) + train_config = util.load_config(DEFAULT_CONFIG_PATH) + filled = train_config.merge(filled) + with make_tempdir() as tmp_dir: + pretrain_dir = tmp_dir / "pretrain" + pretrain_dir.mkdir() + file_path = write_sample_jsonl(pretrain_dir) + filled["paths"]["raw_text"] = file_path + filled["pretraining"]["component"] = "tagger" + filled["pretraining"]["layer"] = "tok2vec" + train_dir = tmp_dir / "train" + train_dir.mkdir() + train_path, dev_path = write_sample_training(train_dir) + filled["paths"]["train"] = train_path + filled["paths"]["dev"] = dev_path + filled = filled.interpolate() + P = filled["pretraining"] + nlp_base = init_nlp(filled) + model_base = ( + nlp_base.get_pipe(P["component"]).model.get_ref(P["layer"]).get_ref("embed") + ) + embed_base = None + for node in model_base.walk(): + if node.name == "hashembed": + embed_base = node + pretrain(filled, pretrain_dir) + pretrained_model = Path(pretrain_dir / "model3.bin") + assert pretrained_model.exists() + filled["initialize"]["init_tok2vec"] = str(pretrained_model) + nlp = init_nlp(filled) + model = nlp.get_pipe(P["component"]).model.get_ref(P["layer"]).get_ref("embed") + embed = None + for node in model.walk(): + if node.name == "hashembed": + embed = node + # ensure that the tok2vec weights are actually changed by the pretraining + assert np.any(np.not_equal(embed.get_param("E"), embed_base.get_param("E"))) + train(nlp, train_dir) + + +def write_sample_jsonl(tmp_dir): + data = [ + { + "meta": {"id": "1"}, + "text": "This is the best TV you'll ever buy!", + "cats": {"pos": 1, "neg": 0}, + }, + { + "meta": {"id": "2"}, + "text": "I wouldn't buy this again.", + "cats": {"pos": 0, "neg": 1}, + }, + ] + file_path = f"{tmp_dir}/text.jsonl" + srsly.write_jsonl(file_path, data) + return file_path + + +def write_sample_training(tmp_dir): + words = ["The", "players", "start", "."] + tags = ["DT", "NN", "VBZ", "."] + doc = Doc(English().vocab, words=words, tags=tags) + doc_bin = DocBin() + doc_bin.add(doc) + train_path = f"{tmp_dir}/train.spacy" + dev_path = f"{tmp_dir}/dev.spacy" + doc_bin.to_disk(train_path) + doc_bin.to_disk(dev_path) + return train_path, dev_path + + +def write_vectors_model(tmp_dir): + import numpy + + vocab = Vocab() + vector_data = { + "dog": numpy.random.uniform(-1, 1, (300,)), + "cat": numpy.random.uniform(-1, 1, (300,)), + "orange": numpy.random.uniform(-1, 1, (300,)), + } + for word, vector in vector_data.items(): + vocab.set_vector(word, vector) + nlp_path = tmp_dir / "vectors_model" + nlp = English(vocab) + nlp.to_disk(nlp_path) + return str(nlp_path) + + +def test_pretrain_default_vectors(): + nlp = English() + nlp.add_pipe("tok2vec") + nlp.initialize() + + # default vectors are supported + nlp.vocab.vectors = Vectors(shape=(10, 10)) + create_pretrain_vectors(1, 1, "cosine")(nlp.vocab, nlp.get_pipe("tok2vec").model) + + # floret vectors are supported + nlp.vocab.vectors = Vectors( + data=get_current_ops().xp.zeros((10, 10)), mode="floret", hash_count=1 + ) + create_pretrain_vectors(1, 1, "cosine")(nlp.vocab, nlp.get_pipe("tok2vec").model) + + # error for no vectors + with pytest.raises(ValueError, match="E875"): + nlp.vocab.vectors = Vectors() + create_pretrain_vectors(1, 1, "cosine")( + nlp.vocab, nlp.get_pipe("tok2vec").model + ) diff --git a/spacy/tests/training/test_readers.py b/spacy/tests/training/test_readers.py new file mode 100644 index 0000000..22cf752 --- /dev/null +++ b/spacy/tests/training/test_readers.py @@ -0,0 +1,122 @@ +from typing import Callable, Dict, Iterable + +import pytest +from thinc.api import Config, fix_random_seed + +from spacy import Language +from spacy.schemas import ConfigSchemaTraining +from spacy.training import Example +from spacy.util import load_model_from_config, registry, resolve_dot_names + + +def test_readers(): + config_string = """ + [training] + + [corpora] + @readers = "myreader.v1" + + [nlp] + lang = "en" + pipeline = ["tok2vec", "textcat"] + + [components] + + [components.tok2vec] + factory = "tok2vec" + + [components.textcat] + factory = "textcat" + """ + + @registry.readers("myreader.v1") + def myreader() -> Dict[str, Callable[[Language], Iterable[Example]]]: + annots = {"cats": {"POS": 1.0, "NEG": 0.0}} + + def reader(nlp: Language): + doc = nlp.make_doc(f"This is an example") + return [Example.from_dict(doc, annots)] + + return {"train": reader, "dev": reader, "extra": reader, "something": reader} + + config = Config().from_str(config_string) + nlp = load_model_from_config(config, auto_fill=True) + T = registry.resolve( + nlp.config.interpolate()["training"], schema=ConfigSchemaTraining + ) + dot_names = [T["train_corpus"], T["dev_corpus"]] + train_corpus, dev_corpus = resolve_dot_names(nlp.config, dot_names) + assert isinstance(train_corpus, Callable) + optimizer = T["optimizer"] + # simulate a training loop + nlp.initialize(lambda: train_corpus(nlp), sgd=optimizer) + for example in train_corpus(nlp): + nlp.update([example], sgd=optimizer) + scores = nlp.evaluate(list(dev_corpus(nlp))) + assert scores["cats_macro_auc"] == 0.0 + # ensure the pipeline runs + doc = nlp("Quick test") + assert doc.cats + corpora = {"corpora": nlp.config.interpolate()["corpora"]} + extra_corpus = registry.resolve(corpora)["corpora"]["extra"] + assert isinstance(extra_corpus, Callable) + + +@pytest.mark.slow +@pytest.mark.parametrize( + "reader,additional_config", + [ + ("ml_datasets.imdb_sentiment.v1", {"train_limit": 10, "dev_limit": 10}), + ("ml_datasets.dbpedia.v1", {"train_limit": 10, "dev_limit": 10}), + ("ml_datasets.cmu_movies.v1", {"limit": 10, "freq_cutoff": 200, "split": 0.8}), + ], +) +def test_cat_readers(reader, additional_config): + nlp_config_string = """ + [training] + seed = 0 + + [training.score_weights] + cats_macro_auc = 1.0 + + [corpora] + @readers = "PLACEHOLDER" + + [nlp] + lang = "en" + pipeline = ["tok2vec", "textcat_multilabel"] + + [components] + + [components.tok2vec] + factory = "tok2vec" + + [components.textcat_multilabel] + factory = "textcat_multilabel" + """ + config = Config().from_str(nlp_config_string) + fix_random_seed(config["training"]["seed"]) + config["corpora"]["@readers"] = reader + config["corpora"].update(additional_config) + nlp = load_model_from_config(config, auto_fill=True) + T = registry.resolve(nlp.config["training"], schema=ConfigSchemaTraining) + dot_names = [T["train_corpus"], T["dev_corpus"]] + train_corpus, dev_corpus = resolve_dot_names(nlp.config, dot_names) + optimizer = T["optimizer"] + # simulate a training loop + nlp.initialize(lambda: train_corpus(nlp), sgd=optimizer) + for example in train_corpus(nlp): + assert example.y.cats + # this shouldn't fail if each training example has at least one positive label + assert sorted(list(set(example.y.cats.values()))) == [0.0, 1.0] + nlp.update([example], sgd=optimizer) + # simulate performance benchmark on dev corpus + dev_examples = list(dev_corpus(nlp)) + for example in dev_examples: + # this shouldn't fail if each dev example has at least one positive label + assert sorted(list(set(example.y.cats.values()))) == [0.0, 1.0] + scores = nlp.evaluate(dev_examples) + assert scores["cats_score"] + # ensure the pipeline runs + doc = nlp("Quick test") + assert doc.cats diff --git a/spacy/tests/training/test_rehearse.py b/spacy/tests/training/test_rehearse.py new file mode 100644 index 0000000..7efe57a --- /dev/null +++ b/spacy/tests/training/test_rehearse.py @@ -0,0 +1,211 @@ +from typing import List + +import pytest + +import spacy +from spacy.training import Example + +TRAIN_DATA = [ + ( + "Who is Kofi Annan?", + { + "entities": [(7, 18, "PERSON")], + "tags": ["PRON", "AUX", "PROPN", "PRON", "PUNCT"], + "heads": [1, 1, 3, 1, 1], + "deps": ["attr", "ROOT", "compound", "nsubj", "punct"], + "morphs": [ + "", + "Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin", + "Number=Sing", + "Number=Sing", + "PunctType=Peri", + ], + "cats": {"question": 1.0}, + }, + ), + ( + "Who is Steve Jobs?", + { + "entities": [(7, 17, "PERSON")], + "tags": ["PRON", "AUX", "PROPN", "PRON", "PUNCT"], + "heads": [1, 1, 3, 1, 1], + "deps": ["attr", "ROOT", "compound", "nsubj", "punct"], + "morphs": [ + "", + "Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin", + "Number=Sing", + "Number=Sing", + "PunctType=Peri", + ], + "cats": {"question": 1.0}, + }, + ), + ( + "Bob is a nice person.", + { + "entities": [(0, 3, "PERSON")], + "tags": ["PROPN", "AUX", "DET", "ADJ", "NOUN", "PUNCT"], + "heads": [1, 1, 4, 4, 1, 1], + "deps": ["nsubj", "ROOT", "det", "amod", "attr", "punct"], + "morphs": [ + "Number=Sing", + "Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin", + "Definite=Ind|PronType=Art", + "Degree=Pos", + "Number=Sing", + "PunctType=Peri", + ], + "cats": {"statement": 1.0}, + }, + ), + ( + "Hi Anil, how are you?", + { + "entities": [(3, 7, "PERSON")], + "tags": ["INTJ", "PROPN", "PUNCT", "ADV", "AUX", "PRON", "PUNCT"], + "deps": ["intj", "npadvmod", "punct", "advmod", "ROOT", "nsubj", "punct"], + "heads": [4, 0, 4, 4, 4, 4, 4], + "morphs": [ + "", + "Number=Sing", + "PunctType=Comm", + "", + "Mood=Ind|Tense=Pres|VerbForm=Fin", + "Case=Nom|Person=2|PronType=Prs", + "PunctType=Peri", + ], + "cats": {"greeting": 1.0, "question": 1.0}, + }, + ), + ( + "I like London and Berlin.", + { + "entities": [(7, 13, "LOC"), (18, 24, "LOC")], + "tags": ["PROPN", "VERB", "PROPN", "CCONJ", "PROPN", "PUNCT"], + "deps": ["nsubj", "ROOT", "dobj", "cc", "conj", "punct"], + "heads": [1, 1, 1, 2, 2, 1], + "morphs": [ + "Case=Nom|Number=Sing|Person=1|PronType=Prs", + "Tense=Pres|VerbForm=Fin", + "Number=Sing", + "ConjType=Cmp", + "Number=Sing", + "PunctType=Peri", + ], + "cats": {"statement": 1.0}, + }, + ), +] + +REHEARSE_DATA = [ + ( + "Hi Anil", + { + "entities": [(3, 7, "PERSON")], + "tags": ["INTJ", "PROPN"], + "deps": ["ROOT", "npadvmod"], + "heads": [0, 0], + "morphs": ["", "Number=Sing"], + "cats": {"greeting": 1.0}, + }, + ), + ( + "Hi Ravish, how you doing?", + { + "entities": [(3, 9, "PERSON")], + "tags": ["INTJ", "PROPN", "PUNCT", "ADV", "AUX", "PRON", "PUNCT"], + "deps": ["intj", "ROOT", "punct", "advmod", "nsubj", "advcl", "punct"], + "heads": [1, 1, 1, 5, 5, 1, 1], + "morphs": [ + "", + "VerbForm=Inf", + "PunctType=Comm", + "", + "Case=Nom|Person=2|PronType=Prs", + "Aspect=Prog|Tense=Pres|VerbForm=Part", + "PunctType=Peri", + ], + "cats": {"greeting": 1.0, "question": 1.0}, + }, + ), + # UTENSIL new label + ( + "Natasha bought new forks.", + { + "entities": [(0, 7, "PERSON"), (19, 24, "UTENSIL")], + "tags": ["PROPN", "VERB", "ADJ", "NOUN", "PUNCT"], + "deps": ["nsubj", "ROOT", "amod", "dobj", "punct"], + "heads": [1, 1, 3, 1, 1], + "morphs": [ + "Number=Sing", + "Tense=Past|VerbForm=Fin", + "Degree=Pos", + "Number=Plur", + "PunctType=Peri", + ], + "cats": {"statement": 1.0}, + }, + ), +] + + +def _add_ner_label(ner, data): + for _, annotations in data: + for ent in annotations["entities"]: + ner.add_label(ent[2]) + + +def _add_tagger_label(tagger, data): + for _, annotations in data: + for tag in annotations["tags"]: + tagger.add_label(tag) + + +def _add_parser_label(parser, data): + for _, annotations in data: + for dep in annotations["deps"]: + parser.add_label(dep) + + +def _add_textcat_label(textcat, data): + for _, annotations in data: + for cat in annotations["cats"]: + textcat.add_label(cat) + + +def _optimize(nlp, component: str, data: List, rehearse: bool): + """Run either train or rehearse.""" + pipe = nlp.get_pipe(component) + if component == "ner": + _add_ner_label(pipe, data) + elif component == "tagger": + _add_tagger_label(pipe, data) + elif component == "parser": + _add_parser_label(pipe, data) + elif component == "textcat_multilabel": + _add_textcat_label(pipe, data) + else: + raise NotImplementedError + + if rehearse: + optimizer = nlp.resume_training() + else: + optimizer = nlp.initialize() + + for _ in range(5): + for text, annotation in data: + doc = nlp.make_doc(text) + example = Example.from_dict(doc, annotation) + if rehearse: + nlp.rehearse([example], sgd=optimizer) + else: + nlp.update([example], sgd=optimizer) + return nlp + + +@pytest.mark.parametrize("component", ["ner", "tagger", "parser", "textcat_multilabel"]) +def test_rehearse(component): + nlp = spacy.blank("en") + nlp.add_pipe(component) + nlp = _optimize(nlp, component, TRAIN_DATA, False) + _optimize(nlp, component, REHEARSE_DATA, True) diff --git a/spacy/tests/training/test_training.py b/spacy/tests/training/test_training.py new file mode 100644 index 0000000..a492a8b --- /dev/null +++ b/spacy/tests/training/test_training.py @@ -0,0 +1,1164 @@ +import random + +import numpy +import pytest +import srsly +from thinc.api import Adam, compounding + +import spacy +from spacy.lang.en import English +from spacy.tokens import Doc, DocBin +from spacy.training import ( + Alignment, + Corpus, + Example, + biluo_tags_to_offsets, + biluo_tags_to_spans, + docs_to_json, + iob_to_biluo, + offsets_to_biluo_tags, +) +from spacy.training.align import get_alignments +from spacy.training.alignment_array import AlignmentArray +from spacy.training.converters import json_to_docs +from spacy.training.loop import train_while_improving +from spacy.util import ( + get_words_and_spaces, + load_config_from_str, + load_model_from_path, + minibatch, +) + +from ..util import make_tempdir + + +@pytest.fixture +def doc(): + nlp = English() # make sure we get a new vocab every time + # fmt: off + words = ["Sarah", "'s", "sister", "flew", "to", "Silicon", "Valley", "via", "London", "."] + tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."] + pos = ["PROPN", "PART", "NOUN", "VERB", "ADP", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] + morphs = ["NounType=prop|Number=sing", "Poss=yes", "Number=sing", "Tense=past|VerbForm=fin", + "", "NounType=prop|Number=sing", "NounType=prop|Number=sing", "", + "NounType=prop|Number=sing", "PunctType=peri"] + # head of '.' is intentionally nonprojective for testing + heads = [2, 0, 3, 3, 3, 6, 4, 3, 7, 5] + deps = ["poss", "case", "nsubj", "ROOT", "prep", "compound", "pobj", "prep", "pobj", "punct"] + lemmas = ["Sarah", "'s", "sister", "fly", "to", "Silicon", "Valley", "via", "London", "."] + ents = ["O"] * len(words) + ents[0] = "B-PERSON" + ents[1] = "I-PERSON" + ents[5] = "B-LOC" + ents[6] = "I-LOC" + ents[8] = "B-GPE" + cats = {"TRAVEL": 1.0, "BAKING": 0.0} + # fmt: on + doc = Doc( + nlp.vocab, + words=words, + tags=tags, + pos=pos, + morphs=morphs, + heads=heads, + deps=deps, + lemmas=lemmas, + ents=ents, + ) + doc.cats = cats + return doc + + +@pytest.fixture() +def merged_dict(): + return { + "ids": [1, 2, 3, 4, 5, 6, 7], + "words": ["Hi", "there", "everyone", "It", "is", "just", "me"], + "spaces": [True, True, True, True, True, True, False], + "tags": ["INTJ", "ADV", "PRON", "PRON", "AUX", "ADV", "PRON"], + "sent_starts": [1, 0, 0, 1, 0, 0, 0], + } + + +@pytest.fixture +def vocab(): + nlp = English() + return nlp.vocab + + +@pytest.mark.issue(999) +def test_issue999(): + """Test that adding entities and resuming training works passably OK. + There are two issues here: + 1) We have to re-add labels. This isn't very nice. + 2) There's no way to set the learning rate for the weight update, so we + end up out-of-scale, causing it to learn too fast. + """ + TRAIN_DATA = [ + ["hey", []], + ["howdy", []], + ["hey there", []], + ["hello", []], + ["hi", []], + ["i'm looking for a place to eat", []], + ["i'm looking for a place in the north of town", [(31, 36, "LOCATION")]], + ["show me chinese restaurants", [(8, 15, "CUISINE")]], + ["show me chines restaurants", [(8, 14, "CUISINE")]], + ] + nlp = English() + ner = nlp.add_pipe("ner") + for _, offsets in TRAIN_DATA: + for start, end, label in offsets: + ner.add_label(label) + nlp.initialize() + for itn in range(20): + random.shuffle(TRAIN_DATA) + for raw_text, entity_offsets in TRAIN_DATA: + example = Example.from_dict( + nlp.make_doc(raw_text), {"entities": entity_offsets} + ) + nlp.update([example]) + + with make_tempdir() as model_dir: + nlp.to_disk(model_dir) + nlp2 = load_model_from_path(model_dir) + + for raw_text, entity_offsets in TRAIN_DATA: + doc = nlp2(raw_text) + ents = {(ent.start_char, ent.end_char): ent.label_ for ent in doc.ents} + for start, end, label in entity_offsets: + if (start, end) in ents: + assert ents[(start, end)] == label + break + else: + if entity_offsets: + raise Exception(ents) + + +@pytest.mark.issue(4402) +def test_issue4402(): + json_data = { + "id": 0, + "paragraphs": [ + { + "raw": "How should I cook bacon in an oven?\nI've heard of people cooking bacon in an oven.", + "sentences": [ + { + "tokens": [ + {"id": 0, "orth": "How", "ner": "O"}, + {"id": 1, "orth": "should", "ner": "O"}, + {"id": 2, "orth": "I", "ner": "O"}, + {"id": 3, "orth": "cook", "ner": "O"}, + {"id": 4, "orth": "bacon", "ner": "O"}, + {"id": 5, "orth": "in", "ner": "O"}, + {"id": 6, "orth": "an", "ner": "O"}, + {"id": 7, "orth": "oven", "ner": "O"}, + {"id": 8, "orth": "?", "ner": "O"}, + ], + "brackets": [], + }, + { + "tokens": [ + {"id": 9, "orth": "\n", "ner": "O"}, + {"id": 10, "orth": "I", "ner": "O"}, + {"id": 11, "orth": "'ve", "ner": "O"}, + {"id": 12, "orth": "heard", "ner": "O"}, + {"id": 13, "orth": "of", "ner": "O"}, + {"id": 14, "orth": "people", "ner": "O"}, + {"id": 15, "orth": "cooking", "ner": "O"}, + {"id": 16, "orth": "bacon", "ner": "O"}, + {"id": 17, "orth": "in", "ner": "O"}, + {"id": 18, "orth": "an", "ner": "O"}, + {"id": 19, "orth": "oven", "ner": "O"}, + {"id": 20, "orth": ".", "ner": "O"}, + ], + "brackets": [], + }, + ], + "cats": [ + {"label": "baking", "value": 1.0}, + {"label": "not_baking", "value": 0.0}, + ], + }, + { + "raw": "What is the difference between white and brown eggs?\n", + "sentences": [ + { + "tokens": [ + {"id": 0, "orth": "What", "ner": "O"}, + {"id": 1, "orth": "is", "ner": "O"}, + {"id": 2, "orth": "the", "ner": "O"}, + {"id": 3, "orth": "difference", "ner": "O"}, + {"id": 4, "orth": "between", "ner": "O"}, + {"id": 5, "orth": "white", "ner": "O"}, + {"id": 6, "orth": "and", "ner": "O"}, + {"id": 7, "orth": "brown", "ner": "O"}, + {"id": 8, "orth": "eggs", "ner": "O"}, + {"id": 9, "orth": "?", "ner": "O"}, + ], + "brackets": [], + }, + {"tokens": [{"id": 10, "orth": "\n", "ner": "O"}], "brackets": []}, + ], + "cats": [ + {"label": "baking", "value": 0.0}, + {"label": "not_baking", "value": 1.0}, + ], + }, + ], + } + nlp = English() + attrs = ["ORTH", "SENT_START", "ENT_IOB", "ENT_TYPE"] + with make_tempdir() as tmpdir: + output_file = tmpdir / "test4402.spacy" + docs = json_to_docs([json_data]) + data = DocBin(docs=docs, attrs=attrs).to_bytes() + with output_file.open("wb") as file_: + file_.write(data) + reader = Corpus(output_file) + train_data = list(reader(nlp)) + assert len(train_data) == 2 + + split_train_data = [] + for eg in train_data: + split_train_data.extend(eg.split_sents()) + assert len(split_train_data) == 4 + + +CONFIG_7029 = """ +[nlp] +lang = "en" +pipeline = ["tok2vec", "tagger"] + +[components] + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v1" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = ${components.tok2vec.model.encode:width} +attrs = ["NORM","PREFIX","SUFFIX","SHAPE"] +rows = [5000,2500,2500,2500] +include_static_vectors = false + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v1" +width = 96 +depth = 4 +window_size = 1 +maxout_pieces = 3 + +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "spacy.Tagger.v2" +nO = null + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode:width} +upstream = "*" +""" + + +@pytest.mark.issue(7029) +def test_issue7029(): + """Test that an empty document doesn't mess up an entire batch.""" + TRAIN_DATA = [ + ("I like green eggs", {"tags": ["N", "V", "J", "N"]}), + ("Eat blue ham", {"tags": ["V", "J", "N"]}), + ] + nlp = English.from_config(load_config_from_str(CONFIG_7029)) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses) + texts = ["first", "second", "third", "fourth", "and", "then", "some", ""] + docs1 = list(nlp.pipe(texts, batch_size=1)) + docs2 = list(nlp.pipe(texts, batch_size=4)) + assert [doc[0].tag_ for doc in docs1[:-1]] == [doc[0].tag_ for doc in docs2[:-1]] + + +def test_gold_biluo_U(en_vocab): + words = ["I", "flew", "to", "London", "."] + spaces = [True, True, True, False, True] + doc = Doc(en_vocab, words=words, spaces=spaces) + entities = [(len("I flew to "), len("I flew to London"), "LOC")] + tags = offsets_to_biluo_tags(doc, entities) + assert tags == ["O", "O", "O", "U-LOC", "O"] + + +def test_gold_biluo_BL(en_vocab): + words = ["I", "flew", "to", "San", "Francisco", "."] + spaces = [True, True, True, True, False, True] + doc = Doc(en_vocab, words=words, spaces=spaces) + entities = [(len("I flew to "), len("I flew to San Francisco"), "LOC")] + tags = offsets_to_biluo_tags(doc, entities) + assert tags == ["O", "O", "O", "B-LOC", "L-LOC", "O"] + + +def test_gold_biluo_BIL(en_vocab): + words = ["I", "flew", "to", "San", "Francisco", "Valley", "."] + spaces = [True, True, True, True, True, False, True] + doc = Doc(en_vocab, words=words, spaces=spaces) + entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] + tags = offsets_to_biluo_tags(doc, entities) + assert tags == ["O", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] + + +def test_gold_biluo_overlap(en_vocab): + words = ["I", "flew", "to", "San", "Francisco", "Valley", "."] + spaces = [True, True, True, True, True, False, True] + doc = Doc(en_vocab, words=words, spaces=spaces) + entities = [ + (len("I flew to "), len("I flew to San Francisco Valley"), "LOC"), + (len("I flew to "), len("I flew to San Francisco"), "LOC"), + ] + with pytest.raises(ValueError): + offsets_to_biluo_tags(doc, entities) + + +def test_gold_biluo_misalign(en_vocab): + words = ["I", "flew", "to", "San", "Francisco", "Valley."] + spaces = [True, True, True, True, True, False] + doc = Doc(en_vocab, words=words, spaces=spaces) + entities = [(len("I flew to "), len("I flew to San Francisco Valley"), "LOC")] + with pytest.warns(UserWarning): + tags = offsets_to_biluo_tags(doc, entities) + assert tags == ["O", "O", "O", "-", "-", "-"] + + +def test_example_constructor(en_vocab): + words = ["I", "like", "stuff"] + tags = ["NOUN", "VERB", "NOUN"] + tag_ids = [en_vocab.strings.add(tag) for tag in tags] + predicted = Doc(en_vocab, words=words) + reference = Doc(en_vocab, words=words) + reference = reference.from_array("TAG", numpy.array(tag_ids, dtype="uint64")) + example = Example(predicted, reference) + tags = example.get_aligned("TAG", as_string=True) + assert tags == ["NOUN", "VERB", "NOUN"] + + +def test_example_from_dict_tags(en_vocab): + words = ["I", "like", "stuff"] + tags = ["NOUN", "VERB", "NOUN"] + predicted = Doc(en_vocab, words=words) + example = Example.from_dict(predicted, {"TAGS": tags}) + tags = example.get_aligned("TAG", as_string=True) + assert tags == ["NOUN", "VERB", "NOUN"] + + +def test_example_from_dict_no_ner(en_vocab): + words = ["a", "b", "c", "d"] + spaces = [True, True, False, True] + predicted = Doc(en_vocab, words=words, spaces=spaces) + example = Example.from_dict(predicted, {"words": words}) + ner_tags = example.get_aligned_ner() + assert ner_tags == [None, None, None, None] + + +def test_example_from_dict_some_ner(en_vocab): + words = ["a", "b", "c", "d"] + spaces = [True, True, False, True] + predicted = Doc(en_vocab, words=words, spaces=spaces) + example = Example.from_dict( + predicted, {"words": words, "entities": ["U-LOC", None, None, None]} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["U-LOC", None, None, None] + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_json_to_docs_no_ner(en_vocab): + data = [ + { + "id": 1, + "paragraphs": [ + { + "sentences": [ + { + "tokens": [ + {"dep": "nn", "head": 1, "tag": "NNP", "orth": "Ms."}, + { + "dep": "nsubj", + "head": 1, + "tag": "NNP", + "orth": "Haag", + }, + { + "dep": "ROOT", + "head": 0, + "tag": "VBZ", + "orth": "plays", + }, + { + "dep": "dobj", + "head": -1, + "tag": "NNP", + "orth": "Elianti", + }, + {"dep": "punct", "head": -2, "tag": ".", "orth": "."}, + ] + } + ] + } + ], + } + ] + docs = list(json_to_docs(data)) + assert len(docs) == 1 + for doc in docs: + assert not doc.has_annotation("ENT_IOB") + for token in doc: + assert token.ent_iob == 0 + eg = Example( + Doc( + doc.vocab, + words=[w.text for w in doc], + spaces=[bool(w.whitespace_) for w in doc], + ), + doc, + ) + ner_tags = eg.get_aligned_ner() + assert ner_tags == [None, None, None, None, None] + + +def test_split_sentences(en_vocab): + # fmt: off + words = ["I", "flew", "to", "San Francisco Valley", "had", "loads of fun"] + gold_words = ["I", "flew", "to", "San", "Francisco", "Valley", "had", "loads", "of", "fun"] + sent_starts = [True, False, False, False, False, False, True, False, False, False] + # fmt: on + doc = Doc(en_vocab, words=words) + example = Example.from_dict(doc, {"words": gold_words, "sent_starts": sent_starts}) + assert example.text == "I flew to San Francisco Valley had loads of fun " + split_examples = example.split_sents() + assert len(split_examples) == 2 + assert split_examples[0].text == "I flew to San Francisco Valley " + assert split_examples[1].text == "had loads of fun " + # fmt: off + words = ["I", "flew", "to", "San", "Francisco", "Valley", "had", "loads", "of fun"] + gold_words = ["I", "flew", "to", "San Francisco", "Valley", "had", "loads of", "fun"] + sent_starts = [True, False, False, False, False, True, False, False] + # fmt: on + doc = Doc(en_vocab, words=words) + example = Example.from_dict(doc, {"words": gold_words, "sent_starts": sent_starts}) + assert example.text == "I flew to San Francisco Valley had loads of fun " + split_examples = example.split_sents() + assert len(split_examples) == 2 + assert split_examples[0].text == "I flew to San Francisco Valley " + assert split_examples[1].text == "had loads of fun " + + +def test_gold_biluo_one_to_many(en_vocab, en_tokenizer): + words = ["Mr and ", "Mrs Smith", "flew to", "San Francisco Valley", "."] + spaces = [True, True, True, False, False] + doc = Doc(en_vocab, words=words, spaces=spaces) + prefix = "Mr and Mrs Smith flew to " + entities = [(len(prefix), len(prefix + "San Francisco Valley"), "LOC")] + gold_words = ["Mr and Mrs Smith", "flew", "to", "San", "Francisco", "Valley", "."] + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "U-LOC", "O"] + + entities = [ + (len("Mr and "), len("Mr and Mrs Smith"), "PERSON"), # "Mrs Smith" is a PERSON + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + # fmt: off + gold_words = ["Mr and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "U-PERSON", "O", "U-LOC", "O"] + + entities = [ + (len("Mr and "), len("Mr and Mrs"), "PERSON"), # "Mrs" is a Person + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + # fmt: off + gold_words = ["Mr and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", None, "O", "U-LOC", "O"] + + +def test_gold_biluo_many_to_one(en_vocab, en_tokenizer): + words = ["Mr and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + spaces = [True, True, True, True, True, True, True, False, False] + doc = Doc(en_vocab, words=words, spaces=spaces) + prefix = "Mr and Mrs Smith flew to " + entities = [(len(prefix), len(prefix + "San Francisco Valley"), "LOC")] + gold_words = ["Mr and Mrs Smith", "flew to", "San Francisco Valley", "."] + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] + + entities = [ + (len("Mr and "), len("Mr and Mrs Smith"), "PERSON"), # "Mrs Smith" is a PERSON + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + gold_words = ["Mr and", "Mrs Smith", "flew to", "San Francisco Valley", "."] + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + expected = ["O", "B-PERSON", "L-PERSON", "O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] + assert ner_tags == expected + + +def test_gold_biluo_misaligned(en_vocab, en_tokenizer): + words = ["Mr and Mrs", "Smith", "flew", "to", "San Francisco", "Valley", "."] + spaces = [True, True, True, True, True, False, False] + doc = Doc(en_vocab, words=words, spaces=spaces) + prefix = "Mr and Mrs Smith flew to " + entities = [(len(prefix), len(prefix + "San Francisco Valley"), "LOC")] + gold_words = ["Mr", "and Mrs Smith", "flew to", "San", "Francisco Valley", "."] + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "O", "B-LOC", "L-LOC", "O"] + + entities = [ + (len("Mr and "), len("Mr and Mrs Smith"), "PERSON"), # "Mrs Smith" is a PERSON + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + gold_words = ["Mr and", "Mrs Smith", "flew to", "San", "Francisco Valley", "."] + example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) + ner_tags = example.get_aligned_ner() + assert ner_tags == [None, None, "O", "O", "B-LOC", "L-LOC", "O"] + + +def test_gold_biluo_additional_whitespace(en_vocab, en_tokenizer): + # additional whitespace tokens in GoldParse words + words, spaces = get_words_and_spaces( + ["I", "flew", "to", "San Francisco", "Valley", "."], + "I flew to San Francisco Valley.", + ) + doc = Doc(en_vocab, words=words, spaces=spaces) + prefix = "I flew to " + entities = [(len(prefix), len(prefix + "San Francisco Valley"), "LOC")] + gold_words = ["I", "flew", " ", "to", "San Francisco Valley", "."] + gold_spaces = [True, True, False, True, False, False] + example = Example.from_dict( + doc, {"words": gold_words, "spaces": gold_spaces, "entities": entities} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "O", "B-LOC", "L-LOC", "O"] + + +def test_gold_biluo_4791(en_vocab, en_tokenizer): + doc = en_tokenizer("I'll return the A54 amount") + gold_words = ["I", "'ll", "return", "the", "A", "54", "amount"] + gold_spaces = [False, True, True, True, False, True, False] + entities = [(16, 19, "MONEY")] + example = Example.from_dict( + doc, {"words": gold_words, "spaces": gold_spaces, "entities": entities} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "O", "U-MONEY", "O"] + + doc = en_tokenizer("I'll return the $54 amount") + gold_words = ["I", "'ll", "return", "the", "$", "54", "amount"] + gold_spaces = [False, True, True, True, False, True, False] + entities = [(16, 19, "MONEY")] + example = Example.from_dict( + doc, {"words": gold_words, "spaces": gold_spaces, "entities": entities} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "O", "O", "O", "B-MONEY", "L-MONEY", "O"] + + +def test_roundtrip_offsets_biluo_conversion(en_tokenizer): + text = "I flew to Silicon Valley via London." + biluo_tags = ["O", "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"] + offsets = [(10, 24, "LOC"), (29, 35, "GPE")] + doc = en_tokenizer(text) + biluo_tags_converted = offsets_to_biluo_tags(doc, offsets) + assert biluo_tags_converted == biluo_tags + offsets_converted = biluo_tags_to_offsets(doc, biluo_tags) + offsets_converted = [ent for ent in offsets if ent[2]] + assert offsets_converted == offsets + + +def test_biluo_spans(en_tokenizer): + doc = en_tokenizer("I flew to Silicon Valley via London.") + biluo_tags = ["O", "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"] + spans = biluo_tags_to_spans(doc, biluo_tags) + spans = [span for span in spans if span.label_] + assert len(spans) == 2 + assert spans[0].text == "Silicon Valley" + assert spans[0].label_ == "LOC" + assert spans[1].text == "London" + assert spans[1].label_ == "GPE" + + +def test_aligned_spans_y2x(en_vocab, en_tokenizer): + words = ["Mr and Mrs Smith", "flew", "to", "San Francisco Valley", "."] + spaces = [True, True, True, False, False] + doc = Doc(en_vocab, words=words, spaces=spaces) + prefix = "Mr and Mrs Smith flew to " + entities = [ + (0, len("Mr and Mrs Smith"), "PERSON"), + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + # fmt: off + tokens_ref = ["Mr", "and", "Mrs", "Smith", "flew", "to", "San", "Francisco", "Valley", "."] + # fmt: on + example = Example.from_dict(doc, {"words": tokens_ref, "entities": entities}) + ents_ref = example.reference.ents + assert [(ent.start, ent.end) for ent in ents_ref] == [(0, 4), (6, 9)] + ents_y2x = example.get_aligned_spans_y2x(ents_ref) + assert [(ent.start, ent.end) for ent in ents_y2x] == [(0, 1), (3, 4)] + + +def test_aligned_spans_x2y(en_vocab, en_tokenizer): + text = "Mr and Mrs Smith flew to San Francisco Valley" + nlp = English() + patterns = [ + {"label": "PERSON", "pattern": "Mr and Mrs Smith"}, + {"label": "LOC", "pattern": "San Francisco Valley"}, + ] + ruler = nlp.add_pipe("entity_ruler") + ruler.add_patterns(patterns) + doc = nlp(text) + assert [(ent.start, ent.end) for ent in doc.ents] == [(0, 4), (6, 9)] + prefix = "Mr and Mrs Smith flew to " + entities = [ + (0, len("Mr and Mrs Smith"), "PERSON"), + (len(prefix), len(prefix + "San Francisco Valley"), "LOC"), + ] + tokens_ref = ["Mr and Mrs", "Smith", "flew", "to", "San Francisco", "Valley"] + example = Example.from_dict(doc, {"words": tokens_ref, "entities": entities}) + assert [(ent.start, ent.end) for ent in example.reference.ents] == [(0, 2), (4, 6)] + # Ensure that 'get_aligned_spans_x2y' has the aligned entities correct + ents_pred = example.predicted.ents + assert [(ent.start, ent.end) for ent in ents_pred] == [(0, 4), (6, 9)] + ents_x2y = example.get_aligned_spans_x2y(ents_pred) + assert [(ent.start, ent.end) for ent in ents_x2y] == [(0, 2), (4, 6)] + + +def test_aligned_spans_y2x_overlap(en_vocab, en_tokenizer): + text = "I flew to San Francisco Valley" + nlp = English() + doc = nlp(text) + # the reference doc has overlapping spans + gold_doc = nlp.make_doc(text) + spans = [] + prefix = "I flew to " + spans.append( + gold_doc.char_span(len(prefix), len(prefix + "San Francisco"), label="CITY") + ) + spans.append( + gold_doc.char_span( + len(prefix), len(prefix + "San Francisco Valley"), label="VALLEY" + ) + ) + spans_key = "overlap_ents" + gold_doc.spans[spans_key] = spans + example = Example(doc, gold_doc) + spans_gold = example.reference.spans[spans_key] + assert [(ent.start, ent.end) for ent in spans_gold] == [(3, 5), (3, 6)] + + # Ensure that 'get_aligned_spans_y2x' has the aligned entities correct + spans_y2x_no_overlap = example.get_aligned_spans_y2x( + spans_gold, allow_overlap=False + ) + assert [(ent.start, ent.end) for ent in spans_y2x_no_overlap] == [(3, 5)] + spans_y2x_overlap = example.get_aligned_spans_y2x(spans_gold, allow_overlap=True) + assert [(ent.start, ent.end) for ent in spans_y2x_overlap] == [(3, 5), (3, 6)] + + +def test_gold_ner_missing_tags(en_tokenizer): + doc = en_tokenizer("I flew to Silicon Valley via London.") + biluo_tags = [None, "O", "O", "B-LOC", "L-LOC", "O", "U-GPE", "O"] + example = Example.from_dict(doc, {"entities": biluo_tags}) + assert example.get_aligned("ENT_IOB") == [0, 2, 2, 3, 1, 2, 3, 2] + + +def test_projectivize(en_tokenizer): + doc = en_tokenizer("He pretty quickly walks away") + heads = [3, 2, 3, 3, 2] + deps = ["dep"] * len(heads) + example = Example.from_dict(doc, {"heads": heads, "deps": deps}) + proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) + nonproj_heads, nonproj_labels = example.get_aligned_parse(projectivize=False) + assert proj_heads == [3, 2, 3, 3, 3] + assert nonproj_heads == [3, 2, 3, 3, 2] + + # Test single token documents + doc = en_tokenizer("Conrail") + heads = [0] + deps = ["dep"] + example = Example.from_dict(doc, {"heads": heads, "deps": deps}) + proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) + assert proj_heads == heads + assert proj_labels == deps + + # Test documents with no alignments + doc_a = Doc( + doc.vocab, words=["Double-Jointed"], spaces=[False], deps=["ROOT"], heads=[0] + ) + doc_b = Doc( + doc.vocab, + words=["Double", "-", "Jointed"], + spaces=[True, True, True], + deps=["amod", "punct", "ROOT"], + heads=[2, 2, 2], + ) + example = Example(doc_a, doc_b) + proj_heads, proj_deps = example.get_aligned_parse(projectivize=True) + assert proj_heads == [None] + assert proj_deps == [None] + + +def test_iob_to_biluo(): + good_iob = ["O", "O", "B-LOC", "I-LOC", "O", "B-PERSON"] + good_biluo = ["O", "O", "B-LOC", "L-LOC", "O", "U-PERSON"] + bad_iob = ["O", "O", '"', "B-LOC", "I-LOC"] + converted_biluo = iob_to_biluo(good_iob) + assert good_biluo == converted_biluo + with pytest.raises(ValueError): + iob_to_biluo(bad_iob) + + +def test_roundtrip_docs_to_docbin(doc): + text = doc.text + idx = [t.idx for t in doc] + tags = [t.tag_ for t in doc] + pos = [t.pos_ for t in doc] + morphs = [str(t.morph) for t in doc] + lemmas = [t.lemma_ for t in doc] + deps = [t.dep_ for t in doc] + heads = [t.head.i for t in doc] + cats = doc.cats + ents = [(e.start_char, e.end_char, e.label_) for e in doc.ents] + # roundtrip to DocBin + with make_tempdir() as tmpdir: + # use a separate vocab to test that all labels are added + reloaded_nlp = English() + json_file = tmpdir / "roundtrip.json" + srsly.write_json(json_file, [docs_to_json(doc)]) + output_file = tmpdir / "roundtrip.spacy" + DocBin(docs=[doc]).to_disk(output_file) + reader = Corpus(output_file) + reloaded_examples = list(reader(reloaded_nlp)) + assert len(doc) == sum(len(eg) for eg in reloaded_examples) + reloaded_example = reloaded_examples[0] + assert text == reloaded_example.reference.text + assert idx == [t.idx for t in reloaded_example.reference] + assert tags == [t.tag_ for t in reloaded_example.reference] + assert pos == [t.pos_ for t in reloaded_example.reference] + assert morphs == [str(t.morph) for t in reloaded_example.reference] + assert lemmas == [t.lemma_ for t in reloaded_example.reference] + assert deps == [t.dep_ for t in reloaded_example.reference] + assert heads == [t.head.i for t in reloaded_example.reference] + assert ents == [ + (e.start_char, e.end_char, e.label_) for e in reloaded_example.reference.ents + ] + assert "TRAVEL" in reloaded_example.reference.cats + assert "BAKING" in reloaded_example.reference.cats + assert cats["TRAVEL"] == reloaded_example.reference.cats["TRAVEL"] + assert cats["BAKING"] == reloaded_example.reference.cats["BAKING"] + + +def test_docbin_user_data_serialized(doc): + doc.user_data["check"] = True + nlp = English() + + with make_tempdir() as tmpdir: + output_file = tmpdir / "userdata.spacy" + DocBin(docs=[doc], store_user_data=True).to_disk(output_file) + reloaded_docs = DocBin().from_disk(output_file).get_docs(nlp.vocab) + reloaded_doc = list(reloaded_docs)[0] + + assert reloaded_doc.user_data["check"] == True + + +def test_docbin_user_data_not_serialized(doc): + # this isn't serializable, but that shouldn't cause an error + doc.user_data["check"] = set() + nlp = English() + + with make_tempdir() as tmpdir: + output_file = tmpdir / "userdata.spacy" + DocBin(docs=[doc], store_user_data=False).to_disk(output_file) + reloaded_docs = DocBin().from_disk(output_file).get_docs(nlp.vocab) + reloaded_doc = list(reloaded_docs)[0] + + assert "check" not in reloaded_doc.user_data + + +@pytest.mark.parametrize( + "tokens_a,tokens_b,expected", + [ + (["a", "b", "c"], ["ab", "c"], ([[0], [0], [1]], [[0, 1], [2]])), + ( + ["a", "b", '"', "c"], + ['ab"', "c"], + ([[0], [0], [0], [1]], [[0, 1, 2], [3]]), + ), + (["a", "bc"], ["ab", "c"], ([[0], [0, 1]], [[0, 1], [1]])), + ( + ["ab", "c", "d"], + ["a", "b", "cd"], + ([[0, 1], [2], [2]], [[0], [0], [1, 2]]), + ), + ( + ["a", "b", "cd"], + ["a", "b", "c", "d"], + ([[0], [1], [2, 3]], [[0], [1], [2], [2]]), + ), + ([" ", "a"], ["a"], ([[], [0]], [[1]])), + ( + ["a", "''", "'", ","], + ["a'", "''", ","], + ([[0], [0, 1], [1], [2]], [[0, 1], [1, 2], [3]]), + ), + ], +) +def test_align(tokens_a, tokens_b, expected): # noqa + a2b, b2a = get_alignments(tokens_a, tokens_b) + assert (a2b, b2a) == expected # noqa + # check symmetry + a2b, b2a = get_alignments(tokens_b, tokens_a) # noqa + assert (b2a, a2b) == expected # noqa + + +def test_goldparse_startswith_space(en_tokenizer): + text = " a" + doc = en_tokenizer(text) + gold_words = ["a"] + entities = ["U-DATE"] + deps = ["ROOT"] + heads = [0] + example = Example.from_dict( + doc, {"words": gold_words, "entities": entities, "deps": deps, "heads": heads} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["O", "U-DATE"] + assert example.get_aligned("DEP", as_string=True) == [None, "ROOT"] + + +def test_goldparse_endswith_space(en_tokenizer): + text = "a\n" + doc = en_tokenizer(text) + gold_words = ["a"] + entities = ["U-DATE"] + deps = ["ROOT"] + heads = [0] + example = Example.from_dict( + doc, {"words": gold_words, "entities": entities, "deps": deps, "heads": heads} + ) + ner_tags = example.get_aligned_ner() + assert ner_tags == ["U-DATE", "O"] + assert example.get_aligned("DEP", as_string=True) == ["ROOT", None] + + +def test_gold_constructor(): + """Test that the Example constructor works fine""" + nlp = English() + doc = nlp("This is a sentence") + example = Example.from_dict(doc, {"cats": {"cat1": 1.0, "cat2": 0.0}}) + assert example.get_aligned("ORTH", as_string=True) == [ + "This", + "is", + "a", + "sentence", + ] + assert example.reference.cats["cat1"] + assert not example.reference.cats["cat2"] + + +def test_tuple_format_implicit(): + """Test tuple format""" + + train_data = [ + ("Uber blew through $1 million a week", {"entities": [(0, 4, "ORG")]}), + ( + "Spotify steps up Asia expansion", + {"entities": [(0, 7, "ORG"), (17, 21, "LOC")]}, + ), + ("Google rebrands its business apps", {"entities": [(0, 6, "ORG")]}), + ] + + _train_tuples(train_data) + + +def test_tuple_format_implicit_invalid(): + """Test that an error is thrown for an implicit invalid field""" + train_data = [ + ("Uber blew through $1 million a week", {"frumble": [(0, 4, "ORG")]}), + ( + "Spotify steps up Asia expansion", + {"entities": [(0, 7, "ORG"), (17, 21, "LOC")]}, + ), + ("Google rebrands its business apps", {"entities": [(0, 6, "ORG")]}), + ] + with pytest.raises(KeyError): + _train_tuples(train_data) + + +def _train_tuples(train_data): + nlp = English() + ner = nlp.add_pipe("ner") + ner.add_label("ORG") + ner.add_label("LOC") + train_examples = [] + for t in train_data: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize() + for i in range(5): + losses = {} + batches = minibatch(train_examples, size=compounding(4.0, 32.0, 1.001)) + for batch in batches: + nlp.update(batch, sgd=optimizer, losses=losses) + + +def test_split_sents(merged_dict): + nlp = English() + example = Example.from_dict( + Doc(nlp.vocab, words=merged_dict["words"], spaces=merged_dict["spaces"]), + merged_dict, + ) + assert example.text == "Hi there everyone It is just me" + split_examples = example.split_sents() + assert len(split_examples) == 2 + assert split_examples[0].text == "Hi there everyone " + assert split_examples[1].text == "It is just me" + token_annotation_1 = split_examples[0].to_dict()["token_annotation"] + assert token_annotation_1["ORTH"] == ["Hi", "there", "everyone"] + assert token_annotation_1["TAG"] == ["INTJ", "ADV", "PRON"] + assert token_annotation_1["SENT_START"] == [1, 0, 0] + token_annotation_2 = split_examples[1].to_dict()["token_annotation"] + assert token_annotation_2["ORTH"] == ["It", "is", "just", "me"] + assert token_annotation_2["TAG"] == ["PRON", "AUX", "ADV", "PRON"] + assert token_annotation_2["SENT_START"] == [1, 0, 0, 0] + + +def test_alignment(): + other_tokens = ["i", "listened", "to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts", "."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [1, 1, 1, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 6] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 1, 1] + assert list(align.y2x.data) == [0, 1, 2, 3, 4, 5, 6, 7] + + +def test_alignment_array(): + a = AlignmentArray([[0, 1, 2], [3], [], [4, 5, 6, 7], [8, 9]]) + assert list(a.data) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + assert list(a.lengths) == [3, 1, 0, 4, 2] + assert list(a[3]) == [4, 5, 6, 7] + assert list(a[2]) == [] + assert list(a[-2]) == [4, 5, 6, 7] + assert list(a[1:4]) == [3, 4, 5, 6, 7] + assert list(a[1:]) == [3, 4, 5, 6, 7, 8, 9] + assert list(a[:3]) == [0, 1, 2, 3] + assert list(a[:]) == list(a.data) + assert list(a[0:0]) == [] + assert list(a[3:3]) == [] + assert list(a[-1:-1]) == [] + with pytest.raises(ValueError, match=r"only supports slicing with a step of 1"): + a[:4:-1] + with pytest.raises( + ValueError, match=r"only supports indexing using an int or a slice" + ): + a[[0, 1, 3]] + + a = AlignmentArray([[], [1, 2, 3], [4, 5]]) + assert list(a[0]) == [] + assert list(a[0:1]) == [] + assert list(a[2]) == [4, 5] + assert list(a[0:2]) == [1, 2, 3] + + a = AlignmentArray([[1, 2, 3], [4, 5], []]) + assert list(a[-1]) == [] + assert list(a[-2:]) == [4, 5] + + +def test_alignment_case_insensitive(): + other_tokens = ["I", "listened", "to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "Obama", "'s", "PODCASTS", "."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [1, 1, 1, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 6] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 1, 1] + assert list(align.y2x.data) == [0, 1, 2, 3, 4, 5, 6, 7] + + +def test_alignment_complex(): + other_tokens = ["i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [0, 0, 0, 1, 2, 3, 4, 5] + + +def test_alignment_complex_example(en_vocab): + other_tokens = ["i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts."] + predicted = Doc( + en_vocab, words=other_tokens, spaces=[True, False, False, True, False, False] + ) + reference = Doc( + en_vocab, words=spacy_tokens, spaces=[True, True, True, False, True, False] + ) + assert predicted.text == "i listened to obama's podcasts." + assert reference.text == "i listened to obama's podcasts." + example = Example(predicted, reference) + align = example.alignment + assert list(align.x2y.lengths) == [3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [0, 0, 0, 1, 2, 3, 4, 5] + + +def test_alignment_different_texts(): + other_tokens = ["she", "listened", "to", "obama", "'s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts", "."] + with pytest.raises(ValueError): + Alignment.from_strings(other_tokens, spacy_tokens) + + +def test_alignment_spaces(en_vocab): + # single leading whitespace + other_tokens = [" ", "i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [0, 3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [1, 1, 1, 2, 3, 4, 5, 6] + + # multiple leading whitespace tokens + other_tokens = [" ", " ", "i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [0, 0, 3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [2, 2, 2, 3, 4, 5, 6, 7] + + # both with leading whitespace, not identical + other_tokens = [" ", " ", "i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = [" ", "i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [1, 0, 3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 5, 5, 6, 6] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [0, 2, 2, 2, 3, 4, 5, 6, 7] + + # same leading whitespace, different tokenization + other_tokens = [" ", " ", "i listened to", "obama", "'", "s", "podcasts", "."] + spacy_tokens = [" ", "i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [1, 1, 3, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 0, 1, 2, 3, 4, 5, 5, 6, 6] + assert list(align.y2x.lengths) == [2, 1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [0, 1, 2, 2, 2, 3, 4, 5, 6, 7] + + # only one with trailing whitespace + other_tokens = ["i listened to", "obama", "'", "s", "podcasts", ".", " "] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts."] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [3, 1, 1, 1, 1, 1, 0] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2] + assert list(align.y2x.data) == [0, 0, 0, 1, 2, 3, 4, 5] + + # different trailing whitespace + other_tokens = ["i listened to", "obama", "'", "s", "podcasts", ".", " ", " "] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts.", " "] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [3, 1, 1, 1, 1, 1, 1, 0] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5, 6] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2, 1] + assert list(align.y2x.data) == [0, 0, 0, 1, 2, 3, 4, 5, 6] + + # same trailing whitespace, different tokenization + other_tokens = ["i listened to", "obama", "'", "s", "podcasts", ".", " ", " "] + spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts.", " "] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.lengths) == [3, 1, 1, 1, 1, 1, 1, 1] + assert list(align.x2y.data) == [0, 1, 2, 3, 4, 4, 5, 5, 6, 6] + assert list(align.y2x.lengths) == [1, 1, 1, 1, 2, 2, 2] + assert list(align.y2x.data) == [0, 0, 0, 1, 2, 3, 4, 5, 6, 7] + + # differing whitespace is allowed + other_tokens = ["a", " \n ", "b", "c"] + spacy_tokens = ["a", "b", " ", "c"] + align = Alignment.from_strings(other_tokens, spacy_tokens) + assert list(align.x2y.data) == [0, 1, 3] + assert list(align.y2x.data) == [0, 2, 3] + + # other differences in whitespace are allowed + other_tokens = [" ", "a"] + spacy_tokens = [" ", "a", " "] + align = Alignment.from_strings(other_tokens, spacy_tokens) + + other_tokens = ["a", " "] + spacy_tokens = ["a", " "] + align = Alignment.from_strings(other_tokens, spacy_tokens) + + +def test_retokenized_docs(doc): + a = doc.to_array(["TAG"]) + doc1 = Doc(doc.vocab, words=[t.text for t in doc]).from_array(["TAG"], a) + doc2 = Doc(doc.vocab, words=[t.text for t in doc]).from_array(["TAG"], a) + example = Example(doc1, doc2) + # fmt: off + expected1 = ["Sarah", "'s", "sister", "flew", "to", "Silicon", "Valley", "via", "London", "."] + expected2 = [None, "sister", "flew", "to", None, "via", "London", "."] + # fmt: on + assert example.get_aligned("ORTH", as_string=True) == expected1 + with doc1.retokenize() as retokenizer: + retokenizer.merge(doc1[0:2]) + retokenizer.merge(doc1[5:7]) + assert example.get_aligned("ORTH", as_string=True) == expected2 + + +def test_training_before_update(doc): + def before_update(nlp, args): + assert args["step"] == 0 + assert args["epoch"] == 1 + + # Raise an error here as the rest of the loop + # will not run to completion due to uninitialized + # models. + raise ValueError("ran_before_update") + + def generate_batch(): + yield 1, [Example(doc, doc)] + + nlp = spacy.blank("en") + nlp.add_pipe("tagger") + optimizer = Adam() + generator = train_while_improving( + nlp, + optimizer, + generate_batch(), + lambda: None, + dropout=0.1, + eval_frequency=100, + accumulate_gradient=10, + patience=10, + max_steps=100, + exclude=[], + annotating_components=[], + before_update=before_update, + ) + + with pytest.raises(ValueError, match="ran_before_update"): + for _ in generator: + pass diff --git a/spacy/tests/util.py b/spacy/tests/util.py new file mode 100644 index 0000000..a554889 --- /dev/null +++ b/spacy/tests/util.py @@ -0,0 +1,104 @@ +import contextlib +import re +import tempfile + +import numpy +import srsly +from thinc.api import get_current_ops + +from spacy.tokens import Doc +from spacy.training import split_bilu_label +from spacy.util import make_tempdir # noqa: F401 +from spacy.vocab import Vocab + + +@contextlib.contextmanager +def make_tempfile(mode="r"): + f = tempfile.TemporaryFile(mode=mode) + yield f + f.close() + + +def get_batch(batch_size): + vocab = Vocab() + docs = [] + start = 0 + for size in range(1, batch_size + 1): + # Make the words numbers, so that they're distinct + # across the batch, and easy to track. + numbers = [str(i) for i in range(start, start + size)] + docs.append(Doc(vocab, words=numbers)) + start += size + return docs + + +def get_random_doc(n_words): + vocab = Vocab() + # Make the words numbers, so that they're easy to track. + numbers = [str(i) for i in range(0, n_words)] + return Doc(vocab, words=numbers) + + +def apply_transition_sequence(parser, doc, sequence): + """Perform a series of pre-specified transitions, to put the parser in a + desired state.""" + for action_name in sequence: + if "-" in action_name: + move, label = split_bilu_label(action_name) + parser.add_label(label) + with parser.step_through(doc) as stepwise: + for transition in sequence: + stepwise.transition(transition) + + +def add_vecs_to_vocab(vocab, vectors): + """Add list of vector tuples to given vocab. All vectors need to have the + same length. Format: [("text", [1, 2, 3])]""" + length = len(vectors[0][1]) + vocab.reset_vectors(width=length) + for word, vec in vectors: + vocab.set_vector(word, vector=vec) + return vocab + + +def get_cosine(vec1, vec2): + """Get cosine for two given vectors""" + OPS = get_current_ops() + v1 = OPS.to_numpy(OPS.asarray(vec1)) + v2 = OPS.to_numpy(OPS.asarray(vec2)) + return numpy.dot(v1, v2) / (numpy.linalg.norm(v1) * numpy.linalg.norm(v2)) + + +def assert_docs_equal(doc1, doc2): + """Compare two Doc objects and assert that they're equal. Tests for tokens, + tags, dependencies and entities.""" + assert [t.orth for t in doc1] == [t.orth for t in doc2] + + assert [t.pos for t in doc1] == [t.pos for t in doc2] + assert [t.tag for t in doc1] == [t.tag for t in doc2] + + assert [t.head.i for t in doc1] == [t.head.i for t in doc2] + assert [t.dep for t in doc1] == [t.dep for t in doc2] + assert [t.is_sent_start for t in doc1] == [t.is_sent_start for t in doc2] + + assert [t.ent_type for t in doc1] == [t.ent_type for t in doc2] + assert [t.ent_iob for t in doc1] == [t.ent_iob for t in doc2] + for ent1, ent2 in zip(doc1.ents, doc2.ents): + assert ent1.start == ent2.start + assert ent1.end == ent2.end + assert ent1.label == ent2.label + assert ent1.kb_id == ent2.kb_id + + +def assert_packed_msg_equal(b1, b2): + """Assert that two packed msgpack messages are equal.""" + msg1 = srsly.msgpack_loads(b1) + msg2 = srsly.msgpack_loads(b2) + assert sorted(msg1.keys()) == sorted(msg2.keys()) + for (k1, v1), (k2, v2) in zip(sorted(msg1.items()), sorted(msg2.items())): + assert k1 == k2 + assert v1 == v2 + + +def normalize_whitespace(s): + return re.sub(r"\s+", " ", s) diff --git a/spacy/tests/vocab_vectors/__init__.py b/spacy/tests/vocab_vectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tests/vocab_vectors/test_lexeme.py b/spacy/tests/vocab_vectors/test_lexeme.py new file mode 100644 index 0000000..3c01055 --- /dev/null +++ b/spacy/tests/vocab_vectors/test_lexeme.py @@ -0,0 +1,83 @@ +import numpy +import pytest + +from spacy.attrs import IS_ALPHA, IS_DIGIT +from spacy.tokens import Doc +from spacy.util import OOV_RANK +from spacy.vocab import Vocab + + +@pytest.mark.issue(361) +@pytest.mark.parametrize("text1,text2", [("cat", "dog")]) +def test_issue361(en_vocab, text1, text2): + """Test Issue #361: Equality of lexemes""" + assert en_vocab[text1] == en_vocab[text1] + assert en_vocab[text1] != en_vocab[text2] + + +@pytest.mark.issue(600) +def test_issue600(): + vocab = Vocab(tag_map={"NN": {"pos": "NOUN"}}) + doc = Doc(vocab, words=["hello"]) + doc[0].tag_ = "NN" + + +@pytest.mark.parametrize("text1,prob1,text2,prob2", [("NOUN", -1, "opera", -2)]) +def test_vocab_lexeme_lt(en_vocab, text1, text2, prob1, prob2): + """More frequent is l.t. less frequent""" + lex1 = en_vocab[text1] + lex1.prob = prob1 + lex2 = en_vocab[text2] + lex2.prob = prob2 + + assert lex1 < lex2 + assert lex2 > lex1 + + +@pytest.mark.parametrize("text1,text2", [("phantom", "opera")]) +def test_vocab_lexeme_hash(en_vocab, text1, text2): + """Test that lexemes are hashable.""" + lex1 = en_vocab[text1] + lex2 = en_vocab[text2] + lexes = {lex1: lex1, lex2: lex2} + assert lexes[lex1].orth_ == text1 + assert lexes[lex2].orth_ == text2 + + +def test_vocab_lexeme_is_alpha(en_vocab): + assert en_vocab["the"].flags & (1 << IS_ALPHA) + assert not en_vocab["1999"].flags & (1 << IS_ALPHA) + assert not en_vocab["hello1"].flags & (1 << IS_ALPHA) + + +def test_vocab_lexeme_is_digit(en_vocab): + assert not en_vocab["the"].flags & (1 << IS_DIGIT) + assert en_vocab["1999"].flags & (1 << IS_DIGIT) + assert not en_vocab["hello1"].flags & (1 << IS_DIGIT) + + +def test_vocab_lexeme_add_flag_auto_id(en_vocab): + is_len4 = en_vocab.add_flag(lambda string: len(string) == 4) + assert en_vocab["1999"].check_flag(is_len4) is True + assert en_vocab["1999"].check_flag(IS_DIGIT) is True + assert en_vocab["199"].check_flag(is_len4) is False + assert en_vocab["199"].check_flag(IS_DIGIT) is True + assert en_vocab["the"].check_flag(is_len4) is False + assert en_vocab["dogs"].check_flag(is_len4) is True + + +def test_vocab_lexeme_add_flag_provided_id(en_vocab): + is_len4 = en_vocab.add_flag(lambda string: len(string) == 4, flag_id=IS_DIGIT) + assert en_vocab["1999"].check_flag(is_len4) is True + assert en_vocab["199"].check_flag(is_len4) is False + assert en_vocab["199"].check_flag(IS_DIGIT) is False + assert en_vocab["the"].check_flag(is_len4) is False + assert en_vocab["dogs"].check_flag(is_len4) is True + en_vocab.add_flag(lambda string: string.isdigit(), flag_id=IS_DIGIT) + + +def test_vocab_lexeme_oov_rank(en_vocab): + """Test that default rank is OOV_RANK.""" + lex = en_vocab["word"] + assert OOV_RANK == numpy.iinfo(numpy.uint64).max + assert lex.rank == OOV_RANK diff --git a/spacy/tests/vocab_vectors/test_lookups.py b/spacy/tests/vocab_vectors/test_lookups.py new file mode 100644 index 0000000..addd3fe --- /dev/null +++ b/spacy/tests/vocab_vectors/test_lookups.py @@ -0,0 +1,142 @@ +import pytest + +from spacy.lookups import Lookups, Table +from spacy.strings import get_string_id +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +def test_lookups_api(): + table_name = "test" + data = {"foo": "bar", "hello": "world"} + lookups = Lookups() + lookups.add_table(table_name, data) + assert len(lookups) == 1 + assert table_name in lookups + assert lookups.has_table(table_name) + table = lookups.get_table(table_name) + assert table.name == table_name + assert len(table) == 2 + assert table["hello"] == "world" + table["a"] = "b" + assert table["a"] == "b" + table = lookups.get_table(table_name) + assert len(table) == 3 + with pytest.raises(KeyError): + lookups.get_table("xyz") + with pytest.raises(ValueError): + lookups.add_table(table_name) + table = lookups.remove_table(table_name) + assert table.name == table_name + assert len(lookups) == 0 + assert table_name not in lookups + with pytest.raises(KeyError): + lookups.get_table(table_name) + + +def test_table_api(): + table = Table(name="table") + assert table.name == "table" + assert len(table) == 0 + assert "abc" not in table + data = {"foo": "bar", "hello": "world"} + table = Table(name="table", data=data) + assert len(table) == len(data) + assert "foo" in table + assert get_string_id("foo") in table + assert table["foo"] == "bar" + assert table[get_string_id("foo")] == "bar" + assert table.get("foo") == "bar" + assert table.get("abc") is None + table["abc"] = 123 + assert table["abc"] == 123 + assert table[get_string_id("abc")] == 123 + table.set("def", 456) + assert table["def"] == 456 + assert table[get_string_id("def")] == 456 + + +def test_table_api_to_from_bytes(): + data = {"foo": "bar", "hello": "world", "abc": 123} + table = Table(name="table", data=data) + table_bytes = table.to_bytes() + new_table = Table().from_bytes(table_bytes) + assert new_table.name == "table" + assert len(new_table) == 3 + assert new_table["foo"] == "bar" + assert new_table[get_string_id("foo")] == "bar" + new_table2 = Table(data={"def": 456}) + new_table2.from_bytes(table_bytes) + assert len(new_table2) == 3 + assert "def" not in new_table2 + + +def test_lookups_to_from_bytes(): + lookups = Lookups() + lookups.add_table("table1", {"foo": "bar", "hello": "world"}) + lookups.add_table("table2", {"a": 1, "b": 2, "c": 3}) + lookups_bytes = lookups.to_bytes() + new_lookups = Lookups() + new_lookups.from_bytes(lookups_bytes) + assert len(new_lookups) == 2 + assert "table1" in new_lookups + assert "table2" in new_lookups + table1 = new_lookups.get_table("table1") + assert len(table1) == 2 + assert table1["foo"] == "bar" + table2 = new_lookups.get_table("table2") + assert len(table2) == 3 + assert table2["b"] == 2 + assert new_lookups.to_bytes() == lookups_bytes + + +def test_lookups_to_from_disk(): + lookups = Lookups() + lookups.add_table("table1", {"foo": "bar", "hello": "world"}) + lookups.add_table("table2", {"a": 1, "b": 2, "c": 3}) + with make_tempdir() as tmpdir: + lookups.to_disk(tmpdir) + new_lookups = Lookups() + new_lookups.from_disk(tmpdir) + assert len(new_lookups) == 2 + assert "table1" in new_lookups + assert "table2" in new_lookups + table1 = new_lookups.get_table("table1") + assert len(table1) == 2 + assert table1["foo"] == "bar" + table2 = new_lookups.get_table("table2") + assert len(table2) == 3 + assert table2["b"] == 2 + + +def test_lookups_to_from_bytes_via_vocab(): + table_name = "test" + vocab = Vocab() + vocab.lookups.add_table(table_name, {"foo": "bar", "hello": "world"}) + assert table_name in vocab.lookups + vocab_bytes = vocab.to_bytes() + new_vocab = Vocab() + new_vocab.from_bytes(vocab_bytes) + assert len(new_vocab.lookups) == len(vocab.lookups) + assert table_name in new_vocab.lookups + table = new_vocab.lookups.get_table(table_name) + assert len(table) == 2 + assert table["hello"] == "world" + assert new_vocab.to_bytes() == vocab_bytes + + +def test_lookups_to_from_disk_via_vocab(): + table_name = "test" + vocab = Vocab() + vocab.lookups.add_table(table_name, {"foo": "bar", "hello": "world"}) + assert table_name in vocab.lookups + with make_tempdir() as tmpdir: + vocab.to_disk(tmpdir) + new_vocab = Vocab() + new_vocab.from_disk(tmpdir) + assert len(new_vocab.lookups) == len(vocab.lookups) + assert table_name in new_vocab.lookups + table = new_vocab.lookups.get_table(table_name) + assert len(table) == 2 + assert table["hello"] == "world" diff --git a/spacy/tests/vocab_vectors/test_memory_zone.py b/spacy/tests/vocab_vectors/test_memory_zone.py new file mode 100644 index 0000000..f718afa --- /dev/null +++ b/spacy/tests/vocab_vectors/test_memory_zone.py @@ -0,0 +1,59 @@ +from spacy.vocab import Vocab + + +def test_memory_zone_no_insertion(): + vocab = Vocab() + with vocab.memory_zone(): + pass + lex = vocab["horse"] + assert lex.text == "horse" + + +def test_memory_zone_insertion(): + vocab = Vocab() + _ = vocab["dog"] + assert "dog" in vocab + assert "horse" not in vocab + with vocab.memory_zone(): + lex = vocab["horse"] + assert lex.text == "horse" + assert "dog" in vocab + assert "horse" not in vocab + + +def test_memory_zone_redundant_insertion(): + """Test that if we insert an already-existing word while + in the memory zone, it stays persistent""" + vocab = Vocab() + _ = vocab["dog"] + assert "dog" in vocab + assert "horse" not in vocab + with vocab.memory_zone(): + lex = vocab["horse"] + assert lex.text == "horse" + _ = vocab["dog"] + assert "dog" in vocab + assert "horse" not in vocab + + +def test_memory_zone_exception_cleanup(): + """Test that if an exception occurs inside a memory zone, the vocab + is properly cleaned up and remains usable afterward.""" + vocab = Vocab() + _ = vocab["dog"] + assert "dog" in vocab + try: + with vocab.memory_zone(): + _ = vocab["horse"] + raise ValueError("simulated error") + except ValueError: + pass + # Vocab should not be stuck in memory zone state + assert not vocab.in_memory_zone + # Pre-existing words should still work + assert "dog" in vocab + # Transient word from failed zone should be cleaned up + assert "horse" not in vocab + # Vocab should be fully usable for new operations + lex = vocab["cat"] + assert lex.text == "cat" diff --git a/spacy/tests/vocab_vectors/test_similarity.py b/spacy/tests/vocab_vectors/test_similarity.py new file mode 100644 index 0000000..5a28f54 --- /dev/null +++ b/spacy/tests/vocab_vectors/test_similarity.py @@ -0,0 +1,111 @@ +import numpy +import pytest + +from spacy.tokens import Doc +from spacy.vocab import Vocab + +from ..util import add_vecs_to_vocab, get_cosine + + +@pytest.fixture +def vectors(): + return [("apple", [1, 2, 3]), ("orange", [-1, -2, -3])] + + +@pytest.fixture() +def vocab(en_vocab, vectors): + add_vecs_to_vocab(en_vocab, vectors) + return en_vocab + + +@pytest.mark.issue(2219) +def test_issue2219(en_vocab): + """Test if indexing issue still occurs during Token-Token similarity""" + vectors = [("a", [1, 2, 3]), ("letter", [4, 5, 6])] + add_vecs_to_vocab(en_vocab, vectors) + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(en_vocab, words=[word1, word2]) + assert doc[0].similarity(doc[1]) == doc[1].similarity(doc[0]) + + +def test_vectors_similarity_LL(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + lex1 = vocab[word1] + lex2 = vocab[word2] + assert lex1.has_vector + assert lex2.has_vector + assert lex1.vector_norm != 0 + assert lex2.vector_norm != 0 + assert lex1.vector[0] != lex2.vector[0] and lex1.vector[1] != lex2.vector[1] + assert isinstance(lex1.similarity(lex2), float) + assert numpy.isclose(lex1.similarity(lex2), get_cosine(vec1, vec2)) + assert numpy.isclose(lex2.similarity(lex2), lex1.similarity(lex1)) + + +def test_vectors_similarity_TT(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(vocab, words=[word1, word2]) + assert doc[0].has_vector + assert doc[1].has_vector + assert doc[0].vector_norm != 0 + assert doc[1].vector_norm != 0 + assert doc[0].vector[0] != doc[1].vector[0] and doc[0].vector[1] != doc[1].vector[1] + assert isinstance(doc[0].similarity(doc[1]), float) + assert numpy.isclose(doc[0].similarity(doc[1]), get_cosine(vec1, vec2)) + assert numpy.isclose(doc[1].similarity(doc[0]), doc[0].similarity(doc[1])) + + +def test_vectors_similarity_SS(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(vocab, words=[word1, word2]) + assert isinstance(doc[0:1].similarity(doc[0:2]), float) + assert doc[0:1].similarity(doc[0:2]) == doc[0:2].similarity(doc[0:1]) + + +def test_vectors_similarity_DD(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc1 = Doc(vocab, words=[word1, word2]) + doc2 = Doc(vocab, words=[word2, word1]) + assert isinstance(doc1.similarity(doc2), float) + assert doc1.similarity(doc2) == doc2.similarity(doc1) + + +def test_vectors_similarity_TD(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(vocab, words=[word1, word2]) + assert isinstance(doc.similarity(doc[0]), float) + assert isinstance(doc[0].similarity(doc), float) + assert doc.similarity(doc[0]) == doc[0].similarity(doc) + + +def test_vectors_similarity_TS(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(vocab, words=[word1, word2]) + assert isinstance(doc[:2].similarity(doc[0]), float) + assert isinstance(doc[0].similarity(doc[:2]), float) + assert doc[:2].similarity(doc[0]) == doc[0].similarity(doc[:2]) + + +def test_vectors_similarity_DS(vocab, vectors): + [(word1, vec1), (word2, vec2)] = vectors + doc = Doc(vocab, words=[word1, word2]) + assert isinstance(doc.similarity(doc[:2]), float) + assert doc.similarity(doc[:2]) == doc[:2].similarity(doc) + + +def test_vectors_similarity_no_vectors(): + vocab = Vocab() + doc1 = Doc(vocab, words=["a", "b"]) + doc2 = Doc(vocab, words=["c", "d", "e"]) + with pytest.warns(UserWarning): + doc1.similarity(doc2) + with pytest.warns(UserWarning): + doc1.similarity(doc2[1]) + with pytest.warns(UserWarning): + doc1.similarity(doc2[:2]) + with pytest.warns(UserWarning): + doc2.similarity(doc1) + with pytest.warns(UserWarning): + doc2[1].similarity(doc1) + with pytest.warns(UserWarning): + doc2[:2].similarity(doc1) diff --git a/spacy/tests/vocab_vectors/test_stringstore.py b/spacy/tests/vocab_vectors/test_stringstore.py new file mode 100644 index 0000000..61039ff --- /dev/null +++ b/spacy/tests/vocab_vectors/test_stringstore.py @@ -0,0 +1,98 @@ +import pytest + +from spacy.strings import StringStore + + +@pytest.fixture +def stringstore(): + return StringStore() + + +def test_string_hash(stringstore): + """Test that string hashing is stable across platforms""" + assert stringstore.add("apple") == 8566208034543834098 + heart = "\U0001f499" + h = stringstore.add(heart) + assert h == 11841826740069053588 + + +def test_stringstore_from_api_docs(stringstore): + apple_hash = stringstore.add("apple") + assert apple_hash == 8566208034543834098 + assert stringstore[apple_hash] == "apple" + assert "apple" in stringstore + assert "cherry" not in stringstore + stringstore.add("orange") + all_strings = [s for s in stringstore] + assert all_strings == ["apple", "orange"] + banana_hash = stringstore.add("banana") + assert len(stringstore) == 3 + assert banana_hash == 2525716904149915114 + assert stringstore[banana_hash] == "banana" + assert stringstore["banana"] == banana_hash + + +@pytest.mark.parametrize("text1,text2,text3", [(b"Hello", b"goodbye", b"hello")]) +def test_stringstore_save_bytes(stringstore, text1, text2, text3): + key = stringstore.add(text1) + assert stringstore[text1] == key + assert stringstore[text2] != key + assert stringstore[text3] != key + + +@pytest.mark.parametrize("text1,text2,text3", [("Hello", "goodbye", "hello")]) +def test_stringstore_save_unicode(stringstore, text1, text2, text3): + key = stringstore.add(text1) + assert stringstore[text1] == key + assert stringstore[text2] != key + assert stringstore[text3] != key + + +@pytest.mark.parametrize("text", [b"A"]) +def test_stringstore_retrieve_id(stringstore, text): + key = stringstore.add(text) + assert len(stringstore) == 1 + assert stringstore[key] == text.decode("utf8") + with pytest.raises(KeyError): + stringstore[20000] + + +@pytest.mark.parametrize("text1,text2", [(b"0123456789", b"A")]) +def test_stringstore_med_string(stringstore, text1, text2): + store = stringstore.add(text1) + assert stringstore[store] == text1.decode("utf8") + stringstore.add(text2) + assert stringstore[text1] == store + + +def test_stringstore_long_string(stringstore): + text = "INFORMATIVE](http://www.google.com/search?as_q=RedditMonkey&hl=en&num=50&btnG=Google+Search&as_epq=&as_oq=&as_eq=&lr=&as_ft=i&as_filetype=&as_qdr=all&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch=&as_rights=&safe=off" + store = stringstore.add(text) + assert stringstore[store] == text + + +@pytest.mark.parametrize("factor", [254, 255, 256]) +def test_stringstore_multiply(stringstore, factor): + text = "a" * factor + store = stringstore.add(text) + assert stringstore[store] == text + + +def test_stringstore_massive_strings(stringstore): + text = "a" * 511 + store = stringstore.add(text) + assert stringstore[store] == text + text2 = "z" * 512 + store = stringstore.add(text2) + assert stringstore[store] == text2 + text3 = "1" * 513 + store = stringstore.add(text3) + assert stringstore[store] == text3 + + +@pytest.mark.parametrize("text", ["qqqqq"]) +def test_stringstore_to_bytes(stringstore, text): + store = stringstore.add(text) + serialized = stringstore.to_bytes() + new_stringstore = StringStore().from_bytes(serialized) + assert new_stringstore[store] == text diff --git a/spacy/tests/vocab_vectors/test_vectors.py b/spacy/tests/vocab_vectors/test_vectors.py new file mode 100644 index 0000000..7172913 --- /dev/null +++ b/spacy/tests/vocab_vectors/test_vectors.py @@ -0,0 +1,678 @@ +import numpy +import pytest +from numpy.testing import assert_allclose, assert_almost_equal, assert_equal +from thinc.api import NumpyOps, get_current_ops + +from spacy.lang.en import English +from spacy.strings import hash_string # type: ignore +from spacy.tokenizer import Tokenizer +from spacy.tokens import Doc +from spacy.training.initialize import convert_vectors +from spacy.vectors import Vectors +from spacy.vocab import Vocab + +from ..util import add_vecs_to_vocab, get_cosine, make_tempdir + +OPS = get_current_ops() + + +@pytest.fixture +def strings(): + return ["apple", "orange"] + + +@pytest.fixture +def vectors(): + return [ + ("apple", OPS.asarray([1, 2, 3])), + ("orange", OPS.asarray([-1, -2, -3])), + ("and", OPS.asarray([-1, -1, -1])), + ("juice", OPS.asarray([5, 5, 10])), + ("pie", OPS.asarray([7, 6.3, 8.9])), + ] + + +@pytest.fixture +def data(): + return numpy.asarray([[0.0, 1.0, 2.0], [3.0, -2.0, 4.0]], dtype="f") + + +@pytest.fixture +def most_similar_vectors_data(): + return numpy.asarray( + [[0.0, 1.0, 2.0], [1.0, -2.0, 4.0], [1.0, 1.0, -1.0], [2.0, 3.0, 1.0]], + dtype="f", + ) + + +@pytest.fixture +def most_similar_vectors_keys(): + return ["a", "b", "c", "d"] + + +@pytest.fixture +def resize_data(): + return numpy.asarray([[0.0, 1.0], [2.0, 3.0]], dtype="f") + + +@pytest.fixture() +def vocab(en_vocab, vectors): + add_vecs_to_vocab(en_vocab, vectors) + return en_vocab + + +@pytest.fixture() +def tokenizer_v(vocab): + return Tokenizer(vocab, {}, None, None, None) + + +@pytest.mark.issue(1518) +def test_issue1518(): + """Test vectors.resize() works.""" + vectors = Vectors(shape=(10, 10)) + vectors.add("hello", row=2) + vectors.resize((5, 9)) + + +@pytest.mark.issue(1539) +def test_issue1539(): + """Ensure vectors.resize() doesn't try to modify dictionary during iteration.""" + v = Vectors(shape=(10, 10), keys=[5, 3, 98, 100]) + v.resize((100, 100)) + + +@pytest.mark.issue(1807) +def test_issue1807(): + """Test vocab.set_vector also adds the word to the vocab.""" + vocab = Vocab(vectors_name="test_issue1807") + assert "hello" not in vocab + vocab.set_vector("hello", numpy.ones((50,), dtype="f")) + assert "hello" in vocab + + +@pytest.mark.issue(2871) +def test_issue2871(): + """Test that vectors recover the correct key for spaCy reserved words.""" + words = ["dog", "cat", "SUFFIX"] + vocab = Vocab(vectors_name="test_issue2871") + vocab.vectors.resize(shape=(3, 10)) + vector_data = numpy.zeros((3, 10), dtype="f") + for word in words: + _ = vocab[word] # noqa: F841 + vocab.set_vector(word, vector_data[0]) + vocab.vectors.name = "dummy_vectors" + assert vocab["dog"].rank == 0 + assert vocab["cat"].rank == 1 + assert vocab["SUFFIX"].rank == 2 + assert vocab.vectors.find(key="dog") == 0 + assert vocab.vectors.find(key="cat") == 1 + assert vocab.vectors.find(key="SUFFIX") == 2 + + +@pytest.mark.issue(3412) +def test_issue3412(): + data = numpy.asarray([[0, 0, 0], [1, 2, 3], [9, 8, 7]], dtype="f") + vectors = Vectors(data=data, keys=["A", "B", "C"]) + keys, best_rows, scores = vectors.most_similar( + numpy.asarray([[9, 8, 7], [0, 0, 0]], dtype="f") + ) + assert best_rows[0] == 2 + + +@pytest.mark.issue(4725) +def test_issue4725_2(): + if isinstance(get_current_ops, NumpyOps): + # ensures that this runs correctly and doesn't hang or crash because of the global vectors + # if it does crash, it's usually because of calling 'spawn' for multiprocessing (e.g. on Windows), + # or because of issues with pickling the NER (cf test_issue4725_1) + vocab = Vocab(vectors_name="test_vocab_add_vector") + data = numpy.ndarray((5, 3), dtype="f") + data[0] = 1.0 + data[1] = 2.0 + vocab.set_vector("cat", data[0]) + vocab.set_vector("dog", data[1]) + nlp = English(vocab=vocab) + nlp.add_pipe("ner") + nlp.initialize() + docs = ["Kurt is in London."] * 10 + for _ in nlp.pipe(docs, batch_size=2, n_process=2): + pass + + +def test_init_vectors_with_resize_shape(strings, resize_data): + v = Vectors(shape=(len(strings), 3)) + v.resize(shape=resize_data.shape) + assert v.shape == resize_data.shape + assert v.shape != (len(strings), 3) + + +def test_init_vectors_with_resize_data(data, resize_data): + v = Vectors(data=data) + v.resize(shape=resize_data.shape) + assert v.shape == resize_data.shape + assert v.shape != data.shape + + +def test_get_vector_resize(strings, data): + strings = [hash_string(s) for s in strings] + + # decrease vector dimension (truncate) + v = Vectors(data=data) + resized_dim = v.shape[1] - 1 + v.resize(shape=(v.shape[0], resized_dim)) + for i, string in enumerate(strings): + v.add(string, row=i) + + assert list(v[strings[0]]) == list(data[0, :resized_dim]) + assert list(v[strings[1]]) == list(data[1, :resized_dim]) + + # increase vector dimension (pad with zeros) + v = Vectors(data=data) + resized_dim = v.shape[1] + 1 + v.resize(shape=(v.shape[0], resized_dim)) + for i, string in enumerate(strings): + v.add(string, row=i) + + assert list(v[strings[0]]) == list(data[0]) + [0] + assert list(v[strings[1]]) == list(data[1]) + [0] + + +def test_init_vectors_with_data(strings, data): + v = Vectors(data=data) + assert v.shape == data.shape + + +def test_init_vectors_with_shape(strings): + v = Vectors(shape=(len(strings), 3)) + assert v.shape == (len(strings), 3) + assert v.is_full is False + + +def test_get_vector(strings, data): + v = Vectors(data=data) + strings = [hash_string(s) for s in strings] + for i, string in enumerate(strings): + v.add(string, row=i) + assert list(v[strings[0]]) == list(data[0]) + assert list(v[strings[0]]) != list(data[1]) + assert list(v[strings[1]]) != list(data[0]) + + +def test_set_vector(strings, data): + orig = data.copy() + v = Vectors(data=data) + strings = [hash_string(s) for s in strings] + for i, string in enumerate(strings): + v.add(string, row=i) + assert list(v[strings[0]]) == list(orig[0]) + assert list(v[strings[0]]) != list(orig[1]) + v[strings[0]] = data[1] + assert list(v[strings[0]]) == list(orig[1]) + assert list(v[strings[0]]) != list(orig[0]) + + +def test_vectors_most_similar(most_similar_vectors_data, most_similar_vectors_keys): + v = Vectors(data=most_similar_vectors_data, keys=most_similar_vectors_keys) + _, best_rows, _ = v.most_similar(v.data, batch_size=2, n=2, sort=True) + assert all(row[0] == i for i, row in enumerate(best_rows)) + + with pytest.raises(ValueError): + v.most_similar(v.data, batch_size=2, n=10, sort=True) + + +def test_vectors_most_similar_identical(): + """Test that most similar identical vectors are assigned a score of 1.0.""" + data = numpy.asarray([[4, 2, 2, 2], [4, 2, 2, 2], [1, 1, 1, 1]], dtype="f") + v = Vectors(data=data, keys=["A", "B", "C"]) + keys, _, scores = v.most_similar(numpy.asarray([[4, 2, 2, 2]], dtype="f")) + assert scores[0][0] == 1.0 # not 1.0000002 + data = numpy.asarray([[1, 2, 3], [1, 2, 3], [1, 1, 1]], dtype="f") + v = Vectors(data=data, keys=["A", "B", "C"]) + keys, _, scores = v.most_similar(numpy.asarray([[1, 2, 3]], dtype="f")) + assert scores[0][0] == 1.0 # not 0.9999999 + + +@pytest.mark.parametrize("text", ["apple and orange"]) +def test_vectors_token_vector(tokenizer_v, vectors, text): + doc = tokenizer_v(text) + assert vectors[0][0] == doc[0].text + assert all([a == b for a, b in zip(vectors[0][1], doc[0].vector)]) + assert vectors[1][0] == doc[2].text + assert all([a == b for a, b in zip(vectors[1][1], doc[2].vector)]) + + +@pytest.mark.parametrize("text", ["apple", "orange"]) +def test_vectors_lexeme_vector(vocab, text): + lex = vocab[text] + assert list(lex.vector) + assert lex.vector_norm + + +@pytest.mark.parametrize("text", [["apple", "and", "orange"]]) +def test_vectors_doc_vector(vocab, text): + doc = Doc(vocab, words=text) + assert list(doc.vector) + assert doc.vector_norm + + +@pytest.mark.parametrize("text", [["apple", "and", "orange"]]) +def test_vectors_span_vector(vocab, text): + span = Doc(vocab, words=text)[0:2] + assert list(span.vector) + assert span.vector_norm + + +@pytest.mark.parametrize("text", ["apple orange"]) +def test_vectors_token_token_similarity(tokenizer_v, text): + doc = tokenizer_v(text) + assert doc[0].similarity(doc[1]) == doc[1].similarity(doc[0]) + assert -1.0 < doc[0].similarity(doc[1]) < 1.0 + + +@pytest.mark.parametrize("text1,text2", [("apple", "orange")]) +def test_vectors_token_lexeme_similarity(tokenizer_v, vocab, text1, text2): + token = tokenizer_v(text1) + lex = vocab[text2] + assert token.similarity(lex) == lex.similarity(token) + assert -1.0 < token.similarity(lex) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_token_span_similarity(vocab, text): + doc = Doc(vocab, words=text) + assert doc[0].similarity(doc[1:3]) == doc[1:3].similarity(doc[0]) + assert -1.0 < doc[0].similarity(doc[1:3]) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_token_doc_similarity(vocab, text): + doc = Doc(vocab, words=text) + assert doc[0].similarity(doc) == doc.similarity(doc[0]) + assert -1.0 < doc[0].similarity(doc) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_lexeme_span_similarity(vocab, text): + doc = Doc(vocab, words=text) + lex = vocab[text[0]] + assert lex.similarity(doc[1:3]) == doc[1:3].similarity(lex) + assert -1.0 < doc.similarity(doc[1:3]) < 1.0 + + +@pytest.mark.parametrize("text1,text2", [("apple", "orange")]) +def test_vectors_lexeme_lexeme_similarity(vocab, text1, text2): + lex1 = vocab[text1] + lex2 = vocab[text2] + assert lex1.similarity(lex2) == lex2.similarity(lex1) + assert -1.0 < lex1.similarity(lex2) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_lexeme_doc_similarity(vocab, text): + doc = Doc(vocab, words=text) + lex = vocab[text[0]] + assert lex.similarity(doc) == doc.similarity(lex) + assert -1.0 < lex.similarity(doc) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_span_span_similarity(vocab, text): + doc = Doc(vocab, words=text) + assert doc[0:2].similarity(doc[1:3]) == doc[1:3].similarity(doc[0:2]) + assert -1.0 < doc[0:2].similarity(doc[1:3]) < 1.0 + + +@pytest.mark.parametrize("text", [["apple", "orange", "juice"]]) +def test_vectors_span_doc_similarity(vocab, text): + doc = Doc(vocab, words=text) + assert doc[0:2].similarity(doc) == doc.similarity(doc[0:2]) + assert -1.0 < doc[0:2].similarity(doc) < 1.0 + + +@pytest.mark.parametrize( + "text1,text2", [(["apple", "and", "apple", "pie"], ["orange", "juice"])] +) +def test_vectors_doc_doc_similarity(vocab, text1, text2): + doc1 = Doc(vocab, words=text1) + doc2 = Doc(vocab, words=text2) + assert doc1.similarity(doc2) == doc2.similarity(doc1) + assert -1.0 < doc1.similarity(doc2) < 1.0 + + +def test_vocab_add_vector(): + vocab = Vocab(vectors_name="test_vocab_add_vector") + data = OPS.xp.ndarray((5, 3), dtype="f") + data[0] = 1.0 + data[1] = 2.0 + vocab.set_vector("cat", data[0]) + vocab.set_vector("dog", data[1]) + cat = vocab["cat"] + assert list(cat.vector) == [1.0, 1.0, 1.0] + dog = vocab["dog"] + assert list(dog.vector) == [2.0, 2.0, 2.0] + + with pytest.raises(ValueError): + vocab.vectors.add(vocab["hamster"].orth, row=1000000) + + +def test_vocab_prune_vectors(): + vocab = Vocab(vectors_name="test_vocab_prune_vectors") + _ = vocab["cat"] # noqa: F841 + _ = vocab["dog"] # noqa: F841 + _ = vocab["kitten"] # noqa: F841 + data = OPS.xp.ndarray((5, 3), dtype="f") + data[0] = OPS.asarray([1.0, 1.2, 1.1]) + data[1] = OPS.asarray([0.3, 1.3, 1.0]) + data[2] = OPS.asarray([0.9, 1.22, 1.05]) + vocab.set_vector("cat", data[0]) + vocab.set_vector("dog", data[1]) + vocab.set_vector("kitten", data[2]) + + remap = vocab.prune_vectors(2, batch_size=2) + assert list(remap.keys()) == ["kitten"] + neighbour, similarity = list(remap.values())[0] + assert neighbour == "cat", remap + cosine = get_cosine(data[0], data[2]) + assert_allclose(float(similarity), cosine, atol=1e-4, rtol=1e-3) + + +def test_vectors_serialize(): + data = OPS.asarray([[4, 2, 2, 2], [4, 2, 2, 2], [1, 1, 1, 1]], dtype="f") + v = Vectors(data=data, keys=["A", "B", "C"]) + b = v.to_bytes() + v_r = Vectors() + v_r.from_bytes(b) + assert_equal(OPS.to_numpy(v.data), OPS.to_numpy(v_r.data)) + assert v.key2row == v_r.key2row + v.resize((5, 4)) + v_r.resize((5, 4)) + row = v.add("D", vector=OPS.asarray([1, 2, 3, 4], dtype="f")) + row_r = v_r.add("D", vector=OPS.asarray([1, 2, 3, 4], dtype="f")) + assert row == row_r + assert_equal(OPS.to_numpy(v.data), OPS.to_numpy(v_r.data)) + assert v.is_full == v_r.is_full + with make_tempdir() as d: + v.to_disk(d) + v_r.from_disk(d) + assert_equal(OPS.to_numpy(v.data), OPS.to_numpy(v_r.data)) + assert v.key2row == v_r.key2row + v.resize((5, 4)) + v_r.resize((5, 4)) + row = v.add("D", vector=OPS.asarray([10, 20, 30, 40], dtype="f")) + row_r = v_r.add("D", vector=OPS.asarray([10, 20, 30, 40], dtype="f")) + assert row == row_r + assert_equal(OPS.to_numpy(v.data), OPS.to_numpy(v_r.data)) + assert v.attr == v_r.attr + + +def test_vector_is_oov(): + vocab = Vocab(vectors_name="test_vocab_is_oov") + data = OPS.xp.ndarray((5, 3), dtype="f") + data[0] = 1.0 + data[1] = 2.0 + vocab.set_vector("cat", data[0]) + vocab.set_vector("dog", data[1]) + assert vocab["cat"].is_oov is False + assert vocab["dog"].is_oov is False + assert vocab["hamster"].is_oov is True + + +def test_init_vectors_unset(): + v = Vectors(shape=(10, 10)) + assert v.is_full is False + assert v.shape == (10, 10) + + with pytest.raises(ValueError): + v = Vectors(shape=(10, 10), mode="floret") + + v = Vectors(data=OPS.xp.zeros((10, 10)), mode="floret", hash_count=1) + assert v.is_full is True + + +def test_vectors_clear(): + data = OPS.asarray([[4, 2, 2, 2], [4, 2, 2, 2], [1, 1, 1, 1]], dtype="f") + v = Vectors(data=data, keys=["A", "B", "C"]) + assert v.is_full is True + assert hash_string("A") in v + v.clear() + # no keys + assert v.key2row == {} + assert list(v) == [] + assert v.is_full is False + assert "A" not in v + with pytest.raises(KeyError): + v["A"] + + +def test_vectors_get_batch(): + data = OPS.asarray([[4, 2, 2, 2], [4, 2, 2, 2], [1, 1, 1, 1]], dtype="f") + v = Vectors(data=data, keys=["A", "B", "C"]) + # check with mixed int/str keys + words = ["C", "B", "A", v.strings["B"]] + rows = v.find(keys=words) + vecs = OPS.as_contig(v.data[rows]) + assert_equal(OPS.to_numpy(vecs), OPS.to_numpy(v.get_batch(words))) + + +def test_vectors_deduplicate(): + data = OPS.asarray([[1, 1], [2, 2], [3, 4], [1, 1], [3, 4]], dtype="f") + v = Vectors(data=data, keys=["a1", "b1", "c1", "a2", "c2"]) + vocab = Vocab() + vocab.vectors = v + # duplicate vectors do not use the same keys + assert ( + vocab.vectors.key2row[v.strings["a1"]] != vocab.vectors.key2row[v.strings["a2"]] + ) + assert ( + vocab.vectors.key2row[v.strings["c1"]] != vocab.vectors.key2row[v.strings["c2"]] + ) + vocab.deduplicate_vectors() + # there are three unique vectors + assert vocab.vectors.shape[0] == 3 + # the uniqued data is the same as the deduplicated data + assert_equal( + numpy.unique(OPS.to_numpy(vocab.vectors.data), axis=0), + OPS.to_numpy(vocab.vectors.data), + ) + # duplicate vectors use the same keys now + assert ( + vocab.vectors.key2row[v.strings["a1"]] == vocab.vectors.key2row[v.strings["a2"]] + ) + assert ( + vocab.vectors.key2row[v.strings["c1"]] == vocab.vectors.key2row[v.strings["c2"]] + ) + # deduplicating again makes no changes + vocab_b = vocab.to_bytes() + vocab.deduplicate_vectors() + assert vocab_b == vocab.to_bytes() + + +@pytest.fixture() +def floret_vectors_hashvec_str(): + """The full hashvec table from floret with the settings: + bucket 10, dim 10, minn 2, maxn 3, hash count 2, hash seed 2166136261, + bow <, eow >""" + return """10 10 2 3 2 2166136261 < > +0 -2.2611 3.9302 2.6676 -11.233 0.093715 -10.52 -9.6463 -0.11853 2.101 -0.10145 +1 -3.12 -1.7981 10.7 -6.171 4.4527 10.967 9.073 6.2056 -6.1199 -2.0402 +2 9.5689 5.6721 -8.4832 -1.2249 2.1871 -3.0264 -2.391 -5.3308 -3.2847 -4.0382 +3 3.6268 4.2759 -1.7007 1.5002 5.5266 1.8716 -12.063 0.26314 2.7645 2.4929 +4 -11.683 -7.7068 2.1102 2.214 7.2202 0.69799 3.2173 -5.382 -2.0838 5.0314 +5 -4.3024 8.0241 2.0714 -1.0174 -0.28369 1.7622 7.8797 -1.7795 6.7541 5.6703 +6 8.3574 -5.225 8.6529 8.5605 -8.9465 3.767 -5.4636 -1.4635 -0.98947 -0.58025 +7 -10.01 3.3894 -4.4487 1.1669 -11.904 6.5158 4.3681 0.79913 -6.9131 -8.687 +8 -5.4576 7.1019 -8.8259 1.7189 4.955 -8.9157 -3.8905 -0.60086 -2.1233 5.892 +9 8.0678 -4.4142 3.6236 4.5889 -2.7611 2.4455 0.67096 -4.2822 2.0875 4.6274 +""" + + +@pytest.fixture() +def floret_vectors_vec_str(): + """The top 10 rows from floret with the settings above, to verify + that the spacy floret vectors are equivalent to the fasttext static + vectors.""" + return """10 10 +, -5.7814 2.6918 0.57029 -3.6985 -2.7079 1.4406 1.0084 1.7463 -3.8625 -3.0565 +. 3.8016 -1.759 0.59118 3.3044 -0.72975 0.45221 -2.1412 -3.8933 -2.1238 -0.47409 +der 0.08224 2.6601 -1.173 1.1549 -0.42821 -0.097268 -2.5589 -1.609 -0.16968 0.84687 +die -2.8781 0.082576 1.9286 -0.33279 0.79488 3.36 3.5609 -0.64328 -2.4152 0.17266 +und 2.1558 1.8606 -1.382 0.45424 -0.65889 1.2706 0.5929 -2.0592 -2.6949 -1.6015 +" -1.1242 1.4588 -1.6263 1.0382 -2.7609 -0.99794 -0.83478 -1.5711 -1.2137 1.0239 +in -0.87635 2.0958 4.0018 -2.2473 -1.2429 2.3474 1.8846 0.46521 -0.506 -0.26653 +von -0.10589 1.196 1.1143 -0.40907 -1.0848 -0.054756 -2.5016 -1.0381 -0.41598 0.36982 +( 0.59263 2.1856 0.67346 1.0769 1.0701 1.2151 1.718 -3.0441 2.7291 3.719 +) 0.13812 3.3267 1.657 0.34729 -3.5459 0.72372 0.63034 -1.6145 1.2733 0.37798 +""" + + +def test_floret_vectors(floret_vectors_vec_str, floret_vectors_hashvec_str): + nlp = English() + nlp_plain = English() + # load both vec and hashvec tables + with make_tempdir() as tmpdir: + p = tmpdir / "test.hashvec" + with open(p, "w") as fileh: + fileh.write(floret_vectors_hashvec_str) + convert_vectors(nlp, p, truncate=0, prune=-1, mode="floret") + p = tmpdir / "test.vec" + with open(p, "w") as fileh: + fileh.write(floret_vectors_vec_str) + convert_vectors(nlp_plain, p, truncate=0, prune=-1) + + word = "der" + # ngrams: full padded word + padded 2-grams + padded 3-grams + ngrams = nlp.vocab.vectors._get_ngrams(word) + assert ngrams == ["", "", ""] + # rows: 2 rows per ngram + rows = OPS.xp.asarray( + [ + h % nlp.vocab.vectors.shape[0] + for ngram in ngrams + for h in nlp.vocab.vectors._get_ngram_hashes(ngram) + ], + dtype="uint32", + ) + assert_equal( + OPS.to_numpy(rows), + numpy.asarray([5, 6, 7, 5, 8, 2, 8, 9, 3, 3, 4, 6, 7, 3, 0, 2]), + ) + assert len(rows) == len(ngrams) * nlp.vocab.vectors.hash_count + # all vectors are equivalent for plain static table vs. hash ngrams + for word in nlp_plain.vocab.vectors: + word = nlp_plain.vocab.strings.as_string(word) + assert_almost_equal( + nlp.vocab[word].vector, nlp_plain.vocab[word].vector, decimal=3 + ) + + # every word has a vector + assert nlp.vocab[word * 5].has_vector + + # n_keys is -1 for floret + assert nlp_plain.vocab.vectors.n_keys > 0 + assert nlp.vocab.vectors.n_keys == -1 + + # check that single and batched vector lookups are identical + words = [s for s in nlp_plain.vocab.vectors] + single_vecs = OPS.to_numpy(OPS.asarray([nlp.vocab[word].vector for word in words])) + batch_vecs = OPS.to_numpy(nlp.vocab.vectors.get_batch(words)) + assert_equal(single_vecs, batch_vecs) + + # an empty key returns 0s + assert_equal( + OPS.to_numpy(nlp.vocab[""].vector), + numpy.zeros((nlp.vocab.vectors.shape[0],)), + ) + # an empty batch returns 0s + assert_equal( + OPS.to_numpy(nlp.vocab.vectors.get_batch([""])), + numpy.zeros((1, nlp.vocab.vectors.shape[0])), + ) + # an empty key within a batch returns 0s + assert_equal( + OPS.to_numpy(nlp.vocab.vectors.get_batch(["a", "", "b"])[1]), + numpy.zeros((nlp.vocab.vectors.shape[0],)), + ) + + # the loaded ngram vector table cannot be modified + # except for clear: warning, then return without modifications + vector = list(range(nlp.vocab.vectors.shape[1])) + orig_bytes = nlp.vocab.vectors.to_bytes(exclude=["strings"]) + with pytest.warns(UserWarning): + nlp.vocab.set_vector("the", vector) + assert orig_bytes == nlp.vocab.vectors.to_bytes(exclude=["strings"]) + with pytest.warns(UserWarning): + nlp.vocab[word].vector = vector + assert orig_bytes == nlp.vocab.vectors.to_bytes(exclude=["strings"]) + with pytest.warns(UserWarning): + nlp.vocab.vectors.add("the", row=6) + assert orig_bytes == nlp.vocab.vectors.to_bytes(exclude=["strings"]) + with pytest.warns(UserWarning): + nlp.vocab.vectors.resize(shape=(100, 10)) + assert orig_bytes == nlp.vocab.vectors.to_bytes(exclude=["strings"]) + with pytest.raises(ValueError): + nlp.vocab.vectors.clear() + + # data and settings are serialized correctly + with make_tempdir() as d: + nlp.vocab.to_disk(d) + vocab_r = Vocab() + vocab_r.from_disk(d) + assert nlp.vocab.vectors.to_bytes() == vocab_r.vectors.to_bytes() + assert_equal( + OPS.to_numpy(nlp.vocab.vectors.data), OPS.to_numpy(vocab_r.vectors.data) + ) + assert_equal(nlp.vocab.vectors._get_cfg(), vocab_r.vectors._get_cfg()) + assert_almost_equal( + OPS.to_numpy(nlp.vocab[word].vector), + OPS.to_numpy(vocab_r[word].vector), + decimal=6, + ) + + +def test_equality(): + vectors1 = Vectors(shape=(10, 10)) + vectors2 = Vectors(shape=(10, 8)) + + assert vectors1 != vectors2 + + vectors2 = Vectors(shape=(10, 10)) + assert vectors1 == vectors2 + + vectors1.add("hello", row=2) + assert vectors1 != vectors2 + + vectors2.add("hello", row=2) + assert vectors1 == vectors2 + + vectors1.resize((5, 9)) + vectors2.resize((5, 9)) + assert vectors1 == vectors2 + + +def test_vectors_attr(): + data = numpy.asarray([[0, 0, 0], [1, 2, 3], [9, 8, 7]], dtype="f") + # default ORTH + nlp = English() + nlp.vocab.vectors = Vectors(data=data, keys=["A", "B", "C"]) + assert nlp.vocab.strings["A"] in nlp.vocab.vectors.key2row + assert nlp.vocab.strings["a"] not in nlp.vocab.vectors.key2row + assert nlp.vocab["A"].has_vector is True + assert nlp.vocab["a"].has_vector is False + assert nlp("A")[0].has_vector is True + assert nlp("a")[0].has_vector is False + + # custom LOWER + nlp = English() + nlp.vocab.vectors = Vectors(data=data, keys=["a", "b", "c"], attr="LOWER") + assert nlp.vocab.strings["A"] not in nlp.vocab.vectors.key2row + assert nlp.vocab.strings["a"] in nlp.vocab.vectors.key2row + assert nlp.vocab["A"].has_vector is True + assert nlp.vocab["a"].has_vector is True + assert nlp("A")[0].has_vector is True + assert nlp("a")[0].has_vector is True + # add a new vectors entry + assert nlp.vocab["D"].has_vector is False + assert nlp.vocab["d"].has_vector is False + nlp.vocab.set_vector("D", numpy.asarray([4, 5, 6])) + assert nlp.vocab["D"].has_vector is True + assert nlp.vocab["d"].has_vector is True diff --git a/spacy/tests/vocab_vectors/test_vocab_api.py b/spacy/tests/vocab_vectors/test_vocab_api.py new file mode 100644 index 0000000..e373b9d --- /dev/null +++ b/spacy/tests/vocab_vectors/test_vocab_api.py @@ -0,0 +1,83 @@ +import os + +import pytest + +from spacy.attrs import IS_ALPHA, LEMMA, ORTH +from spacy.lang.en import English +from spacy.parts_of_speech import NOUN, VERB +from spacy.vocab import Vocab + +from ..util import make_tempdir + + +@pytest.mark.issue(1868) +def test_issue1868(): + """Test Vocab.__contains__ works with int keys.""" + vocab = Vocab() + lex = vocab["hello"] + assert lex.orth in vocab + assert lex.orth_ in vocab + assert "some string" not in vocab + int_id = vocab.strings.add("some string") + assert int_id not in vocab + + +@pytest.mark.parametrize( + "text1,text2", [("Hello", "bye"), ("Hello", "hello"), ("Hello", "Hello,")] +) +def test_vocab_api_neq(en_vocab, text1, text2): + assert en_vocab[text1].orth != en_vocab[text2].orth + + +@pytest.mark.parametrize("text", "Hello") +def test_vocab_api_eq(en_vocab, text): + lex = en_vocab[text] + assert en_vocab[text].orth == lex.orth + + +@pytest.mark.parametrize("text", ["example"]) +def test_vocab_api_shape_attr(en_vocab, text): + lex = en_vocab[text] + assert lex.orth != lex.shape + + +@pytest.mark.parametrize( + "string,symbol", + [ + ("IS_ALPHA", IS_ALPHA), + ("NOUN", NOUN), + ("VERB", VERB), + ("LEMMA", LEMMA), + ("ORTH", ORTH), + ], +) +def test_vocab_api_symbols(en_vocab, string, symbol): + assert en_vocab.strings[string] == symbol + + +@pytest.mark.parametrize("text", "Hello") +def test_vocab_api_contains(en_vocab, text): + _ = en_vocab[text] # noqa: F841 + assert text in en_vocab + assert "LKsdjvlsakdvlaksdvlkasjdvljasdlkfvm" not in en_vocab + + +def test_vocab_writing_system(en_vocab): + assert en_vocab.writing_system["direction"] == "ltr" + assert en_vocab.writing_system["has_case"] is True + + +def test_to_disk(): + nlp = English() + with make_tempdir() as d: + nlp.vocab.to_disk(d) + assert "vectors" in os.listdir(d) + assert "lookups.bin" in os.listdir(d) + + +def test_to_disk_exclude(): + nlp = English() + with make_tempdir() as d: + nlp.vocab.to_disk(d, exclude=("vectors", "lookups")) + assert "vectors" not in os.listdir(d) + assert "lookups.bin" not in os.listdir(d) diff --git a/spacy/tokenizer.pxd b/spacy/tokenizer.pxd new file mode 100644 index 0000000..88e4b06 --- /dev/null +++ b/spacy/tokenizer.pxd @@ -0,0 +1,86 @@ +from cymem.cymem cimport Pool +from libcpp.vector cimport vector +from preshed.maps cimport PreshMap + +from .matcher.phrasematcher cimport PhraseMatcher +from .strings cimport StringStore +from .structs cimport LexemeC, SpanC, TokenC +from .tokens.doc cimport Doc +from .typedefs cimport hash_t +from .vocab cimport LexemesOrTokens, Vocab, _Cached + + +cdef class Tokenizer: + cdef Pool mem + cdef PreshMap _cache + cdef PreshMap _specials + cdef readonly Vocab vocab + + cdef object _token_match + cdef object _url_match + cdef object _prefix_search + cdef object _suffix_search + cdef object _infix_finditer + cdef object _rules + cdef PhraseMatcher _special_matcher + # TODO convert to bool in v4 + cdef int _faster_heuristics + cdef public int max_cache_size + + cdef Doc _tokenize_affixes(self, str string, bint with_special_cases) + cdef int _apply_special_cases(self, Doc doc) except -1 + cdef void _filter_special_spans( + self, + vector[SpanC] &original, + vector[SpanC] &filtered, + int doc_len, + ) nogil + cdef object _prepare_special_spans( + self, + Doc doc, + vector[SpanC] &filtered, + ) + cdef int _retokenize_special_spans( + self, + Doc doc, + TokenC* tokens, + object span_data, + ) + cdef int _try_specials_and_cache( + self, + hash_t key, + Doc tokens, + int* has_special, + bint with_special_cases, + ) except -1 + cdef int _tokenize( + self, + Doc tokens, + str span, + hash_t key, + int* has_special, + bint with_special_cases, + ) except -1 + cdef str _split_affixes( + self, + Pool mem, + str string, + vector[LexemeC*] *prefixes, + vector[LexemeC*] *suffixes, int* has_special, + bint with_special_cases, + ) + cdef int _attach_tokens( + self, + Doc tokens, + str string, + vector[LexemeC*] *prefixes, + vector[LexemeC*] *suffixes, int* has_special, + bint with_special_cases, + ) except -1 + cdef int _save_cached( + self, + const TokenC* tokens, + hash_t key, + int* has_special, + int n, + ) except -1 diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx new file mode 100644 index 0000000..77718a7 --- /dev/null +++ b/spacy/tokenizer.pyx @@ -0,0 +1,877 @@ +# cython: embedsignature=True, binding=True +cimport cython +from cymem.cymem cimport Pool +from cython.operator cimport dereference as deref +from cython.operator cimport preincrement as preinc +from libc.string cimport memcpy, memset +from libcpp.set cimport set as stdset +from preshed.maps cimport PreshMap + +import re +from .lexeme cimport EMPTY_LEXEME +from .strings cimport hash_string +from .tokens.doc cimport Doc + +from . import util +from .attrs import intify_attrs +from .errors import Errors +from .scorer import Scorer +from .symbols import NORM, ORTH +from .tokens import Span +from .training import validate_examples +from .util import get_words_and_spaces + + +cdef class Tokenizer: + """Segment text, and create Doc objects with the discovered segment + boundaries. + + DOCS: https://spacy.io/api/tokenizer + """ + def __init__(self, Vocab vocab, rules=None, prefix_search=None, + suffix_search=None, infix_finditer=None, token_match=None, + url_match=None, faster_heuristics=True, max_cache_size=10000): + """Create a `Tokenizer`, to create `Doc` objects given unicode text. + + vocab (Vocab): A storage container for lexical types. + rules (dict): Exceptions and special-cases for the tokenizer. + prefix_search (callable): A function matching the signature of + `re.compile(string).search` to match prefixes. + suffix_search (callable): A function matching the signature of + `re.compile(string).search` to match suffixes. + infix_finditer (callable): A function matching the signature of + `re.compile(string).finditer` to find infixes. + token_match (callable): A function matching the signature of + `re.compile(string).match`, for matching strings to be + recognized as tokens. + url_match (callable): A function matching the signature of + `re.compile(string).match`, for matching strings to be + recognized as urls. + faster_heuristics (bool): Whether to restrict the final + Matcher-based pass for rules to those containing affixes or space. + Defaults to True. + max_cache_size (int): Maximum number of tokenization chunks to cache. + + EXAMPLE: + >>> tokenizer = Tokenizer(nlp.vocab) + + DOCS: https://spacy.io/api/tokenizer#init + """ + self.mem = Pool() + self._cache = PreshMap() + self._specials = PreshMap() + self.token_match = token_match + self.url_match = url_match + self.prefix_search = prefix_search + self.suffix_search = suffix_search + self.infix_finditer = infix_finditer + self.vocab = vocab + self.faster_heuristics = faster_heuristics + self._rules = {} + self._special_matcher = PhraseMatcher(self.vocab) + self._load_special_cases(rules) + self.max_cache_size = max_cache_size + + @property + def token_match(self): + return self._token_match + + @token_match.setter + def token_match(self, token_match): + self._token_match = token_match + self._reload_special_cases() + + @property + def url_match(self): + return self._url_match + + @url_match.setter + def url_match(self, url_match): + self._url_match = url_match + self._reload_special_cases() + + @property + def prefix_search(self): + return self._prefix_search + + @prefix_search.setter + def prefix_search(self, prefix_search): + self._prefix_search = prefix_search + self._reload_special_cases() + + @property + def suffix_search(self): + return self._suffix_search + + @suffix_search.setter + def suffix_search(self, suffix_search): + self._suffix_search = suffix_search + self._reload_special_cases() + + @property + def infix_finditer(self): + return self._infix_finditer + + @infix_finditer.setter + def infix_finditer(self, infix_finditer): + self._infix_finditer = infix_finditer + self._reload_special_cases() + + @property + def rules(self): + return self._rules + + @rules.setter + def rules(self, rules): + self._rules = {} + self._flush_cache() + self._flush_specials() + self._cache = PreshMap() + self._specials = PreshMap() + self._load_special_cases(rules) + + @property + def faster_heuristics(self): + return bool(self._faster_heuristics) + + @faster_heuristics.setter + def faster_heuristics(self, faster_heuristics): + self._faster_heuristics = bool(faster_heuristics) + self._reload_special_cases() + + def __reduce__(self): + args = (self.vocab, + self.rules, + self.prefix_search, + self.suffix_search, + self.infix_finditer, + self.token_match, + self.url_match) + return (self.__class__, args, None, None) + + def __call__(self, str string): + """Tokenize a string. + + string (str): The string to tokenize. + RETURNS (Doc): A container for linguistic annotations. + + DOCS: https://spacy.io/api/tokenizer#call + """ + doc = self._tokenize_affixes(string, True) + self._apply_special_cases(doc) + return doc + + @cython.boundscheck(False) + cdef Doc _tokenize_affixes(self, str string, bint with_special_cases): + """Tokenize according to affix and token_match settings. + + string (str): The string to tokenize. + RETURNS (Doc): A container for linguistic annotations. + """ + if len(string) >= (2 ** 30): + raise ValueError(Errors.E025.format(length=len(string))) + cdef int length = len(string) + cdef Doc doc = Doc(self.vocab) + if length == 0: + return doc + cdef int i = 0 + cdef int start = 0 + cdef int has_special = 0 + cdef bint in_ws = string[0].isspace() + cdef str span + # The task here is much like string.split, but not quite + # We find spans of whitespace and non-space characters, and ignore + # spans that are exactly ' '. So, our sequences will all be separated + # by either ' ' or nothing. + for uc in string: + if uc.isspace() != in_ws: + if start < i: + # When we want to make this fast, get the data buffer once + # with PyUnicode_AS_DATA, and then maintain a start_byte + # and end_byte, so we can call hash64 directly. That way + # we don't have to create the slice when we hit the cache. + span = string[start:i] + key = hash_string(span) + if not self._try_specials_and_cache(key, doc, &has_special, with_special_cases): + self._tokenize(doc, span, key, &has_special, with_special_cases) + if uc == ' ': + doc.c[doc.length - 1].spacy = True + start = i + 1 + else: + start = i + in_ws = not in_ws + i += 1 + if start < i: + span = string[start:] + key = hash_string(span) + if not self._try_specials_and_cache(key, doc, &has_special, with_special_cases): + self._tokenize(doc, span, key, &has_special, with_special_cases) + doc.c[doc.length - 1].spacy = string[-1] == " " and not in_ws + return doc + + def pipe(self, texts, batch_size=1000): + """Tokenize a stream of texts. + + texts: A sequence of unicode texts. + batch_size (int): Number of texts to accumulate in an internal buffer. + Defaults to 1000. + YIELDS (Doc): A sequence of Doc objects, in order. + + DOCS: https://spacy.io/api/tokenizer#pipe + """ + for text in texts: + yield self(text) + + def _flush_cache(self): + self._reset_cache([key for key in self._cache]) + + def _reset_cache(self, keys): + for k in keys: + cached = <_Cached*>self._cache.get(k) + del self._cache[k] + if cached is not NULL: + self.mem.free(cached) + + def _flush_specials(self): + self._special_matcher = PhraseMatcher(self.vocab) + for k in self._specials: + cached = <_Cached*>self._specials.get(k) + del self._specials[k] + if cached is not NULL: + self.mem.free(cached) + + cdef int _apply_special_cases(self, Doc doc) except -1: + """Retokenize doc according to special cases. + + doc (Doc): Document. + """ + cdef int i + cdef int max_length = 0 + cdef bint modify_in_place + cdef Pool mem = Pool() + cdef vector[SpanC] c_matches + cdef vector[SpanC] c_filtered + cdef int offset + cdef int modified_doc_length + # Find matches for special cases + self._special_matcher.find_matches(doc, 0, doc.length, &c_matches) + # Skip processing if no matches + if c_matches.size() == 0: + return True + self._filter_special_spans(c_matches, c_filtered, doc.length) + # Put span info in span.start-indexed dict and calculate maximum + # intermediate document size + (span_data, max_length, modify_in_place) = self._prepare_special_spans(doc, c_filtered) + # If modifications never increase doc length, can modify in place + if modify_in_place: + tokens = doc.c + # Otherwise create a separate array to store modified tokens + else: + assert max_length > 0 + tokens = mem.alloc(max_length, sizeof(TokenC)) + # Modify tokenization according to filtered special cases + offset = self._retokenize_special_spans(doc, tokens, span_data) + # Allocate more memory for doc if needed + modified_doc_length = doc.length + offset + while modified_doc_length >= doc.max_length: + doc._realloc(doc.max_length * 2) + # If not modified in place, copy tokens back to doc + if not modify_in_place: + memcpy(doc.c, tokens, max_length * sizeof(TokenC)) + for i in range(doc.length + offset, doc.length): + memset(&doc.c[i], 0, sizeof(TokenC)) + doc.c[i].lex = &EMPTY_LEXEME + doc.length = doc.length + offset + return True + + cdef void _filter_special_spans(self, vector[SpanC] &original, vector[SpanC] &filtered, int doc_len) nogil: + + cdef int seen_i + cdef SpanC span + cdef stdset[int] seen_tokens + stdsort(original.begin(), original.end(), len_start_cmp) + cdef int orig_i = original.size() - 1 + while orig_i >= 0: + span = original[orig_i] + if not seen_tokens.count(span.start) and not seen_tokens.count(span.end - 1): + filtered.push_back(span) + for seen_i in range(span.start, span.end): + seen_tokens.insert(seen_i) + orig_i -= 1 + stdsort(filtered.begin(), filtered.end(), start_cmp) + + cdef object _prepare_special_spans(self, Doc doc, vector[SpanC] &filtered): + spans = [doc[match.start:match.end] for match in filtered] + cdef bint modify_in_place = True + cdef int curr_length = doc.length + cdef int max_length = 0 + cdef int span_length_diff = 0 + span_data = {} + for span in spans: + rule = self._rules.get(span.text, None) + span_length_diff = 0 + if rule: + span_length_diff = len(rule) - (span.end - span.start) + if span_length_diff > 0: + modify_in_place = False + curr_length += span_length_diff + if curr_length > max_length: + max_length = curr_length + span_data[span.start] = (span.text, span.start, span.end, span_length_diff) + return (span_data, max_length, modify_in_place) + + cdef int _retokenize_special_spans(self, Doc doc, TokenC* tokens, object span_data): + cdef int i = 0 + cdef int j = 0 + cdef int offset = 0 + cdef _Cached* cached + cdef int idx_offset = 0 + cdef int orig_final_spacy + cdef int orig_idx + cdef int span_start + cdef int span_end + while i < doc.length: + if i not in span_data: + tokens[i + offset] = doc.c[i] + i += 1 + else: + span = span_data[i] + span_start = span[1] + span_end = span[2] + cached = <_Cached*>self._specials.get(hash_string(span[0])) + if cached == NULL: + # Copy original tokens if no rule found + for j in range(span_end - span_start): + tokens[i + offset + j] = doc.c[i + j] + i += span_end - span_start + else: + # Copy special case tokens into doc and adjust token and + # character offsets + idx_offset = 0 + orig_final_spacy = doc.c[span_end - 1].spacy + orig_idx = doc.c[i].idx + for j in range(cached.length): + tokens[i + offset + j] = cached.data.tokens[j] + tokens[i + offset + j].idx = orig_idx + idx_offset + idx_offset += cached.data.tokens[j].lex.length + if cached.data.tokens[j].spacy: + idx_offset += 1 + tokens[i + offset + cached.length - 1].spacy = orig_final_spacy + i += span_end - span_start + offset += span[3] + return offset + + cdef int _try_specials_and_cache(self, hash_t key, Doc tokens, int* has_special, bint with_special_cases) except -1: + cdef bint specials_hit = 0 + cdef bint cache_hit = 0 + cdef int i + if with_special_cases: + cached = <_Cached*>self._specials.get(key) + if cached == NULL: + specials_hit = False + else: + for i in range(cached.length): + tokens.push_back(&cached.data.tokens[i], False) + has_special[0] = 1 + specials_hit = True + if not specials_hit: + cached = <_Cached*>self._cache.get(key) + if cached == NULL: + cache_hit = False + else: + if cached.is_lex: + for i in range(cached.length): + tokens.push_back(cached.data.lexemes[i], False) + else: + for i in range(cached.length): + tokens.push_back(&cached.data.tokens[i], False) + cache_hit = True + if not specials_hit and not cache_hit: + return False + return True + + cdef int _tokenize(self, Doc tokens, str span, hash_t orig_key, int* has_special, bint with_special_cases) except -1: + cdef vector[LexemeC*] prefixes + cdef vector[LexemeC*] suffixes + cdef int orig_size + orig_size = tokens.length + span = self._split_affixes(tokens.mem, span, &prefixes, &suffixes, + has_special, with_special_cases) + self._attach_tokens(tokens, span, &prefixes, &suffixes, has_special, + with_special_cases) + if len(self._cache) < self.max_cache_size: + self._save_cached(&tokens.c[orig_size], orig_key, has_special, + tokens.length - orig_size) + + cdef str _split_affixes( + self, + Pool mem, + str string, + vector[const LexemeC*] *prefixes, + vector[const LexemeC*] *suffixes, + int* has_special, + bint with_special_cases + ): + cdef str prefix + cdef str suffix + cdef str minus_pre + cdef str minus_suf + cdef size_t last_size = 0 + while string and len(string) != last_size: + if self.token_match and self.token_match(string): + break + if with_special_cases and self._specials.get(hash_string(string)) != NULL: + break + last_size = len(string) + pre_len = self.find_prefix(string) + if pre_len != 0: + prefix = string[:pre_len] + minus_pre = string[pre_len:] + if minus_pre and with_special_cases and self._specials.get(hash_string(minus_pre)) != NULL: + string = minus_pre + prefixes.push_back(self.vocab.get(mem, prefix)) + break + suf_len = self.find_suffix(string[pre_len:]) + if suf_len != 0: + suffix = string[-suf_len:] + minus_suf = string[:-suf_len] + if minus_suf and with_special_cases and self._specials.get(hash_string(minus_suf)) != NULL: + string = minus_suf + suffixes.push_back(self.vocab.get(mem, suffix)) + break + if pre_len and suf_len and (pre_len + suf_len) <= len(string): + string = string[pre_len:-suf_len] + prefixes.push_back(self.vocab.get(mem, prefix)) + suffixes.push_back(self.vocab.get(mem, suffix)) + elif pre_len: + string = minus_pre + prefixes.push_back(self.vocab.get(mem, prefix)) + elif suf_len: + string = minus_suf + suffixes.push_back(self.vocab.get(mem, suffix)) + return string + + cdef int _attach_tokens(self, Doc tokens, str string, + vector[const LexemeC*] *prefixes, + vector[const LexemeC*] *suffixes, + int* has_special, + bint with_special_cases) except -1: + cdef const LexemeC* lexeme + cdef str span + cdef int i + if prefixes.size(): + for i in range(prefixes.size()): + tokens.push_back(prefixes[0][i], False) + if string: + if self._try_specials_and_cache(hash_string(string), tokens, has_special, with_special_cases): + pass + elif ( + (self.token_match and self.token_match(string)) or + (self.url_match and self.url_match(string)) + ): + + # We're always saying 'no' to spaces here -- the caller will + # fix up the outermost one, with reference to the original. + # See Issue #859 + tokens.push_back(self.vocab.get(tokens.mem, string), False) + else: + matches = self.find_infix(string) + if not matches: + tokens.push_back(self.vocab.get(tokens.mem, string), False) + else: + # Let's say we have dyn-o-mite-dave - the regex finds the + # start and end positions of the hyphens + start = 0 + start_before_infixes = start + for match in matches: + infix_start = match.start() + infix_end = match.end() + + if infix_start == start_before_infixes: + continue + + if infix_start != start: + span = string[start:infix_start] + tokens.push_back(self.vocab.get(tokens.mem, span), False) + + if infix_start != infix_end: + # If infix_start != infix_end, it means the infix + # token is non-empty. Empty infix tokens are useful + # for tokenization in some languages (see + # https://github.com/explosion/spaCy/issues/768) + infix_span = string[infix_start:infix_end] + tokens.push_back(self.vocab.get(tokens.mem, infix_span), False) + start = infix_end + span = string[start:] + if span: + tokens.push_back(self.vocab.get(tokens.mem, span), False) + cdef vector[const LexemeC*].reverse_iterator it = suffixes.rbegin() + while it != suffixes.rend(): + lexeme = deref(it) + preinc(it) + tokens.push_back(lexeme, False) + + cdef int _save_cached(self, const TokenC* tokens, hash_t key, + int* has_special, int n) except -1: + cdef int i + if n <= 0: + # avoid mem alloc of zero length + return 0 + if self.vocab.in_memory_zone: + return 0 + # See #1250 + if has_special[0]: + return 0 + cached = <_Cached*>self.mem.alloc(1, sizeof(_Cached)) + cached.length = n + cached.is_lex = True + lexemes = self.mem.alloc(n, sizeof(LexemeC**)) + for i in range(n): + lexemes[i] = tokens[i].lex + cached.data.lexemes = lexemes + self._cache.set(key, cached) + + def find_infix(self, str string): + """Find internal split points of the string, such as hyphens. + + string (str): The string to segment. + RETURNS (list): A list of `re.MatchObject` objects that have `.start()` + and `.end()` methods, denoting the placement of internal segment + separators, e.g. hyphens. + + DOCS: https://spacy.io/api/tokenizer#find_infix + """ + if self.infix_finditer is None: + return 0 + return list(self.infix_finditer(string)) + + def find_prefix(self, str string): + """Find the length of a prefix that should be segmented from the + string, or None if no prefix rules match. + + string (str): The string to segment. + RETURNS (int): The length of the prefix if present, otherwise `None`. + + DOCS: https://spacy.io/api/tokenizer#find_prefix + """ + if self.prefix_search is None: + return 0 + match = self.prefix_search(string) + return (match.end() - match.start()) if match is not None else 0 + + def find_suffix(self, str string): + """Find the length of a suffix that should be segmented from the + string, or None if no suffix rules match. + + string (str): The string to segment. + Returns (int): The length of the suffix if present, otherwise `None`. + + DOCS: https://spacy.io/api/tokenizer#find_suffix + """ + if self.suffix_search is None: + return 0 + match = self.suffix_search(string) + return (match.end() - match.start()) if match is not None else 0 + + def _load_special_cases(self, special_cases): + """Add special-case tokenization rules.""" + if special_cases is not None: + for chunk, substrings in sorted(special_cases.items()): + self.add_special_case(chunk, substrings) + + def _validate_special_case(self, chunk, substrings): + """Check whether the `ORTH` fields match the string. Check that + additional features beyond `ORTH` and `NORM` are not set by the + exception. + + chunk (str): The string to specially tokenize. + substrings (iterable): A sequence of dicts, where each dict describes + a token and its attributes. + """ + attrs = [intify_attrs(spec, _do_deprecated=True) for spec in substrings] + orth = "".join([spec[ORTH] for spec in attrs]) + if chunk != orth: + raise ValueError(Errors.E997.format(chunk=chunk, orth=orth, token_attrs=substrings)) + for substring in attrs: + for attr in substring: + if attr not in (ORTH, NORM): + raise ValueError(Errors.E1005.format(attr=self.vocab.strings[attr], chunk=chunk)) + + def add_special_case(self, str string, substrings): + """Add a special-case tokenization rule. + + string (str): The string to specially tokenize. + substrings (iterable): A sequence of dicts, where each dict describes + a token and its attributes. The `ORTH` fields of the attributes + must exactly match the string when they are concatenated. + + DOCS: https://spacy.io/api/tokenizer#add_special_case + """ + self._validate_special_case(string, substrings) + substrings = list(substrings) + cached = <_Cached*>self.mem.alloc(1, sizeof(_Cached)) + cached.length = len(substrings) + cached.is_lex = False + cached.data.tokens = self.vocab.make_fused_token(substrings) + key = hash_string(string) + stale_special = <_Cached*>self._specials.get(key) + self._specials.set(key, cached) + if stale_special is not NULL: + self.mem.free(stale_special) + self._rules[string] = substrings + self._flush_cache() + if not self.faster_heuristics or self.find_prefix(string) or self.find_infix(string) or self.find_suffix(string) or " " in string: + self._special_matcher.add(string, None, self._tokenize_affixes(string, False)) + + def _reload_special_cases(self): + self._flush_cache() + self._flush_specials() + self._load_special_cases(self._rules) + + def explain(self, text): + """A debugging tokenizer that provides information about which + tokenizer rule or pattern was matched for each token. The tokens + produced are identical to `nlp.tokenizer()` except for whitespace + tokens. + + string (str): The string to tokenize. + RETURNS (list): A list of (pattern_string, token_string) tuples + + DOCS: https://spacy.io/api/tokenizer#explain + """ + prefix_search = self.prefix_search + if prefix_search is None: + prefix_search = re.compile("a^").search + suffix_search = self.suffix_search + if suffix_search is None: + suffix_search = re.compile("a^").search + infix_finditer = self.infix_finditer + if infix_finditer is None: + infix_finditer = re.compile("a^").finditer + token_match = self.token_match + if token_match is None: + token_match = re.compile("a^").match + url_match = self.url_match + if url_match is None: + url_match = re.compile("a^").match + special_cases = {} + for orth, special_tokens in self.rules.items(): + special_cases[orth] = [intify_attrs(special_token, strings_map=self.vocab.strings, _do_deprecated=True) for special_token in special_tokens] + tokens = [] + for substring in text.split(): + suffixes = [] + while substring: + if substring in special_cases: + tokens.extend(("SPECIAL-" + str(i + 1), self.vocab.strings[e[ORTH]]) for i, e in enumerate(special_cases[substring])) + substring = '' + continue + while prefix_search(substring) or suffix_search(substring): + if token_match(substring): + tokens.append(("TOKEN_MATCH", substring)) + substring = '' + break + if substring in special_cases: + tokens.extend(("SPECIAL-" + str(i + 1), self.vocab.strings[e[ORTH]]) for i, e in enumerate(special_cases[substring])) + substring = '' + break + if prefix_search(substring): + split = prefix_search(substring).end() + # break if pattern matches the empty string + if split == 0: + break + tokens.append(("PREFIX", substring[:split])) + substring = substring[split:] + if substring in special_cases: + continue + if suffix_search(substring): + split = suffix_search(substring).start() + # break if pattern matches the empty string + if split == len(substring): + break + suffixes.append(("SUFFIX", substring[split:])) + substring = substring[:split] + if len(substring) == 0: + continue + if token_match(substring): + tokens.append(("TOKEN_MATCH", substring)) + substring = '' + elif url_match(substring): + tokens.append(("URL_MATCH", substring)) + substring = '' + elif substring in special_cases: + tokens.extend((f"SPECIAL-{i + 1}", self.vocab.strings[e[ORTH]]) for i, e in enumerate(special_cases[substring])) + substring = '' + elif list(infix_finditer(substring)): + infixes = infix_finditer(substring) + offset = 0 + for match in infixes: + if offset == 0 and match.start() == 0: + continue + if substring[offset : match.start()]: + tokens.append(("TOKEN", substring[offset : match.start()])) + if substring[match.start() : match.end()]: + tokens.append(("INFIX", substring[match.start() : match.end()])) + offset = match.end() + if substring[offset:]: + tokens.append(("TOKEN", substring[offset:])) + substring = '' + elif substring: + tokens.append(("TOKEN", substring)) + substring = '' + tokens.extend(reversed(suffixes)) + # Find matches for special cases handled by special matcher + words, spaces = get_words_and_spaces([t[1] for t in tokens], text) + t_words = [] + t_spaces = [] + for word, space in zip(words, spaces): + if not word.isspace(): + t_words.append(word) + t_spaces.append(space) + doc = Doc(self.vocab, words=t_words, spaces=t_spaces) + matches = self._special_matcher(doc) + spans = [Span(doc, s, e, label=m_id) for m_id, s, e in matches] + spans = util.filter_spans(spans) + # Replace matched tokens with their exceptions + i = 0 + final_tokens = [] + spans_by_start = {s.start: s for s in spans} + while i < len(tokens): + if i in spans_by_start: + span = spans_by_start[i] + exc = [d[ORTH] for d in special_cases[span.label_]] + # The phrase matcher can overmatch for tokens separated by + # spaces in the text but not in the underlying rule, so skip + # cases where the texts aren't identical + if span.text != "".join([self.vocab.strings[orth] for orth in exc]): + final_tokens.append(tokens[i]) + i += 1 + else: + for j, orth in enumerate(exc): + final_tokens.append((f"SPECIAL-{j + 1}", self.vocab.strings[orth])) + i += len(span) + else: + final_tokens.append(tokens[i]) + i += 1 + return final_tokens + + def score(self, examples, **kwargs): + validate_examples(examples, "Tokenizer.score") + return Scorer.score_tokenization(examples) + + def to_disk(self, path, **kwargs): + """Save the current state to a directory. + + path (str / Path): A path to a directory, which will be created if + it doesn't exist. + exclude (list): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/tokenizer#to_disk + """ + path = util.ensure_path(path) + with path.open("wb") as file_: + file_.write(self.to_bytes(**kwargs)) + + def from_disk(self, path, *, exclude=tuple()): + """Loads state from a directory. Modifies the object in place and + returns it. + + path (str / Path): A path to a directory. + exclude (list): String names of serialization fields to exclude. + RETURNS (Tokenizer): The modified `Tokenizer` object. + + DOCS: https://spacy.io/api/tokenizer#from_disk + """ + path = util.ensure_path(path) + with path.open("rb") as file_: + bytes_data = file_.read() + self.from_bytes(bytes_data, exclude=exclude) + return self + + def to_bytes(self, *, exclude=tuple()): + """Serialize the current state to a binary string. + + exclude (list): String names of serialization fields to exclude. + RETURNS (bytes): The serialized form of the `Tokenizer` object. + + DOCS: https://spacy.io/api/tokenizer#to_bytes + """ + serializers = { + "vocab": lambda: self.vocab.to_bytes(exclude=exclude), + "prefix_search": lambda: _get_regex_pattern(self.prefix_search), + "suffix_search": lambda: _get_regex_pattern(self.suffix_search), + "infix_finditer": lambda: _get_regex_pattern(self.infix_finditer), + "token_match": lambda: _get_regex_pattern(self.token_match), + "url_match": lambda: _get_regex_pattern(self.url_match), + "exceptions": lambda: dict(sorted(self._rules.items())), + "faster_heuristics": lambda: self.faster_heuristics, + } + return util.to_bytes(serializers, exclude) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load state from a binary string. + + bytes_data (bytes): The data to load from. + exclude (list): String names of serialization fields to exclude. + RETURNS (Tokenizer): The `Tokenizer` object. + + DOCS: https://spacy.io/api/tokenizer#from_bytes + """ + data = {} + deserializers = { + "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude), + "prefix_search": lambda b: data.setdefault("prefix_search", b), + "suffix_search": lambda b: data.setdefault("suffix_search", b), + "infix_finditer": lambda b: data.setdefault("infix_finditer", b), + "token_match": lambda b: data.setdefault("token_match", b), + "url_match": lambda b: data.setdefault("url_match", b), + "exceptions": lambda b: data.setdefault("rules", b), + "faster_heuristics": lambda b: data.setdefault("faster_heuristics", b), + } + # reset all properties and flush all caches (through rules), + # reset rules first so that _reload_special_cases is trivial/fast as + # the other properties are reset + self.rules = {} + self.prefix_search = None + self.suffix_search = None + self.infix_finditer = None + self.token_match = None + self.url_match = None + util.from_bytes(bytes_data, deserializers, exclude) + if "prefix_search" in data and isinstance(data["prefix_search"], str): + self.prefix_search = re.compile(data["prefix_search"]).search + if "suffix_search" in data and isinstance(data["suffix_search"], str): + self.suffix_search = re.compile(data["suffix_search"]).search + if "infix_finditer" in data and isinstance(data["infix_finditer"], str): + self.infix_finditer = re.compile(data["infix_finditer"]).finditer + if "token_match" in data and isinstance(data["token_match"], str): + self.token_match = re.compile(data["token_match"]).match + if "url_match" in data and isinstance(data["url_match"], str): + self.url_match = re.compile(data["url_match"]).match + if "faster_heuristics" in data: + self.faster_heuristics = data["faster_heuristics"] + # always load rules last so that all other settings are set before the + # internal tokenization for the phrase matcher + if "rules" in data and isinstance(data["rules"], dict): + self.rules = data["rules"] + return self + + +def _get_regex_pattern(regex): + """Get a pattern string for a regex, or None if the pattern is None.""" + return None if regex is None else regex.__self__.pattern + + +cdef extern from "" namespace "std" nogil: + void stdsort "sort"(vector[SpanC].iterator, + vector[SpanC].iterator, + bint (*)(SpanC, SpanC)) + + +cdef bint len_start_cmp(SpanC a, SpanC b) noexcept nogil: + if a.end - a.start == b.end - b.start: + return b.start < a.start + return a.end - a.start < b.end - b.start + + +cdef bint start_cmp(SpanC a, SpanC b) noexcept nogil: + return a.start < b.start diff --git a/spacy/tokens/__init__.pxd b/spacy/tokens/__init__.pxd new file mode 100644 index 0000000..e69de29 diff --git a/spacy/tokens/__init__.py b/spacy/tokens/__init__.py new file mode 100644 index 0000000..3393ca6 --- /dev/null +++ b/spacy/tokens/__init__.py @@ -0,0 +1,8 @@ +from ._serialize import DocBin +from .doc import Doc +from .morphanalysis import MorphAnalysis +from .span import Span +from .span_group import SpanGroup +from .token import Token + +__all__ = ["Doc", "DocBin", "MorphAnalysis", "Span", "SpanGroup", "Token"] diff --git a/spacy/tokens/_dict_proxies.py b/spacy/tokens/_dict_proxies.py new file mode 100644 index 0000000..b2b4963 --- /dev/null +++ b/spacy/tokens/_dict_proxies.py @@ -0,0 +1,110 @@ +import warnings +import weakref +from collections import UserDict +from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union + +import srsly + +from ..errors import Errors, Warnings +from .span_group import SpanGroup + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .doc import Doc # noqa: F401 + from .span import Span # noqa: F401 + + +# Why inherit from UserDict instead of dict here? +# Well, the 'dict' class doesn't necessarily delegate everything nicely, +# for performance reasons. The UserDict is slower but better behaved. +# See https://treyhunner.com/2019/04/why-you-shouldnt-inherit-from-list-and-dict-in-python/ +class SpanGroups(UserDict): + """A dict-like proxy held by the Doc, to control access to span groups.""" + + _EMPTY_BYTES = srsly.msgpack_dumps([]) + + def __init__( + self, doc: "Doc", items: Iterable[Tuple[str, SpanGroup]] = tuple() + ) -> None: + self.doc_ref = weakref.ref(doc) + UserDict.__init__(self, items) # type: ignore[arg-type] + + def __setitem__(self, key: str, value: Union[SpanGroup, Iterable["Span"]]) -> None: + if not isinstance(value, SpanGroup): + value = self._make_span_group(key, value) + assert value.doc is self.doc_ref() + UserDict.__setitem__(self, key, value) + + def _make_span_group(self, name: str, spans: Iterable["Span"]) -> SpanGroup: + doc = self._ensure_doc() + return SpanGroup(doc, name=name, spans=spans) + + def copy(self, doc: Optional["Doc"] = None) -> "SpanGroups": + if doc is None: + doc = self._ensure_doc() + data_copy = ((k, v.copy(doc=doc)) for k, v in self.items()) + return SpanGroups(doc, items=data_copy) + + def setdefault(self, key, default=None): + if not isinstance(default, SpanGroup): + if default is None: + spans = [] + else: + spans = default + default = self._make_span_group(key, spans) + return super().setdefault(key, default=default) + + def to_bytes(self) -> bytes: + # We serialize this as a dict in order to track the key(s) a SpanGroup + # is a value of (in a backward- and forward-compatible way), since + # a SpanGroup can have a key that doesn't match its `.name` (See #10685) + if len(self) == 0: + return self._EMPTY_BYTES + msg: Dict[bytes, List[str]] = {} + for key, value in self.items(): + msg.setdefault(value.to_bytes(), []).append(key) + return srsly.msgpack_dumps(msg) + + def from_bytes(self, bytes_data: bytes) -> "SpanGroups": + # backwards-compatibility: bytes_data may be one of: + # b'', a serialized empty list, a serialized list of SpanGroup bytes + # or a serialized dict of SpanGroup bytes -> keys + msg = ( + [] + if not bytes_data or bytes_data == self._EMPTY_BYTES + else srsly.msgpack_loads(bytes_data) + ) + self.clear() + doc = self._ensure_doc() + if isinstance(msg, list): + # This is either the 1st version of `SpanGroups` serialization + # or there were no SpanGroups serialized + for value_bytes in msg: + group = SpanGroup(doc).from_bytes(value_bytes) + if group.name in self: + # Display a warning if `msg` contains `SpanGroup`s + # that have the same .name (attribute). + # Because, for `SpanGroups` serialized as lists, + # only 1 SpanGroup per .name is loaded. (See #10685) + warnings.warn( + Warnings.W120.format( + group_name=group.name, group_values=self[group.name] + ) + ) + self[group.name] = group + else: + for value_bytes, keys in msg.items(): + group = SpanGroup(doc).from_bytes(value_bytes) + # Deserialize `SpanGroup`s as copies because it's possible for two + # different `SpanGroup`s (pre-serialization) to have the same bytes + # (since they can have the same `.name`). + self[keys[0]] = group + for key in keys[1:]: + self[key] = group.copy() + return self + + def _ensure_doc(self) -> "Doc": + doc = self.doc_ref() + if doc is None: + raise ValueError(Errors.E866) + return doc diff --git a/spacy/tokens/_retokenize.pyi b/spacy/tokens/_retokenize.pyi new file mode 100644 index 0000000..097fbd1 --- /dev/null +++ b/spacy/tokens/_retokenize.pyi @@ -0,0 +1,22 @@ +from typing import Any, Dict, List, Tuple, Union + +from .. import Vocab +from .doc import Doc +from .span import Span +from .token import Token + +class Retokenizer: + def __init__(self, doc: Doc) -> None: ... + def merge(self, span: Span, attrs: Dict[Union[str, int], Any] = ...) -> None: ... + def split( + self, + token: Token, + orths: List[str], + heads: List[Union[Token, Tuple[Token, int]]], + attrs: Dict[Union[str, int], List[Any]] = ..., + ) -> None: ... + def __enter__(self) -> Retokenizer: ... + def __exit__(self, *args: Any) -> None: ... + +def normalize_token_attrs(vocab: Vocab, attrs: Dict): ... +def set_token_attrs(py_token: Token, attrs: Dict): ... diff --git a/spacy/tokens/_retokenize.pyx b/spacy/tokens/_retokenize.pyx new file mode 100644 index 0000000..b0e4ff8 --- /dev/null +++ b/spacy/tokens/_retokenize.pyx @@ -0,0 +1,474 @@ +# cython: infer_types=True, bounds_check=False +from cymem.cymem cimport Pool +from libc.string cimport memset + +import numpy +from thinc.api import get_array_module + +from ..attrs cimport MORPH, NORM +from ..lexeme cimport EMPTY_LEXEME, Lexeme +from ..structs cimport LexemeC, TokenC +from ..vocab cimport Vocab +from .doc cimport Doc, set_children_from_heads, token_by_start +from .span cimport Span +from .token cimport Token + +from ..attrs import intify_attrs +from ..errors import Errors +from ..strings import get_string_id +from ..util import SimpleFrozenDict +from .underscore import is_writable_attr + + +cdef class Retokenizer: + """Helper class for doc.retokenize() context manager. + + DOCS: https://spacy.io/api/doc#retokenize + USAGE: https://spacy.io/usage/linguistic-features#retokenization + """ + cdef Doc doc + cdef list merges + cdef list splits + cdef set tokens_to_merge + cdef list _spans_to_merge + + def __init__(self, doc): + self.doc = doc + self.merges = [] + self.splits = [] + self.tokens_to_merge = set() + self._spans_to_merge = [] # keep a record to filter out duplicates + + def merge(self, Span span, attrs=SimpleFrozenDict()): + """Mark a span for merging. The attrs will be applied to the resulting + token. + + span (Span): The span to merge. + attrs (dict): Attributes to set on the merged token. + + DOCS: https://spacy.io/api/doc#retokenizer.merge + """ + if (span.start, span.end) in self._spans_to_merge: + return + if span.end - span.start <= 0: + raise ValueError(Errors.E199.format(start=span.start, end=span.end)) + for token in span: + if token.i in self.tokens_to_merge: + raise ValueError(Errors.E102.format(token=repr(token))) + self.tokens_to_merge.add(token.i) + self._spans_to_merge.append((span.start, span.end)) + attrs = normalize_token_attrs(self.doc.vocab, attrs) + self.merges.append((span, attrs)) + + def split(self, Token token, orths, heads, attrs=SimpleFrozenDict()): + """Mark a Token for splitting, into the specified orths. The attrs + will be applied to each subtoken. + + token (Token): The token to split. + orths (list): The verbatim text of the split tokens. Needs to match the + text of the original token. + heads (list): List of token or `(token, subtoken)` tuples specifying the + tokens to attach the newly split subtokens to. + attrs (dict): Attributes to set on all split tokens. Attribute names + mapped to list of per-token attribute values. + + DOCS: https://spacy.io/api/doc#retokenizer.split + """ + if ''.join(orths) != token.text: + raise ValueError(Errors.E117.format(new=''.join(orths), old=token.text)) + if "_" in attrs: # Extension attributes + extensions = attrs["_"] + for extension in extensions: + _validate_extensions(extension) + attrs = {key: value for key, value in attrs.items() if key != "_"} + # NB: Since we support {"KEY": [value, value]} syntax here, this + # will only "intify" the keys, not the values + attrs = intify_attrs(attrs, strings_map=self.doc.vocab.strings) + attrs["_"] = extensions + else: + # NB: Since we support {"KEY": [value, value]} syntax here, this + # will only "intify" the keys, not the values + attrs = intify_attrs(attrs, strings_map=self.doc.vocab.strings) + if MORPH in attrs: + for i, morph in enumerate(attrs[MORPH]): + # add and set to normalized value + morph = self.doc.vocab.morphology.add(self.doc.vocab.strings.as_string(morph)) + attrs[MORPH][i] = morph + head_offsets = [] + for head in heads: + if isinstance(head, Token): + head_offsets.append((head.idx, 0)) + else: + head_offsets.append((head[0].idx, head[1])) + self.splits.append((token.idx, orths, head_offsets, attrs)) + + def __enter__(self): + self.merges = [] + self.splits = [] + return self + + def __exit__(self, *args): + # Do the actual merging here + if len(self.merges) >= 1: + _merge(self.doc, self.merges) + # Iterate in order, to keep things simple. + for start_char, orths, heads, attrs in sorted(self.splits): + # Resolve token index + token_index = token_by_start(self.doc.c, self.doc.length, start_char) + # Check we're still able to find tokens starting at the character offsets + # referred to in the splits. If we merged these tokens previously, we + # have to raise an error + if token_index == -1: + raise IndexError(Errors.E122) + head_indices = [] + for head_char, subtoken in heads: + head_index = token_by_start(self.doc.c, self.doc.length, head_char) + if head_index == -1: + raise IndexError(Errors.E123) + # We want to refer to the token index of the head *after* the + # mergery. We need to account for the extra tokens introduced. + # e.g., let's say we have [ab, c] and we want a and b to depend + # on c. The correct index for c will be 2, not 1. + if head_index > token_index: + head_index += len(orths)-1 + head_indices.append(head_index+subtoken) + _split(self.doc, token_index, orths, head_indices, attrs) + + +def _merge(Doc doc, merges): + """Retokenize the document, such that the spans described in 'merges' + are merged into a single token. This method assumes that the merges + are in the same order at which they appear in the doc, and that merges + do not intersect each other in any way. + + merges: Tokens to merge, and corresponding attributes to assign to the + merged token. By default, attributes are inherited from the + syntactic root of the span. + RETURNS (Token): The first newly merged token. + """ + cdef int i, merge_index, start, token_index, current_span_index, current_offset, offset, span_index + cdef Span span + cdef const LexemeC* lex + cdef TokenC* token + cdef Pool mem = Pool() + cdef int merged_iob = 0 + + # merges should not be empty, but make sure to avoid zero-length mem alloc + assert len(merges) > 0 + tokens = mem.alloc(len(merges), sizeof(TokenC)) + spans = [] + + def _get_start(merge): + return merge[0].start + + merges.sort(key=_get_start) + for merge_index, (span, attributes) in enumerate(merges): + start = span.start + spans.append(span) + # House the new merged token where it starts + token = &doc.c[start] + start_ent_iob = doc.c[start].ent_iob + start_ent_type = doc.c[start].ent_type + # Initially set attributes to attributes of span root + token.tag = doc.c[span.root.i].tag + token.pos = doc.c[span.root.i].pos + token.morph = doc.c[span.root.i].morph + token.ent_iob = doc.c[span.root.i].ent_iob + token.ent_type = doc.c[span.root.i].ent_type + merged_iob = token.ent_iob + # If span root is part of an entity, merged token is B-ENT + if token.ent_iob in (1, 3): + merged_iob = 3 + # If start token is I-ENT and previous token is of the same + # type, then I-ENT (could check I-ENT from start to span root) + if start_ent_iob == 1 and start > 0 \ + and start_ent_type == token.ent_type \ + and doc.c[start - 1].ent_type == token.ent_type: + merged_iob = 1 + token.ent_iob = merged_iob + # Set lemma to concatenated lemmas + merged_lemma = "" + for span_token in span: + merged_lemma += span_token.lemma_ + if doc.c[span_token.i].spacy: + merged_lemma += " " + merged_lemma = merged_lemma.strip() + token.lemma = doc.vocab.strings.add(merged_lemma) + # Unset attributes that don't match new token + token.norm = 0 + tokens[merge_index] = token + # Resize the doc.tensor, if it's set. Let the last row for each token stand + # for the merged region. To do this, we create a boolean array indicating + # whether the row is to be deleted, then use numpy.delete + if doc.tensor is not None and doc.tensor.size != 0: + doc.tensor = _resize_tensor( + doc.tensor, [(m[0].start, m[0].end) for m in merges] + ) + # Memorize span roots and sets dependencies of the newly merged + # tokens to the dependencies of their roots. + span_roots = [] + for i, span in enumerate(spans): + span_roots.append(span.root.i) + tokens[i].dep = span.root.dep + # We update token.lex after keeping span root and dep, since + # setting token.lex will change span.start and span.end properties + # as it modifies the character offsets in the doc + for token_index, (span, attributes) in enumerate(merges): + new_orth = ''.join([t.text_with_ws for t in spans[token_index]]) + if spans[token_index][-1].whitespace_: + new_orth = new_orth[:-len(spans[token_index][-1].whitespace_)] + # add the vector of the (merged) entity to the vocab + if not doc.vocab.get_vector(new_orth).any(): + if doc.vocab.vectors_length > 0: + doc.vocab.set_vector(new_orth, span.vector) + token = tokens[token_index] + lex = doc.vocab.get(doc.mem, new_orth) + token.lex = lex + # We set trailing space here too + token.spacy = doc.c[spans[token_index].end-1].spacy + set_token_attrs(span[0], attributes) + # Begin by setting all the head indices to absolute token positions + # This is easier to work with for now than the offsets + # Before thinking of something simpler, beware the case where a + # dependency bridges over the entity. Here the alignment of the + # tokens changes. + for i in range(doc.length): + doc.c[i].head += i + # Set the head of the merged token from the Span + for i in range(len(merges)): + tokens[i].head = doc.c[span_roots[i]].head + # Adjust deps before shrinking tokens + # Tokens which point into the merged token should now point to it + # Subtract the offset from all tokens which point to >= end + offsets = [] + current_span_index = 0 + current_offset = 0 + for i in range(doc.length): + if current_span_index < len(spans) and i == spans[current_span_index].end: + # Last token was the last of the span + current_offset += (spans[current_span_index].end - spans[current_span_index].start) -1 + current_span_index += 1 + if current_span_index < len(spans) and \ + spans[current_span_index].start <= i < spans[current_span_index].end: + offsets.append(spans[current_span_index].start - current_offset) + else: + offsets.append(i - current_offset) + for i in range(doc.length): + doc.c[i].head = offsets[doc.c[i].head] + # Now compress the token array + offset = 0 + in_span = False + span_index = 0 + for i in range(doc.length): + if in_span and i == spans[span_index].end: + # First token after a span + in_span = False + span_index += 1 + if span_index < len(spans) and i == spans[span_index].start: + # First token in a span + doc.c[i - offset] = doc.c[i] # move token to its place + offset += (spans[span_index].end - spans[span_index].start) - 1 + in_span = True + if not in_span: + doc.c[i - offset] = doc.c[i] # move token to its place + + for i in range(doc.length - offset, doc.length): + memset(&doc.c[i], 0, sizeof(TokenC)) + doc.c[i].lex = &EMPTY_LEXEME + doc.length -= offset + # ...And, set heads back to a relative position + for i in range(doc.length): + doc.c[i].head -= i + # Set the left/right children, left/right edges + if doc.has_annotation("DEP"): + set_children_from_heads(doc.c, 0, doc.length) + # Make sure ent_iob remains consistent + make_iob_consistent(doc.c, doc.length) + # Return the merged Python object + return doc[spans[0].start] + + +def _resize_tensor(tensor, ranges): + delete = [] + for start, end in ranges: + for i in range(start, end-1): + delete.append(i) + xp = get_array_module(tensor) + if xp is numpy: + return xp.delete(tensor, delete, axis=0) + else: + offset = 0 + copy_start = 0 + resized_shape = (tensor.shape[0] - len(delete), tensor.shape[1]) + for start, end in ranges: + if copy_start > 0: + tensor[copy_start - offset:start - offset] = tensor[copy_start: start] + offset += end - start - 1 + copy_start = end - 1 + tensor[copy_start - offset:resized_shape[0]] = tensor[copy_start:] + return xp.asarray(tensor[:resized_shape[0]]) + + +def _split(Doc doc, int token_index, orths, heads, attrs): + """Retokenize the document, such that the token at + `doc[token_index]` is split into tokens with the orth 'orths' + token_index(int): token index of the token to split. + + orths: IDs of the verbatim text content of the tokens to create + **attributes: Attributes to assign to each of the newly created tokens. By default, + attributes are inherited from the original token. + RETURNS (Token): The first newly created token. + """ + cdef int nb_subtokens = len(orths) + cdef const LexemeC* lex + cdef TokenC* token + cdef TokenC orig_token = doc.c[token_index] + cdef int orig_length = len(doc) + + if(len(heads) != nb_subtokens): + raise ValueError(Errors.E115) + # First, make the dependencies absolutes + for i in range(doc.length): + doc.c[i].head += i + # Adjust dependencies, so they refer to post-split indexing + offset = nb_subtokens - 1 + for i in range(doc.length): + if doc.c[i].head > token_index: + doc.c[i].head += offset + # Double doc.c max_length if necessary (until big enough for all new tokens) + while doc.length + nb_subtokens - 1 >= doc.max_length: + doc._realloc(doc.max_length * 2) + # Move tokens after the split to create space for the new tokens + doc.length = len(doc) + nb_subtokens -1 + to_process_tensor = (doc.tensor is not None and doc.tensor.size != 0) + if to_process_tensor: + xp = get_array_module(doc.tensor) + if xp is numpy: + doc.tensor = xp.append( + doc.tensor, + xp.zeros((nb_subtokens, doc.tensor.shape[1]), dtype="float32"), + axis=0 + ) + else: + shape = (doc.tensor.shape[0] + nb_subtokens, doc.tensor.shape[1]) + resized_array = xp.zeros(shape, dtype="float32") + resized_array[:doc.tensor.shape[0]] = doc.tensor[:doc.tensor.shape[0]] + doc.tensor = resized_array + for token_to_move in range(orig_length - 1, token_index, -1): + doc.c[token_to_move + nb_subtokens - 1] = doc.c[token_to_move] + if to_process_tensor: + doc.tensor[token_to_move + nb_subtokens - 1] = doc.tensor[token_to_move] + # Host the tokens in the newly created space + cdef int idx_offset = 0 + for i, orth in enumerate(orths): + token = &doc.c[token_index + i] + lex = doc.vocab.get(doc.mem, orth) + token.lex = lex + # If lemma is currently set, set default lemma to orth + if token.lemma != 0: + token.lemma = lex.orth + token.norm = 0 # reset norm + if to_process_tensor: + # setting the tensors of the split tokens to array of zeros + doc.tensor[token_index + i:token_index + i + 1] = \ + xp.zeros((1, doc.tensor.shape[1]), dtype="float32") + # Update the character offset of the subtokens + if i != 0: + token.idx = orig_token.idx + idx_offset + idx_offset += len(orth) + # Set token.spacy to False for all non-last split tokens, and + # to origToken.spacy for the last token + if (i < nb_subtokens - 1): + token.spacy = False + else: + token.spacy = orig_token.spacy + # Make IOB consistent + if (orig_token.ent_iob == 3): + if i == 0: + token.ent_iob = 3 + else: + token.ent_iob = 1 + else: + # In all other cases subtokens inherit iob from origToken + token.ent_iob = orig_token.ent_iob + # Apply attrs to each subtoken + for attr_name, attr_values in attrs.items(): + for i, attr_value in enumerate(attr_values): + token = &doc.c[token_index + i] + if attr_name == "_": + for ext_attr_key, ext_attr_value in attr_value.items(): + doc[token_index + i]._.set(ext_attr_key, ext_attr_value) + # NB: We need to call get_string_id here because only the keys are + # "intified" (since we support "KEY": [value, value] syntax here). + else: + # Set attributes on both token and lexeme to take care of token + # attribute vs. lexical attribute without having to enumerate + # them. If an attribute name is not valid, set_struct_attr will + # ignore it. Exception: set NORM only on tokens. + Token.set_struct_attr(token, attr_name, get_string_id(attr_value)) + if attr_name != NORM: + Lexeme.set_struct_attr(token.lex, attr_name, get_string_id(attr_value)) + # Assign correct dependencies to the inner token + for i, head in enumerate(heads): + doc.c[token_index + i].head = head + # Transform the dependencies into relative ones again + for i in range(doc.length): + doc.c[i].head -= i + # set children from head + if doc.has_annotation("DEP"): + set_children_from_heads(doc.c, 0, doc.length) + + +def _validate_extensions(extensions): + if not isinstance(extensions, dict): + raise ValueError(Errors.E120.format(value=repr(extensions))) + for key, value in extensions.items(): + # Get the extension and make sure it's available and writable + extension = Token.get_extension(key) + if not extension: # Extension attribute doesn't exist + raise ValueError(Errors.E118.format(attr=key)) + if not is_writable_attr(extension): + raise ValueError(Errors.E119.format(attr=key)) + + +cdef make_iob_consistent(TokenC* tokens, int length): + cdef int i + if tokens[0].ent_iob == 1: + tokens[0].ent_iob = 3 + for i in range(1, length): + if tokens[i].ent_iob == 1 and tokens[i - 1].ent_type != tokens[i].ent_type: + tokens[i].ent_iob = 3 + + +def normalize_token_attrs(Vocab vocab, attrs): + if "_" in attrs: # Extension attributes + extensions = attrs["_"] + _validate_extensions(extensions) + attrs = {key: value for key, value in attrs.items() if key != "_"} + attrs = intify_attrs(attrs, strings_map=vocab.strings) + attrs["_"] = extensions + else: + attrs = intify_attrs(attrs, strings_map=vocab.strings) + if MORPH in attrs: + # add and set to normalized value + morph = vocab.morphology.add(vocab.strings.as_string(attrs[MORPH])) + attrs[MORPH] = morph + return attrs + + +def set_token_attrs(Token py_token, attrs): + cdef TokenC* token = py_token.c + cdef const LexemeC* lex = token.lex + # Assign attributes + for attr_name, attr_value in attrs.items(): + if attr_name == "_": # Set extension attributes + for ext_attr_key, ext_attr_value in attr_value.items(): + py_token._.set(ext_attr_key, ext_attr_value) + else: + # Set attributes on both token and lexeme to take care of token + # attribute vs. lexical attribute without having to enumerate + # them. If an attribute name is not valid, set_struct_attr will + # ignore it. Exception: set NORM only on tokens. + Token.set_struct_attr(token, attr_name, attr_value) + if attr_name != NORM: + Lexeme.set_struct_attr(lex, attr_name, attr_value) diff --git a/spacy/tokens/_serialize.py b/spacy/tokens/_serialize.py new file mode 100644 index 0000000..51f5740 --- /dev/null +++ b/spacy/tokens/_serialize.py @@ -0,0 +1,306 @@ +import zlib +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional, Set, Union + +import numpy +import srsly +from numpy import ndarray +from thinc.api import NumpyOps + +from ..attrs import IDS, ORTH, SPACY, intify_attr +from ..compat import copy_reg +from ..errors import Errors +from ..util import SimpleFrozenList, ensure_path +from ..vocab import Vocab +from ._dict_proxies import SpanGroups +from .doc import DOCBIN_ALL_ATTRS as ALL_ATTRS, Doc + + +class DocBin: + """Pack Doc objects for binary serialization. + + The DocBin class lets you efficiently serialize the information from a + collection of Doc objects. You can control which information is serialized + by passing a list of attribute IDs, and optionally also specify whether the + user data is serialized. The DocBin is faster and produces smaller data + sizes than pickle, and allows you to deserialize without executing arbitrary + Python code. + + The serialization format is gzipped msgpack, where the msgpack object has + the following structure: + + { + "attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE] + "tokens": bytes, # Serialized numpy uint64 array with the token data + "spans": List[Dict[str, bytes]], # SpanGroups data for each doc + "spaces": bytes, # Serialized numpy boolean array with spaces data + "lengths": bytes, # Serialized numpy int32 array with the doc lengths + "strings": List[str] # List of unique strings in the token data + "version": str, # DocBin version number + } + + Strings for the words, tags, labels etc are represented by 64-bit hashes in + the token data, and every string that occurs at least once is passed via the + strings object. This means the storage is more efficient if you pack more + documents together, because you have less duplication in the strings. + + A notable downside to this format is that you can't easily extract just one + document from the DocBin. + """ + + def __init__( + self, + attrs: Iterable[str] = ALL_ATTRS, + store_user_data: bool = False, + docs: Iterable[Doc] = SimpleFrozenList(), + ) -> None: + """Create a DocBin object to hold serialized annotations. + + attrs (Iterable[str]): List of attributes to serialize. 'orth' and + 'spacy' are always serialized, so they're not required. + store_user_data (bool): Whether to write the `Doc.user_data` to bytes/file. + docs (Iterable[Doc]): Docs to add. + + DOCS: https://spacy.io/api/docbin#init + """ + int_attrs = [intify_attr(attr) for attr in attrs] + if None in int_attrs: + non_valid = [attr for attr in attrs if intify_attr(attr) is None] + raise KeyError( + Errors.E983.format(dict="attrs", key=non_valid, keys=IDS.keys()) + ) from None + attrs = sorted(int_attrs) + self.version = "0.1" + self.attrs = [attr for attr in attrs if attr != ORTH and attr != SPACY] + self.attrs.insert(0, ORTH) # Ensure ORTH is always attrs[0] + self.tokens: List[ndarray] = [] + self.spaces: List[ndarray] = [] + self.cats: List[Dict] = [] + self.span_groups: List[bytes] = [] + self.user_data: List[Optional[bytes]] = [] + self.flags: List[Dict] = [] + self.strings: Set[str] = set() + self.store_user_data = store_user_data + for doc in docs: + self.add(doc) + + def __len__(self) -> int: + """RETURNS: The number of Doc objects added to the DocBin.""" + return len(self.tokens) + + def add(self, doc: Doc) -> None: + """Add a Doc's annotations to the DocBin for serialization. + + doc (Doc): The Doc object to add. + + DOCS: https://spacy.io/api/docbin#add + """ + array = doc.to_array(self.attrs) + if len(array.shape) == 1: + array = array.reshape((array.shape[0], 1)) + self.tokens.append(array) + spaces = doc.to_array(SPACY) + assert array.shape[0] == spaces.shape[0] # this should never happen + spaces = spaces.reshape((spaces.shape[0], 1)) + self.spaces.append(numpy.asarray(spaces, dtype=bool)) + self.flags.append({"has_unknown_spaces": doc.has_unknown_spaces}) + for token in doc: + self.strings.add(token.text) + self.strings.add(token.tag_) + self.strings.add(token.lemma_) + self.strings.add(token.norm_) + self.strings.add(str(token.morph)) + self.strings.add(token.dep_) + self.strings.add(token.ent_type_) + self.strings.add(token.ent_kb_id_) + self.strings.add(token.ent_id_) + self.cats.append(doc.cats) + if self.store_user_data: + self.user_data.append(srsly.msgpack_dumps(doc.user_data)) + self.span_groups.append(doc.spans.to_bytes()) + for key, group in doc.spans.items(): + for span in group: + self.strings.add(span.label_) + if span.kb_id in span.doc.vocab.strings: + self.strings.add(span.kb_id_) + if span.id in span.doc.vocab.strings: + self.strings.add(span.id_) + + def get_docs(self, vocab: Vocab) -> Iterator[Doc]: + """Recover Doc objects from the annotations, using the given vocab. + Note that the user data of each doc will be read (if available) and returned, + regardless of the setting of 'self.store_user_data'. + + vocab (Vocab): The shared vocab. + YIELDS (Doc): The Doc objects. + + DOCS: https://spacy.io/api/docbin#get_docs + """ + for string in self.strings: + vocab[string] + orth_col = self.attrs.index(ORTH) + for i in range(len(self.tokens)): + flags = self.flags[i] + tokens = self.tokens[i] + spaces: Optional[ndarray] = self.spaces[i] + if flags.get("has_unknown_spaces"): + spaces = None + doc = Doc(vocab, words=tokens[:, orth_col], spaces=spaces) # type: ignore + doc = doc.from_array(self.attrs, tokens) # type: ignore + doc.cats = self.cats[i] + # backwards-compatibility: may be b'' or serialized empty list + if self.span_groups[i] and self.span_groups[i] != SpanGroups._EMPTY_BYTES: + doc.spans.from_bytes(self.span_groups[i]) + else: + doc.spans.clear() + if i < len(self.user_data) and self.user_data[i] is not None: + user_data = srsly.msgpack_loads(self.user_data[i], use_list=False) + doc.user_data.update(user_data) + yield doc + + def merge(self, other: "DocBin") -> None: + """Extend the annotations of this DocBin with the annotations from + another. Will raise an error if the pre-defined attrs of the two + DocBins don't match, or if they differ in whether or not to store + user data. + + other (DocBin): The DocBin to merge into the current bin. + + DOCS: https://spacy.io/api/docbin#merge + """ + if self.attrs != other.attrs: + raise ValueError( + Errors.E166.format(param="attrs", current=self.attrs, other=other.attrs) + ) + if self.store_user_data != other.store_user_data: + raise ValueError( + Errors.E166.format( + param="store_user_data", + current=self.store_user_data, + other=other.store_user_data, + ) + ) + self.tokens.extend(other.tokens) + self.spaces.extend(other.spaces) + self.strings.update(other.strings) + self.cats.extend(other.cats) + self.span_groups.extend(other.span_groups) + self.flags.extend(other.flags) + self.user_data.extend(other.user_data) + + def to_bytes(self) -> bytes: + """Serialize the DocBin's annotations to a bytestring. + + RETURNS (bytes): The serialized DocBin. + + DOCS: https://spacy.io/api/docbin#to_bytes + """ + for tokens in self.tokens: + assert len(tokens.shape) == 2, tokens.shape # this should never happen + lengths = [len(tokens) for tokens in self.tokens] + tokens = numpy.vstack(self.tokens) if self.tokens else numpy.asarray([]) + spaces = numpy.vstack(self.spaces) if self.spaces else numpy.asarray([]) + msg = { + "version": self.version, + "attrs": self.attrs, + "tokens": tokens.tobytes("C"), + "spaces": spaces.tobytes("C"), + "lengths": numpy.asarray(lengths, dtype="int32").tobytes("C"), + "strings": sorted(self.strings), + "cats": self.cats, + "flags": self.flags, + "span_groups": self.span_groups, + } + if self.store_user_data: + msg["user_data"] = self.user_data + return zlib.compress(srsly.msgpack_dumps(msg)) + + def from_bytes(self, bytes_data: bytes) -> "DocBin": + """Deserialize the DocBin's annotations from a bytestring. + + bytes_data (bytes): The data to load from. + RETURNS (DocBin): The loaded DocBin. + + DOCS: https://spacy.io/api/docbin#from_bytes + """ + try: + msg = srsly.msgpack_loads(zlib.decompress(bytes_data)) + except zlib.error: + raise ValueError(Errors.E1014) + self.attrs = msg["attrs"] + self.strings = set(msg["strings"]) + lengths = numpy.frombuffer(msg["lengths"], dtype="int32") + flat_spaces = numpy.frombuffer(msg["spaces"], dtype=bool) + flat_tokens = numpy.frombuffer(msg["tokens"], dtype="uint64") + shape = (flat_tokens.size // len(self.attrs), len(self.attrs)) + flat_tokens = flat_tokens.reshape(shape) + flat_spaces = flat_spaces.reshape((flat_spaces.size, 1)) + self.tokens = NumpyOps().unflatten(flat_tokens, lengths) + self.spaces = NumpyOps().unflatten(flat_spaces, lengths) + self.cats = msg["cats"] + self.span_groups = msg.get("span_groups", [b"" for _ in lengths]) + self.flags = msg.get("flags", [{} for _ in lengths]) + if "user_data" in msg: + self.user_data = list(msg["user_data"]) + else: + self.user_data = [None] * len(self) + for tokens in self.tokens: + assert len(tokens.shape) == 2, tokens.shape # this should never happen + return self + + def to_disk(self, path: Union[str, Path]) -> None: + """Save the DocBin to a file (typically called .spacy). + + path (str / Path): The file path. + + DOCS: https://spacy.io/api/docbin#to_disk + """ + path = ensure_path(path) + with path.open("wb") as file_: + try: + file_.write(self.to_bytes()) + except ValueError: + raise ValueError(Errors.E870) + + def from_disk(self, path: Union[str, Path]) -> "DocBin": + """Load the DocBin from a file (typically called .spacy). + + path (str / Path): The file path. + RETURNS (DocBin): The loaded DocBin. + + DOCS: https://spacy.io/api/docbin#to_disk + """ + path = ensure_path(path) + with path.open("rb") as file_: + self.from_bytes(file_.read()) + return self + + +def merge_bins(bins): + merged = None + for byte_string in bins: + if byte_string is not None: + doc_bin = DocBin(store_user_data=True).from_bytes(byte_string) + if merged is None: + merged = doc_bin + else: + merged.merge(doc_bin) + if merged is not None: + return merged.to_bytes() + else: + return b"" + + +def pickle_bin(doc_bin): + return (unpickle_bin, (doc_bin.to_bytes(),)) + + +def unpickle_bin(byte_string): + return DocBin().from_bytes(byte_string) + + +copy_reg.pickle(DocBin, pickle_bin, unpickle_bin) +# Compatibility, as we had named it this previously. +Binder = DocBin + +__all__ = ["DocBin"] diff --git a/spacy/tokens/doc.pxd b/spacy/tokens/doc.pxd new file mode 100644 index 0000000..4541660 --- /dev/null +++ b/spacy/tokens/doc.pxd @@ -0,0 +1,70 @@ +cimport numpy as np +from cymem.cymem cimport Pool + +from ..attrs cimport attr_id_t +from ..structs cimport LexemeC, SpanC, TokenC +from ..typedefs cimport attr_t +from ..vocab cimport Vocab + + +cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) noexcept nogil +cdef attr_t get_token_attr_for_matcher(const TokenC* token, attr_id_t feat_name) noexcept nogil + + +ctypedef const LexemeC* const_Lexeme_ptr +ctypedef const TokenC* const_TokenC_ptr + +ctypedef fused LexemeOrToken: + const_Lexeme_ptr + const_TokenC_ptr + + +cdef int set_children_from_heads(TokenC* tokens, int start, int end) except -1 + + +cdef int _set_lr_kids_and_edges(TokenC* tokens, int start, int end, int loop_count) except -1 + + +cdef int token_by_start(const TokenC* tokens, int length, int start_char) except -2 + + +cdef int token_by_end(const TokenC* tokens, int length, int end_char) except -2 + + +cdef int [:, :] _get_lca_matrix(Doc, int start, int end) + + +cdef class Doc: + cdef readonly Pool mem + cdef readonly Vocab vocab + + cdef public object _vector + cdef public object _vector_norm + + cdef public object tensor + cdef public object cats + cdef public object user_data + cdef readonly object spans + + cdef TokenC* c + + cdef public float sentiment + + cdef public dict user_hooks + cdef public dict user_token_hooks + cdef public dict user_span_hooks + + cdef public bint has_unknown_spaces + + cdef public object _context + + cdef int length + cdef int max_length + + cdef public object noun_chunks_iterator + + cdef object __weakref__ + + cdef int push_back(self, LexemeOrToken lex_or_tok, bint has_space) except -1 + + cpdef np.ndarray to_array(self, object features) diff --git a/spacy/tokens/doc.pyi b/spacy/tokens/doc.pyi new file mode 100644 index 0000000..b8b26ce --- /dev/null +++ b/spacy/tokens/doc.pyi @@ -0,0 +1,198 @@ +from pathlib import Path +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Protocol, + Sequence, + Tuple, + Union, + overload, +) + +import numpy as np +from cymem.cymem import Pool +from thinc.types import Floats1d, Floats2d, Ints2d + +from ..lexeme import Lexeme +from ..vocab import Vocab +from ._dict_proxies import SpanGroups +from ._retokenize import Retokenizer +from .span import Span +from .token import Token +from .underscore import Underscore + +DOCBIN_ALL_ATTRS: Tuple[str, ...] + +class DocMethod(Protocol): + def __call__(self: Doc, *args: Any, **kwargs: Any) -> Any: ... # type: ignore[misc] + +class Doc: + vocab: Vocab + mem: Pool + spans: SpanGroups + max_length: int + length: int + sentiment: float + cats: Dict[str, float] + user_hooks: Dict[str, Callable[..., Any]] + user_token_hooks: Dict[str, Callable[..., Any]] + user_span_hooks: Dict[str, Callable[..., Any]] + tensor: np.ndarray[Any, np.dtype[np.float64]] + user_data: Dict[str, Any] + has_unknown_spaces: bool + _context: Any + @classmethod + def set_extension( + cls, + name: str, + default: Optional[Any] = ..., + getter: Optional[Callable[[Doc], Any]] = ..., + setter: Optional[Callable[[Doc, Any], None]] = ..., + method: Optional[DocMethod] = ..., + force: bool = ..., + ) -> None: ... + @classmethod + def get_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[DocMethod], + Optional[Callable[[Doc], Any]], + Optional[Callable[[Doc, Any], None]], + ]: ... + @classmethod + def has_extension(cls, name: str) -> bool: ... + @classmethod + def remove_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[DocMethod], + Optional[Callable[[Doc], Any]], + Optional[Callable[[Doc, Any], None]], + ]: ... + def __init__( + self, + vocab: Vocab, + words: Optional[List[str]] = ..., + spaces: Optional[List[bool]] = ..., + user_data: Optional[Dict[Any, Any]] = ..., + tags: Optional[List[str]] = ..., + pos: Optional[List[str]] = ..., + morphs: Optional[List[str]] = ..., + lemmas: Optional[List[str]] = ..., + heads: Optional[List[int]] = ..., + deps: Optional[List[str]] = ..., + sent_starts: Optional[List[Union[bool, int, None]]] = ..., + ents: Optional[List[str]] = ..., + ) -> None: ... + @property + def _(self) -> Underscore: ... + @property + def is_tagged(self) -> bool: ... + @property + def is_parsed(self) -> bool: ... + @property + def is_nered(self) -> bool: ... + @property + def is_sentenced(self) -> bool: ... + def has_annotation( + self, attr: Union[int, str], *, require_complete: bool = ... + ) -> bool: ... + @overload + def __getitem__(self, i: int) -> Token: ... + @overload + def __getitem__(self, i: slice) -> Span: ... + def __iter__(self) -> Iterator[Token]: ... + def __len__(self) -> int: ... + def __unicode__(self) -> str: ... + def __bytes__(self) -> bytes: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... + @property + def doc(self) -> Doc: ... + def char_span( + self, + start_idx: int, + end_idx: int, + label: Union[int, str] = ..., + kb_id: Union[int, str] = ..., + vector: Optional[Floats1d] = ..., + alignment_mode: str = ..., + span_id: Union[int, str] = ..., + ) -> Optional[Span]: ... + def similarity(self, other: Union[Doc, Span, Token, Lexeme]) -> float: ... + @property + def has_vector(self) -> bool: ... + vector: Floats1d + vector_norm: float + @property + def text(self) -> str: ... + @property + def text_with_ws(self) -> str: ... + # Ideally the getter would output Tuple[Span] + # see https://github.com/python/mypy/issues/3004 + @property + def ents(self) -> Sequence[Span]: ... + @ents.setter + def ents(self, value: Sequence[Span]) -> None: ... + def set_ents( + self, + entities: List[Span], + *, + blocked: Optional[List[Span]] = ..., + missing: Optional[List[Span]] = ..., + outside: Optional[List[Span]] = ..., + default: str = ..., + ) -> None: ... + @property + def noun_chunks(self) -> Iterator[Span]: ... + @property + def sents(self) -> Iterator[Span]: ... + @property + def lang(self) -> int: ... + @property + def lang_(self) -> str: ... + def count_by( + self, attr_id: int, exclude: Optional[Any] = ..., counts: Optional[Any] = ... + ) -> Dict[Any, int]: ... + def from_array( + self, attrs: Union[int, str, List[Union[int, str]]], array: Ints2d + ) -> Doc: ... + def to_array( + self, py_attr_ids: Union[int, str, List[Union[int, str]]] + ) -> np.ndarray[Any, np.dtype[np.float64]]: ... + @staticmethod + def from_docs( + docs: List[Doc], + ensure_whitespace: bool = ..., + attrs: Optional[Union[Tuple[Union[str, int]], List[Union[int, str]]]] = ..., + ) -> Doc: ... + def get_lca_matrix(self) -> Ints2d: ... + def copy(self) -> Doc: ... + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = ... + ) -> None: ... + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = ... + ) -> Doc: ... + def to_bytes(self, *, exclude: Iterable[str] = ...) -> bytes: ... + def from_bytes(self, bytes_data: bytes, *, exclude: Iterable[str] = ...) -> Doc: ... + def to_dict(self, *, exclude: Iterable[str] = ...) -> Dict[str, Any]: ... + def from_dict( + self, msg: Dict[str, Any], *, exclude: Iterable[str] = ... + ) -> Doc: ... + def extend_tensor(self, tensor: Floats2d) -> None: ... + def retokenize(self) -> Retokenizer: ... + def to_json(self, underscore: Optional[List[str]] = ...) -> Dict[str, Any]: ... + def from_json( + self, doc_json: Dict[str, Any] = ..., validate: bool = False + ) -> Doc: ... + def to_utf8_array(self, nr_char: int = ...) -> Ints2d: ... + @staticmethod + def _get_array_attrs() -> Tuple[Any]: ... diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx new file mode 100644 index 0000000..0a90a67 --- /dev/null +++ b/spacy/tokens/doc.pyx @@ -0,0 +1,2009 @@ +# cython: infer_types=True, bounds_check=False +from typing import Set + +cimport cython +cimport numpy as np +from libc.math cimport sqrt +from libc.stdint cimport int32_t, uint64_t +from libc.string cimport memcpy + +import copy +import itertools +import warnings +from collections import Counter, defaultdict +from enum import Enum + +import numpy +import srsly +from thinc.api import get_array_module, get_current_ops +from thinc.util import copy_array + +from .span cimport Span +from .token cimport MISSING_DEP + +from ._dict_proxies import SpanGroups + +from ..attrs cimport ( + DEP, + ENT_ID, + ENT_IOB, + ENT_KB_ID, + ENT_TYPE, + HEAD, + IDX, + LEMMA, + LENGTH, + MORPH, + NORM, + ORTH, + POS, + SENT_START, + SPACY, + TAG, + attr_id_t, +) +from ..lexeme cimport EMPTY_LEXEME, Lexeme +from ..typedefs cimport attr_t +from .token cimport Token + +from .. import parts_of_speech, schemas, util +from ..attrs import IDS, intify_attr +from ..compat import copy_reg +from ..errors import Errors, Warnings +from ..util import get_words_and_spaces +from ._retokenize import Retokenizer +from .underscore import Underscore, get_ext_args + +DEF PADDING = 5 + + +# We store the docbin attrs here rather than in _serialize to avoid +# import cycles. + +# fmt: off +DOCBIN_ALL_ATTRS = ("ORTH", "NORM", "TAG", "HEAD", "DEP", "ENT_IOB", "ENT_TYPE", "ENT_KB_ID", "ENT_ID", "LEMMA", "MORPH", "POS", "SENT_START") +# fmt: on + +cdef int bounds_check(int i, int length, int padding) except -1: + if (i + padding) < 0: + raise IndexError(Errors.E026.format(i=i, length=length)) + if (i - padding) >= length: + raise IndexError(Errors.E026.format(i=i, length=length)) + + +cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) noexcept nogil: + if feat_name == LEMMA: + return token.lemma + elif feat_name == NORM: + if not token.norm: + return token.lex.norm + return token.norm + elif feat_name == POS: + return token.pos + elif feat_name == TAG: + return token.tag + elif feat_name == MORPH: + return token.morph + elif feat_name == DEP: + return token.dep + elif feat_name == HEAD: + return token.head + elif feat_name == SENT_START: + return token.sent_start + elif feat_name == SPACY: + return token.spacy + elif feat_name == ENT_IOB: + return token.ent_iob + elif feat_name == ENT_TYPE: + return token.ent_type + elif feat_name == ENT_ID: + return token.ent_id + elif feat_name == ENT_KB_ID: + return token.ent_kb_id + elif feat_name == IDX: + return token.idx + else: + return Lexeme.get_struct_attr(token.lex, feat_name) + + +cdef attr_t get_token_attr_for_matcher(const TokenC* token, attr_id_t feat_name) noexcept nogil: + if feat_name == SENT_START: + if token.sent_start == 1: + return True + else: + return False + else: + return get_token_attr(token, feat_name) + + +class SetEntsDefault(str, Enum): + blocked = "blocked" + missing = "missing" + outside = "outside" + unmodified = "unmodified" + + @classmethod + def values(cls): + return list(cls.__members__.keys()) + + +cdef class Doc: + """A sequence of Token objects. Access sentences and named entities, export + annotations to numpy arrays, losslessly serialize to compressed binary + strings. The `Doc` object holds an array of `TokenC` structs. The + Python-level `Token` and `Span` objects are views of this array, i.e. + they don't own the data themselves. + + EXAMPLE: + Construction 1 + >>> doc = nlp(u'Some text') + + Construction 2 + >>> from spacy.tokens import Doc + >>> doc = Doc(nlp.vocab, words=["hello", "world", "!"], spaces=[True, False, False]) + + DOCS: https://spacy.io/api/doc + """ + + @classmethod + def set_extension(cls, name, **kwargs): + """Define a custom attribute which becomes available as `Doc._`. + + name (str): Name of the attribute to set. + default: Optional default value of the attribute. + getter (callable): Optional getter function. + setter (callable): Optional setter function. + method (callable): Optional method for method extension. + force (bool): Force overwriting existing attribute. + + DOCS: https://spacy.io/api/doc#set_extension + USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes + """ + if cls.has_extension(name) and not kwargs.get("force", False): + raise ValueError(Errors.E090.format(name=name, obj="Doc")) + Underscore.doc_extensions[name] = get_ext_args(**kwargs) + + @classmethod + def get_extension(cls, name): + """Look up a previously registered extension by name. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple. + + DOCS: https://spacy.io/api/doc#get_extension + """ + return Underscore.doc_extensions.get(name) + + @classmethod + def has_extension(cls, name): + """Check whether an extension has been registered. + + name (str): Name of the extension. + RETURNS (bool): Whether the extension has been registered. + + DOCS: https://spacy.io/api/doc#has_extension + """ + return name in Underscore.doc_extensions + + @classmethod + def remove_extension(cls, name): + """Remove a previously registered extension. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple of the + removed extension. + + DOCS: https://spacy.io/api/doc#remove_extension + """ + if not cls.has_extension(name): + raise ValueError(Errors.E046.format(name=name)) + return Underscore.doc_extensions.pop(name) + + def __init__( + self, + Vocab vocab, + words=None, + spaces=None, + *, + user_data=None, + tags=None, + pos=None, + morphs=None, + lemmas=None, + heads=None, + deps=None, + sent_starts=None, + ents=None, + ): + """Create a Doc object. + + vocab (Vocab): A vocabulary object, which must match any models you + want to use (e.g. tokenizer, parser, entity recognizer). + words (Optional[List[Union[str, int]]]): A list of unicode strings or + hash values to add to the document as words. If `None`, defaults to + empty list. + spaces (Optional[List[bool]]): A list of boolean values, of the same + length as `words`. `True` means that the word is followed by a space, + `False` means it is not. If `None`, defaults to `[True]*len(words)` + user_data (dict or None): Optional extra data to attach to the Doc. + tags (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.tag. Defaults to None. + pos (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.pos. Defaults to None. + morphs (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.morph. Defaults to None. + lemmas (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.lemma. Defaults to None. + heads (Optional[List[int]]): A list of values, of the same length as + words, to assign as heads. Head indices are the position of the + head in the doc. Defaults to None. + deps (Optional[List[str]]): A list of unicode strings, of the same + length as words, to assign as token.dep. Defaults to None. + sent_starts (Optional[List[Union[bool, int, None]]]): A list of values, + of the same length as words, to assign as token.is_sent_start. Will + be overridden by heads if heads is provided. Defaults to None. + ents (Optional[List[str]]): A list of unicode strings, of the same + length as words, as IOB tags to assign as token.ent_iob and + token.ent_type. Defaults to None. + + DOCS: https://spacy.io/api/doc#init + """ + self.vocab = vocab + size = max(20, (len(words) if words is not None else 0)) + self.mem = Pool() + self.spans = SpanGroups(self) + # Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds + # However, we need to remember the true starting places, so that we can + # realloc. + assert size + (PADDING*2) > 0 + data_start = self.mem.alloc(size + (PADDING*2), sizeof(TokenC)) + cdef int i + for i in range(size + (PADDING*2)): + data_start[i].lex = &EMPTY_LEXEME + data_start[i].l_edge = i + data_start[i].r_edge = i + self.c = data_start + PADDING + self.max_length = size + self.length = 0 + self.sentiment = 0.0 + self.cats = {} + self.user_hooks = {} + self.user_token_hooks = {} + self.user_span_hooks = {} + self.tensor = numpy.zeros((0,), dtype="float32") + self.user_data = {} if user_data is None else user_data + self._vector = None + self.noun_chunks_iterator = self.vocab.get_noun_chunks + cdef bint has_space + if words is None and spaces is not None: + raise ValueError(Errors.E908) + elif spaces is None and words is not None: + self.has_unknown_spaces = True + else: + self.has_unknown_spaces = False + words = words if words is not None else [] + spaces = spaces if spaces is not None else ([True] * len(words)) + if len(spaces) != len(words): + raise ValueError(Errors.E027) + cdef const LexemeC* lexeme + for word, has_space in zip(words, spaces): + if isinstance(word, str): + lexeme = self.vocab.get(self.mem, word) + elif isinstance(word, bytes): + raise ValueError(Errors.E028.format(value=word)) + else: + try: + lexeme = self.vocab.get_by_orth(self.mem, word) + except TypeError: + raise TypeError(Errors.E1022.format(wtype=type(word))) + self.push_back(lexeme, has_space) + + if heads is not None: + heads = [head - i if head is not None else 0 for i, head in enumerate(heads)] + if deps is not None: + MISSING_DEP_ = self.vocab.strings[MISSING_DEP] + deps = [dep if dep is not None else MISSING_DEP_ for dep in deps] + if deps and not heads: + heads = [0] * len(deps) + if heads and not deps: + raise ValueError(Errors.E1017) + sent_starts = list(sent_starts) if sent_starts is not None else None + if sent_starts is not None: + for i in range(len(sent_starts)): + if sent_starts[i] is True: + sent_starts[i] = 1 + elif sent_starts[i] is False: + sent_starts[i] = -1 + elif sent_starts[i] is None or sent_starts[i] not in [-1, 0, 1]: + sent_starts[i] = 0 + if pos is not None: + for pp in set(pos): + if pp not in parts_of_speech.IDS: + raise ValueError(Errors.E1021.format(pp=pp)) + ent_iobs = None + ent_types = None + if ents is not None: + ents = [ent if ent != "" else None for ent in ents] + iob_strings = Token.iob_strings() + # make valid IOB2 out of IOB1 or IOB2 + for i, ent in enumerate(ents): + if ent is not None and not isinstance(ent, str): + raise ValueError(Errors.E177.format(tag=ent)) + if i < len(ents) - 1: + # OI -> OB + if (ent is None or ent.startswith("O")) and \ + (ents[i+1] is not None and ents[i+1].startswith("I")): + ents[i+1] = "B" + ents[i+1][1:] + # B-TYPE1 I-TYPE2 or I-TYPE1 I-TYPE2 -> B/I-TYPE1 B-TYPE2 + if ent is not None and ents[i+1] is not None and \ + (ent.startswith("B") or ent.startswith("I")) and \ + ents[i+1].startswith("I") and \ + ent[1:] != ents[i+1][1:]: + ents[i+1] = "B" + ents[i+1][1:] + ent_iobs = [] + ent_types = [] + for ent in ents: + if ent is None: + ent_iobs.append(iob_strings.index("")) + ent_types.append("") + elif ent == "O": + ent_iobs.append(iob_strings.index(ent)) + ent_types.append("") + else: + if len(ent) < 3 or ent[1] != "-": + raise ValueError(Errors.E177.format(tag=ent)) + ent_iob, ent_type = ent.split("-", 1) + if ent_iob not in iob_strings: + raise ValueError(Errors.E177.format(tag=ent)) + ent_iob = iob_strings.index(ent_iob) + ent_iobs.append(ent_iob) + ent_types.append(ent_type) + headings = [] + values = [] + annotations = [pos, heads, deps, lemmas, tags, morphs, sent_starts, ent_iobs, ent_types] + possible_headings = [POS, HEAD, DEP, LEMMA, TAG, MORPH, SENT_START, ENT_IOB, ENT_TYPE] + for a, annot in enumerate(annotations): + if annot is not None: + if len(annot) != len(words): + raise ValueError(Errors.E189) + headings.append(possible_headings[a]) + if annot is not heads and annot is not sent_starts and annot is not ent_iobs: + values.extend(annot) + for value in values: + if value is not None: + self.vocab.strings.add(value) + + # if there are any other annotations, set them + if headings: + attrs = self.to_array(headings) + + j = 0 + for annot in annotations: + if annot: + if annot is heads or annot is sent_starts or annot is ent_iobs: + annot = numpy.array(annot, dtype=numpy.int32).astype(numpy.uint64) + for i in range(len(words)): + if attrs.ndim == 1: + attrs[i] = annot[i] + else: + attrs[i, j] = annot[i] + elif annot is morphs: + for i in range(len(words)): + morph_key = vocab.morphology.add(morphs[i]) + if attrs.ndim == 1: + attrs[i] = morph_key + else: + attrs[i, j] = morph_key + else: + for i in range(len(words)): + if attrs.ndim == 1: + attrs[i] = self.vocab.strings[annot[i]] + else: + attrs[i, j] = self.vocab.strings[annot[i]] + j += 1 + self.from_array(headings, attrs) + + @property + def _(self): + """Custom extension attributes registered via `set_extension`.""" + return Underscore(Underscore.doc_extensions, self) + + @property + def is_tagged(self): + warnings.warn(Warnings.W107.format(prop="is_tagged", attr="TAG"), DeprecationWarning) + return self.has_annotation("TAG") + + @property + def is_parsed(self): + warnings.warn(Warnings.W107.format(prop="is_parsed", attr="DEP"), DeprecationWarning) + return self.has_annotation("DEP") + + @property + def is_nered(self): + warnings.warn(Warnings.W107.format(prop="is_nered", attr="ENT_IOB"), DeprecationWarning) + return self.has_annotation("ENT_IOB") + + @property + def is_sentenced(self): + warnings.warn(Warnings.W107.format(prop="is_sentenced", attr="SENT_START"), DeprecationWarning) + return self.has_annotation("SENT_START") + + def has_annotation(self, attr, *, require_complete=False): + """Check whether the doc contains annotation on a token attribute. + + attr (Union[int, str]): The attribute string name or int ID. + require_complete (bool): Whether to check that the attribute is set on + every token in the doc. + RETURNS (bool): Whether annotation is present. + + DOCS: https://spacy.io/api/doc#has_annotation + """ + + # empty docs are always annotated + input_attr = attr + if self.length == 0: + return True + cdef int i + cdef int range_start = 0 + if attr == "IS_SENT_START" or attr == self.vocab.strings["IS_SENT_START"]: + attr = SENT_START + elif attr == "IS_SENT_END" or attr == self.vocab.strings["IS_SENT_END"]: + attr = SENT_START + attr = intify_attr(attr) + if attr is None: + raise ValueError( + Errors.E1037.format(attr=input_attr) + ) + # adjust attributes + if attr == HEAD: + # HEAD does not have an unset state, so rely on DEP + attr = DEP + # special cases for sentence boundaries + if attr == SENT_START: + if "sents" in self.user_hooks: + return True + # docs of length 1 always have sentence boundaries + if self.length == 1: + return True + range_start = 1 + if require_complete: + return all(Token.get_struct_attr(&self.c[i], attr) for i in range(range_start, self.length)) + else: + return any(Token.get_struct_attr(&self.c[i], attr) for i in range(range_start, self.length)) + + def __getitem__(self, object i): + """Get a `Token` or `Span` object. + + i (int or tuple) The index of the token, or the slice of the document + to get. + RETURNS (Token or Span): The token at `doc[i]]`, or the span at + `doc[start : end]`. + + EXAMPLE: + >>> doc[i] + Get the `Token` object at position `i`, where `i` is an integer. + Negative indexing is supported, and follows the usual Python + semantics, i.e. `doc[-2]` is `doc[len(doc) - 2]`. + + >>> doc[start : end]] + Get a `Span` object, starting at position `start` and ending at + position `end`, where `start` and `end` are token indices. For + instance, `doc[2:5]` produces a span consisting of tokens 2, 3 and + 4. Stepped slices (e.g. `doc[start : end : step]`) are not + supported, as `Span` objects must be contiguous (cannot have gaps). + You can use negative indices and open-ended ranges, which have + their normal Python semantics. + + DOCS: https://spacy.io/api/doc#getitem + """ + if isinstance(i, slice): + start, stop = util.normalize_slice(len(self), i.start, i.stop, i.step) + return Span(self, start, stop, label=0) + if i < 0: + i = self.length + i + bounds_check(i, self.length, PADDING) + return Token.cinit(self.vocab, &self.c[i], i, self) + + def __iter__(self): + """Iterate over `Token` objects, from which the annotations can be + easily accessed. This is the main way of accessing `Token` objects, + which are the main way annotations are accessed from Python. If faster- + than-Python speeds are required, you can instead access the annotations + as a numpy array, or access the underlying C data directly from Cython. + + DOCS: https://spacy.io/api/doc#iter + """ + cdef int i + for i in range(self.length): + yield Token.cinit(self.vocab, &self.c[i], i, self) + + def __len__(self): + """The number of tokens in the document. + + RETURNS (int): The number of tokens in the document. + + DOCS: https://spacy.io/api/doc#len + """ + return self.length + + def __unicode__(self): + return "".join([t.text_with_ws for t in self]) + + def __bytes__(self): + return "".join([t.text_with_ws for t in self]).encode("utf-8") + + def __str__(self): + return self.__unicode__() + + def __repr__(self): + return self.__str__() + + @property + def doc(self): + return self + + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, alignment_mode="strict", span_id=0): + """Create a `Span` object from the slice + `doc.text[start_idx : end_idx]`. Returns None if no valid `Span` can be + created. + + doc (Doc): The parent document. + start_idx (int): The index of the first character of the span. + end_idx (int): The index of the first character after the span. + label (Union[int, str]): A label to attach to the Span, e.g. for + named entities. + kb_id (Union[int, str]): An ID from a KB to capture the meaning of a + named entity. + vector (ndarray[ndim=1, dtype='float32']): A meaning representation of + the span. + alignment_mode (str): How character indices are aligned to token + boundaries. Options: "strict" (character indices must be aligned + with token boundaries), "contract" (span of all tokens completely + within the character span), "expand" (span of all tokens at least + partially covered by the character span). Defaults to "strict". + span_id (Union[int, str]): An identifier to associate with the span. + RETURNS (Span): The newly constructed object. + + DOCS: https://spacy.io/api/doc#char_span + """ + alignment_modes = ("strict", "contract", "expand") + if alignment_mode not in alignment_modes: + raise ValueError( + Errors.E202.format( + name="alignment", + mode=alignment_mode, + modes=", ".join(alignment_modes), + ) + ) + cdef int start = token_by_char(self.c, self.length, start_idx) + if start < 0 or (alignment_mode == "strict" and start_idx != self[start].idx): + return None + # end_idx is exclusive, so find the token at one char before + cdef int end = token_by_char(self.c, self.length, end_idx - 1) + if end < 0 or (alignment_mode == "strict" and end_idx != self[end].idx + len(self[end])): + return None + # Adjust start and end by alignment_mode + if alignment_mode == "contract": + if self[start].idx < start_idx: + start += 1 + if end_idx < self[end].idx + len(self[end]): + end -= 1 + # if no tokens are completely within the span, return None + if end < start: + return None + elif alignment_mode == "expand": + # Don't consider the trailing whitespace to be part of the previous + # token + if start_idx == self[start].idx + len(self[start]): + start += 1 + # Currently we have the token index, we want the range-end index + end += 1 + cdef Span span = Span(self, start, end, label=label, kb_id=kb_id, span_id=span_id, vector=vector) + return span + + def similarity(self, other): + """Make a semantic similarity estimate. The default estimate is cosine + similarity using an average of word vectors. + + other (object): The object to compare with. By default, accepts `Doc`, + `Span`, `Token` and `Lexeme` objects. + RETURNS (float): A scalar similarity score. Higher is more similar. + + DOCS: https://spacy.io/api/doc#similarity + """ + if "similarity" in self.user_hooks: + return self.user_hooks["similarity"](self, other) + attr = getattr(self.vocab.vectors, "attr", ORTH) + cdef Token this_token + cdef Token other_token + cdef Lexeme other_lex + if len(self) == 1 and isinstance(other, Token): + this_token = self[0] + other_token = other + if Token.get_struct_attr(this_token.c, attr) == Token.get_struct_attr(other_token.c, attr): + return 1.0 + elif len(self) == 1 and isinstance(other, Lexeme): + this_token = self[0] + other_lex = other + if Token.get_struct_attr(this_token.c, attr) == Lexeme.get_struct_attr(other_lex.c, attr): + return 1.0 + elif isinstance(other, (Doc, Span)) and len(self) == len(other): + similar = True + for i in range(len(self)): + this_token = self[i] + other_token = other[i] + if Token.get_struct_attr(this_token.c, attr) != Token.get_struct_attr(other_token.c, attr): + similar = False + break + if similar: + return 1.0 + if self.vocab.vectors.n_keys == 0: + warnings.warn(Warnings.W007.format(obj="Doc")) + if self.vector_norm == 0 or other.vector_norm == 0: + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Doc")) + return 0.0 + vector = self.vector + xp = get_array_module(vector) + result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm) + # ensure we get a scalar back (numpy does this automatically but cupy doesn't) + return result.item() + + @property + def has_vector(self): + """A boolean value indicating whether a word vector is associated with + the object. + + RETURNS (bool): Whether a word vector is associated with the object. + + DOCS: https://spacy.io/api/doc#has_vector + """ + if "has_vector" in self.user_hooks: + return self.user_hooks["has_vector"](self) + elif self.vocab.vectors.size: + return any(token.has_vector for token in self) + elif self.tensor.size: + return True + else: + return False + + @property + def vector(self): + """A real-valued meaning representation. Defaults to an average of the + token vectors. + + RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array + representing the document's semantics. + + DOCS: https://spacy.io/api/doc#vector + """ + if "vector" in self.user_hooks: + return self.user_hooks["vector"](self) + if self._vector is not None: + return self._vector + xp = get_array_module(self.vocab.vectors.data) + if not len(self): + self._vector = xp.zeros((self.vocab.vectors_length,), dtype="f") + return self._vector + elif self.vocab.vectors.size > 0: + self._vector = sum(t.vector for t in self) / len(self) + return self._vector + elif self.tensor.size > 0: + self._vector = self.tensor.mean(axis=0) + return self._vector + else: + return xp.zeros((self.vocab.vectors_length,), dtype="float32") + + @vector.setter + def vector(self, value): + self._vector = value + + @property + def vector_norm(self): + """The L2 norm of the document's vector representation. + + RETURNS (float): The L2 norm of the vector representation. + + DOCS: https://spacy.io/api/doc#vector_norm + """ + if "vector_norm" in self.user_hooks: + return self.user_hooks["vector_norm"](self) + cdef float value + cdef double norm = 0 + if self._vector_norm is None: + norm = 0.0 + for value in self.vector: + norm += value * value + self._vector_norm = sqrt(norm) if norm != 0 else 0 + return self._vector_norm + + @vector_norm.setter + def vector_norm(self, value): + self._vector_norm = value + + @property + def text(self): + """A unicode representation of the document text. + + RETURNS (str): The original verbatim text of the document. + """ + return "".join(t.text_with_ws for t in self) + + @property + def text_with_ws(self): + """An alias of `Doc.text`, provided for duck-type compatibility with + `Span` and `Token`. + + RETURNS (str): The original verbatim text of the document. + """ + return self.text + + @property + def ents(self): + """The named entities in the document. Returns a tuple of named entity + `Span` objects, if the entity recognizer has been applied. + + RETURNS (tuple): Entities in the document, one `Span` per entity. + + DOCS: https://spacy.io/api/doc#ents + """ + cdef int i + cdef const TokenC* token + cdef int start = -1 + cdef attr_t label = 0 + cdef attr_t kb_id = 0 + cdef attr_t ent_id = 0 + output = [] + for i in range(self.length): + token = &self.c[i] + if token.ent_iob == 1: + if start == -1: + seq = [f"{t.text}|{t.ent_iob_}" for t in self[i-5:i+5]] + raise ValueError(Errors.E093.format(seq=" ".join(seq))) + elif token.ent_iob == 2 or token.ent_iob == 0 or \ + (token.ent_iob == 3 and token.ent_type == 0): + if start != -1: + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) + start = -1 + label = 0 + kb_id = 0 + ent_id = 0 + elif token.ent_iob == 3: + if start != -1: + output.append(Span(self, start, i, label=label, kb_id=kb_id, span_id=ent_id)) + start = i + label = token.ent_type + kb_id = token.ent_kb_id + ent_id = token.ent_id + if start != -1: + output.append(Span(self, start, self.length, label=label, kb_id=kb_id, span_id=ent_id)) + # remove empty-label spans + output = [o for o in output if o.label_ != ""] + return tuple(output) + + @ents.setter + def ents(self, ents): + # TODO: + # 1. Test basic data-driven ORTH gazetteer + # 2. Test more nuanced date and currency regex + cdef attr_t kb_id, ent_id + cdef int ent_start, ent_end + ent_spans = [] + for ent_info in ents: + entity_type_, kb_id, ent_start, ent_end, ent_id = get_entity_info(ent_info) + if isinstance(entity_type_, str): + self.vocab.strings.add(entity_type_) + span = Span(self, ent_start, ent_end, label=entity_type_, kb_id=kb_id, span_id=ent_id) + ent_spans.append(span) + self.set_ents(ent_spans, default=SetEntsDefault.outside) + + def set_ents(self, entities, *, blocked=None, missing=None, outside=None, default=SetEntsDefault.outside): + """Set entity annotation. + + entities (List[Span]): Spans with labels to set as entities. + blocked (Optional[List[Span]]): Spans to set as 'blocked' (never an + entity) for spacy's built-in NER component. Other components may + ignore this setting. + missing (Optional[List[Span]]): Spans with missing/unknown entity + information. + outside (Optional[List[Span]]): Spans outside of entities (O in IOB). + default (str): How to set entity annotation for tokens outside of any + provided spans. Options: "blocked", "missing", "outside" and + "unmodified" (preserve current state). Defaults to "outside". + """ + if default not in SetEntsDefault.values(): + raise ValueError(Errors.E1011.format(default=default, modes=", ".join(SetEntsDefault))) + + # Ignore spans with missing labels + entities = [ent for ent in entities if ent.label > 0] + + if blocked is None: + blocked = tuple() + if missing is None: + missing = tuple() + if outside is None: + outside = tuple() + + # Find all tokens covered by spans and check that none are overlapping + cdef int i + seen_tokens = set() + for span in itertools.chain.from_iterable([entities, blocked, missing, outside]): + if not isinstance(span, Span): + raise ValueError(Errors.E1012.format(span=span)) + for i in range(span.start, span.end): + if i in seen_tokens: + raise ValueError(Errors.E1010.format(i=i)) + seen_tokens.add(i) + + # Set all specified entity information + for span in entities: + for i in range(span.start, span.end): + if i == span.start: + self.c[i].ent_iob = 3 + else: + self.c[i].ent_iob = 1 + self.c[i].ent_type = span.label + self.c[i].ent_kb_id = span.kb_id + # for backwards compatibility in v3, only set ent_id from + # span.id if it's set, otherwise don't override + self.c[i].ent_id = span.id if span.id else self.c[i].ent_id + for span in blocked: + for i in range(span.start, span.end): + self.c[i].ent_iob = 3 + self.c[i].ent_type = 0 + for span in missing: + for i in range(span.start, span.end): + self.c[i].ent_iob = 0 + self.c[i].ent_type = 0 + for span in outside: + for i in range(span.start, span.end): + self.c[i].ent_iob = 2 + self.c[i].ent_type = 0 + + # Set tokens outside of all provided spans + if default != SetEntsDefault.unmodified: + for i in range(self.length): + if i not in seen_tokens: + self.c[i].ent_type = 0 + if default == SetEntsDefault.outside: + self.c[i].ent_iob = 2 + elif default == SetEntsDefault.missing: + self.c[i].ent_iob = 0 + elif default == SetEntsDefault.blocked: + self.c[i].ent_iob = 3 + + # Fix any resulting inconsistent annotation + for i in range(self.length - 1): + # I must follow B or I: convert I to B + if (self.c[i].ent_iob == 0 or self.c[i].ent_iob == 2) and \ + self.c[i+1].ent_iob == 1: + self.c[i+1].ent_iob = 3 + # Change of type with BI or II: convert second I to B + if self.c[i].ent_type != self.c[i+1].ent_type and \ + (self.c[i].ent_iob == 3 or self.c[i].ent_iob == 1) and \ + self.c[i+1].ent_iob == 1: + self.c[i+1].ent_iob = 3 + + @property + def noun_chunks(self): + """Iterate over the base noun phrases in the document. Yields base + noun-phrase #[code Span] objects, if the language has a noun chunk iterator. + Raises a NotImplementedError otherwise. + + A base noun phrase, or "NP chunk", is a noun + phrase that does not permit other NPs to be nested within it – so no + NP-level coordination, no prepositional phrases, and no relative + clauses. + + YIELDS (Span): Noun chunks in the document. + + DOCS: https://spacy.io/api/doc#noun_chunks + """ + if self.noun_chunks_iterator is None: + raise NotImplementedError(Errors.E894.format(lang=self.vocab.lang)) + + # Accumulate the result before beginning to iterate over it. This + # prevents the tokenization from being changed out from under us + # during the iteration. The tricky thing here is that Span accepts + # its tokenization changing, so it's okay once we have the Span + # objects. See Issue #375. + spans = [] + for start, end, label in self.noun_chunks_iterator(self): + spans.append(Span(self, start, end, label=label)) + for span in spans: + yield span + + @property + def sents(self): + """Iterate over the sentences in the document. Yields sentence `Span` + objects. Sentence spans have no label. + + YIELDS (Span): Sentences in the document. + + DOCS: https://spacy.io/api/doc#sents + """ + if not self.has_annotation("SENT_START"): + raise ValueError(Errors.E030) + if "sents" in self.user_hooks: + yield from self.user_hooks["sents"](self) + else: + start = 0 + for i in range(1, self.length): + if self.c[i].sent_start == 1: + yield Span(self, start, i) + start = i + if start != self.length: + yield Span(self, start, self.length) + + @property + def lang(self): + """RETURNS (uint64): ID of the language of the doc's vocabulary.""" + return self.vocab.strings[self.vocab.lang] + + @property + def lang_(self): + """RETURNS (str): Language of the doc's vocabulary, e.g. 'en'.""" + return self.vocab.lang + + cdef int push_back(self, LexemeOrToken lex_or_tok, bint has_space) except -1: + if self.length == self.max_length: + self._realloc(self.length * 2) + cdef TokenC* t = &self.c[self.length] + if LexemeOrToken is const_TokenC_ptr: + t[0] = lex_or_tok[0] + else: + t.lex = lex_or_tok + if self.length == 0: + t.idx = 0 + else: + t.idx = (t-1).idx + (t-1).lex.length + (t-1).spacy + t.l_edge = self.length + t.r_edge = self.length + if t.lex.orth == 0: + raise ValueError(Errors.E031.format(i=self.length)) + t.spacy = has_space + self.length += 1 + if self.length == 1: + # Set token.sent_start to 1 for first token. See issue #2869 + self.c[0].sent_start = 1 + return t.idx + t.lex.length + t.spacy + + @cython.boundscheck(False) + cpdef np.ndarray to_array(self, object py_attr_ids): + """Export given token attributes to a numpy `ndarray`. + If `attr_ids` is a sequence of M attributes, the output array will be + of shape `(N, M)`, where N is the length of the `Doc` (in tokens). If + `attr_ids` is a single attribute, the output shape will be (N,). You + can specify attributes by integer ID (e.g. spacy.attrs.LEMMA) or + string name (e.g. 'LEMMA' or 'lemma'). + + py_attr_ids (list[]): A list of attributes (int IDs or string names). + RETURNS (numpy.ndarray[long, ndim=2]): A feature matrix, with one row + per word, and one column per attribute indicated in the input + `attr_ids`. + + EXAMPLE: + >>> from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA + >>> doc = nlp(text) + >>> # All strings mapped to integers, for easy export to numpy + >>> np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA]) + """ + cdef int i, j + cdef np.ndarray[attr_t, ndim=2] output + # Handle scalar/list inputs of strings/ints for py_attr_ids + # See also #3064 + if isinstance(py_attr_ids, str): + # Handle inputs like doc.to_array('ORTH') + py_attr_ids = [py_attr_ids] + elif not hasattr(py_attr_ids, "__iter__"): + # Handle inputs like doc.to_array(ORTH) + py_attr_ids = [py_attr_ids] + # Allow strings, e.g. 'lemma' or 'LEMMA' + try: + py_attr_ids = [ + (IDS[id_.upper()] if hasattr(id_, "upper") else id_) + for id_ in py_attr_ids + ] + except KeyError as msg: + keys = [k for k in IDS.keys() if not k.startswith("FLAG")] + raise KeyError(Errors.E983.format(dict="IDS", key=msg, keys=keys)) from None + # Make an array from the attributes --- otherwise our inner loop is + # Python dict iteration. + cdef np.ndarray attr_ids = numpy.asarray(py_attr_ids, dtype="i") + output = numpy.ndarray(shape=(self.length, len(attr_ids)), dtype=numpy.uint64) + c_output = output.data + c_attr_ids = attr_ids.data + cdef TokenC* token + cdef int nr_attr = attr_ids.shape[0] + for i in range(self.length): + token = &self.c[i] + for j in range(nr_attr): + c_output[i*nr_attr + j] = get_token_attr(token, c_attr_ids[j]) + # Handle 1d case + return output if len(attr_ids) >= 2 else output.reshape((self.length,)) + + def count_by(self, attr_id_t attr_id, exclude=None, object counts=None): + """Count the frequencies of a given attribute. Produces a dict of + `{attribute (int): count (ints)}` frequencies, keyed by the values of + the given attribute ID. + + attr_id (int): The attribute ID to key the counts. + RETURNS (dict): A dictionary mapping attributes to integer counts. + + DOCS: https://spacy.io/api/doc#count_by + """ + cdef int i + + if counts is None: + counts = Counter() + output_dict = True + else: + output_dict = False + # Take this check out of the loop, for a bit of extra speed + if exclude is None: + for i in range(self.length): + counts[get_token_attr(&self.c[i], attr_id)] += 1 + else: + for i in range(self.length): + if not exclude(self[i]): + counts[get_token_attr(&self.c[i], attr_id)] += 1 + if output_dict: + return dict(counts) + + def _realloc(self, new_size): + if new_size < self.max_length: + return + self.max_length = new_size + n = new_size + (PADDING * 2) + # What we're storing is a "padded" array. We've jumped forward PADDING + # places, and are storing the pointer to that. This way, we can access + # words out-of-bounds, and get out-of-bounds markers. + # Now that we want to realloc, we need the address of the true start, + # so we jump the pointer back PADDING places. + cdef TokenC* data_start = self.c - PADDING + data_start = self.mem.realloc(data_start, n * sizeof(TokenC)) + self.c = data_start + PADDING + cdef int i + for i in range(self.length, self.max_length + PADDING): + self.c[i].lex = &EMPTY_LEXEME + + def from_array(self, attrs, array): + """Load attributes from a numpy array. Write to a `Doc` object, from an + `(M, N)` array of attributes. + + attrs (list) A list of attribute ID ints. + array (numpy.ndarray[ndim=2, dtype='int32']): The attribute values. + RETURNS (Doc): Itself. + + DOCS: https://spacy.io/api/doc#from_array + """ + # Handle scalar/list inputs of strings/ints for py_attr_ids + # See also #3064 + if isinstance(attrs, str): + # Handle inputs like doc.to_array('ORTH') + attrs = [attrs] + elif not hasattr(attrs, "__iter__"): + # Handle inputs like doc.to_array(ORTH) + attrs = [attrs] + # Allow strings, e.g. 'lemma' or 'LEMMA' + attrs = [(IDS[id_.upper()] if hasattr(id_, "upper") else id_) + for id_ in attrs] + if array.dtype != numpy.uint64: + warnings.warn(Warnings.W028.format(type=array.dtype)) + + cdef int i, col + cdef int32_t abs_head_index + cdef attr_id_t attr_id + cdef int length = len(array) + if length != len(self): + raise ValueError(Errors.E971.format(array_length=length, doc_length=len(self))) + + # Get set up for fast loading + cdef Pool mem = Pool() + cdef int n_attrs = len(attrs) + # attrs should not be empty, but make sure to avoid zero-length mem alloc + assert n_attrs > 0 + attr_ids = mem.alloc(n_attrs, sizeof(attr_id_t)) + for i, attr_id in enumerate(attrs): + attr_ids[i] = attr_id + if len(array.shape) == 1: + array = array.reshape((array.size, 1)) + cdef np.ndarray transposed_array = numpy.ascontiguousarray(array.T) + values = transposed_array.data + stride = transposed_array.shape[1] + # Check that all heads are within the document bounds + if HEAD in attrs: + col = attrs.index(HEAD) + for i in range(length): + # cast index to signed int + abs_head_index = values[col * stride + i] + abs_head_index += i + if abs_head_index < 0 or abs_head_index >= length: + raise ValueError( + Errors.E190.format( + index=i, + value=array[i, col], + rel_head_index=abs_head_index-i + ) + ) + # Verify ENT_IOB are proper integers + if ENT_IOB in attrs: + iob_strings = Token.iob_strings() + col = attrs.index(ENT_IOB) + n_iob_strings = len(iob_strings) + for i in range(length): + value = values[col * stride + i] + if value < 0 or value >= n_iob_strings: + raise ValueError( + Errors.E982.format( + values=iob_strings, + value=value + ) + ) + # Now load the data + for i in range(length): + token = &self.c[i] + for j in range(n_attrs): + value = values[j * stride + i] + if attr_ids[j] == MORPH: + # add morph to morphology table + self.vocab.morphology.add(self.vocab.strings[value]) + Token.set_struct_attr(token, attr_ids[j], value) + # If document is parsed, set children and sentence boundaries + if HEAD in attrs and DEP in attrs: + col = attrs.index(DEP) + if array[:, col].any(): + set_children_from_heads(self.c, 0, length) + return self + + @staticmethod + def from_docs(docs, ensure_whitespace=True, attrs=None, *, exclude=tuple()): + """Concatenate multiple Doc objects to form a new one. Raises an error + if the `Doc` objects do not all share the same `Vocab`. + + docs (list): A list of Doc objects. + ensure_whitespace (bool): Insert a space between two adjacent docs + whenever the first doc does not end in whitespace. + attrs (list): Optional list of attribute ID ints or attribute name + strings. + exclude (Iterable[str]): Doc attributes to exclude. Supported + attributes: `spans`, `tensor`, `user_data`. + RETURNS (Doc): A doc that contains the concatenated docs, or None if no + docs were given. + + DOCS: https://spacy.io/api/doc#from_docs + """ + if not docs: + return None + + vocab = {doc.vocab for doc in docs} + if len(vocab) > 1: + raise ValueError(Errors.E999) + (vocab,) = vocab + + if attrs is None: + attrs = list(Doc._get_array_attrs()) + else: + if any(isinstance(attr, str) for attr in attrs): # resolve attribute names + attrs = [intify_attr(attr) for attr in attrs] # intify_attr returns None for invalid attrs + attrs = list(attr for attr in set(attrs) if attr) # filter duplicates, remove None if present + if SPACY not in attrs: + attrs.append(SPACY) + + concat_words = [] + concat_spaces = [] + concat_user_data = {} + concat_spans = defaultdict(list) + char_offset = 0 + for doc in docs: + concat_words.extend(t.text for t in doc) + concat_spaces.extend(bool(t.whitespace_) for t in doc) + + if "user_data" not in exclude: + for key, value in doc.user_data.items(): + if isinstance(key, tuple) and len(key) == 4 and key[0] == "._.": + data_type, name, start, end = key + if start is not None or end is not None: + start += char_offset + if end is not None: + end += char_offset + concat_user_data[(data_type, name, start, end)] = copy.copy(value) + else: + warnings.warn(Warnings.W101.format(name=name)) + else: + warnings.warn(Warnings.W102.format(key=key, value=value)) + if "spans" not in exclude: + for key in doc.spans: + # if a spans key is in any doc, include it in the merged doc + # even if it is empty + if key not in concat_spans: + concat_spans[key] = [] + for span in doc.spans[key]: + concat_spans[key].append(( + span.start_char + char_offset, + span.end_char + char_offset, + span.label, + span.kb_id, + span.id, + span.text, # included as a check + )) + char_offset += len(doc.text) + if len(doc) > 0 and ensure_whitespace and not doc[-1].is_space and not bool(doc[-1].whitespace_): + char_offset += 1 + + arrays = [doc.to_array(attrs) for doc in docs] + + if ensure_whitespace: + spacy_index = attrs.index(SPACY) + for i, array in enumerate(arrays[:-1]): + if len(array) > 0 and not docs[i][-1].is_space: + array[-1][spacy_index] = 1 + if len(concat_spaces) > 0: + token_offset = -1 + for doc in docs[:-1]: + token_offset += len(doc) + if len(doc) > 0 and not doc[-1].is_space: + concat_spaces[token_offset] = True + + concat_array = numpy.concatenate(arrays) + + concat_doc = Doc(vocab, words=concat_words, spaces=concat_spaces, user_data=concat_user_data) + + concat_doc.from_array(attrs, concat_array) + + for key in concat_spans: + if key not in concat_doc.spans: + concat_doc.spans[key] = [] + for span_tuple in concat_spans[key]: + span = concat_doc.char_span( + span_tuple[0], + span_tuple[1], + label=span_tuple[2], + kb_id=span_tuple[3], + span_id=span_tuple[4], + ) + text = span_tuple[5] + if span is not None and span.text == text: + concat_doc.spans[key].append(span) + else: + raise ValueError(Errors.E873.format(key=key, text=text)) + + if "tensor" not in exclude and any(len(doc) for doc in docs): + ops = get_current_ops() + concat_doc.tensor = ops.xp.vstack([ops.asarray(doc.tensor) for doc in docs if len(doc)]) + + return concat_doc + + def get_lca_matrix(self): + """Calculates a matrix of Lowest Common Ancestors (LCA) for a given + `Doc`, where LCA[i, j] is the index of the lowest common ancestor among + token i and j. + + RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape + (n, n), where n = len(self). + + DOCS: https://spacy.io/api/doc#get_lca_matrix + """ + return numpy.asarray(_get_lca_matrix(self, 0, len(self))) + + def copy(self): + cdef Doc other = Doc(self.vocab) + other._vector = copy.deepcopy(self._vector) + other._vector_norm = copy.deepcopy(self._vector_norm) + other.tensor = copy.deepcopy(self.tensor) + other.cats = copy.deepcopy(self.cats) + other.user_data = copy.deepcopy(self.user_data) + other.sentiment = self.sentiment + other.has_unknown_spaces = self.has_unknown_spaces + other.user_hooks = dict(self.user_hooks) + other.user_token_hooks = dict(self.user_token_hooks) + other.user_span_hooks = dict(self.user_span_hooks) + other.length = self.length + other.max_length = self.max_length + buff_size = other.max_length + (PADDING*2) + assert buff_size > 0 + tokens = other.mem.alloc(buff_size, sizeof(TokenC)) + memcpy(tokens, self.c - PADDING, buff_size * sizeof(TokenC)) + other.c = &tokens[PADDING] + # copy spans after setting tokens so that SpanGroup.copy can verify + # that the start/end offsets are valid + other.spans = self.spans.copy(doc=other) + return other + + def to_disk(self, path, *, exclude=tuple()): + """Save the current state to a directory. + + path (str / Path): A path to a directory, which will be created if + it doesn't exist. Paths may be either strings or Path-like objects. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/doc#to_disk + """ + path = util.ensure_path(path) + with path.open("wb") as file_: + file_.write(self.to_bytes(exclude=exclude)) + + def from_disk(self, path, *, exclude=tuple()): + """Loads state from a directory. Modifies the object in place and + returns it. + + path (str / Path): A path to a directory. Paths may be either + strings or `Path`-like objects. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Doc): The modified `Doc` object. + + DOCS: https://spacy.io/api/doc#from_disk + """ + path = util.ensure_path(path) + with path.open("rb") as file_: + bytes_data = file_.read() + return self.from_bytes(bytes_data, exclude=exclude) + + def to_bytes(self, *, exclude=tuple()): + """Serialize, i.e. export the document contents to a binary string. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): A losslessly serialized copy of the `Doc`, including + all annotations. + + DOCS: https://spacy.io/api/doc#to_bytes + """ + return srsly.msgpack_dumps(self.to_dict(exclude=exclude)) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Deserialize, i.e. import the document contents from a binary string. + + data (bytes): The string to load from. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Doc): Itself. + + DOCS: https://spacy.io/api/doc#from_bytes + """ + return self.from_dict(srsly.msgpack_loads(bytes_data), exclude=exclude) + + def to_dict(self, *, exclude=tuple()): + """Export the document contents to a dictionary for serialization. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Dict[str, Any]): A dictionary representation of the `Doc` + """ + array_head = Doc._get_array_attrs() + strings = set() + for token in self: + strings.add(token.tag_) + strings.add(token.lemma_) + strings.add(str(token.morph)) + strings.add(token.dep_) + strings.add(token.ent_type_) + strings.add(token.ent_kb_id_) + strings.add(token.ent_id_) + strings.add(token.norm_) + for group in self.spans.values(): + for span in group: + strings.add(span.label_) + if span.kb_id in span.doc.vocab.strings: + strings.add(span.kb_id_) + if span.id in span.doc.vocab.strings: + strings.add(span.id_) + # Msgpack doesn't distinguish between lists and tuples, which is + # vexing for user data. As a best guess, we *know* that within + # keys, we must have tuples. In values we just have to hope + # users don't mind getting a list instead of a tuple. + serializers = { + "text": lambda: self.text, + "array_head": lambda: array_head, + "array_body": lambda: self.to_array(array_head), + "sentiment": lambda: self.sentiment, + "tensor": lambda: self.tensor, + "cats": lambda: self.cats, + "spans": lambda: self.spans.to_bytes(), + "strings": lambda: list(strings), + "has_unknown_spaces": lambda: self.has_unknown_spaces + } + if "user_data" not in exclude and self.user_data: + user_data_keys, user_data_values = list(zip(*self.user_data.items())) + if "user_data_keys" not in exclude: + serializers["user_data_keys"] = lambda: srsly.msgpack_dumps(user_data_keys) + if "user_data_values" not in exclude: + serializers["user_data_values"] = lambda: srsly.msgpack_dumps(user_data_values) + if "user_hooks" not in exclude and any((self.user_hooks, self.user_token_hooks, self.user_span_hooks)): + warnings.warn(Warnings.W109) + return util.to_dict(serializers, exclude) + + def from_dict(self, msg, *, exclude=tuple()): + """Deserialize the document contents from a dictionary representation. + + msg (Dict[str, Any]): The dictionary to load from. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Doc): Itself. + """ + if self.length != 0: + raise ValueError(Errors.E033.format(length=self.length)) + # Msgpack doesn't distinguish between lists and tuples, which is + # vexing for user data. As a best guess, we *know* that within + # keys, we must have tuples. In values we just have to hope + # users don't mind getting a list instead of a tuple. + if "user_data" not in exclude and "user_data_keys" in msg: + user_data_keys = srsly.msgpack_loads(msg["user_data_keys"], use_list=False) + user_data_values = srsly.msgpack_loads(msg["user_data_values"]) + for key, value in zip(user_data_keys, user_data_values): + self.user_data[key] = value + cdef int i, start, end, has_space + if "sentiment" not in exclude and "sentiment" in msg: + self.sentiment = msg["sentiment"] + if "tensor" not in exclude and "tensor" in msg: + self.tensor = msg["tensor"] + if "cats" not in exclude and "cats" in msg: + self.cats = msg["cats"] + if "strings" not in exclude and "strings" in msg: + for s in msg["strings"]: + self.vocab.strings.add(s) + if "has_unknown_spaces" not in exclude and "has_unknown_spaces" in msg: + self.has_unknown_spaces = msg["has_unknown_spaces"] + start = 0 + cdef const LexemeC* lex + cdef str orth_ + text = msg["text"] + attrs = msg["array_body"] + for i in range(attrs.shape[0]): + end = start + attrs[i, 0] + has_space = attrs[i, 1] + orth_ = text[start:end] + lex = self.vocab.get(self.mem, orth_) + self.push_back(lex, has_space) + start = end + has_space + self.from_array(msg["array_head"][2:], attrs[:, 2:]) + if "spans" in msg: + self.spans.from_bytes(msg["spans"]) + else: + self.spans.clear() + return self + + def extend_tensor(self, tensor): + """Concatenate a new tensor onto the doc.tensor object. + + The doc.tensor attribute holds dense feature vectors + computed by the models in the pipeline. Let's say a + document with 30 words has a tensor with 128 dimensions + per word. doc.tensor.shape will be (30, 128). After + calling doc.extend_tensor with an array of shape (30, 64), + doc.tensor == (30, 192). + """ + xp = get_array_module(self.tensor) + if self.tensor.size == 0: + self.tensor.resize(tensor.shape, refcheck=False) + copy_array(self.tensor, tensor) + else: + self.tensor = xp.hstack((self.tensor, tensor)) + + def retokenize(self): + """Context manager to handle retokenization of the Doc. + Modifications to the Doc's tokenization are stored, and then + made all at once when the context manager exits. This is + much more efficient, and less error-prone. + + All views of the Doc (Span and Token) created before the + retokenization are invalidated, although they may accidentally + continue to work. + + DOCS: https://spacy.io/api/doc#retokenize + USAGE: https://spacy.io/usage/linguistic-features#retokenization + """ + return Retokenizer(self) + + def _bulk_merge(self, spans, attributes): + """Retokenize the document, such that the spans given as arguments + are merged into single tokens. The spans need to be in document + order, and no span intersection is allowed. + + spans (Span[]): Spans to merge, in document order, with all span + intersections empty. Cannot be empty. + attributes (Dictionary[]): Attributes to assign to the merged tokens. By default, + must be the same length as spans, empty dictionaries are allowed. + attributes are inherited from the syntactic root of the span. + RETURNS (Token): The first newly merged token. + """ + attr_len = len(attributes) + span_len = len(spans) + if not attr_len == span_len: + raise ValueError(Errors.E121.format(attr_len=attr_len, span_len=span_len)) + with self.retokenize() as retokenizer: + for i, span in enumerate(spans): + fix_attributes(self, attributes[i]) + remove_label_if_necessary(attributes[i]) + retokenizer.merge(span, attributes[i]) + + def from_json(self, doc_json, *, validate=False): + """Convert a JSON document generated by Doc.to_json() to a Doc. + + doc_json (Dict): JSON representation of doc object to load. + validate (bool): Whether to validate `doc_json` against the expected schema. + Defaults to False. + RETURNS (Doc): A doc instance corresponding to the specified JSON representation. + """ + + if validate: + schema_validation_message = schemas.validate(schemas.DocJSONSchema, doc_json) + if schema_validation_message: + raise ValueError(Errors.E1038.format(message=schema_validation_message)) + + ### Token-level properties ### + + words = [] + token_attrs_ids = (POS, HEAD, DEP, LEMMA, TAG, MORPH) + # Map annotation type IDs to their string equivalents. + token_attrs = {t: self.vocab.strings[t].lower() for t in token_attrs_ids} + token_annotations = {} + + # Gather token-level properties. + for token_json in doc_json["tokens"]: + words.append(doc_json["text"][token_json["start"]:token_json["end"]]) + for attr, attr_json in token_attrs.items(): + if attr_json in token_json: + if token_json["id"] == 0 and attr not in token_annotations: + token_annotations[attr] = [] + elif attr not in token_annotations: + raise ValueError(Errors.E1040.format(partial_attrs=attr)) + token_annotations[attr].append(token_json[attr_json]) + + # Initialize doc instance. + start = 0 + cdef const LexemeC* lex + cdef bint has_space + reconstructed_words, spaces = get_words_and_spaces(words, doc_json["text"]) + assert words == reconstructed_words + + for word, has_space in zip(words, spaces): + lex = self.vocab.get(self.mem, word) + self.push_back(lex, has_space) + + # Set remaining token-level attributes via Doc.from_array(). + if HEAD in token_annotations: + token_annotations[HEAD] = [ + head - i for i, head in enumerate(token_annotations[HEAD]) + ] + + if DEP in token_annotations and HEAD not in token_annotations: + token_annotations[HEAD] = [0] * len(token_annotations[DEP]) + if HEAD in token_annotations and DEP not in token_annotations: + raise ValueError(Errors.E1017) + if POS in token_annotations: + for pp in set(token_annotations[POS]): + if pp not in parts_of_speech.IDS: + raise ValueError(Errors.E1021.format(pp=pp)) + + # Collect token attributes, assert all tokens have exactly the same set of attributes. + attrs = [] + partial_attrs: Set[str] = set() + for attr in token_attrs.keys(): + if attr in token_annotations: + if len(token_annotations[attr]) != len(words): + partial_attrs.add(token_attrs[attr]) + attrs.append(attr) + if len(partial_attrs): + raise ValueError(Errors.E1040.format(partial_attrs=partial_attrs)) + + # If there are any other annotations, set them. + if attrs: + array = self.to_array(attrs) + if array.ndim == 1: + array = numpy.reshape(array, (array.size, 1)) + j = 0 + + for j, (attr, annot) in enumerate(token_annotations.items()): + if attr is HEAD: + annot = numpy.array(annot, dtype=numpy.int32).astype(numpy.uint64) + for i in range(len(words)): + array[i, j] = annot[i] + elif attr is MORPH: + for i in range(len(words)): + array[i, j] = self.vocab.morphology.add(annot[i]) + else: + for i in range(len(words)): + array[i, j] = self.vocab.strings.add(annot[i]) + self.from_array(attrs, array) + + ### Span/document properties ### + + # Complement other document-level properties (cats, spans, ents). + self.cats = doc_json.get("cats", {}) + + # Set sentence boundaries, if dependency parser not available but sentences are specified in JSON. + if not self.has_annotation("DEP"): + for sent in doc_json.get("sents", {}): + char_span = self.char_span(sent["start"], sent["end"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="sentence", start=sent["start"], end=sent["end"])) + char_span[0].is_sent_start = True + for token in char_span[1:]: + token.is_sent_start = False + + for span_group in doc_json.get("spans", {}): + spans = [] + for span in doc_json["spans"][span_group]: + char_span = self.char_span(span["start"], span["end"], span["label"], span["kb_id"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="span", start=span["start"], end=span["end"])) + spans.append(char_span) + self.spans[span_group] = spans + + if "ents" in doc_json: + ents = [] + for ent in doc_json["ents"]: + char_span = self.char_span(ent["start"], ent["end"], ent["label"]) + if char_span is None: + raise ValueError(Errors.E1039.format(obj="entity"), start=ent["start"], end=ent["end"]) + ents.append(char_span) + self.ents = ents + + # Add custom attributes for the whole Doc object. + for attr in doc_json.get("_", {}): + if not Doc.has_extension(attr): + Doc.set_extension(attr) + self._.set(attr, doc_json["_"][attr]) + + for token_attr in doc_json.get("underscore_token", {}): + if not Token.has_extension(token_attr): + Token.set_extension(token_attr) + for token_data in doc_json["underscore_token"][token_attr]: + start = token_by_char(self.c, self.length, token_data["start"]) + value = token_data["value"] + self[start]._.set(token_attr, value) + + for span_attr in doc_json.get("underscore_span", {}): + if not Span.has_extension(span_attr): + Span.set_extension(span_attr) + for span_data in doc_json["underscore_span"][span_attr]: + value = span_data["value"] + self.char_span(span_data["start"], span_data["end"])._.set(span_attr, value) + return self + + def to_json(self, underscore=None): + """Convert a Doc to JSON. + + underscore (list): Optional list of string names of custom doc._. + attributes. Attribute values need to be JSON-serializable. Values will + be added to an "_" key in the data, e.g. "_": {"foo": "bar"}. + RETURNS (dict): The data in JSON format. + """ + data = {"text": self.text} + if self.has_annotation("ENT_IOB"): + data["ents"] = [{"start": ent.start_char, "end": ent.end_char, "label": ent.label_} for ent in self.ents] + if self.has_annotation("SENT_START"): + sents = list(self.sents) + data["sents"] = [{"start": sent.start_char, "end": sent.end_char} for sent in sents] + if self.cats: + data["cats"] = self.cats + data["tokens"] = [] + attrs = ["TAG", "MORPH", "POS", "LEMMA", "DEP"] + include_annotation = {attr: self.has_annotation(attr) for attr in attrs} + for token in self: + token_data = {"id": token.i, "start": token.idx, "end": token.idx + len(token)} + if include_annotation["TAG"]: + token_data["tag"] = token.tag_ + if include_annotation["POS"]: + token_data["pos"] = token.pos_ + if include_annotation["MORPH"]: + token_data["morph"] = token.morph.to_json() + if include_annotation["LEMMA"]: + token_data["lemma"] = token.lemma_ + if include_annotation["DEP"]: + token_data["dep"] = token.dep_ + token_data["head"] = token.head.i + data["tokens"].append(token_data) + + if self.spans: + data["spans"] = {} + for span_group in self.spans: + data["spans"][span_group] = [] + for span in self.spans[span_group]: + span_data = {"start": span.start_char, "end": span.end_char, "label": span.label_, "kb_id": span.kb_id_} + data["spans"][span_group].append(span_data) + + if underscore: + user_keys = set() + # Handle doc attributes with .get to include values from getters + # and not only values stored in user_data, for backwards + # compatibility + for attr in underscore: + if self.has_extension(attr): + if "_" not in data: + data["_"] = {} + value = self._.get(attr) + if not srsly.is_json_serializable(value): + raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) + data["_"][attr] = value + user_keys.add(attr) + # Token and span attributes only include values stored in user_data + # and not values generated by getters + if self.user_data: + for data_key, value in self.user_data.copy().items(): + if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.": + attr = data_key[1] + start = data_key[2] + end = data_key[3] + if attr in underscore: + user_keys.add(attr) + if not srsly.is_json_serializable(value): + raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) + # Token attribute + if start is not None and end is None: + if "underscore_token" not in data: + data["underscore_token"] = {} + if attr not in data["underscore_token"]: + data["underscore_token"][attr] = [] + data["underscore_token"][attr].append({"start": start, "value": value}) + # Span attribute + elif start is not None and end is not None: + if "underscore_span" not in data: + data["underscore_span"] = {} + if attr not in data["underscore_span"]: + data["underscore_span"][attr] = [] + data["underscore_span"][attr].append({"start": start, "end": end, "value": value}) + + for attr in underscore: + if attr not in user_keys: + raise ValueError(Errors.E106.format(attr=attr, opts=underscore)) + return data + + def to_utf8_array(self, int nr_char=-1): + """Encode word strings to utf8, and export to a fixed-width array + of characters. Characters are placed into the array in the order: + 0, -1, 1, -2, etc + For example, if the array is sliced array[:, :8], the array will + contain the first 4 characters and last 4 characters of each word --- + with the middle characters clipped out. The value 255 is used as a pad + value. + """ + byte_strings = [token.orth_.encode('utf8') for token in self] + if nr_char == -1: + nr_char = max(len(bs) for bs in byte_strings) + cdef np.ndarray output = numpy.zeros((len(byte_strings), nr_char), dtype='uint8') + output.fill(255) + cdef int i, j, start_idx, end_idx + cdef bytes byte_string + for i, byte_string in enumerate(byte_strings): + j = 0 + start_idx = 0 + end_idx = len(byte_string) - 1 + while j < nr_char and start_idx <= end_idx: + output[i, j] = byte_string[start_idx] + start_idx += 1 + j += 1 + if j < nr_char and start_idx <= end_idx: + output[i, j] = byte_string[end_idx] + end_idx -= 1 + j += 1 + return output + + @staticmethod + def _get_array_attrs(): + attrs = [LENGTH, SPACY] + attrs.extend(intify_attr(x) for x in DOCBIN_ALL_ATTRS) + return tuple(attrs) + + +cdef int token_by_start(const TokenC* tokens, int length, int start_char) except -2: + cdef int i = token_by_char(tokens, length, start_char) + if i >= 0 and tokens[i].idx == start_char: + return i + else: + return -1 + + +cdef int token_by_end(const TokenC* tokens, int length, int end_char) except -2: + # end_char is exclusive, so find the token at one char before + cdef int i = token_by_char(tokens, length, end_char - 1) + if i >= 0 and tokens[i].idx + tokens[i].lex.length == end_char: + return i + else: + return -1 + + +cdef int token_by_char(const TokenC* tokens, int length, int char_idx) except -2: + cdef int start = 0, mid, end = length - 1 + while start <= end: + mid = (start + end) / 2 + if char_idx < tokens[mid].idx: + end = mid - 1 + elif char_idx >= tokens[mid].idx + tokens[mid].lex.length + tokens[mid].spacy: + start = mid + 1 + else: + return mid + return -1 + +cdef int set_children_from_heads(TokenC* tokens, int start, int end) except -1: + # note: end is exclusive + cdef int i + # Set number of left/right children to 0. We'll increment it in the loops. + for i in range(start, end): + tokens[i].l_kids = 0 + tokens[i].r_kids = 0 + tokens[i].l_edge = i + tokens[i].r_edge = i + cdef int loop_count = 0 + cdef bint heads_within_sents = False + # Try up to 10 iterations of adjusting lr_kids and lr_edges in order to + # handle non-projective dependency parses, stopping when all heads are + # within their respective sentence boundaries. We have documented cases + # that need at least 4 iterations, so this is to be on the safe side + # without risking getting stuck in an infinite loop if something is + # terribly malformed. + while not heads_within_sents: + heads_within_sents = _set_lr_kids_and_edges(tokens, start, end, loop_count) + if loop_count > 10: + util.logger.debug(Warnings.W026) + break + loop_count += 1 + # Set sentence starts + for i in range(start, end): + tokens[i].sent_start = -1 + for i in range(start, end): + if tokens[i].head == 0 and not Token.missing_head(&tokens[i]): + tokens[tokens[i].l_edge].sent_start = 1 + + +cdef int _set_lr_kids_and_edges(TokenC* tokens, int start, int end, int loop_count) except -1: + # May be called multiple times due to non-projectivity. See issues #3170 + # and #4688. + # Set left edges + cdef TokenC* head + cdef TokenC* child + cdef int i, j + for i in range(start, end): + child = &tokens[i] + head = &tokens[i + child.head] + if loop_count == 0 and child < head: + head.l_kids += 1 + if child.l_edge < head.l_edge: + head.l_edge = child.l_edge + if child.r_edge > head.r_edge: + head.r_edge = child.r_edge + # Set right edges - same as above, but iterate in reverse + for i in range(end-1, start-1, -1): + child = &tokens[i] + head = &tokens[i + child.head] + if loop_count == 0 and child > head: + head.r_kids += 1 + if child.r_edge > head.r_edge: + head.r_edge = child.r_edge + if child.l_edge < head.l_edge: + head.l_edge = child.l_edge + # Get sentence start positions according to current state + sent_starts = set() + for i in range(start, end): + if tokens[i].head == 0: + sent_starts.add(tokens[i].l_edge) + cdef int curr_sent_start = 0 + cdef int curr_sent_end = 0 + # Check whether any heads are not within the current sentence + for i in range(start, end): + if (i > 0 and i in sent_starts) or i == end - 1: + curr_sent_end = i + for j in range(curr_sent_start, curr_sent_end): + if tokens[j].head + j < curr_sent_start or tokens[j].head + j >= curr_sent_end + 1: + return False + curr_sent_start = i + return True + + +cdef int _get_tokens_lca(Token token_j, Token token_k): + """Given two tokens, returns the index of the lowest common ancestor + (LCA) among the two. If they have no common ancestor, -1 is returned. + + token_j (Token): a token. + token_k (Token): another token. + RETURNS (int): index of lowest common ancestor, or -1 if the tokens + have no common ancestor. + """ + if token_j == token_k: + return token_j.i + elif token_j.head == token_k: + return token_k.i + elif token_k.head == token_j: + return token_j.i + token_j_ancestors = set(token_j.ancestors) + if token_k in token_j_ancestors: + return token_k.i + for token_k_ancestor in token_k.ancestors: + if token_k_ancestor == token_j: + return token_j.i + if token_k_ancestor in token_j_ancestors: + return token_k_ancestor.i + return -1 + + +cdef int [:, :] _get_lca_matrix(Doc doc, int start, int end): + """Given a doc and a start and end position defining a set of contiguous + tokens within it, returns a matrix of Lowest Common Ancestors (LCA), where + LCA[i, j] is the index of the lowest common ancestor among token i and j. + If the tokens have no common ancestor within the specified span, + LCA[i, j] will be -1. + + doc (Doc): The index of the token, or the slice of the document + start (int): First token to be included in the LCA matrix. + end (int): Position of next to last token included in the LCA matrix. + RETURNS (int [:, :]): memoryview of numpy.array[ndim=2, dtype=numpy.int32], + with shape (n, n), where n = len(doc). + """ + cdef int [:, :] lca_matrix + cdef int j, k + n_tokens= end - start + lca_mat = numpy.empty((n_tokens, n_tokens), dtype=numpy.int32) + lca_mat.fill(-1) + lca_matrix = lca_mat + for j in range(n_tokens): + token_j = doc[start + j] + # the common ancestor of token and itself is itself: + lca_matrix[j, j] = j + # we will only iterate through tokens in the same sentence + sent = token_j.sent + sent_start = sent.start + j_idx_in_sent = start + j - sent_start + n_missing_tokens_in_sent = len(sent) - j_idx_in_sent + # make sure we do not go past `end`, in cases where `end` < sent.end + max_range = min(j + n_missing_tokens_in_sent, end - start) + for k in range(j + 1, max_range): + lca = _get_tokens_lca(token_j, doc[start + k]) + # if lca is outside of span, we set it to -1 + if not start <= lca < end: + lca_matrix[j, k] = -1 + lca_matrix[k, j] = -1 + else: + lca_matrix[j, k] = lca - start + lca_matrix[k, j] = lca - start + return lca_matrix + + +def pickle_doc(doc): + bytes_data = doc.to_bytes(exclude=["vocab", "user_data", "user_hooks"]) + hooks_and_data = (doc.user_data, doc.user_hooks, doc.user_span_hooks, + doc.user_token_hooks) + return (unpickle_doc, (doc.vocab, srsly.pickle_dumps(hooks_and_data), bytes_data)) + + +def unpickle_doc(vocab, hooks_and_data, bytes_data): + user_data, doc_hooks, span_hooks, token_hooks = srsly.pickle_loads(hooks_and_data) + + doc = Doc(vocab, user_data=user_data).from_bytes(bytes_data, exclude=["user_data"]) + doc.user_hooks.update(doc_hooks) + doc.user_span_hooks.update(span_hooks) + doc.user_token_hooks.update(token_hooks) + return doc + + +copy_reg.pickle(Doc, pickle_doc, unpickle_doc) + + +def remove_label_if_necessary(attributes): + # More deprecated attribute handling =/ + if "label" in attributes: + attributes["ent_type"] = attributes.pop("label") + + +def fix_attributes(doc, attributes): + if "label" in attributes and "ent_type" not in attributes: + if isinstance(attributes["label"], int): + attributes[ENT_TYPE] = attributes["label"] + else: + attributes[ENT_TYPE] = doc.vocab.strings[attributes["label"]] + if "ent_type" in attributes: + attributes[ENT_TYPE] = attributes["ent_type"] + + +def get_entity_info(ent_info): + ent_kb_id = 0 + ent_id = 0 + if isinstance(ent_info, Span): + ent_type = ent_info.label + ent_kb_id = ent_info.kb_id + start = ent_info.start + end = ent_info.end + ent_id = ent_info.id + elif len(ent_info) == 3: + ent_type, start, end = ent_info + elif len(ent_info) == 4: + ent_type, ent_kb_id, start, end = ent_info + else: + ent_id, ent_kb_id, ent_type, start, end = ent_info + return ent_type, ent_kb_id, start, end, ent_id diff --git a/spacy/tokens/graph.pxd b/spacy/tokens/graph.pxd new file mode 100644 index 0000000..083ef65 --- /dev/null +++ b/spacy/tokens/graph.pxd @@ -0,0 +1,14 @@ +from cymem.cymem cimport Pool +from libcpp.vector cimport vector +from preshed.maps cimport PreshMap + +from ..structs cimport EdgeC, GraphC + + +cdef class Graph: + cdef GraphC c + cdef Pool mem + cdef PreshMap node_map + cdef PreshMap edge_map + cdef object doc_ref + cdef public str name diff --git a/spacy/tokens/graph.pyx b/spacy/tokens/graph.pyx new file mode 100644 index 0000000..6c4ce6c --- /dev/null +++ b/spacy/tokens/graph.pyx @@ -0,0 +1,714 @@ +# cython: infer_types=True, cdivision=True, boundscheck=False, binding=True +# cython: profile=False +from typing import Generator, List, Tuple + +cimport cython +from cython.operator cimport dereference +from libc.stdint cimport int32_t +from libcpp.pair cimport pair +from libcpp.unordered_map cimport unordered_map +from libcpp.unordered_set cimport unordered_set + +import weakref + +from murmurhash.mrmr cimport hash64 + +from .. import Errors + +from ..typedefs cimport hash_t + +from ..strings import get_string_id + +from ..structs cimport EdgeC, GraphC + +from .token import Token + + +@cython.freelist(8) +cdef class Edge: + cdef readonly Graph graph + cdef readonly int i + + def __init__(self, Graph graph, int i): + self.graph = graph + self.i = i + + @property + def is_none(self) -> bool: + return False + + @property + def doc(self) -> "Doc": + return self.graph.doc + + @property + def head(self) -> "Node": + return Node(self.graph, self.graph.c.edges[self.i].head) + + @property + def tail(self) -> "Tail": + return Node(self.graph, self.graph.c.edges[self.i].tail) + + @property + def label(self) -> int: + return self.graph.c.edges[self.i].label + + @property + def weight(self) -> float: + return self.graph.c.weights[self.i] + + @property + def label_(self) -> str: + return self.doc.vocab.strings[self.label] + + +@cython.freelist(8) +cdef class Node: + cdef readonly Graph graph + cdef readonly int i + + def __init__(self, Graph graph, int i): + """A reference to a node of an annotation graph. Each node is made up of + an ordered set of zero or more token indices. + + Node references are usually created by the Graph object itself, or from + the Node or Edge objects. You usually won't need to instantiate this + class yourself. + """ + cdef int length = graph.c.nodes.size() + if i >= length or -i >= length: + raise IndexError(Errors.E1034.format(i=i, length=length)) + if i < 0: + i += length + self.graph = graph + self.i = i + + def __eq__(self, other): + if self.graph is not other.graph: + return False + else: + return self.i == other.i + + def __iter__(self) -> Generator[int]: + for i in self.graph.c.nodes[self.i]: + yield i + + def __getitem__(self, int i) -> int: + """Get a token index from the node's set of tokens.""" + length = self.graph.c.nodes[self.i].size() + if i >= length or -i >= length: + raise IndexError(Errors.E1035.format(i=i, length=length)) + if i < 0: + i += length + return self.graph.c.nodes[self.i][i] + + def __len__(self) -> int: + """The number of tokens that make up the node.""" + return self.graph.c.nodes[self.i].size() + + @property + def is_none(self) -> bool: + """Whether the node is a special value, indicating 'none'. + + The NoneNode type is returned by the Graph, Edge and Node objects when + there is no match to a query. It has the same API as Node, but it always + returns NoneNode, NoneEdge or empty lists for its queries. + """ + return False + + @property + def doc(self) -> "Doc": + """The Doc object that the graph refers to.""" + return self.graph.doc + + @property + def tokens(self) -> Tuple[Token]: + """A tuple of Token objects that make up the node.""" + doc = self.doc + return tuple([doc[i] for i in self]) + + def head(self, i=None, label=None) -> "Node": + """Get the head of the first matching edge, searching by index, label, + both or neither. + + For instance, `node.head(i=1)` will get the head of the second edge that + this node is a tail of. `node.head(i=1, label="ARG0")` will further + check that the second edge has the label `"ARG0"`. + + If no matching node can be found, the graph's NoneNode is returned. + """ + return self.headed(i=i, label=label) + + def tail(self, i=None, label=None) -> "Node": + """Get the tail of the first matching edge, searching by index, label, + both or neither. + + If no matching node can be found, the graph's NoneNode is returned. + """ + return self.tailed(i=i, label=label).tail + + def sibling(self, i=None, label=None): + """Get the first matching sibling node. Two nodes are siblings if they + are both tails of the same head. + If no matching node can be found, the graph's NoneNode is returned. + """ + if i is None: + siblings = self.siblings(label=label) + return siblings[0] if siblings else NoneNode(self) + else: + edges = [] + for h in self.headed(): + edges.extend([e for e in h.tailed() if e.tail.i != self.i]) + if i >= len(edges): + return NoneNode(self) + elif label is not None and edges[i].label != label: + return NoneNode(self) + else: + return edges[i].tail + + def heads(self, label=None) -> List["Node"]: + """Find all matching heads of this node.""" + cdef vector[int] edge_indices + self._find_edges(edge_indices, "head", label) + return [Node(self.graph, self.graph.c.edges[i].head) for i in edge_indices] + + def tails(self, label=None) -> List["Node"]: + """Find all matching tails of this node.""" + cdef vector[int] edge_indices + self._find_edges(edge_indices, "tail", label) + return [Node(self.graph, self.graph.c.edges[i].tail) for i in edge_indices] + + def siblings(self, label=None) -> List["Node"]: + """Find all maching siblings of this node. Two nodes are siblings if they + are tails of the same head. + """ + edges = [] + for h in self.headed(): + edges.extend([e for e in h.tailed() if e.tail.i != self.i]) + if label is None: + return [e.tail for e in edges] + else: + return [e.tail for e in edges if e.label == label] + + def headed(self, i=None, label=None) -> Edge: + """Find the first matching edge headed by this node. + If no matching edge can be found, the graph's NoneEdge is returned. + """ + start, end = self._find_range(i, self.c.n_head[self.i]) + idx = self._find_edge("head", start, end, label) + if idx == -1: + return NoneEdge(self.graph) + else: + return Edge(self.graph, idx) + + def tailed(self, i=None, label=None) -> Edge: + """Find the first matching edge tailed by this node. + If no matching edge can be found, the graph's NoneEdge is returned. + """ + start, end = self._find_range(i, self.c.n_tail[self.i]) + idx = self._find_edge("tail", start, end, label) + if idx == -1: + return NoneEdge(self.graph) + else: + return Edge(self.graph, idx) + + def headeds(self, label=None) -> List[Edge]: + """Find all matching edges headed by this node.""" + cdef vector[int] edge_indices + self._find_edges(edge_indices, "head", label) + return [Edge(self.graph, i) for i in edge_indices] + + def taileds(self, label=None) -> List["Edge"]: + """Find all matching edges headed by this node.""" + cdef vector[int] edge_indices + self._find_edges(edge_indices, "tail", label) + return [Edge(self.graph, i) for i in edge_indices] + + def walk_heads(self): + cdef vector[int] node_indices + walk_head_nodes(node_indices, &self.graph.c, self.i) + for i in node_indices: + yield Node(self.graph, i) + + def walk_tails(self): + cdef vector[int] node_indices + walk_tail_nodes(node_indices, &self.graph.c, self.i) + for i in node_indices: + yield Node(self.graph, i) + + cdef (int, int) _get_range(self, i, n): + if i is None: + return (0, n) + elif i < n: + return (i, i+1) + else: + return (0, 0) + + cdef int _find_edge(self, str direction, int start, int end, label) except -2: + if direction == "head": + get_edges = get_head_edges + else: + get_edges = get_tail_edges + cdef vector[int] edge_indices + get_edges(edge_indices, &self.graph.c, self.i) + if label is None: + return edge_indices[start] + for edge_index in edge_indices[start:end]: + if self.graph.c.edges[edge_index].label == label: + return edge_index + else: + return -1 + + cdef int _find_edges(self, vector[int]& edge_indices, str direction, label): + if direction == "head": + get_edges = get_head_edges + else: + get_edges = get_tail_edges + if label is None: + get_edges(edge_indices, &self.graph.c, self.i) + return edge_indices.size() + cdef vector[int] unfiltered + get_edges(unfiltered, &self.graph.c, self.i) + for edge_index in unfiltered: + if self.graph.c.edges[edge_index].label == label: + edge_indices.push_back(edge_index) + return edge_indices.size() + + +cdef class NoneEdge(Edge): + """An Edge subclass, representing a non-result. The NoneEdge has the same + API as other Edge instances, but always returns NoneEdge, NoneNode, or empty + lists. + """ + def __init__(self, graph): + self.graph = graph + self.i = -1 + + @property + def doc(self) -> "Doc": + return self.graph.doc + + @property + def head(self) -> "NoneNode": + return NoneNode(self.graph) + + @property + def tail(self) -> "NoneNode": + return NoneNode(self.graph) + + @property + def label(self) -> int: + return 0 + + @property + def weight(self) -> float: + return 0.0 + + @property + def label_(self) -> str: + return "" + + +cdef class NoneNode(Node): + def __init__(self, graph): + self.graph = graph + self.i = -1 + + def __getitem__(self, int i): + raise IndexError(Errors.E1036) + + def __len__(self): + return 0 + + @property + def is_none(self): + return -1 + + @property + def doc(self): + return self.graph.doc + + @property + def tokens(self): + return tuple() + + def head(self, i=None, label=None): + return self + + def tail(self, i=None, label=None): + return self + + def walk_heads(self): + yield from [] + + def walk_tails(self): + yield from [] + + +cdef class Graph: + """A set of directed labelled relationships between sets of tokens. + + EXAMPLE: + Construction 1 + >>> graph = Graph(doc, name="srl") + + Construction 2 + >>> graph = Graph( + doc, + name="srl", + nodes=[(0,), (1, 3), (,)], + edges=[(0, 2), (2, 1)] + ) + + Construction 3 + >>> graph = Graph( + doc, + name="srl", + nodes=[(0,), (1, 3), (,)], + edges=[(2, 0), (0, 1)], + labels=["word sense ID 1675", "agent"], + weights=[-42.6, -1.7] + ) + >>> assert graph.has_node((0,)) + >>> assert graph.has_edge((0,), (1,3), label="agent") + """ + def __init__( + self, doc, *, name="", nodes=[], edges=[], labels=None, weights=None # no-cython-lint + ): + """Create a Graph object. + + doc (Doc): The Doc object the graph will refer to. + name (str): A string name to help identify the graph. Defaults to "". + nodes (List[Tuple[int]]): A list of token-index tuples to add to the graph + as nodes. Defaults to []. + edges (List[Tuple[int, int]]): A list of edges between the provided nodes. + Each edge should be a (head, tail) tuple, where `head` and `tail` + are integers pointing into the `nodes` list. Defaults to []. + labels (Optional[List[str]]): A list of labels for the provided edges. + If None, all of the edges specified by the edges argument will have + be labelled with the empty string (""). If `labels` is not `None`, + it must have the same length as the `edges` argument. + weights (Optional[List[float]]): A list of weights for the provided edges. + If None, all of the edges specified by the edges argument will + have the weight 0.0. If `weights` is not `None`, it must have the + same length as the `edges` argument. + """ + if weights is not None: + assert len(weights) == len(edges) + else: + weights = [0.0] * len(edges) + if labels is not None: + assert len(labels) == len(edges) + else: + labels = [""] * len(edges) + self.c.node_map = new unordered_map[hash_t, int]() + self.c.edge_map = new unordered_map[hash_t, int]() + self.c.roots = new unordered_set[int]() + self.name = name + self.doc_ref = weakref.ref(doc) + for node in nodes: + self.add_node(node) + for (head, tail), label, weight in zip(edges, labels, weights): + self.add_edge( + Node(self, head), + Node(self, tail), + label=label, + weight=weight + ) + + def __dealloc__(self): + del self.c.node_map + del self.c.edge_map + del self.c.roots + + @property + def doc(self) -> "Doc": + """The Doc object the graph refers to.""" + return self.doc_ref() + + @property + def edges(self) -> Generator[Edge]: + """Iterate over the edges in the graph.""" + for i in range(self.c.edges.size()): + yield Edge(self, i) + + @property + def nodes(self) -> Generator[Node]: + """Iterate over the nodes in the graph.""" + for i in range(self.c.nodes.size()): + yield Node(self, i) + + def add_edge(self, head, tail, *, label="", weight=None) -> Edge: + """Add an edge to the graph, connecting two groups of tokens. + + If there is already an edge for the (head, tail, label) triple, it will + be returned, and no new edge will be created. The weight of the edge + will be updated if a weight is specified. + """ + edge_index = add_edge( + &self.c, + EdgeC( + head=self.add_node(head).i, + tail=self.add_node(tail).i, + label=self.doc.vocab.strings.as_int(label), + ), + weight=weight if weight is not None else 0.0 + ) + return Edge(self, edge_index) + + def get_edge(self, head, tail, *, label="") -> Edge: + """Look up an edge in the graph. If the graph has no matching edge, + the NoneEdge object is returned. + """ + head_node = self.get_node(head) + if head_node.is_none: + return NoneEdge(self) + tail_node = self.get_node(tail) + if tail_node.is_none: + return NoneEdge(self) + edge_index = get_edge( + &self.c, + EdgeC(head=head_node.i, tail=tail_node.i, label=get_string_id(label)) + ) + if edge_index < 0: + return NoneEdge(self) + else: + return Edge(self, edge_index) + + def has_edge(self, head, tail, label) -> bool: + """Check whether a (head, tail, label) triple is an edge in the graph.""" + return not self.get_edge(head, tail, label=label).is_none + + def add_node(self, indices) -> Node: + """Add a node to the graph and return it. Nodes refer to ordered sets + of token indices. + + This method is idempotent: if there is already a node for the given + indices, it is returned without a new node being created. + """ + if isinstance(indices, Node): + return indices + cdef vector[int32_t] node + node.reserve(len(indices)) + for idx in indices: + node.push_back(idx) + i = add_node(&self.c, node) + return Node(self, i) + + def get_node(self, indices) -> Node: + """Get a node from the graph, or the NoneNode if there is no node for + the given indices. + """ + if isinstance(indices, Node): + return indices + cdef vector[int32_t] node + node.reserve(len(indices)) + for idx in indices: + node.push_back(idx) + node_index = get_node(&self.c, node) + if node_index < 0: + return NoneNode(self) + else: + return Node(self, node_index) + + def has_node(self, tuple indices) -> bool: + """Check whether the graph has a node for the given indices.""" + return not self.get_node(indices).is_none + + +cdef int add_edge(GraphC* graph, EdgeC edge, float weight) nogil: + key = hash64(&edge, sizeof(edge), 0) + it = graph.edge_map.find(key) + if it != graph.edge_map.end(): + edge_index = dereference(it).second + graph.weights[edge_index] = weight + return edge_index + else: + edge_index = graph.edges.size() + graph.edge_map.insert(pair[hash_t, int](key, edge_index)) + graph.edges.push_back(edge) + if graph.n_tails[edge.head] == 0: + graph.first_tail[edge.head] = edge_index + if graph.n_heads[edge.tail] == 0: + graph.first_head[edge.tail] = edge_index + graph.n_tails[edge.head] += 1 + graph.n_heads[edge.tail] += 1 + graph.weights.push_back(weight) + # If we had the tail marked as a root, remove it. + tail_root_index = graph.roots.find(edge.tail) + if tail_root_index != graph.roots.end(): + graph.roots.erase(tail_root_index) + return edge_index + + +cdef int get_edge(const GraphC* graph, EdgeC edge) nogil: + key = hash64(&edge, sizeof(edge), 0) + it = graph.edge_map.find(key) + if it == graph.edge_map.end(): + return -1 + else: + return dereference(it).second + + +cdef int has_edge(const GraphC* graph, EdgeC edge) nogil: + return get_edge(graph, edge) >= 0 + + +cdef int add_node(GraphC* graph, vector[int32_t]& node) nogil: + key = hash64(&node[0], node.size() * sizeof(node[0]), 0) + it = graph.node_map.find(key) + if it != graph.node_map.end(): + # Item found. Convert the iterator to an index value. + return dereference(it).second + else: + index = graph.nodes.size() + graph.nodes.push_back(node) + graph.n_heads.push_back(0) + graph.n_tails.push_back(0) + graph.first_head.push_back(0) + graph.first_tail.push_back(0) + graph.roots.insert(index) + graph.node_map.insert(pair[hash_t, int](key, index)) + return index + + +cdef int get_node(const GraphC* graph, vector[int32_t] node) nogil: + key = hash64(&node[0], node.size() * sizeof(node[0]), 0) + it = graph.node_map.find(key) + if it == graph.node_map.end(): + return -1 + else: + return dereference(it).second + + +cdef int has_node(const GraphC* graph, vector[int32_t] node) nogil: + return get_node(graph, node) >= 0 + + +cdef int get_head_nodes(vector[int]& output, const GraphC* graph, int node) nogil: + todo = graph.n_heads[node] + if todo == 0: + return 0 + output.reserve(output.size() + todo) + start = graph.first_head[node] + end = graph.edges.size() + for i in range(start, end): + if todo <= 0: + break + elif graph.edges[i].tail == node: + output.push_back(graph.edges[i].head) + todo -= 1 + return todo + + +cdef int get_tail_nodes(vector[int]& output, const GraphC* graph, int node) nogil: + todo = graph.n_tails[node] + if todo == 0: + return 0 + output.reserve(output.size() + todo) + start = graph.first_tail[node] + end = graph.edges.size() + for i in range(start, end): + if todo <= 0: + break + elif graph.edges[i].head == node: + output.push_back(graph.edges[i].tail) + todo -= 1 + return todo + + +cdef int get_sibling_nodes(vector[int]& output, const GraphC* graph, int node) nogil: + cdef vector[int] heads + cdef vector[int] tails + get_head_nodes(heads, graph, node) + for i in range(heads.size()): + get_tail_nodes(tails, graph, heads[i]) + for j in range(tails.size()): + if tails[j] != node: + output.push_back(tails[j]) + tails.clear() + return output.size() + + +cdef int get_head_edges(vector[int]& output, const GraphC* graph, int node) nogil: + todo = graph.n_heads[node] + if todo == 0: + return 0 + output.reserve(output.size() + todo) + start = graph.first_head[node] + end = graph.edges.size() + for i in range(start, end): + if todo <= 0: + break + elif graph.edges[i].tail == node: + output.push_back(i) + todo -= 1 + return todo + + +cdef int get_tail_edges(vector[int]& output, const GraphC* graph, int node) nogil: + todo = graph.n_tails[node] + if todo == 0: + return 0 + output.reserve(output.size() + todo) + start = graph.first_tail[node] + end = graph.edges.size() + for i in range(start, end): + if todo <= 0: + break + elif graph.edges[i].head == node: + output.push_back(i) + todo -= 1 + return todo + + +cdef int walk_head_nodes(vector[int]& output, const GraphC* graph, int node) nogil: + cdef unordered_set[int] seen = unordered_set[int]() + get_head_nodes(output, graph, node) + seen.insert(node) + i = 0 + while i < output.size(): + if seen.find(output[i]) == seen.end(): + seen.insert(output[i]) + get_head_nodes(output, graph, output[i]) + i += 1 + return i + + +cdef int walk_tail_nodes(vector[int]& output, const GraphC* graph, int node) nogil: + cdef unordered_set[int] seen = unordered_set[int]() + get_tail_nodes(output, graph, node) + seen.insert(node) + i = 0 + while i < output.size(): + if seen.find(output[i]) == seen.end(): + seen.insert(output[i]) + get_tail_nodes(output, graph, output[i]) + i += 1 + return i + + +cdef int walk_head_edges(vector[int]& output, const GraphC* graph, int node) nogil: + cdef unordered_set[int] seen = unordered_set[int]() + get_head_edges(output, graph, node) + seen.insert(node) + i = 0 + while i < output.size(): + if seen.find(output[i]) == seen.end(): + seen.insert(output[i]) + get_head_edges(output, graph, output[i]) + i += 1 + return i + + +cdef int walk_tail_edges(vector[int]& output, const GraphC* graph, int node) nogil: + cdef unordered_set[int] seen = unordered_set[int]() + get_tail_edges(output, graph, node) + seen.insert(node) + i = 0 + while i < output.size(): + if seen.find(output[i]) == seen.end(): + seen.insert(output[i]) + get_tail_edges(output, graph, output[i]) + i += 1 + return i diff --git a/spacy/tokens/morphanalysis.pxd b/spacy/tokens/morphanalysis.pxd new file mode 100644 index 0000000..728f0aa --- /dev/null +++ b/spacy/tokens/morphanalysis.pxd @@ -0,0 +1,9 @@ +from ..structs cimport MorphAnalysisC +from ..typedefs cimport hash_t +from ..vocab cimport Vocab + + +cdef class MorphAnalysis: + cdef readonly Vocab vocab + cdef readonly hash_t key + cdef MorphAnalysisC c diff --git a/spacy/tokens/morphanalysis.pyi b/spacy/tokens/morphanalysis.pyi new file mode 100644 index 0000000..b35ff36 --- /dev/null +++ b/spacy/tokens/morphanalysis.pyi @@ -0,0 +1,21 @@ +from typing import Any, Dict, Iterator, List, Optional, Union + +from ..vocab import Vocab + +class MorphAnalysis: + def __init__( + self, vocab: Vocab, features: Union[Dict[str, str], str] = ... + ) -> None: ... + @classmethod + def from_id(cls, vocab: Vocab, key: Any) -> MorphAnalysis: ... + def __contains__(self, feature: str) -> bool: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + def __hash__(self) -> int: ... + def __eq__(self, other: MorphAnalysis) -> bool: ... # type: ignore[override] + def __ne__(self, other: MorphAnalysis) -> bool: ... # type: ignore[override] + def get(self, field: Any, default: Optional[List[str]]) -> List[str]: ... + def to_json(self) -> str: ... + def to_dict(self) -> Dict[str, str]: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... diff --git a/spacy/tokens/morphanalysis.pyx b/spacy/tokens/morphanalysis.pyx new file mode 100644 index 0000000..ea5d07f --- /dev/null +++ b/spacy/tokens/morphanalysis.pyx @@ -0,0 +1,92 @@ +# cython: profile=False +cimport numpy as np +from libc.string cimport memset + +from ..errors import Errors +from ..morphology import Morphology + +from ..morphology cimport check_feature, get_by_field, list_features +from ..typedefs cimport attr_t, hash_t +from ..vocab cimport Vocab + + +cdef class MorphAnalysis: + """Control access to morphological features for a token.""" + def __init__(self, Vocab vocab, features=dict()): + self.vocab = vocab + self.key = self.vocab.morphology.add(features) + analysis = self.vocab.morphology.tags.get(self.key) + if analysis is not NULL: + self.c = analysis[0] + else: + memset(&self.c, 0, sizeof(self.c)) + + @classmethod + def from_id(cls, Vocab vocab, hash_t key): + """Create a morphological analysis from a given ID.""" + cdef MorphAnalysis morph = MorphAnalysis.__new__(MorphAnalysis, vocab) + morph.vocab = vocab + morph.key = key + analysis = vocab.morphology.tags.get(key) + if analysis is not NULL: + morph.c = analysis[0] + else: + memset(&morph.c, 0, sizeof(morph.c)) + return morph + + def __contains__(self, feature): + """Test whether the morphological analysis contains some feature.""" + cdef attr_t feat_id = self.vocab.strings.as_int(feature) + return check_feature(&self.c, feat_id) + + def __iter__(self): + """Iterate over the features in the analysis.""" + cdef attr_t feature + for feature in list_features(&self.c): + yield self.vocab.strings[feature] + + def __len__(self): + """The number of features in the analysis.""" + return self.c.length + + def __hash__(self): + return self.key + + def __eq__(self, other): + if isinstance(other, str): + raise ValueError(Errors.E977) + return self.key == other.key + + def __ne__(self, other): + return self.key != other.key + + def get(self, field, default=None): + """Retrieve feature values by field.""" + cdef attr_t field_id = self.vocab.strings.as_int(field) + cdef np.ndarray results = get_by_field(&self.c, field_id) + if len(results) == 0: + if default is None: + default = [] + return default + features = [self.vocab.strings[result] for result in results] + return [f.split(Morphology.FIELD_SEP)[1] for f in features] + + def to_json(self): + """Produce a json serializable representation as a UD FEATS-style + string. + """ + morph_string = self.vocab.strings[self.c.key] + if morph_string == self.vocab.morphology.EMPTY_MORPH: + return "" + return morph_string + + def to_dict(self): + """Produce a dict representation. + """ + return self.vocab.morphology.feats_to_dict(self.to_json()) + + def __str__(self): + return self.to_json() + + def __repr__(self): + return self.to_json() diff --git a/spacy/tokens/span.pxd b/spacy/tokens/span.pxd new file mode 100644 index 0000000..d77bbea --- /dev/null +++ b/spacy/tokens/span.pxd @@ -0,0 +1,25 @@ +cimport numpy as np + +from ..structs cimport SpanC +from ..typedefs cimport attr_t +from .doc cimport Doc + + +cdef class Span: + cdef readonly Doc doc + cdef SpanC c + cdef public _vector + cdef public _vector_norm + + @staticmethod + cdef inline Span cinit(Doc doc, SpanC span): + cdef Span self = Span.__new__( + Span, + doc, + start=span.start, + end=span.end + ) + self.c = span + return self + + cpdef np.ndarray to_array(self, object features) diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi new file mode 100644 index 0000000..b982eb8 --- /dev/null +++ b/spacy/tokens/span.pyi @@ -0,0 +1,139 @@ +from typing import Any, Callable, Iterator, Optional, Protocol, Tuple, Union, overload + +from thinc.types import Floats1d, FloatsXd, Ints2d + +from ..lexeme import Lexeme +from ..vocab import Vocab +from .doc import Doc +from .token import Token +from .underscore import Underscore + +class SpanMethod(Protocol): + def __call__(self: Span, *args: Any, **kwargs: Any) -> Any: ... # type: ignore[misc] + +class Span: + @classmethod + def set_extension( + cls, + name: str, + default: Optional[Any] = ..., + getter: Optional[Callable[[Span], Any]] = ..., + setter: Optional[Callable[[Span, Any], None]] = ..., + method: Optional[SpanMethod] = ..., + force: bool = ..., + ) -> None: ... + @classmethod + def get_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[SpanMethod], + Optional[Callable[[Span], Any]], + Optional[Callable[[Span, Any], None]], + ]: ... + @classmethod + def has_extension(cls, name: str) -> bool: ... + @classmethod + def remove_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[SpanMethod], + Optional[Callable[[Span], Any]], + Optional[Callable[[Span, Any], None]], + ]: ... + def __init__( + self, + doc: Doc, + start: int, + end: int, + label: Union[str, int] = ..., + vector: Optional[Floats1d] = ..., + vector_norm: Optional[float] = ..., + kb_id: Union[str, int] = ..., + span_id: Union[str, int] = ..., + ) -> None: ... + def __lt__(self, other: Any) -> bool: ... + def __le__(self, other: Any) -> bool: ... + def __eq__(self, other: Any) -> bool: ... + def __ne__(self, other: Any) -> bool: ... + def __gt__(self, other: Any) -> bool: ... + def __ge__(self, other: Any) -> bool: ... + def __hash__(self) -> int: ... + def __len__(self) -> int: ... + def __repr__(self) -> str: ... + @overload + def __getitem__(self, i: int) -> Token: ... + @overload + def __getitem__(self, i: slice) -> Span: ... + def __iter__(self) -> Iterator[Token]: ... + @property + def _(self) -> Underscore: ... + def as_doc(self, *, copy_user_data: bool = ...) -> Doc: ... + def get_lca_matrix(self) -> Ints2d: ... + def similarity(self, other: Union[Doc, Span, Token, Lexeme]) -> float: ... + @property + def doc(self) -> Doc: ... + @property + def vocab(self) -> Vocab: ... + @property + def sent(self) -> Span: ... + @property + def ents(self) -> Tuple[Span]: ... + @property + def has_vector(self) -> bool: ... + @property + def vector(self) -> Floats1d: ... + @property + def vector_norm(self) -> float: ... + @property + def tensor(self) -> FloatsXd: ... + @property + def sentiment(self) -> float: ... + @property + def text(self) -> str: ... + @property + def text_with_ws(self) -> str: ... + @property + def noun_chunks(self) -> Iterator[Span]: ... + @property + def root(self) -> Token: ... + def char_span( + self, + start_idx: int, + end_idx: int, + label: Union[int, str] = ..., + kb_id: Union[int, str] = ..., + vector: Optional[Floats1d] = ..., + id: Union[int, str] = ..., + alignment_mode: str = ..., + span_id: Union[int, str] = ..., + ) -> Span: ... + @property + def conjuncts(self) -> Tuple[Token]: ... + @property + def lefts(self) -> Iterator[Token]: ... + @property + def rights(self) -> Iterator[Token]: ... + @property + def n_lefts(self) -> int: ... + @property + def n_rights(self) -> int: ... + @property + def subtree(self) -> Iterator[Token]: ... + start: int + end: int + start_char: int + end_char: int + label: int + kb_id: int + id: int + ent_id: int + ent_id_: str + @property + def orth_(self) -> str: ... + @property + def lemma_(self) -> str: ... + label_: str + kb_id_: str + id_: str diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx new file mode 100644 index 0000000..a7faf0d --- /dev/null +++ b/spacy/tokens/span.pyx @@ -0,0 +1,897 @@ +# cython: profile=False +cimport numpy as np + +import copy +import warnings + +import numpy +from thinc.api import get_array_module + +from ..attrs cimport * +from ..attrs cimport ORTH, attr_id_t +from ..lexeme cimport Lexeme +from ..structs cimport TokenC +from ..symbols cimport dep +from ..typedefs cimport attr_t, hash_t +from .doc cimport _get_lca_matrix, get_token_attr +from .token cimport Token + +from ..errors import Errors, Warnings +from ..util import normalize_slice +from .underscore import Underscore, get_ext_args + + +cdef class Span: + """A slice from a Doc object. + + DOCS: https://spacy.io/api/span + """ + @classmethod + def set_extension(cls, name, **kwargs): + """Define a custom attribute which becomes available as `Span._`. + + name (str): Name of the attribute to set. + default: Optional default value of the attribute. + getter (callable): Optional getter function. + setter (callable): Optional setter function. + method (callable): Optional method for method extension. + force (bool): Force overwriting existing attribute. + + DOCS: https://spacy.io/api/span#set_extension + USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes + """ + if cls.has_extension(name) and not kwargs.get("force", False): + raise ValueError(Errors.E090.format(name=name, obj="Span")) + Underscore.span_extensions[name] = get_ext_args(**kwargs) + + @classmethod + def get_extension(cls, name): + """Look up a previously registered extension by name. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple. + + DOCS: https://spacy.io/api/span#get_extension + """ + return Underscore.span_extensions.get(name) + + @classmethod + def has_extension(cls, name): + """Check whether an extension has been registered. + + name (str): Name of the extension. + RETURNS (bool): Whether the extension has been registered. + + DOCS: https://spacy.io/api/span#has_extension + """ + return name in Underscore.span_extensions + + @classmethod + def remove_extension(cls, name): + """Remove a previously registered extension. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple of the + removed extension. + + DOCS: https://spacy.io/api/span#remove_extension + """ + if not cls.has_extension(name): + raise ValueError(Errors.E046.format(name=name)) + return Underscore.span_extensions.pop(name) + + def __cinit__(self, Doc doc, int start, int end, label=0, vector=None, + vector_norm=None, kb_id=0, span_id=0): + """Create a `Span` object from the slice `doc[start : end]`. + + doc (Doc): The parent document. + start (int): The index of the first token of the span. + end (int): The index of the first token after the span. + label (Union[int, str]): A label to attach to the Span, e.g. for named + entities. + vector (ndarray[ndim=1, dtype='float32']): A meaning representation + of the span. + vector_norm (float): The L2 norm of the span's vector representation. + kb_id (Union[int, str]): An identifier from a Knowledge Base to capture + the meaning of a named entity. + span_id (Union[int, str]): An identifier to associate with the span. + + DOCS: https://spacy.io/api/span#init + """ + if not (0 <= start <= end <= len(doc)): + raise IndexError(Errors.E035.format(start=start, end=end, length=len(doc))) + self.doc = doc + if isinstance(label, str): + label = doc.vocab.strings.add(label) + if isinstance(kb_id, str): + kb_id = doc.vocab.strings.add(kb_id) + if isinstance(span_id, str): + span_id = doc.vocab.strings.add(span_id) + if label not in doc.vocab.strings: + raise ValueError(Errors.E084.format(label=label)) + + start_char = doc[start].idx if start < doc.length else len(doc.text) + if start == end: + end_char = start_char + else: + end_char = doc[end - 1].idx + len(doc[end - 1]) + self.c = SpanC( + label=label, + kb_id=kb_id, + id=span_id, + start=start, + end=end, + start_char=start_char, + end_char=end_char, + ) + self._vector = vector + self._vector_norm = vector_norm + + def __richcmp__(self, object other, int op): + if other is None: + if op == 0 or op == 1 or op == 2: + return False + else: + return True + if not isinstance(other, Span): + return False + cdef Span other_span = other + self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.id, self.doc) + other_tuple = (other_span.c.start_char, other_span.c.end_char, other_span.c.label, other_span.c.kb_id, other_span.id, other_span.doc) + # < + if op == 0: + return self_tuple < other_tuple + # <= + elif op == 1: + return self_tuple <= other_tuple + # == + elif op == 2: + return self_tuple == other_tuple + # != + elif op == 3: + return self_tuple != other_tuple + # > + elif op == 4: + return self_tuple > other_tuple + # >= + elif op == 5: + return self_tuple >= other_tuple + + def __hash__(self): + return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.c.id)) + + def __len__(self): + """Get the number of tokens in the span. + + RETURNS (int): The number of tokens in the span. + + DOCS: https://spacy.io/api/span#len + """ + if self.c.end < self.c.start: + return 0 + return self.c.end - self.c.start + + def __repr__(self): + return self.text + + def __getitem__(self, object i): + """Get a `Token` or a `Span` object + + i (int or tuple): The index of the token within the span, or slice of + the span to get. + RETURNS (Token or Span): The token at `span[i]`. + + DOCS: https://spacy.io/api/span#getitem + """ + if isinstance(i, slice): + start, end = normalize_slice(len(self), i.start, i.stop, i.step) + return Span(self.doc, start + self.start, end + self.start) + else: + if i < 0: + token_i = self.c.end + i + else: + token_i = self.c.start + i + if self.c.start <= token_i < self.c.end: + return self.doc[token_i] + else: + raise IndexError(Errors.E1002) + + def __iter__(self): + """Iterate over `Token` objects. + + YIELDS (Token): A `Token` object. + + DOCS: https://spacy.io/api/span#iter + """ + for i in range(self.c.start, self.c.end): + yield self.doc[i] + + def __reduce__(self): + raise NotImplementedError(Errors.E112) + + @property + def _(self): + """Custom extension attributes registered via `set_extension`.""" + return Underscore(Underscore.span_extensions, self, + start=self.c.start_char, end=self.c.end_char) + + def as_doc(self, *, bint copy_user_data=False, array_head=None, array=None): + """Create a `Doc` object with a copy of the `Span`'s data. + + copy_user_data (bool): Whether or not to copy the original doc's user data. + array_head (tuple): `Doc` array attrs, can be passed in to speed up computation. + array (ndarray): `Doc` as array, can be passed in to speed up computation. + RETURNS (Doc): The `Doc` copy of the span. + + DOCS: https://spacy.io/api/span#as_doc + """ + words = [t.text for t in self] + spaces = [bool(t.whitespace_) for t in self] + cdef Doc doc = Doc(self.doc.vocab, words=words, spaces=spaces) + if array_head is None: + array_head = self.doc._get_array_attrs() + if array is None: + array = self.doc.to_array(array_head) + array = array[self.start : self.end] + self._fix_dep_copy(array_head, array) + # Fix initial IOB so the entities are valid for doc.ents below. + if len(array) > 0 and ENT_IOB in array_head: + ent_iob_col = array_head.index(ENT_IOB) + if array[0][ent_iob_col] == 1: + array[0][ent_iob_col] = 3 + doc.from_array(array_head, array) + # Set partial entities at the beginning or end of the span to have + # missing entity annotation. Note: the initial partial entity could be + # detected from the IOB annotation but the final partial entity can't, + # so detect and remove both in the same way by checking self.ents. + span_ents = {(ent.start, ent.end) for ent in self.ents} + doc_ents = doc.ents + if len(doc_ents) > 0: + # Remove initial partial ent + if (doc_ents[0].start + self.start, doc_ents[0].end + self.start) not in span_ents: + doc.set_ents([], missing=[doc_ents[0]], default="unmodified") + # Remove final partial ent + if (doc_ents[-1].start + self.start, doc_ents[-1].end + self.start) not in span_ents: + doc.set_ents([], missing=[doc_ents[-1]], default="unmodified") + doc.noun_chunks_iterator = self.doc.noun_chunks_iterator + doc.user_hooks = self.doc.user_hooks + doc.user_span_hooks = self.doc.user_span_hooks + doc.user_token_hooks = self.doc.user_token_hooks + doc.vector = self.vector + doc.vector_norm = self.vector_norm + doc.tensor = self.doc.tensor[self.start : self.end] + for key, value in self.doc.cats.items(): + if hasattr(key, "__len__") and len(key) == 3: + cat_start, cat_end, cat_label = key + if cat_start == self.start_char and cat_end == self.end_char: + doc.cats[cat_label] = value + if copy_user_data: + user_data = {} + char_offset = self.start_char + for key, value in self.doc.user_data.items(): + if isinstance(key, tuple) and len(key) == 4 and key[0] == "._.": + data_type, name, start, end = key + if start is not None or end is not None: + start -= char_offset + if end is not None: + end -= char_offset + user_data[(data_type, name, start, end)] = copy.copy(value) + else: + user_data[key] = copy.copy(value) + doc.user_data = user_data + return doc + + def _fix_dep_copy(self, attrs, array): + """ Rewire dependency links to make sure their heads fall into the span + while still keeping the correct number of sentences. """ + cdef int length = len(array) + cdef attr_t value + cdef int i, head_col, ancestor_i + old_to_new_root = dict() + if HEAD in attrs: + head_col = attrs.index(HEAD) + for i in range(length): + # if the HEAD refers to a token outside this span, find a more appropriate ancestor + token = self[i] + ancestor_i = token.head.i - self.c.start # span offset + if ancestor_i not in range(length): + if DEP in attrs: + array[i, attrs.index(DEP)] = dep + + # try finding an ancestor within this span + ancestors = token.ancestors + for ancestor in ancestors: + ancestor_i = ancestor.i - self.c.start + if ancestor_i in range(length): + array[i, head_col] = numpy.int32(ancestor_i - i).astype(numpy.uint64) + + # if there is no appropriate ancestor, define a new artificial root + value = array[i, head_col] + if (i+value) not in range(length): + new_root = old_to_new_root.get(ancestor_i, None) + if new_root is not None: + # take the same artificial root as a previous token from the same sentence + array[i, head_col] = numpy.int32(new_root - i).astype(numpy.uint64) + else: + # set this token as the new artificial root + array[i, head_col] = 0 + old_to_new_root[ancestor_i] = i + + return array + + def get_lca_matrix(self): + """Calculates a matrix of Lowest Common Ancestors (LCA) for a given + `Span`, where LCA[i, j] is the index of the lowest common ancestor among + the tokens span[i] and span[j]. If they have no common ancestor within + the span, LCA[i, j] will be -1. + + RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape + (n, n), where n = len(self). + + DOCS: https://spacy.io/api/span#get_lca_matrix + """ + return numpy.asarray(_get_lca_matrix(self.doc, self.c.start, self.c.end)) + + def similarity(self, other): + """Make a semantic similarity estimate. The default estimate is cosine + similarity using an average of word vectors. + + other (object): The object to compare with. By default, accepts `Doc`, + `Span`, `Token` and `Lexeme` objects. + RETURNS (float): A scalar similarity score. Higher is more similar. + + DOCS: https://spacy.io/api/span#similarity + """ + if "similarity" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["similarity"](self, other) + attr = getattr(self.doc.vocab.vectors, "attr", ORTH) + cdef Token this_token + cdef Token other_token + cdef Lexeme other_lex + if len(self) == 1 and isinstance(other, Token): + this_token = self[0] + other_token = other + if Token.get_struct_attr(this_token.c, attr) == Token.get_struct_attr(other_token.c, attr): + return 1.0 + elif len(self) == 1 and isinstance(other, Lexeme): + this_token = self[0] + other_lex = other + if Token.get_struct_attr(this_token.c, attr) == Lexeme.get_struct_attr(other_lex.c, attr): + return 1.0 + elif isinstance(other, (Doc, Span)) and len(self) == len(other): + similar = True + for i in range(len(self)): + this_token = self[i] + other_token = other[i] + if Token.get_struct_attr(this_token.c, attr) != Token.get_struct_attr(other_token.c, attr): + similar = False + break + if similar: + return 1.0 + if self.vocab.vectors.n_keys == 0: + warnings.warn(Warnings.W007.format(obj="Span")) + if self.vector_norm == 0.0 or other.vector_norm == 0.0: + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Span")) + return 0.0 + vector = self.vector + xp = get_array_module(vector) + result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm) + # ensure we get a scalar back (numpy does this automatically but cupy doesn't) + return result.item() + + cpdef np.ndarray to_array(self, object py_attr_ids): + """Given a list of M attribute IDs, export the tokens to a numpy + `ndarray` of shape `(N, M)`, where `N` is the length of the document. + The values will be 32-bit integers. + + attr_ids (list[int]): A list of attribute ID ints. + RETURNS (numpy.ndarray[long, ndim=2]): A feature matrix, with one row + per word, and one column per attribute indicated in the input + `attr_ids`. + """ + cdef int i, j + cdef attr_id_t feature + cdef np.ndarray[attr_t, ndim=2] output + # Make an array from the attributes - otherwise our inner loop is Python + # dict iteration + cdef np.ndarray[attr_t, ndim=1] attr_ids = numpy.asarray(py_attr_ids, dtype=numpy.uint64) + cdef int length = self.end - self.start + output = numpy.ndarray(shape=(length, len(attr_ids)), dtype=numpy.uint64) + for i in range(self.start, self.end): + for j, feature in enumerate(attr_ids): + output[i-self.start, j] = get_token_attr(&self.doc.c[i], feature) + return output + + @property + def vocab(self): + """RETURNS (Vocab): The Span's Doc's vocab.""" + return self.doc.vocab + + @property + def sent(self): + """Obtain the sentence that contains this span. If the given span + crosses sentence boundaries, return only the first sentence + to which it belongs. + + RETURNS (Span): The sentence span that the span is a part of. + """ + if "sent" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["sent"](self) + elif "sents" in self.doc.user_hooks: + for sentence in self.doc.user_hooks["sents"](self.doc): + if sentence.start <= self.start < sentence.end: + return sentence + # Use `sent_start` token attribute to find sentence boundaries + cdef int n = 0 + if self.doc.has_annotation("SENT_START"): + # Find start of the sentence + start = self.start + while self.doc.c[start].sent_start != 1 and start > 0: + start += -1 + # Find end of the sentence - can be within the entity + end = self.start + 1 + while end < self.doc.length and self.doc.c[end].sent_start != 1: + end += 1 + n += 1 + if n >= self.doc.length: + break + return self.doc[start:end] + else: + raise ValueError(Errors.E030) + + @property + def sents(self): + """Obtain the sentences that contain this span. If the given span + crosses sentence boundaries, return all sentences it is a part of. + + RETURNS (Iterable[Span]): All sentences that the span is a part of. + + DOCS: https://spacy.io/api/span#sents + """ + cdef int start + cdef int i + + if "sents" in self.doc.user_span_hooks: + yield from self.doc.user_span_hooks["sents"](self) + elif "sents" in self.doc.user_hooks: + for sentence in self.doc.user_hooks["sents"](self.doc): + if sentence.end > self.start: + if sentence.start < self.end or sentence.start == self.start == self.end: + yield sentence + else: + break + else: + if not self.doc.has_annotation("SENT_START"): + raise ValueError(Errors.E030) + # Use `sent_start` token attribute to find sentence boundaries + # Find start of the 1st sentence of the Span + start = self.start + while self.doc.c[start].sent_start != 1 and start > 0: + start -= 1 + + # Now, find all the sentences in the span + for i in range(start + 1, self.doc.length): + if self.doc.c[i].sent_start == 1: + yield Span(self.doc, start, i) + start = i + if start >= self.end: + break + elif i == self.doc.length - 1: + yield Span(self.doc, start, self.doc.length) + else: + # Ensure that trailing parts of the Span instance are included in last element of .sents. + # We only want to do this if we didn't break above + if start == self.doc.length - 1: + yield Span(self.doc, start, self.doc.length) + + @property + def ents(self): + """The named entities that fall completely within the span. Returns + a tuple of `Span` objects. + + RETURNS (tuple): Entities in the span, one `Span` per entity. + + DOCS: https://spacy.io/api/span#ents + """ + cdef Span ent + ents = [] + for ent in self.doc.ents: + if ent.c.start >= self.c.start: + if ent.c.end <= self.c.end: + ents.append(ent) + else: + break + return ents + + @property + def has_vector(self): + """A boolean value indicating whether a word vector is associated with + the object. + + RETURNS (bool): Whether a word vector is associated with the object. + + DOCS: https://spacy.io/api/span#has_vector + """ + if "has_vector" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["has_vector"](self) + elif self.vocab.vectors.size > 0: + return any(token.has_vector for token in self) + elif self.doc.tensor.size > 0: + return True + else: + return False + + @property + def vector(self): + """A real-valued meaning representation. Defaults to an average of the + token vectors. + + RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array + representing the span's semantics. + + DOCS: https://spacy.io/api/span#vector + """ + if "vector" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["vector"](self) + if self._vector is None: + if not len(self): + xp = get_array_module(self.vocab.vectors.data) + self._vector = xp.zeros((self.vocab.vectors_length,), dtype="f") + else: + self._vector = sum(t.vector for t in self) / len(self) + return self._vector + + @property + def vector_norm(self): + """The L2 norm of the span's vector representation. + + RETURNS (float): The L2 norm of the vector representation. + + DOCS: https://spacy.io/api/span#vector_norm + """ + if "vector_norm" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["vector"](self) + if self._vector_norm is None: + vector = self.vector + total = (vector*vector).sum() + xp = get_array_module(vector) + self._vector_norm = xp.sqrt(total) if total != 0. else 0. + return self._vector_norm + + @property + def tensor(self): + """The span's slice of the doc's tensor. + + RETURNS (ndarray[ndim=2, dtype='float32']): A 2D numpy or cupy array + representing the span's semantics. + """ + if self.doc.tensor is None: + return None + return self.doc.tensor[self.start : self.end] + + @property + def sentiment(self): + """RETURNS (float): A scalar value indicating the positivity or + negativity of the span. + """ + if "sentiment" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["sentiment"](self) + else: + return sum([token.sentiment for token in self]) / len(self) + + @property + def text(self): + """RETURNS (str): The original verbatim text of the span.""" + text = self.text_with_ws + if len(self) > 0 and self[-1].whitespace_: + text = text[:-1] + return text + + @property + def text_with_ws(self): + """The text content of the span with a trailing whitespace character if + the last token has one. + + RETURNS (str): The text content of the span (with trailing + whitespace). + """ + return "".join([t.text_with_ws for t in self]) + + @property + def noun_chunks(self): + """Iterate over the base noun phrases in the span. Yields base + noun-phrase #[code Span] objects, if the language has a noun chunk iterator. + Raises a NotImplementedError otherwise. + + A base noun phrase, or "NP chunk", is a noun + phrase that does not permit other NPs to be nested within it – so no + NP-level coordination, no prepositional phrases, and no relative + clauses. + + YIELDS (Span): Noun chunks in the span. + + DOCS: https://spacy.io/api/span#noun_chunks + """ + for span in self.doc.noun_chunks: + if span.start >= self.start and span.end <= self.end: + yield span + + @property + def root(self): + """The token with the shortest path to the root of the + sentence (or the root itself). If multiple tokens are equally + high in the tree, the first token is taken. + + RETURNS (Token): The root token. + + DOCS: https://spacy.io/api/span#root + """ + if "root" in self.doc.user_span_hooks: + return self.doc.user_span_hooks["root"](self) + # This should probably be called 'head', and the other one called + # 'gov'. But we went with 'head' elsewhere, and now we're stuck =/ + cdef int i + # First, we scan through the Span, and check whether there's a word + # with head==0, i.e. a sentence root. If so, we can return it. The + # longer the span, the more likely it contains a sentence root, and + # in this case we return in linear time. + for i in range(self.c.start, self.c.end): + if self.doc.c[i].head == 0: + return self.doc[i] + # If we don't have a sentence root, we do something that's not so + # algorithmically clever, but I think should be quite fast, + # especially for short spans. + # For each word, we count the path length, and arg min this measure. + # We could use better tree logic to save steps here...But I + # think this should be okay. + cdef int current_best = self.doc.length + cdef int root = -1 + for i in range(self.c.start, self.c.end): + if self.c.start <= (i+self.doc.c[i].head) < self.c.end: + continue + words_to_root = _count_words_to_root(&self.doc.c[i], self.doc.length) + if words_to_root < current_best: + current_best = words_to_root + root = i + if root == -1: + return self.doc[self.c.start] + else: + return self.doc[root] + + def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0, alignment_mode="strict", span_id=0): + """Create a `Span` object from the slice `span.text[start : end]`. + + start (int): The index of the first character of the span. + end (int): The index of the first character after the span. + label (Union[int, str]): A label to attach to the Span, e.g. for + named entities. + kb_id (Union[int, str]): An ID from a KB to capture the meaning of a named entity. + vector (ndarray[ndim=1, dtype='float32']): A meaning representation of + the span. + id (Union[int, str]): Unused. + alignment_mode (str): How character indices are aligned to token + boundaries. Options: "strict" (character indices must be aligned + with token boundaries), "contract" (span of all tokens completely + within the character span), "expand" (span of all tokens at least + partially covered by the character span). Defaults to "strict". + span_id (Union[int, str]): An identifier to associate with the span. + RETURNS (Span): The newly constructed object. + """ + start_idx += self.c.start_char + end_idx += self.c.start_char + return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector, alignment_mode=alignment_mode, span_id=span_id) + + @property + def conjuncts(self): + """Tokens that are conjoined to the span's root. + + RETURNS (tuple): A tuple of Token objects. + + DOCS: https://spacy.io/api/span#lefts + """ + return self.root.conjuncts + + @property + def lefts(self): + """Tokens that are to the left of the span, whose head is within the + `Span`. + + YIELDS (Token):A left-child of a token of the span. + + DOCS: https://spacy.io/api/span#lefts + """ + for token in reversed(self): # Reverse, so we get tokens in order + for left in token.lefts: + if left.i < self.start: + yield left + + @property + def rights(self): + """Tokens that are to the right of the Span, whose head is within the + `Span`. + + YIELDS (Token): A right-child of a token of the span. + + DOCS: https://spacy.io/api/span#rights + """ + for token in self: + for right in token.rights: + if right.i >= self.end: + yield right + + @property + def n_lefts(self): + """The number of tokens that are to the left of the span, whose + heads are within the span. + + RETURNS (int): The number of leftward immediate children of the + span, in the syntactic dependency parse. + + DOCS: https://spacy.io/api/span#n_lefts + """ + return len(list(self.lefts)) + + @property + def n_rights(self): + """The number of tokens that are to the right of the span, whose + heads are within the span. + + RETURNS (int): The number of rightward immediate children of the + span, in the syntactic dependency parse. + + DOCS: https://spacy.io/api/span#n_rights + """ + return len(list(self.rights)) + + @property + def subtree(self): + """Tokens within the span and tokens which descend from them. + + YIELDS (Token): A token within the span, or a descendant from it. + + DOCS: https://spacy.io/api/span#subtree + """ + for word in self.lefts: + yield from word.subtree + yield from self + for word in self.rights: + yield from word.subtree + + @property + def start(self): + return self.c.start + + @start.setter + def start(self, int start): + if start < 0: + raise IndexError(Errors.E1032.format(var="start", forbidden="< 0", value=start)) + self.c.start = start + + @property + def end(self): + return self.c.end + + @end.setter + def end(self, int end): + if end < 0: + raise IndexError(Errors.E1032.format(var="end", forbidden="< 0", value=end)) + self.c.end = end + + @property + def start_char(self): + return self.c.start_char + + @start_char.setter + def start_char(self, int start_char): + if start_char < 0: + raise IndexError(Errors.E1032.format(var="start_char", forbidden="< 0", value=start_char)) + self.c.start_char = start_char + + @property + def end_char(self): + return self.c.end_char + + @end_char.setter + def end_char(self, int end_char): + if end_char < 0: + raise IndexError(Errors.E1032.format(var="end_char", forbidden="< 0", value=end_char)) + self.c.end_char = end_char + + @property + def label(self): + return self.c.label + + @label.setter + def label(self, attr_t label): + self.c.label = label + + @property + def kb_id(self): + return self.c.kb_id + + @kb_id.setter + def kb_id(self, attr_t kb_id): + self.c.kb_id = kb_id + + @property + def id(self): + return self.c.id + + @id.setter + def id(self, attr_t id): + self.c.id = id + + @property + def ent_id(self): + """RETURNS (uint64): The entity ID.""" + return self.root.ent_id + + @ent_id.setter + def ent_id(self, hash_t key): + raise NotImplementedError(Errors.E200.format(attr="ent_id")) + + @property + def ent_id_(self): + """RETURNS (str): The (string) entity ID.""" + return self.root.ent_id_ + + @ent_id_.setter + def ent_id_(self, str key): + raise NotImplementedError(Errors.E200.format(attr="ent_id_")) + + @property + def orth_(self): + """Verbatim text content (identical to `Span.text`). Exists mostly for + consistency with other attributes. + + RETURNS (str): The span's text.""" + return self.text + + @property + def lemma_(self): + """RETURNS (str): The span's lemma.""" + return "".join([t.lemma_ + t.whitespace_ for t in self]).strip() + + @property + def label_(self): + """RETURNS (str): The span's label.""" + return self.doc.vocab.strings[self.label] + + @label_.setter + def label_(self, str label_): + self.label = self.doc.vocab.strings.add(label_) + + @property + def kb_id_(self): + """RETURNS (str): The span's KB ID.""" + return self.doc.vocab.strings[self.kb_id] + + @kb_id_.setter + def kb_id_(self, str kb_id_): + self.kb_id = self.doc.vocab.strings.add(kb_id_) + + @property + def id_(self): + """RETURNS (str): The span's ID.""" + return self.doc.vocab.strings[self.id] + + @id_.setter + def id_(self, str id_): + self.id = self.doc.vocab.strings.add(id_) + + +cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1: + # Don't allow spaces to be the root, if there are + # better candidates + if Lexeme.c_check_flag(token.lex, IS_SPACE) and token.l_kids == 0 and token.r_kids == 0: + return sent_length-1 + if Lexeme.c_check_flag(token.lex, IS_PUNCT) and token.l_kids == 0 and token.r_kids == 0: + return sent_length-1 + cdef int n = 0 + while token.head != 0: + token += token.head + n += 1 + if n >= sent_length: + raise RuntimeError(Errors.E039) + return n diff --git a/spacy/tokens/span_group.pxd b/spacy/tokens/span_group.pxd new file mode 100644 index 0000000..7f41456 --- /dev/null +++ b/spacy/tokens/span_group.pxd @@ -0,0 +1,12 @@ +from libcpp.vector cimport vector + +from ..structs cimport SpanC + + +cdef class SpanGroup: + cdef public object _doc_ref + cdef public str name + cdef public dict attrs + cdef vector[SpanC] c + + cdef void push_back(self, SpanC span) nogil diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi new file mode 100644 index 0000000..3bd2b67 --- /dev/null +++ b/spacy/tokens/span_group.pyi @@ -0,0 +1,29 @@ +from typing import Any, Dict, Iterable, Iterator, Optional + +from .doc import Doc +from .span import Span + +class SpanGroup: + name: str + attrs: Dict[str, Any] + def __init__( + self, + doc: Doc, + *, + name: str = ..., + attrs: Dict[str, Any] = ..., + spans: Iterable[Span] = ..., + ) -> None: ... + def __repr__(self) -> str: ... + @property + def doc(self) -> Doc: ... + @property + def has_overlap(self) -> bool: ... + def __iter__(self) -> Iterator[Span]: ... + def __len__(self) -> int: ... + def append(self, span: Span) -> None: ... + def extend(self, spans: Iterable[Span]) -> None: ... + def __getitem__(self, i: int) -> Span: ... + def to_bytes(self) -> bytes: ... + def from_bytes(self, bytes_data: bytes) -> SpanGroup: ... + def copy(self, doc: Optional[Doc] = ...) -> SpanGroup: ... diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx new file mode 100644 index 0000000..257c907 --- /dev/null +++ b/spacy/tokens/span_group.pyx @@ -0,0 +1,342 @@ +# cython: profile=False +import struct +import weakref +from copy import deepcopy +from typing import Iterable, Optional, Union + +import srsly + +from spacy.errors import Errors + +from .span cimport Span + + +cdef class SpanGroup: + """A group of spans that all belong to the same Doc object. The group + can be named, and you can attach additional attributes to it. Span groups + are generally accessed via the `doc.spans` attribute. The `doc.spans` + attribute will convert lists of spans into a `SpanGroup` object for you + automatically on assignment. + + Example: + Construction 1 + >>> doc = nlp("Their goi ng home") + >>> doc.spans["errors"] = SpanGroup( + doc, + name="errors", + spans=[doc[0:1], doc[1:3]], + attrs={"annotator": "matt"} + ) + + Construction 2 + >>> doc = nlp("Their goi ng home") + >>> doc.spans["errors"] = [doc[0:1], doc[1:3]] + >>> assert isinstance(doc.spans["errors"], SpanGroup) + + DOCS: https://spacy.io/api/spangroup + """ + def __init__(self, doc, *, name="", attrs={}, spans=[]): # no-cython-lint + """Create a SpanGroup. + + doc (Doc): The reference Doc object. + name (str): The group name. + attrs (Dict[str, Any]): Optional JSON-serializable attributes to attach. + spans (Iterable[Span]): The spans to add to the group. + + DOCS: https://spacy.io/api/spangroup#init + """ + # We need to make this a weak reference, so that the Doc object can + # own the SpanGroup without circular references. We do want to get + # the Doc though, because otherwise the API gets annoying. + self._doc_ref = weakref.ref(doc) + self.name = name + self.attrs = dict(attrs) if attrs is not None else {} + cdef Span span + if len(spans) : + self.c.reserve(len(spans)) + for span in spans: + if doc is not span.doc: + raise ValueError(Errors.E855.format(obj="span")) + self.push_back(span.c) + + def __repr__(self): + return str(list(self)) + + @property + def doc(self): + """RETURNS (Doc): The reference document. + + DOCS: https://spacy.io/api/spangroup#doc + """ + doc = self._doc_ref() + if doc is None: + # referent has been garbage collected + raise RuntimeError(Errors.E865) + return doc + + @property + def has_overlap(self): + """RETURNS (bool): Whether the group contains overlapping spans. + + DOCS: https://spacy.io/api/spangroup#has_overlap + """ + if not len(self): + return False + sorted_spans = list(sorted(self)) + last_end = sorted_spans[0].end + for span in sorted_spans[1:]: + if span.start < last_end: + return True + last_end = span.end + return False + + def __len__(self): + """RETURNS (int): The number of spans in the group. + + DOCS: https://spacy.io/api/spangroup#len + """ + return self.c.size() + + def __getitem__(self, int i) -> Span: + """Get a span from the group. Note that a copy of the span is returned, + so if any changes are made to this span, they are not reflected in the + corresponding member of the span group. + + i (int): The item index. + RETURNS (Span): The span at the given index. + + DOCS: https://spacy.io/api/spangroup#getitem + """ + i = self._normalize_index(i) + return Span.cinit(self.doc, self.c[i]) + + def __delitem__(self, int i): + """Delete a span from the span group at index i. + + i (int): The item index. + + DOCS: https://spacy.io/api/spangroup#delitem + """ + i = self._normalize_index(i) + self.c.erase(self.c.begin() + i - 1) + + def __setitem__(self, int i, Span span): + """Set a span in the span group. + + i (int): The item index. + span (Span): The span. + + DOCS: https://spacy.io/api/spangroup#setitem + """ + if span.doc is not self.doc: + raise ValueError(Errors.E855.format(obj="span")) + + i = self._normalize_index(i) + self.c[i] = span.c + + def __iadd__(self, other: Union[SpanGroup, Iterable["Span"]]) -> SpanGroup: + """Operator +=. Append a span group or spans to this group and return + the current span group. + + other (Union[SpanGroup, Iterable["Span"]]): The SpanGroup or spans to + add. + + RETURNS (SpanGroup): The current span group. + + DOCS: https://spacy.io/api/spangroup#iadd + """ + return self._concat(other, inplace=True) + + def __add__(self, other: SpanGroup) -> SpanGroup: + """Operator +. Concatenate a span group with this group and return a + new span group. + + other (SpanGroup): The SpanGroup to add. + + RETURNS (SpanGroup): The concatenated SpanGroup. + + DOCS: https://spacy.io/api/spangroup#add + """ + # For Cython 0.x and __add__, you cannot rely on `self` as being `self` + # or being the right type, so both types need to be checked explicitly. + if isinstance(self, SpanGroup) and isinstance(other, SpanGroup): + return self._concat(other) + return NotImplemented + + def __iter__(self): + """ + Iterate over the spans in this SpanGroup. + YIELDS (Span): A span in this SpanGroup. + + DOCS: https://spacy.io/api/spangroup#iter + """ + for i in range(self.c.size()): + yield self[i] + + def append(self, Span span): + """Add a span to the group. The span must refer to the same Doc + object as the span group. + + span (Span): The span to append. + + DOCS: https://spacy.io/api/spangroup#append + """ + if span.doc is not self.doc: + raise ValueError(Errors.E855.format(obj="span")) + self.push_back(span.c) + + def extend(self, spans_or_span_group: Union[SpanGroup, Iterable["Span"]]): + """Add multiple spans or contents of another SpanGroup to the group. + All spans must refer to the same Doc object as the span group. + + spans (Union[SpanGroup, Iterable["Span"]]): The spans to add. + + DOCS: https://spacy.io/api/spangroup#extend + """ + self._concat(spans_or_span_group, inplace=True) + + def to_bytes(self): + """Serialize the SpanGroup's contents to a byte string. + + RETURNS (bytes): The serialized span group. + + DOCS: https://spacy.io/api/spangroup#to_bytes + """ + output = {"name": self.name, "attrs": self.attrs, "spans": []} + cdef int i + for i in range(self.c.size()): + span = self.c[i] + # The struct.pack here is probably overkill, but it might help if + # you're saving tonnes of spans, and it doesn't really add any + # complexity. We do take care to specify little-endian byte order + # though, to ensure the message can be loaded back on a different + # arch. + # Q: uint64_t + # q: int64_t + # L: uint32_t + # l: int32_t + output["spans"].append(struct.pack( + ">QQQllll", + span.id, + span.kb_id, + span.label, + span.start, + span.end, + span.start_char, + span.end_char + )) + return srsly.msgpack_dumps(output) + + def from_bytes(self, bytes_data): + """Deserialize the SpanGroup's contents from a byte string. + + bytes_data (bytes): The span group to load. + RETURNS (SpanGroup): The deserialized span group. + + DOCS: https://spacy.io/api/spangroup#from_bytes + """ + msg = srsly.msgpack_loads(bytes_data) + self.name = msg["name"] + self.attrs = dict(msg["attrs"]) + self.c.clear() + self.c.reserve(len(msg["spans"])) + cdef SpanC span + for span_data in msg["spans"]: + items = struct.unpack(">QQQllll", span_data) + span.id = items[0] + span.kb_id = items[1] + span.label = items[2] + span.start = items[3] + span.end = items[4] + span.start_char = items[5] + span.end_char = items[6] + self.c.push_back(span) + return self + + cdef void push_back(self, SpanC span) nogil: + self.c.push_back(span) + + def copy(self, doc: Optional["Doc"] = None) -> SpanGroup: + """Clones the span group. + + doc (Doc): New reference document to which the copy is bound. + RETURNS (SpanGroup): A copy of the span group. + + DOCS: https://spacy.io/api/spangroup#copy + """ + if doc is None: + doc = self.doc + if doc is self.doc: + spans = list(self) + else: + spans = [doc.char_span(span.start_char, span.end_char, label=span.label_, kb_id=span.kb_id, span_id=span.id) for span in self] + for i, span in enumerate(spans): + if span is None: + raise ValueError(Errors.E1052.format(i=i)) + if span.kb_id in self.doc.vocab.strings: + doc.vocab.strings.add(span.kb_id_) + if span.id in span.doc.vocab.strings: + doc.vocab.strings.add(span.id_) + return SpanGroup( + doc, + name=self.name, + attrs=deepcopy(self.attrs), + spans=spans, + ) + + def _concat( + self, + other: Union[SpanGroup, Iterable["Span"]], + *, + inplace: bool = False, + ) -> SpanGroup: + """Concatenates the current span group with the provided span group or + spans, either in place or creating a copy. Preserves the name of self, + updates attrs only with values that are not in self. + + other (Union[SpanGroup, Iterable[Span]]): The spans to append. + inplace (bool): Indicates whether the operation should be performed in + place on the current span group. + + RETURNS (SpanGroup): Either a new SpanGroup or the current SpanGroup + depending on the value of inplace. + """ + cdef SpanGroup span_group = self if inplace else self.copy() + cdef SpanGroup other_group + cdef Span span + + if isinstance(other, SpanGroup): + other_group = other + if other_group.doc is not self.doc: + raise ValueError(Errors.E855.format(obj="span group")) + + other_attrs = deepcopy(other_group.attrs) + span_group.attrs.update({ + key: value for key, value in other_attrs.items() + if key not in span_group.attrs + }) + if len(other_group): + span_group.c.reserve(span_group.c.size() + other_group.c.size()) + span_group.c.insert(span_group.c.end(), other_group.c.begin(), other_group.c.end()) + else: + if len(other): + span_group.c.reserve(self.c.size() + len(other)) + for span in other: + if span.doc is not self.doc: + raise ValueError(Errors.E855.format(obj="span")) + span_group.c.push_back(span.c) + + return span_group + + def _normalize_index(self, int i) -> int: + """Checks list index boundaries and adjusts the index if negative. + + i (int): The index. + RETURNS (int): The adjusted index. + """ + cdef int length = self.c.size() + if i < -length or i >= length: + raise IndexError(Errors.E856.format(i=i, length=length)) + if i < 0: + i += length + return i diff --git a/spacy/tokens/token.pxd b/spacy/tokens/token.pxd new file mode 100644 index 0000000..e3e270a --- /dev/null +++ b/spacy/tokens/token.pxd @@ -0,0 +1,107 @@ +from numpy cimport ndarray + +from ..attrs cimport * +from ..lexeme cimport Lexeme +from ..parts_of_speech cimport univ_pos_t +from ..structs cimport TokenC +from ..typedefs cimport attr_t, flags_t +from ..vocab cimport Vocab +from .doc cimport Doc + +from ..errors import Errors + + +cdef const int MISSING_DEP = 0 + +cdef class Token: + cdef readonly Vocab vocab + cdef TokenC* c + cdef readonly int i + cdef readonly Doc doc + + @staticmethod + cdef inline Token cinit(Vocab vocab, const TokenC* token, int offset, Doc doc): + if offset < 0 or offset >= doc.length: + raise IndexError(Errors.E040.format(i=offset, max_length=doc.length)) + cdef Token self = Token.__new__(Token, vocab, doc, offset) + return self + + # cdef inline TokenC struct_from_attrs(Vocab vocab, attrs): + # cdef TokenC token + # attrs = normalize_attrs(attrs) + + cpdef bint check_flag(self, attr_id_t flag_id) except -1 + + @staticmethod + cdef inline attr_t get_struct_attr(const TokenC* token, attr_id_t feat_name) noexcept nogil: + if feat_name < (sizeof(flags_t) * 8): + return Lexeme.c_check_flag(token.lex, feat_name) + elif feat_name == LEMMA: + return token.lemma + elif feat_name == NORM: + if token.norm == 0: + return token.lex.norm + else: + return token.norm + elif feat_name == POS: + return token.pos + elif feat_name == TAG: + return token.tag + elif feat_name == MORPH: + return token.morph + elif feat_name == DEP: + return token.dep + elif feat_name == HEAD: + return token.head + elif feat_name == SPACY: + return token.spacy + elif feat_name == ENT_IOB: + return token.ent_iob + elif feat_name == ENT_TYPE: + return token.ent_type + elif feat_name == ENT_ID: + return token.ent_id + elif feat_name == ENT_KB_ID: + return token.ent_kb_id + elif feat_name == SENT_START: + return token.sent_start + else: + return Lexeme.get_struct_attr(token.lex, feat_name) + + @staticmethod + cdef inline attr_t set_struct_attr(TokenC* token, attr_id_t feat_name, + attr_t value) noexcept nogil: + if feat_name == LEMMA: + token.lemma = value + elif feat_name == NORM: + token.norm = value + elif feat_name == POS: + token.pos = value + elif feat_name == TAG: + token.tag = value + elif feat_name == MORPH: + token.morph = value + elif feat_name == DEP: + token.dep = value + elif feat_name == HEAD: + token.head = value + elif feat_name == SPACY: + token.spacy = value + elif feat_name == ENT_IOB: + token.ent_iob = value + elif feat_name == ENT_TYPE: + token.ent_type = value + elif feat_name == ENT_ID: + token.ent_id = value + elif feat_name == ENT_KB_ID: + token.ent_kb_id = value + elif feat_name == SENT_START: + token.sent_start = value + + @staticmethod + cdef inline int missing_dep(const TokenC* token) noexcept nogil: + return token.dep == MISSING_DEP + + @staticmethod + cdef inline int missing_head(const TokenC* token) noexcept nogil: + return Token.missing_dep(token) diff --git a/spacy/tokens/token.pyi b/spacy/tokens/token.pyi new file mode 100644 index 0000000..435ace5 --- /dev/null +++ b/spacy/tokens/token.pyi @@ -0,0 +1,207 @@ +from typing import Any, Callable, Iterator, Optional, Protocol, Tuple, Union + +from thinc.types import Floats1d, FloatsXd + +from ..lexeme import Lexeme +from ..vocab import Vocab +from .doc import Doc +from .morphanalysis import MorphAnalysis +from .span import Span +from .underscore import Underscore + +class TokenMethod(Protocol): + def __call__(self: Token, *args: Any, **kwargs: Any) -> Any: ... # type: ignore[misc] + +class Token: + i: int + doc: Doc + vocab: Vocab + @classmethod + def set_extension( + cls, + name: str, + default: Optional[Any] = ..., + getter: Optional[Callable[[Token], Any]] = ..., + setter: Optional[Callable[[Token, Any], None]] = ..., + method: Optional[TokenMethod] = ..., + force: bool = ..., + ) -> None: ... + @classmethod + def get_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[TokenMethod], + Optional[Callable[[Token], Any]], + Optional[Callable[[Token, Any], None]], + ]: ... + @classmethod + def has_extension(cls, name: str) -> bool: ... + @classmethod + def remove_extension( + cls, name: str + ) -> Tuple[ + Optional[Any], + Optional[TokenMethod], + Optional[Callable[[Token], Any]], + Optional[Callable[[Token, Any], None]], + ]: ... + def __init__(self, vocab: Vocab, doc: Doc, offset: int) -> None: ... + def __hash__(self) -> int: ... + def __len__(self) -> int: ... + def __unicode__(self) -> str: ... + def __bytes__(self) -> bytes: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... + def __lt__(self, other: Any) -> bool: ... + def __le__(self, other: Any) -> bool: ... + def __eq__(self, other: Any) -> bool: ... + def __ne__(self, other: Any) -> bool: ... + def __gt__(self, other: Any) -> bool: ... + def __ge__(self, other: Any) -> bool: ... + @property + def _(self) -> Underscore: ... + def nbor(self, i: int = ...) -> Token: ... + def similarity(self, other: Union[Doc, Span, Token, Lexeme]) -> float: ... + def has_morph(self) -> bool: ... + morph: MorphAnalysis + @property + def lex(self) -> Lexeme: ... + @property + def lex_id(self) -> int: ... + @property + def rank(self) -> int: ... + @property + def text(self) -> str: ... + @property + def text_with_ws(self) -> str: ... + @property + def prob(self) -> float: ... + @property + def sentiment(self) -> float: ... + @property + def lang(self) -> int: ... + @property + def idx(self) -> int: ... + @property + def cluster(self) -> int: ... + @property + def orth(self) -> int: ... + @property + def lower(self) -> int: ... + @property + def norm(self) -> int: ... + @property + def shape(self) -> int: ... + @property + def prefix(self) -> int: ... + @property + def suffix(self) -> int: ... + lemma: int + pos: int + tag: int + dep: int + @property + def has_vector(self) -> bool: ... + @property + def vector(self) -> Floats1d: ... + @property + def vector_norm(self) -> float: ... + @property + def tensor(self) -> Optional[FloatsXd]: ... + @property + def n_lefts(self) -> int: ... + @property + def n_rights(self) -> int: ... + @property + def sent(self) -> Span: ... + sent_start: bool + is_sent_start: Optional[bool] + is_sent_end: Optional[bool] + @property + def lefts(self) -> Iterator[Token]: ... + @property + def rights(self) -> Iterator[Token]: ... + @property + def children(self) -> Iterator[Token]: ... + @property + def subtree(self) -> Iterator[Token]: ... + @property + def left_edge(self) -> Token: ... + @property + def right_edge(self) -> Token: ... + @property + def ancestors(self) -> Iterator[Token]: ... + def is_ancestor(self, descendant: Token) -> bool: ... + def has_head(self) -> bool: ... + head: Token + @property + def conjuncts(self) -> Tuple[Token]: ... + ent_type: int + ent_type_: str + @property + def ent_iob(self) -> int: ... + @classmethod + def iob_strings(cls) -> Tuple[str]: ... + @property + def ent_iob_(self) -> str: ... + ent_id: int + ent_id_: str + ent_kb_id: int + ent_kb_id_: str + @property + def whitespace_(self) -> str: ... + @property + def orth_(self) -> str: ... + @property + def lower_(self) -> str: ... + norm_: str + @property + def shape_(self) -> str: ... + @property + def prefix_(self) -> str: ... + @property + def suffix_(self) -> str: ... + @property + def lang_(self) -> str: ... + lemma_: str + pos_: str + tag_: str + def has_dep(self) -> bool: ... + dep_: str + @property + def is_oov(self) -> bool: ... + @property + def is_stop(self) -> bool: ... + @property + def is_alpha(self) -> bool: ... + @property + def is_ascii(self) -> bool: ... + @property + def is_digit(self) -> bool: ... + @property + def is_lower(self) -> bool: ... + @property + def is_upper(self) -> bool: ... + @property + def is_title(self) -> bool: ... + @property + def is_punct(self) -> bool: ... + @property + def is_space(self) -> bool: ... + @property + def is_bracket(self) -> bool: ... + @property + def is_quote(self) -> bool: ... + @property + def is_left_punct(self) -> bool: ... + @property + def is_right_punct(self) -> bool: ... + @property + def is_currency(self) -> bool: ... + @property + def like_url(self) -> bool: ... + @property + def like_num(self) -> bool: ... + @property + def like_email(self) -> bool: ... diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx new file mode 100644 index 0000000..a3efd58 --- /dev/null +++ b/spacy/tokens/token.pyx @@ -0,0 +1,1052 @@ +# cython: infer_types=True +# cython: profile=False +# Compiler crashes on memory view coercion without this. Should report bug. +cimport numpy as np + +np.import_array() + +import warnings + +from thinc.api import get_array_module + +from ..attrs cimport ( + IS_ALPHA, + IS_ASCII, + IS_BRACKET, + IS_CURRENCY, + IS_DIGIT, + IS_LEFT_PUNCT, + IS_LOWER, + IS_PUNCT, + IS_QUOTE, + IS_RIGHT_PUNCT, + IS_SPACE, + IS_STOP, + IS_TITLE, + IS_UPPER, + LIKE_EMAIL, + LIKE_NUM, + LIKE_URL, + ORTH, +) +from ..lexeme cimport Lexeme +from ..symbols cimport conj +from ..typedefs cimport hash_t +from .doc cimport set_children_from_heads +from .morphanalysis cimport MorphAnalysis + +from .. import parts_of_speech +from ..attrs import IOB_STRINGS +from ..errors import Errors, Warnings +from .underscore import Underscore, get_ext_args + + +cdef class Token: + """An individual token – i.e. a word, punctuation symbol, whitespace, + etc. + + DOCS: https://spacy.io/api/token + """ + @classmethod + def set_extension(cls, name, **kwargs): + """Define a custom attribute which becomes available as `Token._`. + + name (str): Name of the attribute to set. + default: Optional default value of the attribute. + getter (callable): Optional getter function. + setter (callable): Optional setter function. + method (callable): Optional method for method extension. + force (bool): Force overwriting existing attribute. + + DOCS: https://spacy.io/api/token#set_extension + USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes + """ + if cls.has_extension(name) and not kwargs.get("force", False): + raise ValueError(Errors.E090.format(name=name, obj="Token")) + Underscore.token_extensions[name] = get_ext_args(**kwargs) + + @classmethod + def get_extension(cls, name): + """Look up a previously registered extension by name. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple. + + DOCS: https://spacy.io/api/token#get_extension + """ + return Underscore.token_extensions.get(name) + + @classmethod + def has_extension(cls, name): + """Check whether an extension has been registered. + + name (str): Name of the extension. + RETURNS (bool): Whether the extension has been registered. + + DOCS: https://spacy.io/api/token#has_extension + """ + return name in Underscore.token_extensions + + @classmethod + def remove_extension(cls, name): + """Remove a previously registered extension. + + name (str): Name of the extension. + RETURNS (tuple): A `(default, method, getter, setter)` tuple of the + removed extension. + + DOCS: https://spacy.io/api/token#remove_extension + """ + if not cls.has_extension(name): + raise ValueError(Errors.E046.format(name=name)) + return Underscore.token_extensions.pop(name) + + def __cinit__(self, Vocab vocab, Doc doc, int offset): + """Construct a `Token` object. + + vocab (Vocab): A storage container for lexical types. + doc (Doc): The parent document. + offset (int): The index of the token within the document. + + DOCS: https://spacy.io/api/token#init + """ + self.vocab = vocab + self.doc = doc + self.c = &self.doc.c[offset] + self.i = offset + + def __hash__(self): + return hash((self.doc, self.i)) + + def __len__(self): + """The number of unicode characters in the token, i.e. `token.text`. + + RETURNS (int): The number of unicode characters in the token. + + DOCS: https://spacy.io/api/token#len + """ + return self.c.lex.length + + def __unicode__(self): + return self.text + + def __bytes__(self): + return self.text.encode('utf8') + + def __str__(self): + return self.__unicode__() + + def __repr__(self): + return self.__str__() + + def __richcmp__(self, object other, int op): + # http://cython.readthedocs.io/en/latest/src/userguide/special_methods.html + if other is None: + if op in (0, 1, 2): + return False + else: + return True + if not isinstance(other, Token): + return False + cdef Token other_token = other + cdef Doc my_doc = self.doc + cdef Doc other_doc = other_token.doc + my = self.idx + their = other_token.idx + if op == 0: + return my < their + elif op == 2: + if my_doc is other_doc: + return my == their + else: + return False + elif op == 4: + return my > their + elif op == 1: + return my <= their + elif op == 3: + if my_doc is other_doc: + return my != their + else: + return True + elif op == 5: + return my >= their + else: + raise ValueError(Errors.E041.format(op=op)) + + def __reduce__(self): + raise NotImplementedError(Errors.E111) + + @property + def _(self): + """Custom extension attributes registered via `set_extension`.""" + return Underscore(Underscore.token_extensions, self, + start=self.idx, end=None) + + cpdef bint check_flag(self, attr_id_t flag_id) except -1: + """Check the value of a boolean flag. + + flag_id (int): The ID of the flag attribute. + RETURNS (bool): Whether the flag is set. + + DOCS: https://spacy.io/api/token#check_flag + """ + return Lexeme.c_check_flag(self.c.lex, flag_id) + + def nbor(self, int i=1): + """Get a neighboring token. + + i (int): The relative position of the token to get. Defaults to 1. + RETURNS (Token): The token at position `self.doc[self.i+i]`. + + DOCS: https://spacy.io/api/token#nbor + """ + if self.i+i < 0 or (self.i+i >= len(self.doc)): + raise IndexError(Errors.E042.format(i=self.i, j=i, length=len(self.doc))) + return self.doc[self.i+i] + + def similarity(self, other): + """Make a semantic similarity estimate. The default estimate is cosine + similarity using an average of word vectors. + + other (object): The object to compare with. By default, accepts `Doc`, + `Span`, `Token` and `Lexeme` objects. + RETURNS (float): A scalar similarity score. Higher is more similar. + + DOCS: https://spacy.io/api/token#similarity + """ + if "similarity" in self.doc.user_token_hooks: + return self.doc.user_token_hooks["similarity"](self, other) + attr = getattr(self.doc.vocab.vectors, "attr", ORTH) + cdef Token this_token = self + cdef Token other_token + cdef Lexeme other_lex + if isinstance(other, Token): + other_token = other + if Token.get_struct_attr(this_token.c, attr) == Token.get_struct_attr(other_token.c, attr): + return 1.0 + elif isinstance(other, Lexeme): + other_lex = other + if Token.get_struct_attr(this_token.c, attr) == Lexeme.get_struct_attr(other_lex.c, attr): + return 1.0 + if self.vocab.vectors.n_keys == 0: + warnings.warn(Warnings.W007.format(obj="Token")) + if self.vector_norm == 0 or other.vector_norm == 0: + if not self.has_vector or not other.has_vector: + warnings.warn(Warnings.W008.format(obj="Token")) + return 0.0 + vector = self.vector + xp = get_array_module(vector) + result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm) + # ensure we get a scalar back (numpy does this automatically but cupy doesn't) + return result.item() + + def has_morph(self): + """Check whether the token has annotated morph information. + Return False when the morph annotation is unset/missing. + + RETURNS (bool): Whether the morph annotation is set. + """ + return not self.c.morph == 0 + + @property + def morph(self): + return MorphAnalysis.from_id(self.vocab, self.c.morph) + + @morph.setter + def morph(self, MorphAnalysis morph): + # Check that the morph has the same vocab + if self.vocab != morph.vocab: + raise ValueError(Errors.E1013) + self.c.morph = morph.c.key + + def set_morph(self, features): + cdef hash_t key + if features is None: + self.c.morph = 0 + elif isinstance(features, MorphAnalysis): + self.morph = features + else: + if isinstance(features, int): + features = self.vocab.strings[features] + key = self.vocab.morphology.add(features) + self.c.morph = key + + @property + def lex(self): + """RETURNS (Lexeme): The underlying lexeme.""" + return self.vocab[self.c.lex.orth] + + @property + def lex_id(self): + """RETURNS (int): Sequential ID of the token's lexical type.""" + return self.c.lex.id + + @property + def rank(self): + """RETURNS (int): Sequential ID of the token's lexical type, used to + index into tables, e.g. for word vectors.""" + return self.c.lex.id + + @property + def text(self): + """RETURNS (str): The original verbatim text of the token.""" + return self.orth_ + + @property + def text_with_ws(self): + """RETURNS (str): The text content of the span (with trailing + whitespace). + """ + cdef str orth = self.vocab.strings[self.c.lex.orth] + if self.c.spacy: + return orth + " " + else: + return orth + + @property + def prob(self): + """RETURNS (float): Smoothed log probability estimate of token type.""" + return self.vocab[self.c.lex.orth].prob + + @property + def sentiment(self): + """RETURNS (float): A scalar value indicating the positivity or + negativity of the token.""" + if "sentiment" in self.doc.user_token_hooks: + return self.doc.user_token_hooks["sentiment"](self) + return self.vocab[self.c.lex.orth].sentiment + + @property + def lang(self): + """RETURNS (uint64): ID of the language of the parent document's + vocabulary. + """ + return self.c.lex.lang + + @property + def idx(self): + """RETURNS (int): The character offset of the token within the parent + document. + """ + return self.c.idx + + @property + def cluster(self): + """RETURNS (int): Brown cluster ID.""" + return self.vocab[self.c.lex.orth].cluster + + @property + def orth(self): + """RETURNS (uint64): ID of the verbatim text content.""" + return self.c.lex.orth + + @property + def lower(self): + """RETURNS (uint64): ID of the lowercase token text.""" + return self.c.lex.lower + + @property + def norm(self): + """RETURNS (uint64): ID of the token's norm, i.e. a normalised form of + the token text. Usually set in the language's tokenizer exceptions + or norm exceptions. + """ + if self.c.norm == 0: + return self.c.lex.norm + else: + return self.c.norm + + @property + def shape(self): + """RETURNS (uint64): ID of the token's shape, a transform of the + token's string, to show orthographic features (e.g. "Xxxx", "dd"). + """ + return self.c.lex.shape + + @property + def prefix(self): + """RETURNS (uint64): ID of a length-N substring from the start of the + token. Defaults to `N=1`. + """ + return self.c.lex.prefix + + @property + def suffix(self): + """RETURNS (uint64): ID of a length-N substring from the end of the + token. Defaults to `N=3`. + """ + return self.c.lex.suffix + + @property + def lemma(self): + """RETURNS (uint64): ID of the base form of the word, with no + inflectional suffixes. + """ + return self.c.lemma + + @lemma.setter + def lemma(self, attr_t lemma): + self.c.lemma = lemma + + @property + def pos(self): + """RETURNS (uint64): ID of coarse-grained part-of-speech tag.""" + return self.c.pos + + @pos.setter + def pos(self, pos): + self.c.pos = pos + + @property + def tag(self): + """RETURNS (uint64): ID of fine-grained part-of-speech tag.""" + return self.c.tag + + @tag.setter + def tag(self, attr_t tag): + self.c.tag = tag + + @property + def dep(self): + """RETURNS (uint64): ID of syntactic dependency label.""" + return self.c.dep + + @dep.setter + def dep(self, attr_t label): + self.c.dep = label + + @property + def has_vector(self): + """A boolean value indicating whether a word vector is associated with + the object. + + RETURNS (bool): Whether a word vector is associated with the object. + + DOCS: https://spacy.io/api/token#has_vector + """ + if "has_vector" in self.doc.user_token_hooks: + return self.doc.user_token_hooks["has_vector"](self) + if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0: + return True + return self.vocab.has_vector(Token.get_struct_attr(self.c, self.vocab.vectors.attr)) + + @property + def vector(self): + """A real-valued meaning representation. + + RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array + representing the token's semantics. + + DOCS: https://spacy.io/api/token#vector + """ + if "vector" in self.doc.user_token_hooks: + return self.doc.user_token_hooks["vector"](self) + if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0: + return self.doc.tensor[self.i] + else: + return self.vocab.get_vector(Token.get_struct_attr(self.c, self.vocab.vectors.attr)) + + @property + def vector_norm(self): + """The L2 norm of the token's vector representation. + + RETURNS (float): The L2 norm of the vector representation. + + DOCS: https://spacy.io/api/token#vector_norm + """ + if "vector_norm" in self.doc.user_token_hooks: + return self.doc.user_token_hooks["vector_norm"](self) + vector = self.vector + xp = get_array_module(vector) + total = (vector ** 2).sum() + return xp.sqrt(total) if total != 0. else 0. + + @property + def tensor(self): + if self.doc.tensor is None: + return None + return self.doc.tensor[self.i] + + @property + def n_lefts(self): + """The number of leftward immediate children of the word, in the + syntactic dependency parse. + + RETURNS (int): The number of leftward immediate children of the + word, in the syntactic dependency parse. + + DOCS: https://spacy.io/api/token#n_lefts + """ + return self.c.l_kids + + @property + def n_rights(self): + """The number of rightward immediate children of the word, in the + syntactic dependency parse. + + RETURNS (int): The number of rightward immediate children of the + word, in the syntactic dependency parse. + + DOCS: https://spacy.io/api/token#n_rights + """ + return self.c.r_kids + + @property + def sent(self): + """RETURNS (Span): The sentence span that the token is a part of.""" + if 'sent' in self.doc.user_token_hooks: + return self.doc.user_token_hooks["sent"](self) + return self.doc[self.i : self.i+1].sent + + @property + def sent_start(self): + """Deprecated: use Token.is_sent_start instead.""" + # Raising a deprecation warning here causes errors for autocomplete + # Handle broken backwards compatibility case: doc[0].sent_start + # was False. + if self.i == 0: + return False + else: + return self.c.sent_start + + @sent_start.setter + def sent_start(self, value): + self.is_sent_start = value + + @property + def is_sent_start(self): + """A boolean value indicating whether the token starts a sentence. + `None` if unknown. Defaults to `True` for the first token in the `Doc`. + + RETURNS (bool / None): Whether the token starts a sentence. + None if unknown. + """ + if self.c.sent_start == 0: + return None + elif self.c.sent_start < 0: + return False + else: + return True + + @is_sent_start.setter + def is_sent_start(self, value): + if self.doc.has_annotation("DEP"): + raise ValueError(Errors.E043) + if value is None: + self.c.sent_start = 0 + elif value is True: + self.c.sent_start = 1 + elif value is False: + self.c.sent_start = -1 + else: + raise ValueError(Errors.E044.format(value=value)) + + @property + def is_sent_end(self): + """A boolean value indicating whether the token ends a sentence. + `None` if unknown. Defaults to `True` for the last token in the `Doc`. + + RETURNS (bool / None): Whether the token ends a sentence. + None if unknown. + + DOCS: https://spacy.io/api/token#is_sent_end + """ + if self.i + 1 == len(self.doc): + return True + elif self.doc[self.i+1].is_sent_start is None: + return None + elif self.doc[self.i+1].is_sent_start is True: + return True + else: + return False + + @is_sent_end.setter + def is_sent_end(self, value): + raise ValueError(Errors.E196) + + @property + def lefts(self): + """The leftward immediate children of the word, in the syntactic + dependency parse. + + YIELDS (Token): A left-child of the token. + + DOCS: https://spacy.io/api/token#lefts + """ + cdef int nr_iter = 0 + cdef const TokenC* ptr = self.c - (self.i - self.c.l_edge) + while ptr < self.c: + if ptr + ptr.head == self.c: + yield self.doc[ptr - (self.c - self.i)] + ptr += 1 + nr_iter += 1 + # This is ugly, but it's a way to guard out infinite loops + if nr_iter >= 10000000: + raise RuntimeError(Errors.E045.format(attr="token.lefts")) + + @property + def rights(self): + """The rightward immediate children of the word, in the syntactic + dependency parse. + + YIELDS (Token): A right-child of the token. + + DOCS: https://spacy.io/api/token#rights + """ + cdef const TokenC* ptr = self.c + (self.c.r_edge - self.i) + tokens = [] + cdef int nr_iter = 0 + while ptr > self.c: + if ptr + ptr.head == self.c: + tokens.append(self.doc[ptr - (self.c - self.i)]) + ptr -= 1 + nr_iter += 1 + if nr_iter >= 10000000: + raise RuntimeError(Errors.E045.format(attr="token.rights")) + tokens.reverse() + for t in tokens: + yield t + + @property + def children(self): + """A sequence of the token's immediate syntactic children. + + YIELDS (Token): A child token such that `child.head==self`. + + DOCS: https://spacy.io/api/token#children + """ + yield from self.lefts + yield from self.rights + + @property + def subtree(self): + """A sequence containing the token and all the token's syntactic + descendants. + + YIELDS (Token): A descendent token such that + `self.is_ancestor(descendent) or token == self`. + + DOCS: https://spacy.io/api/token#subtree + """ + for word in self.lefts: + yield from word.subtree + yield self + for word in self.rights: + yield from word.subtree + + @property + def left_edge(self) -> int: + """The leftmost token of this token's syntactic descendents. + + RETURNS (Token): The first token such that `self.is_ancestor(token)`. + """ + return self.doc[self.c.l_edge] + + @property + def right_edge(self) -> int: + """The rightmost token of this token's syntactic descendents. + + RETURNS (Token): The last token such that `self.is_ancestor(token)`. + """ + return self.doc[self.c.r_edge] + + @property + def ancestors(self): + """A sequence of this token's syntactic ancestors. + + YIELDS (Token): A sequence of ancestor tokens such that + `ancestor.is_ancestor(self)`. + + DOCS: https://spacy.io/api/token#ancestors + """ + cdef const TokenC* head_ptr = self.c + # Guard against infinite loop, no token can have + # more ancestors than tokens in the tree. + cdef int i = 0 + while head_ptr.head != 0 and i < self.doc.length: + head_ptr += head_ptr.head + yield self.doc[head_ptr - (self.c - self.i)] + i += 1 + + def is_ancestor(self, descendant): + """Check whether this token is a parent, grandparent, etc. of another + in the dependency tree. + + descendant (Token): Another token. + RETURNS (bool): Whether this token is the ancestor of the descendant. + + DOCS: https://spacy.io/api/token#is_ancestor + """ + if self.doc is not descendant.doc: + return False + return any(ancestor.i == self.i for ancestor in descendant.ancestors) + + def has_head(self): + """Check whether the token has annotated head information. + Return False when the head annotation is unset/missing. + + RETURNS (bool): Whether the head annotation is valid or not. + """ + return not Token.missing_head(self.c) + + @property + def head(self): + """The syntactic parent, or "governor", of this token. + If token.has_head() is `False`, this method will return itself. + + RETURNS (Token): The token predicted by the parser to be the head of + the current token. + """ + if not self.has_head(): + return self + else: + return self.doc[self.i + self.c.head] + + @head.setter + def head(self, Token new_head): + # This function sets the head of self to new_head and updates the + # counters for left/right dependents and left/right corner for the + # new and the old head + # Check that token is from the same document + if self.doc != new_head.doc: + raise ValueError(Errors.E191) + # Do nothing if old head is new head + if self.i + self.c.head == new_head.i: + return + # Find the widest l/r_edges of the roots of the two tokens involved + # to limit the number of tokens for set_children_from_heads + cdef Token self_root, new_head_root + self_root = ([self] + list(self.ancestors))[-1] + new_head_ancestors = list(new_head.ancestors) + new_head_root = new_head_ancestors[-1] if new_head_ancestors else new_head + start = self_root.c.l_edge if self_root.c.l_edge < new_head_root.c.l_edge else new_head_root.c.l_edge + end = self_root.c.r_edge if self_root.c.r_edge > new_head_root.c.r_edge else new_head_root.c.r_edge + # Set new head + self.c.head = new_head.i - self.i + # Adjust parse properties and sentence starts + set_children_from_heads(self.doc.c, start, end + 1) + + @property + def conjuncts(self): + """A sequence of coordinated tokens, including the token itself. + + RETURNS (tuple): The coordinated tokens. + + DOCS: https://spacy.io/api/token#conjuncts + """ + cdef Token word, child + if "conjuncts" in self.doc.user_token_hooks: + return tuple(self.doc.user_token_hooks["conjuncts"](self)) + start = self + while start.i != start.head.i: + if start.dep == conj: + start = start.head + else: + break + queue = [start] + output = [start] + for word in queue: + for child in word.rights: + if child.c.dep == conj: + output.append(child) + queue.append(child) + return tuple([w for w in output if w.i != self.i]) + + @property + def ent_type(self): + """RETURNS (uint64): Named entity type.""" + return self.c.ent_type + + @ent_type.setter + def ent_type(self, ent_type): + self.c.ent_type = ent_type + + @property + def ent_type_(self): + """RETURNS (str): Named entity type.""" + return self.vocab.strings[self.c.ent_type] + + @ent_type_.setter + def ent_type_(self, ent_type): + self.c.ent_type = self.vocab.strings.add(ent_type) + + @property + def ent_iob(self): + """IOB code of named entity tag. `1="I", 2="O", 3="B"`. 0 means no tag + is assigned. + + RETURNS (uint64): IOB code of named entity tag. + """ + return self.c.ent_iob + + @classmethod + def iob_strings(cls): + return IOB_STRINGS + + @property + def ent_iob_(self): + """IOB code of named entity tag. "B" means the token begins an entity, + "I" means it is inside an entity, "O" means it is outside an entity, + and "" means no entity tag is set. "B" with an empty ent_type + means that the token is blocked from further processing by NER. + + RETURNS (str): IOB code of named entity tag. + """ + return self.iob_strings()[self.c.ent_iob] + + @property + def ent_id(self): + """RETURNS (uint64): ID of the entity the token is an instance of, + if any. + """ + return self.c.ent_id + + @ent_id.setter + def ent_id(self, hash_t key): + self.c.ent_id = key + + @property + def ent_id_(self): + """RETURNS (str): ID of the entity the token is an instance of, + if any. + """ + return self.vocab.strings[self.c.ent_id] + + @ent_id_.setter + def ent_id_(self, name): + self.c.ent_id = self.vocab.strings.add(name) + + @property + def ent_kb_id(self): + """RETURNS (uint64): Named entity KB ID.""" + return self.c.ent_kb_id + + @ent_kb_id.setter + def ent_kb_id(self, attr_t ent_kb_id): + self.c.ent_kb_id = ent_kb_id + + @property + def ent_kb_id_(self): + """RETURNS (str): Named entity KB ID.""" + return self.vocab.strings[self.c.ent_kb_id] + + @ent_kb_id_.setter + def ent_kb_id_(self, ent_kb_id): + self.c.ent_kb_id = self.vocab.strings.add(ent_kb_id) + + @property + def whitespace_(self): + """RETURNS (str): The trailing whitespace character, if present.""" + return " " if self.c.spacy else "" + + @property + def orth_(self): + """RETURNS (str): Verbatim text content (identical to + `Token.text`). Exists mostly for consistency with the other + attributes. + """ + return self.vocab.strings[self.c.lex.orth] + + @property + def lower_(self): + """RETURNS (str): The lowercase token text. Equivalent to + `Token.text.lower()`. + """ + return self.vocab.strings[self.c.lex.lower] + + @property + def norm_(self): + """RETURNS (str): The token's norm, i.e. a normalised form of the + token text. Usually set in the language's tokenizer exceptions or + norm exceptions. + """ + return self.vocab.strings[self.norm] + + @norm_.setter + def norm_(self, str norm_): + self.c.norm = self.vocab.strings.add(norm_) + + @property + def shape_(self): + """RETURNS (str): Transform of the token's string, to show + orthographic features. For example, "Xxxx" or "dd". + """ + return self.vocab.strings[self.c.lex.shape] + + @property + def prefix_(self): + """RETURNS (str): A length-N substring from the start of the token. + Defaults to `N=1`. + """ + return self.vocab.strings[self.c.lex.prefix] + + @property + def suffix_(self): + """RETURNS (str): A length-N substring from the end of the token. + Defaults to `N=3`. + """ + return self.vocab.strings[self.c.lex.suffix] + + @property + def lang_(self): + """RETURNS (str): Language of the parent document's vocabulary, + e.g. 'en'. + """ + return self.vocab.strings[self.c.lex.lang] + + @property + def lemma_(self): + """RETURNS (str): The token lemma, i.e. the base form of the word, + with no inflectional suffixes. + """ + return self.vocab.strings[self.c.lemma] + + @lemma_.setter + def lemma_(self, str lemma_): + self.c.lemma = self.vocab.strings.add(lemma_) + + @property + def pos_(self): + """RETURNS (str): Coarse-grained part-of-speech tag.""" + return parts_of_speech.NAMES[self.c.pos] + + @pos_.setter + def pos_(self, pos_name): + if pos_name not in parts_of_speech.IDS: + raise ValueError(Errors.E1021.format(pp=pos_name)) + self.c.pos = parts_of_speech.IDS[pos_name] + + @property + def tag_(self): + """RETURNS (str): Fine-grained part-of-speech tag.""" + return self.vocab.strings[self.c.tag] + + @tag_.setter + def tag_(self, tag): + self.tag = self.vocab.strings.add(tag) + + def has_dep(self): + """Check whether the token has annotated dep information. + Returns False when the dep label is unset/missing. + + RETURNS (bool): Whether the dep label is valid or not. + """ + return not Token.missing_dep(self.c) + + @property + def dep_(self): + """RETURNS (str): The syntactic dependency label.""" + return self.vocab.strings[self.c.dep] + + @dep_.setter + def dep_(self, str label): + self.c.dep = self.vocab.strings.add(label) + + @property + def is_oov(self): + """RETURNS (bool): Whether the token is out-of-vocabulary.""" + return self.c.lex.orth not in self.vocab.vectors + + @property + def is_stop(self): + """RETURNS (bool): Whether the token is a stop word, i.e. part of a + "stop list" defined by the language data. + """ + return Lexeme.c_check_flag(self.c.lex, IS_STOP) + + @property + def is_alpha(self): + """RETURNS (bool): Whether the token consists of alpha characters. + Equivalent to `token.text.isalpha()`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_ALPHA) + + @property + def is_ascii(self): + """RETURNS (bool): Whether the token consists of ASCII characters. + Equivalent to `[any(ord(c) >= 128 for c in token.text)]`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_ASCII) + + @property + def is_digit(self): + """RETURNS (bool): Whether the token consists of digits. Equivalent to + `token.text.isdigit()`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_DIGIT) + + @property + def is_lower(self): + """RETURNS (bool): Whether the token is in lowercase. Equivalent to + `token.text.islower()`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_LOWER) + + @property + def is_upper(self): + """RETURNS (bool): Whether the token is in uppercase. Equivalent to + `token.text.isupper()` + """ + return Lexeme.c_check_flag(self.c.lex, IS_UPPER) + + @property + def is_title(self): + """RETURNS (bool): Whether the token is in titlecase. Equivalent to + `token.text.istitle()`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_TITLE) + + @property + def is_punct(self): + """RETURNS (bool): Whether the token is punctuation.""" + return Lexeme.c_check_flag(self.c.lex, IS_PUNCT) + + @property + def is_space(self): + """RETURNS (bool): Whether the token consists of whitespace characters. + Equivalent to `token.text.isspace()`. + """ + return Lexeme.c_check_flag(self.c.lex, IS_SPACE) + + @property + def is_bracket(self): + """RETURNS (bool): Whether the token is a bracket.""" + return Lexeme.c_check_flag(self.c.lex, IS_BRACKET) + + @property + def is_quote(self): + """RETURNS (bool): Whether the token is a quotation mark.""" + return Lexeme.c_check_flag(self.c.lex, IS_QUOTE) + + @property + def is_left_punct(self): + """RETURNS (bool): Whether the token is a left punctuation mark.""" + return Lexeme.c_check_flag(self.c.lex, IS_LEFT_PUNCT) + + @property + def is_right_punct(self): + """RETURNS (bool): Whether the token is a right punctuation mark.""" + return Lexeme.c_check_flag(self.c.lex, IS_RIGHT_PUNCT) + + @property + def is_currency(self): + """RETURNS (bool): Whether the token is a currency symbol.""" + return Lexeme.c_check_flag(self.c.lex, IS_CURRENCY) + + @property + def like_url(self): + """RETURNS (bool): Whether the token resembles a URL.""" + return Lexeme.c_check_flag(self.c.lex, LIKE_URL) + + @property + def like_num(self): + """RETURNS (bool): Whether the token resembles a number, e.g. "10.9", + "10", "ten", etc. + """ + return Lexeme.c_check_flag(self.c.lex, LIKE_NUM) + + @property + def like_email(self): + """RETURNS (bool): Whether the token resembles an email address.""" + return Lexeme.c_check_flag(self.c.lex, LIKE_EMAIL) diff --git a/spacy/tokens/underscore.py b/spacy/tokens/underscore.py new file mode 100644 index 0000000..0aa0c1e --- /dev/null +++ b/spacy/tokens/underscore.py @@ -0,0 +1,139 @@ +import copy +import functools +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from ..errors import Errors + +if TYPE_CHECKING: + from .doc import Doc + from .span import Span + from .token import Token + + +class Underscore: + mutable_types = (dict, list, set) + doc_extensions: Dict[Any, Any] = {} + span_extensions: Dict[Any, Any] = {} + token_extensions: Dict[Any, Any] = {} + _extensions: Dict[str, Any] + _obj: Union["Doc", "Span", "Token"] + _start: Optional[int] + _end: Optional[int] + + def __init__( + self, + extensions: Dict[str, Any], + obj: Union["Doc", "Span", "Token"], + start: Optional[int] = None, + end: Optional[int] = None, + ): + object.__setattr__(self, "_extensions", extensions) + object.__setattr__(self, "_obj", obj) + # Assumption is that for doc values, _start and _end will both be None + # Span will set non-None values for _start and _end + # Token will have _start be non-None, _end be None + # This lets us key everything into the doc.user_data dictionary, + # (see _get_key), and lets us use a single Underscore class. + object.__setattr__(self, "_doc", obj.doc) + object.__setattr__(self, "_start", start) + object.__setattr__(self, "_end", end) + + def __dir__(self) -> List[str]: + # Hack to enable autocomplete on custom extensions + extensions = list(self._extensions.keys()) + return ["set", "get", "has"] + extensions + + def __getattr__(self, name: str) -> Any: + if name not in self._extensions: + raise AttributeError(Errors.E046.format(name=name)) + default, method, getter, setter = self._extensions[name] + if getter is not None: + return getter(self._obj) + elif method is not None: + method_partial = functools.partial(method, self._obj) + # Hack to port over docstrings of the original function + # See https://stackoverflow.com/q/27362727/6400719 + method_docstring = method.__doc__ or "" + method_docstring_prefix = ( + "This method is a partial function and its first argument " + "(the object it's called on) will be filled automatically. " + ) + method_partial.__doc__ = method_docstring_prefix + method_docstring + return method_partial + else: + key = self._get_key(name) + if key in self._doc.user_data: + return self._doc.user_data[key] + elif isinstance(default, self.mutable_types): + # Handle mutable default arguments (see #2581) + new_default = copy.copy(default) + self.__setattr__(name, new_default) + return new_default + return default + + def __setattr__(self, name: str, value: Any): + if name not in self._extensions: + raise AttributeError(Errors.E047.format(name=name)) + default, method, getter, setter = self._extensions[name] + if setter is not None: + return setter(self._obj, value) + else: + self._doc.user_data[self._get_key(name)] = value + + def set(self, name: str, value: Any): + return self.__setattr__(name, value) + + def get(self, name: str) -> Any: + return self.__getattr__(name) + + def has(self, name: str) -> bool: + return name in self._extensions + + def _get_key(self, name: str) -> Tuple[str, str, Optional[int], Optional[int]]: + return ("._.", name, self._start, self._end) + + @classmethod + def get_state(cls) -> Tuple[Dict[Any, Any], Dict[Any, Any], Dict[Any, Any]]: + return cls.token_extensions, cls.span_extensions, cls.doc_extensions + + @classmethod + def load_state( + cls, state: Tuple[Dict[Any, Any], Dict[Any, Any], Dict[Any, Any]] + ) -> None: + cls.token_extensions, cls.span_extensions, cls.doc_extensions = state + + +def get_ext_args(**kwargs: Any): + """Validate and convert arguments. Reused in Doc, Token and Span.""" + default = kwargs.get("default") + getter = kwargs.get("getter") + setter = kwargs.get("setter") + method = kwargs.get("method") + if getter is None and setter is not None: + raise ValueError(Errors.E089) + valid_opts = ("default" in kwargs, method is not None, getter is not None) + nr_defined = sum(t is True for t in valid_opts) + if nr_defined != 1: + raise ValueError(Errors.E083.format(nr_defined=nr_defined)) + if setter is not None and not hasattr(setter, "__call__"): + raise ValueError(Errors.E091.format(name="setter", value=repr(setter))) + if getter is not None and not hasattr(getter, "__call__"): + raise ValueError(Errors.E091.format(name="getter", value=repr(getter))) + if method is not None and not hasattr(method, "__call__"): + raise ValueError(Errors.E091.format(name="method", value=repr(method))) + return (default, method, getter, setter) + + +def is_writable_attr(ext): + """Check if an extension attribute is writable. + ext (tuple): The (default, getter, setter, method) tuple available via + {Doc,Span,Token}.get_extension. + RETURNS (bool): Whether the attribute is writable. + """ + default, method, getter, setter = ext + # Extension is writable if it has a setter (getter + setter), if it has a + # default value (or, if its default value is none, none of the other values + # should be set). + if setter is not None or default is not None or all(e is None for e in ext): + return True + return False diff --git a/spacy/training/__init__.pxd b/spacy/training/__init__.pxd new file mode 100644 index 0000000..e69de29 diff --git a/spacy/training/__init__.py b/spacy/training/__init__.py new file mode 100644 index 0000000..5c2ba99 --- /dev/null +++ b/spacy/training/__init__.py @@ -0,0 +1,43 @@ +from .alignment import Alignment # noqa: F401 +from .augment import dont_augment, orth_variants_augmenter # noqa: F401 +from .batchers import minibatch_by_padded_size, minibatch_by_words # noqa: F401 +from .callbacks import create_copy_from_base_model # noqa: F401 +from .corpus import Corpus, JsonlCorpus, PlainTextCorpus # noqa: F401 +from .example import Example, validate_examples, validate_get_examples # noqa: F401 +from .gold_io import docs_to_json, read_json_file # noqa: F401 +from .iob_utils import ( # noqa: F401 + biluo_tags_to_offsets, + biluo_tags_to_spans, + biluo_to_iob, + iob_to_biluo, + offsets_to_biluo_tags, + remove_bilu_prefix, + split_bilu_label, + tags_to_entities, +) +from .loggers import console_logger # noqa: F401 + +__all__ = [ + "Alignment", + "Corpus", + "Example", + "JsonlCorpus", + "PlainTextCorpus", + "biluo_tags_to_offsets", + "biluo_tags_to_spans", + "biluo_to_iob", + "create_copy_from_base_model", + "docs_to_json", + "dont_augment", + "iob_to_biluo", + "minibatch_by_padded_size", + "minibatch_by_words", + "offsets_to_biluo_tags", + "orth_variants_augmenter", + "read_json_file", + "remove_bilu_prefix", + "split_bilu_label", + "tags_to_entities", + "validate_get_examples", + "validate_examples", +] diff --git a/spacy/training/align.pyx b/spacy/training/align.pyx new file mode 100644 index 0000000..c68110e --- /dev/null +++ b/spacy/training/align.pyx @@ -0,0 +1,77 @@ +# cython: profile=False +import re +from itertools import chain +from typing import List, Tuple + +from ..errors import Errors + + +def get_alignments(A: List[str], B: List[str]) -> Tuple[List[List[int]], List[List[int]]]: + # Create character-to-token mappings + char_to_token_a = tuple(chain(*((i,) * len(x.lower()) for i, x in enumerate(A)))) + char_to_token_b = tuple(chain(*((i,) * len(x.lower()) for i, x in enumerate(B)))) + str_a = "".join(A).lower() + str_b = "".join(B).lower() + cdef int len_str_a = len(str_a) + cdef int len_str_b = len(str_b) + # Check that the two texts only differ in whitespace and capitalization + if re.sub(r"\s+", "", str_a) != re.sub(r"\s+", "", str_b) or \ + len_str_a != len(char_to_token_a) or \ + len_str_b != len(char_to_token_b): + raise ValueError(Errors.E949.format(x=str(A[:10]), y=str(B[:10]))) + cdef int char_idx_a = 0 + cdef int char_idx_b = 0 + cdef int token_idx_a = 0 + cdef int token_idx_b = 0 + cdef int prev_token_idx_a = -1 + cdef int prev_token_idx_b = -1 + a2b = [] + b2a = [] + while char_idx_a < len_str_a and char_idx_b < len_str_b: + # Find the current token position from the character position + token_idx_a = char_to_token_a[char_idx_a] + token_idx_b = char_to_token_b[char_idx_b] + # Add a set for the next token if a token boundary has been crossed + if prev_token_idx_a != token_idx_a: + a2b.append(set()) + if prev_token_idx_b != token_idx_b: + b2a.append(set()) + # Process the alignment at the current position + if A[token_idx_a] == B[token_idx_b] and \ + ( + char_idx_a == 0 or + char_to_token_a[char_idx_a - 1] < token_idx_a + ) and \ + ( + char_idx_b == 0 or + char_to_token_b[char_idx_b - 1] < token_idx_b + ): + # Current tokens are identical and both character offsets are the + # start of a token (either at the beginning of the document or the + # previous character belongs to a different token) + a2b[-1].add(token_idx_b) + b2a[-1].add(token_idx_a) + char_idx_a += len(A[token_idx_a]) + char_idx_b += len(B[token_idx_b]) + elif str_a[char_idx_a] == str_b[char_idx_b]: + # Current chars are identical + a2b[-1].add(token_idx_b) + b2a[-1].add(token_idx_a) + char_idx_a += 1 + char_idx_b += 1 + elif str_a[char_idx_a].isspace(): + # Skip unaligned whitespace char in A + char_idx_a += 1 + elif str_b[char_idx_b].isspace(): + # Skip unaligned whitespace char in B + char_idx_b += 1 + else: + # This should never happen + raise ValueError(Errors.E949.format(x=str(A[:10]), y=str(B[:10]))) + prev_token_idx_a = token_idx_a + prev_token_idx_b = token_idx_b + # Process unaligned trailing whitespace + a2b.extend([set()] * len(set(char_to_token_a[char_idx_a:]))) + b2a.extend([set()] * len(set(char_to_token_b[char_idx_b:]))) + # Return values as sorted lists per token position + return [sorted(x) for x in a2b], [sorted(x) for x in b2a] diff --git a/spacy/training/alignment.py b/spacy/training/alignment.py new file mode 100644 index 0000000..3f615d1 --- /dev/null +++ b/spacy/training/alignment.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import List + +from .align import get_alignments +from .alignment_array import AlignmentArray + + +@dataclass +class Alignment: + x2y: AlignmentArray + y2x: AlignmentArray + + @classmethod + def from_indices(cls, x2y: List[List[int]], y2x: List[List[int]]) -> "Alignment": + x2y = AlignmentArray(x2y) + y2x = AlignmentArray(y2x) + return Alignment(x2y=x2y, y2x=y2x) + + @classmethod + def from_strings(cls, A: List[str], B: List[str]) -> "Alignment": + x2y, y2x = get_alignments(A, B) + return Alignment.from_indices(x2y=x2y, y2x=y2x) diff --git a/spacy/training/alignment_array.pxd b/spacy/training/alignment_array.pxd new file mode 100644 index 0000000..bb28f3a --- /dev/null +++ b/spacy/training/alignment_array.pxd @@ -0,0 +1,8 @@ +cimport numpy as np +from libcpp.vector cimport vector + + +cdef class AlignmentArray: + cdef np.ndarray _data + cdef np.ndarray _lengths + cdef np.ndarray _starts_ends diff --git a/spacy/training/alignment_array.pyx b/spacy/training/alignment_array.pyx new file mode 100644 index 0000000..f0eb5cf --- /dev/null +++ b/spacy/training/alignment_array.pyx @@ -0,0 +1,78 @@ +# cython: profile=False +from typing import List + +import numpy + +from ..errors import Errors + +from libc.stdint cimport int32_t + + +cdef class AlignmentArray: + """AlignmentArray is similar to Thinc's Ragged with two simplfications: + indexing returns numpy arrays and this type can only be used for CPU arrays. + However, these changes make AlignmentArray more efficient for indexing in a + tight loop.""" + + __slots__ = [] + + def __init__(self, alignment: List[List[int]]): + cdef int data_len = 0 + cdef int outer_len + cdef int idx + + self._starts_ends = numpy.zeros(len(alignment) + 1, dtype='int32') + cdef int32_t* starts_ends_ptr = self._starts_ends.data + + for idx, outer in enumerate(alignment): + outer_len = len(outer) + starts_ends_ptr[idx + 1] = starts_ends_ptr[idx] + outer_len + data_len += outer_len + + self._lengths = None + self._data = numpy.empty(data_len, dtype="int32") + + idx = 0 + cdef int32_t* data_ptr = self._data.data + + for outer in alignment: + for inner in outer: + data_ptr[idx] = inner + idx += 1 + + def __getitem__(self, idx): + starts = self._starts_ends[:-1] + ends = self._starts_ends[1:] + if isinstance(idx, int): + start = starts[idx] + end = ends[idx] + elif isinstance(idx, slice): + if not (idx.step is None or idx.step == 1): + raise ValueError(Errors.E1027) + start = starts[idx] + if len(start) == 0: + return self._data[0:0] + start = start[0] + end = ends[idx][-1] + else: + raise ValueError(Errors.E1028) + + return self._data[start:end] + + @property + def data(self): + return self._data + + @property + def lengths(self): + if self._lengths is None: + self._lengths = self.ends - self.starts + return self._lengths + + @property + def ends(self): + return self._starts_ends[1:] + + @property + def starts(self): + return self._starts_ends[:-1] diff --git a/spacy/training/augment.py b/spacy/training/augment.py new file mode 100644 index 0000000..ba4368a --- /dev/null +++ b/spacy/training/augment.py @@ -0,0 +1,344 @@ +import itertools +import random +from functools import partial +from typing import TYPE_CHECKING, Callable, Dict, Iterator, List, Optional, Tuple + +from .example import Example +from .iob_utils import _doc_to_biluo_tags_with_partial, split_bilu_label + +if TYPE_CHECKING: + from ..language import Language # noqa: F401 + + +def create_combined_augmenter( + lower_level: float, + orth_level: float, + orth_variants: Optional[Dict[str, List[Dict]]], + whitespace_level: float, + whitespace_per_token: float, + whitespace_variants: Optional[List[str]], +) -> Callable[["Language", Example], Iterator[Example]]: + """Create a data augmentation callback that uses orth-variant replacement. + The callback can be added to a corpus or other data iterator during training. + + lower_level (float): The percentage of texts that will be lowercased. + orth_level (float): The percentage of texts that will be augmented. + orth_variants (Optional[Dict[str, List[Dict]]]): A dictionary containing the + single and paired orth variants. Typically loaded from a JSON file. + whitespace_level (float): The percentage of texts that will have whitespace + tokens inserted. + whitespace_per_token (float): The number of whitespace tokens to insert in + the modified doc as a percentage of the doc length. + whitespace_variants (Optional[List[str]]): The whitespace token texts. + RETURNS (Callable[[Language, Example], Iterator[Example]]): The augmenter. + """ + return partial( + combined_augmenter, + lower_level=lower_level, + orth_level=orth_level, + orth_variants=orth_variants, + whitespace_level=whitespace_level, + whitespace_per_token=whitespace_per_token, + whitespace_variants=whitespace_variants, + ) + + +def combined_augmenter( + nlp: "Language", + example: Example, + *, + lower_level: float = 0.0, + orth_level: float = 0.0, + orth_variants: Optional[Dict[str, List[Dict]]] = None, + whitespace_level: float = 0.0, + whitespace_per_token: float = 0.0, + whitespace_variants: Optional[List[str]] = None, +) -> Iterator[Example]: + if random.random() < lower_level: + example = make_lowercase_variant(nlp, example) + if orth_variants and random.random() < orth_level: + raw_text = example.text + orig_dict = example.to_dict() + orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) + variant_text, variant_token_annot = make_orth_variants( + nlp, + raw_text, + orig_dict["token_annotation"], + orth_variants, + lower=False, + ) + orig_dict["token_annotation"] = variant_token_annot + example = example.from_dict(nlp.make_doc(variant_text), orig_dict) + if whitespace_variants and random.random() < whitespace_level: + for _ in range(int(len(example.reference) * whitespace_per_token)): + example = make_whitespace_variant( + nlp, + example, + random.choice(whitespace_variants), + random.randrange(0, len(example.reference)), + ) + yield example + + +def create_orth_variants_augmenter( + level: float, lower: float, orth_variants: Dict[str, List[Dict]] +) -> Callable[["Language", Example], Iterator[Example]]: + """Create a data augmentation callback that uses orth-variant replacement. + The callback can be added to a corpus or other data iterator during training. + + level (float): The percentage of texts that will be augmented. + lower (float): The percentage of texts that will be lowercased. + orth_variants (Dict[str, List[Dict]]): A dictionary containing + the single and paired orth variants. Typically loaded from a JSON file. + RETURNS (Callable[[Language, Example], Iterator[Example]]): The augmenter. + """ + return partial( + orth_variants_augmenter, orth_variants=orth_variants, level=level, lower=lower + ) + + +def create_lower_casing_augmenter( + level: float, +) -> Callable[["Language", Example], Iterator[Example]]: + """Create a data augmentation callback that converts documents to lowercase. + The callback can be added to a corpus or other data iterator during training. + + level (float): The percentage of texts that will be augmented. + RETURNS (Callable[[Language, Example], Iterator[Example]]): The augmenter. + """ + return partial(lower_casing_augmenter, level=level) + + +def dont_augment(nlp: "Language", example: Example) -> Iterator[Example]: + yield example + + +def lower_casing_augmenter( + nlp: "Language", example: Example, *, level: float +) -> Iterator[Example]: + if random.random() >= level: + yield example + else: + yield make_lowercase_variant(nlp, example) + + +def make_lowercase_variant(nlp: "Language", example: Example): + example_dict = example.to_dict() + example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) + doc = nlp.make_doc(example.text.lower()) + example_dict["token_annotation"]["ORTH"] = [t.lower_ for t in example.reference] + return example.from_dict(doc, example_dict) + + +def orth_variants_augmenter( + nlp: "Language", + example: Example, + orth_variants: Dict[str, List[Dict]], + *, + level: float = 0.0, + lower: float = 0.0, +) -> Iterator[Example]: + if random.random() >= level: + yield example + else: + raw_text = example.text + orig_dict = example.to_dict() + orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) + variant_text, variant_token_annot = make_orth_variants( + nlp, + raw_text, + orig_dict["token_annotation"], + orth_variants, + lower=raw_text is not None and random.random() < lower, + ) + orig_dict["token_annotation"] = variant_token_annot + yield example.from_dict(nlp.make_doc(variant_text), orig_dict) + + +def make_orth_variants( + nlp: "Language", + raw: str, + token_dict: Dict[str, List[str]], + orth_variants: Dict[str, List[Dict[str, List[str]]]], + *, + lower: bool = False, +) -> Tuple[str, Dict[str, List[str]]]: + words = token_dict.get("ORTH", []) + tags = token_dict.get("TAG", []) + # keep unmodified if words are not defined + if not words: + return raw, token_dict + if lower: + words = [w.lower() for w in words] + raw = raw.lower() + # if no tags, only lowercase + if not tags: + token_dict["ORTH"] = words + return raw, token_dict + # single variants + ndsv = orth_variants.get("single", []) + punct_choices = [random.choice(x["variants"]) for x in ndsv] + for word_idx in range(len(words)): + for punct_idx in range(len(ndsv)): + if ( + tags[word_idx] in ndsv[punct_idx]["tags"] + and words[word_idx] in ndsv[punct_idx]["variants"] + ): + words[word_idx] = punct_choices[punct_idx] + # paired variants + ndpv = orth_variants.get("paired", []) + punct_choices = [random.choice(x["variants"]) for x in ndpv] + for word_idx in range(len(words)): + for punct_idx in range(len(ndpv)): + if tags[word_idx] in ndpv[punct_idx]["tags"] and words[ + word_idx + ] in itertools.chain.from_iterable(ndpv[punct_idx]["variants"]): + # backup option: random left vs. right from pair + pair_idx = random.choice([0, 1]) + # best option: rely on paired POS tags like `` / '' + if len(ndpv[punct_idx]["tags"]) == 2: + pair_idx = ndpv[punct_idx]["tags"].index(tags[word_idx]) + # next best option: rely on position in variants + # (may not be unambiguous, so order of variants matters) + else: + for pair in ndpv[punct_idx]["variants"]: + if words[word_idx] in pair: + pair_idx = pair.index(words[word_idx]) + words[word_idx] = punct_choices[punct_idx][pair_idx] + token_dict["ORTH"] = words + raw = construct_modified_raw_text(token_dict) + return raw, token_dict + + +def make_whitespace_variant( + nlp: "Language", + example: Example, + whitespace: str, + position: int, +) -> Example: + """Insert the whitespace token at the specified token offset in the doc. + This is primarily intended for v2-compatible training data that doesn't + include links or spans. If the document includes links, spans, or partial + dependency annotation, it is returned without modifications. + + The augmentation follows the basics of the v2 space attachment policy, but + without a distinction between "real" and other tokens, so space tokens + may be attached to space tokens: + - at the beginning of a sentence attach the space token to the following + token + - otherwise attach the space token to the preceding token + + The augmenter does not attempt to consolidate adjacent whitespace in the + same way that the tokenizer would. + + The following annotation is used for the space token: + TAG: "_SP" + MORPH: "" + POS: "SPACE" + LEMMA: ORTH + DEP: "dep" + SENT_START: False + + The annotation for each attribute is only set for the space token if there + is already at least partial annotation for that attribute in the original + example. + + RETURNS (Example): Example with one additional space token. + """ + example_dict = example.to_dict() + example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) + doc_dict = example_dict.get("doc_annotation", {}) + token_dict = example_dict.get("token_annotation", {}) + # returned unmodified if: + # - doc is empty + # - words are not defined + # - links are defined (only character-based offsets, which is more a quirk + # of Example.to_dict than a technical constraint) + # - spans are defined + # - there are partial dependencies + if ( + len(example.reference) == 0 + or "ORTH" not in token_dict + or len(doc_dict.get("links", [])) > 0 + or len(example.reference.spans) > 0 + or ( + example.reference.has_annotation("DEP") + and not example.reference.has_annotation("DEP", require_complete=True) + ) + ): + return example + words = token_dict.get("ORTH", []) + length = len(words) + assert 0 <= position <= length + if example.reference.has_annotation("ENT_TYPE"): + # I-ENTITY if between B/I-ENTITY and I/L-ENTITY otherwise O + entity = "O" + if position > 1 and position < length: + ent_prev = doc_dict["entities"][position - 1] + ent_next = doc_dict["entities"][position] + if "-" in ent_prev and "-" in ent_next: + ent_iob_prev, ent_type_prev = split_bilu_label(ent_prev) + ent_iob_next, ent_type_next = split_bilu_label(ent_next) + if ( + ent_iob_prev in ("B", "I") + and ent_iob_next in ("I", "L") + and ent_type_prev == ent_type_next + ): + entity = f"I-{ent_type_prev}" + doc_dict["entities"].insert(position, entity) + else: + del doc_dict["entities"] + token_dict["ORTH"].insert(position, whitespace) + token_dict["SPACY"].insert(position, False) + if example.reference.has_annotation("TAG"): + token_dict["TAG"].insert(position, "_SP") + else: + del token_dict["TAG"] + if example.reference.has_annotation("LEMMA"): + token_dict["LEMMA"].insert(position, whitespace) + else: + del token_dict["LEMMA"] + if example.reference.has_annotation("POS"): + token_dict["POS"].insert(position, "SPACE") + else: + del token_dict["POS"] + if example.reference.has_annotation("MORPH"): + token_dict["MORPH"].insert(position, "") + else: + del token_dict["MORPH"] + if example.reference.has_annotation("DEP", require_complete=True): + if position == 0: + token_dict["HEAD"].insert(position, 0) + else: + token_dict["HEAD"].insert(position, position - 1) + for i in range(len(token_dict["HEAD"])): + if token_dict["HEAD"][i] >= position: + token_dict["HEAD"][i] += 1 + token_dict["DEP"].insert(position, "dep") + else: + del token_dict["HEAD"] + del token_dict["DEP"] + if example.reference.has_annotation("SENT_START"): + token_dict["SENT_START"].insert(position, False) + else: + del token_dict["SENT_START"] + raw = construct_modified_raw_text(token_dict) + return Example.from_dict(nlp.make_doc(raw), example_dict) + + +def construct_modified_raw_text(token_dict): + """Construct modified raw text from words and spaces.""" + raw = "" + for orth, spacy in zip(token_dict["ORTH"], token_dict["SPACY"]): + raw += orth + if spacy: + raw += " " + return raw diff --git a/spacy/training/batchers.py b/spacy/training/batchers.py new file mode 100644 index 0000000..40e437d --- /dev/null +++ b/spacy/training/batchers.py @@ -0,0 +1,237 @@ +import itertools +from functools import partial +from typing import ( + Any, + Callable, + Iterable, + Iterator, + List, + Optional, + Sequence, + TypeVar, + Union, +) + +from ..util import minibatch + +Sizing = Union[Sequence[int], int] +ItemT = TypeVar("ItemT") +BatcherT = Callable[[Iterable[ItemT]], Iterable[List[ItemT]]] + + +def configure_minibatch_by_padded_size( + *, + size: Sizing, + buffer: int, + discard_oversize: bool, + get_length: Optional[Callable[[ItemT], int]] = None, +) -> BatcherT: + """Create a batcher that uses the `batch_by_padded_size` strategy. + + The padded size is defined as the maximum length of sequences within the + batch multiplied by the number of sequences in the batch. + + size (int or Sequence[int]): The largest padded size to batch sequences into. + Can be a single integer, or a sequence, allowing for variable batch sizes. + buffer (int): The number of sequences to accumulate before sorting by length. + A larger buffer will result in more even sizing, but if the buffer is + very large, the iteration order will be less random, which can result + in suboptimal training. + discard_oversize (bool): Whether to discard sequences that are by themselves + longer than the largest padded batch size. + get_length (Callable or None): Function to get the length of a sequence item. + The `len` function is used by default. + """ + # Avoid displacing optional values from the underlying function. + optionals = {"get_length": get_length} if get_length is not None else {} + return partial( + minibatch_by_padded_size, + size=size, + buffer=buffer, + discard_oversize=discard_oversize, + **optionals, + ) + + +def configure_minibatch_by_words( + *, + size: Sizing, + tolerance: float, + discard_oversize: bool, + get_length: Optional[Callable[[ItemT], int]] = None, +) -> BatcherT: + """Create a batcher that uses the "minibatch by words" strategy. + + size (int or Sequence[int]): The target number of words per batch. + Can be a single integer, or a sequence, allowing for variable batch sizes. + tolerance (float): What percentage of the size to allow batches to exceed. + discard_oversize (bool): Whether to discard sequences that by themselves + exceed the tolerated size. + get_length (Callable or None): Function to get the length of a sequence + item. The `len` function is used by default. + """ + optionals = {"get_length": get_length} if get_length is not None else {} + return partial( + minibatch_by_words, + size=size, + tolerance=tolerance, + discard_oversize=discard_oversize, + **optionals, + ) + + +def configure_minibatch( + size: Sizing, get_length: Optional[Callable[[ItemT], int]] = None +) -> BatcherT: + """Create a batcher that creates batches of the specified size. + + size (int or Sequence[int]): The target number of items per batch. + Can be a single integer, or a sequence, allowing for variable batch sizes. + """ + optionals = {"get_length": get_length} if get_length is not None else {} + return partial(minibatch, size=size, **optionals) + + +def minibatch_by_padded_size( + seqs: Iterable[ItemT], + size: Sizing, + buffer: int = 256, + discard_oversize: bool = False, + get_length: Callable = len, +) -> Iterable[List[ItemT]]: + """Minibatch a sequence by the size of padded batches that would result, + with sequences binned by length within a window. + + The padded size is defined as the maximum length of sequences within the + batch multiplied by the number of sequences in the batch. + + size (int or Sequence[int]): The largest padded size to batch sequences into. + buffer (int): The number of sequences to accumulate before sorting by length. + A larger buffer will result in more even sizing, but if the buffer is + very large, the iteration order will be less random, which can result + in suboptimal training. + discard_oversize (bool): Whether to discard sequences that are by themselves + longer than the largest padded batch size. + get_length (Callable or None): Function to get the length of a sequence item. + The `len` function is used by default. + """ + if isinstance(size, int): + size_ = itertools.repeat(size) # type: Iterator[int] + else: + size_ = iter(size) + for outer_batch in minibatch(seqs, size=buffer): + outer_batch = list(outer_batch) + target_size = next(size_) + for indices in _batch_by_length(outer_batch, target_size, get_length): + subbatch = [outer_batch[i] for i in indices] + padded_size = max(len(seq) for seq in subbatch) * len(subbatch) + if discard_oversize and padded_size >= target_size: + pass + else: + yield subbatch + + +def minibatch_by_words( + seqs: Iterable[ItemT], + size: Sizing, + tolerance=0.2, + discard_oversize=False, + get_length=len, +) -> Iterable[List[ItemT]]: + """Create minibatches of roughly a given number of words. If any examples + are longer than the specified batch length, they will appear in a batch by + themselves, or be discarded if discard_oversize=True. + + seqs (Iterable[Sequence]): The sequences to minibatch. + size (int or Sequence[int]): The target number of words per batch. + Can be a single integer, or a sequence, allowing for variable batch sizes. + tolerance (float): What percentage of the size to allow batches to exceed. + discard_oversize (bool): Whether to discard sequences that by themselves + exceed the tolerated size. + get_length (Callable or None): Function to get the length of a sequence + item. The `len` function is used by default. + """ + if isinstance(size, int): + size_ = itertools.repeat(size) # type: Iterator[int] + else: + size_ = iter(size) + target_size = next(size_) + tol_size = target_size * tolerance + batch = [] + overflow = [] + batch_size = 0 + overflow_size = 0 + for seq in seqs: + n_words = get_length(seq) + # if the current example exceeds the maximum batch size, it is returned separately + # but only if discard_oversize=False. + if n_words > target_size + tol_size: + if not discard_oversize: + yield [seq] + # add the example to the current batch if there's no overflow yet and it still fits + elif overflow_size == 0 and (batch_size + n_words) <= target_size: + batch.append(seq) + batch_size += n_words + # add the example to the overflow buffer if it fits in the tolerance margin + elif (batch_size + overflow_size + n_words) <= (target_size + tol_size): + overflow.append(seq) + overflow_size += n_words + # yield the previous batch and start a new one. The new one gets the overflow examples. + else: + if batch: + yield batch + target_size = next(size_) + tol_size = target_size * tolerance + batch = overflow + batch_size = overflow_size + overflow = [] + overflow_size = 0 + # this example still fits + if (batch_size + n_words) <= target_size: + batch.append(seq) + batch_size += n_words + # this example fits in overflow + elif (batch_size + n_words) <= (target_size + tol_size): + overflow.append(seq) + overflow_size += n_words + # this example does not fit with the previous overflow: start another new batch + else: + if batch: + yield batch + target_size = next(size_) + tol_size = target_size * tolerance + batch = [seq] + batch_size = n_words + batch.extend(overflow) + if batch: + yield batch + + +def _batch_by_length( + seqs: Sequence[Any], max_words: int, get_length=len +) -> List[List[Any]]: + """Given a list of sequences, return a batched list of indices into the + list, where the batches are grouped by length, in descending order. + + Batches may be at most max_words in size, defined as max sequence length * size. + """ + # Use negative index so we can get sort by position ascending. + lengths_indices = [(get_length(seq), i) for i, seq in enumerate(seqs)] + lengths_indices.sort() + batches = [] + batch: List[int] = [] + for length, i in lengths_indices: + if not batch: + batch.append(i) + elif length * (len(batch) + 1) <= max_words: + batch.append(i) + else: + batches.append(batch) + batch = [i] + if batch: + batches.append(batch) + # Check lengths match + assert sum(len(b) for b in batches) == len(seqs) + batches = [sorted(batch) for batch in batches] + batches.reverse() + return batches diff --git a/spacy/training/callbacks.py b/spacy/training/callbacks.py new file mode 100644 index 0000000..1938275 --- /dev/null +++ b/spacy/training/callbacks.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING, Callable, Optional + +from ..errors import Errors +from ..util import load_model, logger + +if TYPE_CHECKING: + from ..language import Language + + +def create_copy_from_base_model( + tokenizer: Optional[str] = None, + vocab: Optional[str] = None, +) -> Callable[["Language"], "Language"]: + def copy_from_base_model(nlp): + if tokenizer: + logger.info("Copying tokenizer from: %s", tokenizer) + base_nlp = load_model(tokenizer) + if nlp.config["nlp"]["tokenizer"] == base_nlp.config["nlp"]["tokenizer"]: + nlp.tokenizer.from_bytes(base_nlp.tokenizer.to_bytes(exclude=["vocab"])) + else: + raise ValueError( + Errors.E872.format( + curr_config=nlp.config["nlp"]["tokenizer"], + base_config=base_nlp.config["nlp"]["tokenizer"], + ) + ) + if vocab: + logger.info("Copying vocab from: %s", vocab) + # only reload if the vocab is from a different model + if tokenizer != vocab: + base_nlp = load_model(vocab) + nlp.vocab.from_bytes(base_nlp.vocab.to_bytes()) + + return copy_from_base_model diff --git a/spacy/training/converters/__init__.py b/spacy/training/converters/__init__.py new file mode 100644 index 0000000..8173da6 --- /dev/null +++ b/spacy/training/converters/__init__.py @@ -0,0 +1,4 @@ +from .conll_ner_to_docs import conll_ner_to_docs # noqa: F401 +from .conllu_to_docs import conllu_to_docs # noqa: F401 +from .iob_to_docs import iob_to_docs # noqa: F401 +from .json_to_docs import json_to_docs # noqa: F401 diff --git a/spacy/training/converters/conll_ner_to_docs.py b/spacy/training/converters/conll_ner_to_docs.py new file mode 100644 index 0000000..e66a8a8 --- /dev/null +++ b/spacy/training/converters/conll_ner_to_docs.py @@ -0,0 +1,170 @@ +from wasabi import Printer + +from ...errors import Errors +from ...tokens import Doc, Span +from ...training import iob_to_biluo +from ...util import get_lang_class, load_model +from .. import tags_to_entities + + +def conll_ner_to_docs( + input_data, n_sents=10, seg_sents=False, model=None, no_print=False, **kwargs +): + """ + Convert files in the CoNLL-2003 NER format and similar + whitespace-separated columns into Doc objects. + + The first column is the tokens, the final column is the IOB tags. If an + additional second column is present, the second column is the tags. + + Sentences are separated with whitespace and documents can be separated + using the line "-DOCSTART- -X- O O". + + Sample format: + + -DOCSTART- -X- O O + + I O + like O + London B-GPE + and O + New B-GPE + York I-GPE + City I-GPE + . O + + """ + msg = Printer(no_print=no_print) + doc_delimiter = "-DOCSTART- -X- O O" + # check for existing delimiters, which should be preserved + if "\n\n" in input_data and seg_sents: + msg.warn( + "Sentence boundaries found, automatic sentence segmentation with " + "`-s` disabled." + ) + seg_sents = False + if doc_delimiter in input_data and n_sents: + msg.warn( + "Document delimiters found, automatic document segmentation with " + "`-n` disabled." + ) + n_sents = 0 + # do document segmentation with existing sentences + if "\n\n" in input_data and doc_delimiter not in input_data and n_sents: + n_sents_info(msg, n_sents) + input_data = segment_docs(input_data, n_sents, doc_delimiter) + # do sentence segmentation with existing documents + if "\n\n" not in input_data and doc_delimiter in input_data and seg_sents: + input_data = segment_sents_and_docs(input_data, 0, "", model=model, msg=msg) + # do both sentence segmentation and document segmentation according + # to options + if "\n\n" not in input_data and doc_delimiter not in input_data: + # sentence segmentation required for document segmentation + if n_sents > 0 and not seg_sents: + msg.warn( + f"No sentence boundaries found to use with option `-n {n_sents}`. " + f"Use `-s` to automatically segment sentences or `-n 0` " + f"to disable." + ) + else: + n_sents_info(msg, n_sents) + input_data = segment_sents_and_docs( + input_data, n_sents, doc_delimiter, model=model, msg=msg + ) + # provide warnings for problematic data + if "\n\n" not in input_data: + msg.warn( + "No sentence boundaries found. Use `-s` to automatically segment sentences." + ) + if doc_delimiter not in input_data: + msg.warn( + "No document delimiters found. Use `-n` to automatically group " + "sentences into documents." + ) + + if model: + nlp = load_model(model) + else: + nlp = get_lang_class("xx")() + for conll_doc in input_data.strip().split(doc_delimiter): + conll_doc = conll_doc.strip() + if not conll_doc: + continue + words = [] + sent_starts = [] + pos_tags = [] + biluo_tags = [] + for conll_sent in conll_doc.split("\n\n"): + conll_sent = conll_sent.strip() + if not conll_sent: + continue + lines = [line.strip() for line in conll_sent.split("\n") if line.strip()] + cols = list(zip(*[line.split() for line in lines])) + if len(cols) < 2: + raise ValueError(Errors.E903) + length = len(cols[0]) + words.extend(cols[0]) + sent_starts.extend([True] + [False] * (length - 1)) + biluo_tags.extend(iob_to_biluo(cols[-1])) + pos_tags.extend(cols[1] if len(cols) > 2 else ["-"] * length) + + doc = Doc(nlp.vocab, words=words) + for i, token in enumerate(doc): + token.tag_ = pos_tags[i] + token.is_sent_start = sent_starts[i] + entities = tags_to_entities(biluo_tags) + doc.ents = [Span(doc, start=s, end=e + 1, label=L) for L, s, e in entities] + yield doc + + +def segment_sents_and_docs(doc, n_sents, doc_delimiter, model=None, msg=None): + sentencizer = None + if model: + nlp = load_model(model) + if "parser" in nlp.pipe_names: + msg.info(f"Segmenting sentences with parser from model '{model}'.") + for name, proc in nlp.pipeline: + if "parser" in getattr(proc, "listening_components", []): + nlp.replace_listeners(name, "parser", ["model.tok2vec"]) + sentencizer = nlp.get_pipe("parser") + if not sentencizer: + msg.info( + "Segmenting sentences with sentencizer. (Use `-b model` for " + "improved parser-based sentence segmentation.)" + ) + nlp = get_lang_class("xx")() + sentencizer = nlp.create_pipe("sentencizer") + lines = doc.strip().split("\n") + words = [line.strip().split()[0] for line in lines] + nlpdoc = Doc(nlp.vocab, words=words) + sentencizer(nlpdoc) + lines_with_segs = [] + sent_count = 0 + for i, token in enumerate(nlpdoc): + if token.is_sent_start: + if n_sents and sent_count % n_sents == 0: + lines_with_segs.append(doc_delimiter) + lines_with_segs.append("") + sent_count += 1 + lines_with_segs.append(lines[i]) + return "\n".join(lines_with_segs) + + +def segment_docs(input_data, n_sents, doc_delimiter): + sent_delimiter = "\n\n" + sents = input_data.split(sent_delimiter) + docs = [sents[i : i + n_sents] for i in range(0, len(sents), n_sents)] + input_data = "" + for doc in docs: + input_data += sent_delimiter + doc_delimiter + input_data += sent_delimiter.join(doc) + return input_data + + +def n_sents_info(msg, n_sents): + msg.info(f"Grouping every {n_sents} sentences into a document.") + if n_sents == 1: + msg.warn( + "To generate better training data, you may want to group " + "sentences into documents with `-n 10`." + ) diff --git a/spacy/training/converters/conllu_to_docs.py b/spacy/training/converters/conllu_to_docs.py new file mode 100644 index 0000000..3a60c4e --- /dev/null +++ b/spacy/training/converters/conllu_to_docs.py @@ -0,0 +1,306 @@ +import re + +from wasabi import Printer + +from ...tokens import Doc, Span, Token +from ...training import biluo_tags_to_spans, iob_to_biluo +from ...vocab import Vocab +from .conll_ner_to_docs import n_sents_info + + +def conllu_to_docs( + input_data, + n_sents=10, + append_morphology=False, + ner_map=None, + merge_subtokens=False, + no_print=False, + **_, +): + """ + Convert conllu files into JSON format for use with train cli. + append_morphology parameter enables appending morphology to tags, which is + useful for languages such as Spanish, where UD tags are not so rich. + + Extract NER tags if available and convert them so that they follow + BILUO and the Wikipedia scheme + """ + MISC_NER_PATTERN = "^((?:name|NE)=)?([BILU])-([A-Z_]+)|O$" + msg = Printer(no_print=no_print) + n_sents_info(msg, n_sents) + sent_docs = read_conllx( + input_data, + append_morphology=append_morphology, + ner_tag_pattern=MISC_NER_PATTERN, + ner_map=ner_map, + merge_subtokens=merge_subtokens, + ) + sent_docs_to_merge = [] + for sent_doc in sent_docs: + sent_docs_to_merge.append(sent_doc) + if len(sent_docs_to_merge) % n_sents == 0: + yield Doc.from_docs(sent_docs_to_merge) + sent_docs_to_merge = [] + if sent_docs_to_merge: + yield Doc.from_docs(sent_docs_to_merge) + + +def has_ner(input_data, ner_tag_pattern): + """ + Check the MISC column for NER tags. + """ + for sent in input_data.strip().split("\n\n"): + lines = sent.strip().split("\n") + if lines: + while lines[0].startswith("#"): + lines.pop(0) + for line in lines: + parts = line.split("\t") + id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts + for misc_part in misc.split("|"): + if re.match(ner_tag_pattern, misc_part): + return True + return False + + +def read_conllx( + input_data, + append_morphology=False, + merge_subtokens=False, + ner_tag_pattern="", + ner_map=None, +): + """Yield docs, one for each sentence""" + vocab = Vocab() # need vocab to make a minimal Doc + set_ents = has_ner(input_data, ner_tag_pattern) + for sent in input_data.strip().split("\n\n"): + lines = sent.strip().split("\n") + if lines: + while lines[0].startswith("#"): + lines.pop(0) + doc = conllu_sentence_to_doc( + vocab, + lines, + ner_tag_pattern, + merge_subtokens=merge_subtokens, + append_morphology=append_morphology, + ner_map=ner_map, + set_ents=set_ents, + ) + yield doc + + +def get_entities(lines, tag_pattern, ner_map=None): + """Find entities in the MISC column according to the pattern and map to + final entity type with `ner_map` if mapping present. Entity tag is 'O' if + the pattern is not matched. + + lines (str): CONLL-U lines for one sentences + tag_pattern (str): Regex pattern for entity tag + ner_map (dict): Map old NER tag names to new ones, '' maps to O. + RETURNS (list): List of BILUO entity tags + """ + miscs = [] + for line in lines: + parts = line.split("\t") + id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts + if "-" in id_ or "." in id_: + continue + miscs.append(misc) + + iob = [] + for misc in miscs: + iob_tag = "O" + for misc_part in misc.split("|"): + tag_match = re.match(tag_pattern, misc_part) + if tag_match: + prefix = tag_match.group(2) + suffix = tag_match.group(3) + if prefix and suffix: + iob_tag = prefix + "-" + suffix + if ner_map: + suffix = ner_map.get(suffix, suffix) + if suffix == "": + iob_tag = "O" + else: + iob_tag = prefix + "-" + suffix + break + iob.append(iob_tag) + return iob_to_biluo(iob) + + +def conllu_sentence_to_doc( + vocab, + lines, + ner_tag_pattern, + merge_subtokens=False, + append_morphology=False, + ner_map=None, + set_ents=False, +): + """Create an Example from the lines for one CoNLL-U sentence, merging + subtokens and appending morphology to tags if required. + + lines (str): The non-comment lines for a CoNLL-U sentence + ner_tag_pattern (str): The regex pattern for matching NER in MISC col + RETURNS (Example): An example containing the annotation + """ + # create a Doc with each subtoken as its own token + # if merging subtokens, each subtoken orth is the merged subtoken form + if not Token.has_extension("merged_orth"): + Token.set_extension("merged_orth", default="") + if not Token.has_extension("merged_lemma"): + Token.set_extension("merged_lemma", default="") + if not Token.has_extension("merged_morph"): + Token.set_extension("merged_morph", default="") + if not Token.has_extension("merged_spaceafter"): + Token.set_extension("merged_spaceafter", default="") + words, spaces, tags, poses, morphs, lemmas = [], [], [], [], [], [] + heads, deps = [], [] + subtok_word = "" + in_subtok = False + for i in range(len(lines)): + line = lines[i] + parts = line.split("\t") + id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts + if "." in id_: + continue + if "-" in id_: + in_subtok = True + if "-" in id_: + in_subtok = True + subtok_word = word + subtok_start, subtok_end = id_.split("-") + subtok_spaceafter = "SpaceAfter=No" not in misc + continue + if merge_subtokens and in_subtok: + words.append(subtok_word) + else: + words.append(word) + if in_subtok: + if id_ == subtok_end: + spaces.append(subtok_spaceafter) + else: + spaces.append(False) + elif "SpaceAfter=No" in misc: + spaces.append(False) + else: + spaces.append(True) + if in_subtok and id_ == subtok_end: + subtok_word = "" + in_subtok = False + id_ = int(id_) - 1 + head = (int(head) - 1) if head not in ("0", "_") else id_ + tag = pos if tag == "_" else tag + pos = pos if pos != "_" else "" + morph = morph if morph != "_" else "" + dep = "ROOT" if dep == "root" else dep + lemmas.append(lemma) + poses.append(pos) + tags.append(tag) + morphs.append(morph) + heads.append(head) + deps.append(dep) + + doc = Doc( + vocab, + words=words, + spaces=spaces, + tags=tags, + pos=poses, + deps=deps, + lemmas=lemmas, + morphs=morphs, + heads=heads, + ) + for i in range(len(doc)): + doc[i]._.merged_orth = words[i] + doc[i]._.merged_morph = morphs[i] + doc[i]._.merged_lemma = lemmas[i] + doc[i]._.merged_spaceafter = spaces[i] + ents = None + if set_ents: + ents = get_entities(lines, ner_tag_pattern, ner_map) + doc.ents = biluo_tags_to_spans(doc, ents) + + if merge_subtokens: + doc = merge_conllu_subtokens(lines, doc) + + # create final Doc from custom Doc annotation + words, spaces, tags, morphs, lemmas, poses = [], [], [], [], [], [] + heads, deps = [], [] + for i, t in enumerate(doc): + words.append(t._.merged_orth) + lemmas.append(t._.merged_lemma) + spaces.append(t._.merged_spaceafter) + morphs.append(t._.merged_morph) + if append_morphology and t._.merged_morph: + tags.append(t.tag_ + "__" + t._.merged_morph) + else: + tags.append(t.tag_) + poses.append(t.pos_) + heads.append(t.head.i) + deps.append(t.dep_) + + doc_x = Doc( + vocab, + words=words, + spaces=spaces, + tags=tags, + morphs=morphs, + lemmas=lemmas, + pos=poses, + deps=deps, + heads=heads, + ) + if set_ents: + doc_x.ents = [ + Span(doc_x, ent.start, ent.end, label=ent.label) for ent in doc.ents + ] + + return doc_x + + +def merge_conllu_subtokens(lines, doc): + # identify and process all subtoken spans to prepare attrs for merging + subtok_spans = [] + for line in lines: + parts = line.split("\t") + id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts + if "-" in id_: + subtok_start, subtok_end = id_.split("-") + subtok_span = doc[int(subtok_start) - 1 : int(subtok_end)] + subtok_spans.append(subtok_span) + # create merged tag, morph, and lemma values + tags = [] + morphs = {} + lemmas = [] + for token in subtok_span: + tags.append(token.tag_) + lemmas.append(token.lemma_) + if token._.merged_morph: + for feature in token._.merged_morph.split("|"): + field, values = feature.split("=", 1) + if field not in morphs: + morphs[field] = set() + for value in values.split(","): + morphs[field].add(value) + # create merged features for each morph field + for field, values in morphs.items(): + morphs[field] = field + "=" + ",".join(sorted(values)) + # set the same attrs on all subtok tokens so that whatever head the + # retokenizer chooses, the final attrs are available on that token + for token in subtok_span: + token._.merged_orth = token.orth_ + token._.merged_lemma = " ".join(lemmas) + token.tag_ = "_".join(tags) + token._.merged_morph = "|".join(sorted(morphs.values())) + token._.merged_spaceafter = ( + True if subtok_span[-1].whitespace_ else False + ) + + with doc.retokenize() as retokenizer: + for span in subtok_spans: + retokenizer.merge(span) + + return doc diff --git a/spacy/training/converters/iob_to_docs.py b/spacy/training/converters/iob_to_docs.py new file mode 100644 index 0000000..45bb656 --- /dev/null +++ b/spacy/training/converters/iob_to_docs.py @@ -0,0 +1,62 @@ +from wasabi import Printer + +from ...errors import Errors +from ...tokens import Doc, Span +from ...training import iob_to_biluo, tags_to_entities +from ...util import minibatch +from ...vocab import Vocab +from .conll_ner_to_docs import n_sents_info + + +def iob_to_docs(input_data, n_sents=10, no_print=False, *args, **kwargs): + """ + Convert IOB files with one sentence per line and tags separated with '|' + into Doc objects so they can be saved. IOB and IOB2 are accepted. + + Sample formats: + + I|O like|O London|I-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O + I|O like|O London|B-GPE and|O New|B-GPE York|I-GPE City|I-GPE .|O + I|PRP|O like|VBP|O London|NNP|I-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O + I|PRP|O like|VBP|O London|NNP|B-GPE and|CC|O New|NNP|B-GPE York|NNP|I-GPE City|NNP|I-GPE .|.|O + """ + vocab = Vocab() # need vocab to make a minimal Doc + msg = Printer(no_print=no_print) + if n_sents > 0: + n_sents_info(msg, n_sents) + yield from read_iob(input_data.split("\n"), vocab, n_sents) + + +def read_iob(raw_sents, vocab, n_sents): + for group in minibatch(raw_sents, size=n_sents): + tokens = [] + words = [] + tags = [] + iob = [] + sent_starts = [] + for line in group: + if not line.strip(): + continue + sent_tokens = [t.split("|") for t in line.split()] + if len(sent_tokens[0]) == 3: + sent_words, sent_tags, sent_iob = zip(*sent_tokens) + elif len(sent_tokens[0]) == 2: + sent_words, sent_iob = zip(*sent_tokens) + sent_tags = ["-"] * len(sent_words) + else: + raise ValueError(Errors.E902) + words.extend(sent_words) + tags.extend(sent_tags) + iob.extend(sent_iob) + tokens.extend(sent_tokens) + sent_starts.append(True) + sent_starts.extend([False for _ in sent_words[1:]]) + doc = Doc(vocab, words=words) + for i, tag in enumerate(tags): + doc[i].tag_ = tag + for i, sent_start in enumerate(sent_starts): + doc[i].is_sent_start = sent_start + biluo = iob_to_biluo(iob) + entities = tags_to_entities(biluo) + doc.ents = [Span(doc, start=s, end=e + 1, label=L) for (L, s, e) in entities] + yield doc diff --git a/spacy/training/converters/json_to_docs.py b/spacy/training/converters/json_to_docs.py new file mode 100644 index 0000000..b4beedd --- /dev/null +++ b/spacy/training/converters/json_to_docs.py @@ -0,0 +1,24 @@ +import srsly + +from ...lang.xx import MultiLanguage +from ...util import load_model +from ..example import ( + _fix_legacy_dict_data, + _parse_example_dict_data, + annotations_to_doc, +) +from ..gold_io import json_iterate, json_to_annotations + + +def json_to_docs(input_data, model=None, **kwargs): + nlp = load_model(model) if model is not None else MultiLanguage() + if not isinstance(input_data, bytes): + if not isinstance(input_data, str): + input_data = srsly.json_dumps(input_data) + input_data = input_data.encode("utf8") + for json_doc in json_iterate(input_data): + for json_para in json_to_annotations(json_doc): + example_dict = _fix_legacy_dict_data(json_para) + tok_dict, doc_dict = _parse_example_dict_data(example_dict) + doc = annotations_to_doc(nlp.vocab, tok_dict, doc_dict) + yield doc diff --git a/spacy/training/corpus.py b/spacy/training/corpus.py new file mode 100644 index 0000000..30e3291 --- /dev/null +++ b/spacy/training/corpus.py @@ -0,0 +1,331 @@ +import random +import warnings +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, List, Optional, Union + +import srsly + +from .. import util +from ..errors import Errors, Warnings +from ..tokens import Doc, DocBin +from ..vocab import Vocab +from .augment import dont_augment +from .example import Example + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from ..language import Language # noqa: F401 + +FILE_TYPE = ".spacy" + + +@util.registry.readers("spacy.Corpus.v1") +def create_docbin_reader( + path: Optional[Path], + gold_preproc: bool, + max_length: int = 0, + limit: int = 0, + augmenter: Optional[Callable] = None, +) -> Callable[["Language"], Iterable[Example]]: + if path is None: + raise ValueError(Errors.E913) + util.logger.debug("Loading corpus from path: %s", path) + return Corpus( + path, + gold_preproc=gold_preproc, + max_length=max_length, + limit=limit, + augmenter=augmenter, + ) + + +@util.registry.readers("spacy.JsonlCorpus.v1") +def create_jsonl_reader( + path: Optional[Union[str, Path]], + min_length: int = 0, + max_length: int = 0, + limit: int = 0, +) -> Callable[["Language"], Iterable[Example]]: + return JsonlCorpus(path, min_length=min_length, max_length=max_length, limit=limit) + + +@util.registry.readers("spacy.read_labels.v1") +def read_labels(path: Union[str, Path], *, require: bool = False): + # I decided not to give this a generic name, because I don't want people to + # use it for arbitrary stuff, as I want this require arg with default False. + path = Path(path) + if not require and not path.exists(): + return None + return srsly.read_json(path) + + +@util.registry.readers("spacy.PlainTextCorpus.v1") +def create_plain_text_reader( + path: Optional[Path], + min_length: int = 0, + max_length: int = 0, +) -> Callable[["Language"], Iterable[Example]]: + """Iterate Example objects from a file or directory of plain text + UTF-8 files with one line per doc. + + path (Path): The directory or filename to read from. + min_length (int): Minimum document length (in tokens). Shorter documents + will be skipped. Defaults to 0, which indicates no limit. + max_length (int): Maximum document length (in tokens). Longer documents will + be skipped. Defaults to 0, which indicates no limit. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus + """ + if path is None: + raise ValueError(Errors.E913) + return PlainTextCorpus(path, min_length=min_length, max_length=max_length) + + +def walk_corpus(path: Union[str, Path], file_type) -> List[Path]: + path = util.ensure_path(path) + if not path.is_dir() and path.parts[-1].endswith(file_type): + return [path] + orig_path = path + paths = [path] + locs = [] + seen = set() + for path in paths: + if str(path) in seen: + continue + seen.add(str(path)) + if path.parts and path.parts[-1].startswith("."): + continue + elif path.is_dir(): + paths.extend(path.iterdir()) + elif path.parts[-1].endswith(file_type): + locs.append(path) + if len(locs) == 0: + warnings.warn(Warnings.W090.format(path=orig_path, format=file_type)) + # It's good to sort these, in case the ordering messes up a cache. + locs.sort() + return locs + + +class Corpus: + """Iterate Example objects from a file or directory of DocBin (.spacy) + formatted data files. + + path (Path): The directory or filename to read from. + gold_preproc (bool): Whether to set up the Example object with gold-standard + sentences and tokens for the predictions. Gold preprocessing helps + the annotations align to the tokenization, and may result in sequences + of more consistent length. However, it may reduce run-time accuracy due + to train/test skew. Defaults to False. + max_length (int): Maximum document length. Longer documents will be + split into sentences, if sentence boundaries are available. Defaults to + 0, which indicates no limit. + limit (int): Limit corpus to a subset of examples, e.g. for debugging. + Defaults to 0, which indicates no limit. + augment (Callable[Example, Iterable[Example]]): Optional data augmentation + function, to extrapolate additional examples from your annotations. + shuffle (bool): Whether to shuffle the examples. + + DOCS: https://spacy.io/api/corpus + """ + + def __init__( + self, + path: Union[str, Path], + *, + limit: int = 0, + gold_preproc: bool = False, + max_length: int = 0, + augmenter: Optional[Callable] = None, + shuffle: bool = False, + ) -> None: + self.path = util.ensure_path(path) + self.gold_preproc = gold_preproc + self.max_length = max_length + self.limit = limit + self.augmenter = augmenter if augmenter is not None else dont_augment + self.shuffle = shuffle + + def __call__(self, nlp: "Language") -> Iterator[Example]: + """Yield examples from the data. + + nlp (Language): The current nlp object. + YIELDS (Example): The examples. + + DOCS: https://spacy.io/api/corpus#call + """ + ref_docs = self.read_docbin(nlp.vocab, walk_corpus(self.path, FILE_TYPE)) + if self.shuffle: + ref_docs = list(ref_docs) # type: ignore + random.shuffle(ref_docs) # type: ignore + + if self.gold_preproc: + examples = self.make_examples_gold_preproc(nlp, ref_docs) + else: + examples = self.make_examples(nlp, ref_docs) + for real_eg in examples: + for augmented_eg in self.augmenter(nlp, real_eg): # type: ignore[operator] + yield augmented_eg + + def _make_example( + self, nlp: "Language", reference: Doc, gold_preproc: bool + ) -> Example: + if gold_preproc or reference.has_unknown_spaces: + return Example( + Doc( + nlp.vocab, + words=[word.text for word in reference], + spaces=[bool(word.whitespace_) for word in reference], + ), + reference, + ) + else: + return Example(nlp.make_doc(reference.text), reference) + + def make_examples( + self, nlp: "Language", reference_docs: Iterable[Doc] + ) -> Iterator[Example]: + for reference in reference_docs: + if len(reference) == 0: + continue + elif self.max_length == 0 or len(reference) < self.max_length: + yield self._make_example(nlp, reference, False) + elif reference.has_annotation("SENT_START"): + for ref_sent in reference.sents: + if len(ref_sent) == 0: + continue + elif self.max_length == 0 or len(ref_sent) < self.max_length: + yield self._make_example(nlp, ref_sent.as_doc(), False) + + def make_examples_gold_preproc( + self, nlp: "Language", reference_docs: Iterable[Doc] + ) -> Iterator[Example]: + for reference in reference_docs: + if reference.has_annotation("SENT_START"): + ref_sents = [sent.as_doc() for sent in reference.sents] + else: + ref_sents = [reference] + for ref_sent in ref_sents: + eg = self._make_example(nlp, ref_sent, True) + if len(eg.x): + yield eg + + def read_docbin( + self, vocab: Vocab, locs: Iterable[Union[str, Path]] + ) -> Iterator[Doc]: + """Yield training examples as example dicts""" + i = 0 + for loc in locs: + loc = util.ensure_path(loc) + if loc.parts[-1].endswith(FILE_TYPE): # type: ignore[union-attr] + doc_bin = DocBin().from_disk(loc) + docs = doc_bin.get_docs(vocab) + for doc in docs: + if len(doc): + yield doc + i += 1 + if self.limit >= 1 and i >= self.limit: + break + + +class JsonlCorpus: + """Iterate Example objects from a file or directory of jsonl + formatted raw text files. + + path (Path): The directory or filename to read from. + min_length (int): Minimum document length (in tokens). Shorter documents + will be skipped. Defaults to 0, which indicates no limit. + + max_length (int): Maximum document length (in tokens). Longer documents will + be skipped. Defaults to 0, which indicates no limit. + limit (int): Limit corpus to a subset of examples, e.g. for debugging. + Defaults to 0, which indicates no limit. + + DOCS: https://spacy.io/api/corpus#jsonlcorpus + """ + + file_type = "jsonl" + + def __init__( + self, + path: Optional[Union[str, Path]], + *, + limit: int = 0, + min_length: int = 0, + max_length: int = 0, + ) -> None: + self.path = util.ensure_path(path) + self.min_length = min_length + self.max_length = max_length + self.limit = limit + + def __call__(self, nlp: "Language") -> Iterator[Example]: + """Yield examples from the data. + + nlp (Language): The current nlp object. + YIELDS (Example): The example objects. + + DOCS: https://spacy.io/api/corpus#jsonlcorpus-call + """ + for loc in walk_corpus(self.path, ".jsonl"): + records = srsly.read_jsonl(loc) + for record in records: + doc = nlp.make_doc(record["text"]) + if self.min_length >= 1 and len(doc) < self.min_length: + continue + elif self.max_length >= 1 and len(doc) >= self.max_length: + continue + else: + words = [w.text for w in doc] + spaces = [bool(w.whitespace_) for w in doc] + # We don't *need* an example here, but it seems nice to + # make it match the Corpus signature. + yield Example(doc, Doc(nlp.vocab, words=words, spaces=spaces)) + + +class PlainTextCorpus: + """Iterate Example objects from a file or directory of plain text + UTF-8 files with one line per doc. + + path (Path): The directory or filename to read from. + min_length (int): Minimum document length (in tokens). Shorter documents + will be skipped. Defaults to 0, which indicates no limit. + max_length (int): Maximum document length (in tokens). Longer documents will + be skipped. Defaults to 0, which indicates no limit. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus + """ + + file_type = "txt" + + def __init__( + self, + path: Optional[Union[str, Path]], + *, + min_length: int = 0, + max_length: int = 0, + ) -> None: + self.path = util.ensure_path(path) + self.min_length = min_length + self.max_length = max_length + + def __call__(self, nlp: "Language") -> Iterator[Example]: + """Yield examples from the data. + + nlp (Language): The current nlp object. + YIELDS (Example): The example objects. + + DOCS: https://spacy.io/api/corpus#plaintextcorpus-call + """ + for loc in walk_corpus(self.path, ".txt"): + with open(loc, encoding="utf-8") as f: + for text in f: + text = text.rstrip("\r\n") + if len(text): + doc = nlp.make_doc(text) + if self.min_length >= 1 and len(doc) < self.min_length: + continue + elif self.max_length >= 1 and len(doc) > self.max_length: + continue + # We don't *need* an example here, but it seems nice to + # make it match the Corpus signature. + yield Example(doc, doc.copy()) diff --git a/spacy/training/example.pxd b/spacy/training/example.pxd new file mode 100644 index 0000000..a7c71fa --- /dev/null +++ b/spacy/training/example.pxd @@ -0,0 +1,13 @@ +from libc.stdint cimport uint64_t + +from ..tokens.doc cimport Doc + + +cdef class Example: + cdef readonly Doc x + cdef readonly Doc y + cdef readonly object _cached_alignment + cdef readonly object _cached_words_x + cdef readonly object _cached_words_y + cdef readonly uint64_t _x_sig + cdef readonly uint64_t _y_sig diff --git a/spacy/training/example.pyi b/spacy/training/example.pyi new file mode 100644 index 0000000..06639d7 --- /dev/null +++ b/spacy/training/example.pyi @@ -0,0 +1,66 @@ +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +from ..tokens import Doc, Span +from ..vocab import Vocab +from .alignment import Alignment + +def annotations_to_doc( + vocab: Vocab, + tok_annot: Dict[str, Any], + doc_annot: Dict[str, Any], +) -> Doc: ... +def validate_examples( + examples: Iterable[Example], + method: str, +) -> None: ... +def validate_get_examples( + get_examples: Callable[[], Iterable[Example]], + method: str, +): ... + +class Example: + x: Doc + y: Doc + + def __init__( + self, + predicted: Doc, + reference: Doc, + *, + alignment: Optional[Alignment] = None, + ): ... + def __len__(self) -> int: ... + @property + def predicted(self) -> Doc: ... + @predicted.setter + def predicted(self, doc: Doc) -> None: ... + @property + def reference(self) -> Doc: ... + @reference.setter + def reference(self, doc: Doc) -> None: ... + def copy(self) -> Example: ... + @classmethod + def from_dict(cls, predicted: Doc, example_dict: Dict[str, Any]) -> Example: ... + @property + def alignment(self) -> Alignment: ... + def get_aligned(self, field: str, as_string=False): ... + def get_aligned_parse(self, projectivize=True): ... + def get_aligned_sent_starts(self): ... + def get_aligned_spans_x2y( + self, x_spans: Iterable[Span], allow_overlap=False + ) -> List[Span]: ... + def get_aligned_spans_y2x( + self, y_spans: Iterable[Span], allow_overlap=False + ) -> List[Span]: ... + def get_aligned_ents_and_ner(self) -> Tuple[List[Span], List[str]]: ... + def get_aligned_ner(self) -> List[str]: ... + def get_matching_ents(self, check_label: bool = True) -> List[Span]: ... + def to_dict(self) -> Dict[str, Any]: ... + def split_sents(self) -> List[Example]: ... + @property + def text(self) -> str: ... + def __str__(self) -> str: ... + def __repr__(self) -> str: ... + +def _parse_example_dict_data(example_dict): ... +def _fix_legacy_dict_data(example_dict): ... diff --git a/spacy/training/example.pyx b/spacy/training/example.pyx new file mode 100644 index 0000000..2c1ff34 --- /dev/null +++ b/spacy/training/example.pyx @@ -0,0 +1,671 @@ +# cython: profile=False +from collections.abc import Iterable as IterableInstance + +import numpy + +from murmurhash.mrmr cimport hash64 + +from ..tokens.doc cimport Doc +from ..tokens.span cimport Span + +from ..attrs import IDS +from ..errors import Errors, Warnings +from ..pipeline._parser_internals import nonproj +from ..tokens.span import Span +from .alignment import Alignment +from .iob_utils import ( + biluo_tags_to_spans, + biluo_to_iob, + doc_to_biluo_tags, + offsets_to_biluo_tags, + remove_bilu_prefix, +) + +from ..tokens.token cimport MISSING_DEP + +from ..util import all_equal, logger, to_ternary_int + + +cpdef Doc annotations_to_doc(vocab, tok_annot, doc_annot): + """ Create a Doc from dictionaries with token and doc annotations. """ + attrs, array = _annot2array(vocab, tok_annot, doc_annot) + output = Doc(vocab, words=tok_annot["ORTH"], spaces=tok_annot["SPACY"]) + if "entities" in doc_annot: + _add_entities_to_doc(output, doc_annot["entities"]) + if "spans" in doc_annot: + _add_spans_to_doc(output, doc_annot["spans"]) + if array.size: + output = output.from_array(attrs, array) + # links are currently added with ENT_KB_ID on the token level + output.cats.update(doc_annot.get("cats", {})) + return output + + +def validate_examples(examples, method): + """Check that a batch of examples received during processing is valid. + This function lives here to prevent circular imports. + + examples (Iterable[Examples]): A batch of examples. + method (str): The method name to show in error messages. + """ + if not isinstance(examples, IterableInstance): + err = Errors.E978.format(name=method, types=type(examples)) + raise TypeError(err) + wrong = set([type(eg) for eg in examples if not isinstance(eg, Example)]) + if wrong: + err = Errors.E978.format(name=method, types=wrong) + raise TypeError(err) + + +def validate_get_examples(get_examples, method): + """Check that a generator of a batch of examples received during processing is valid: + the callable produces a non-empty list of Example objects. + This function lives here to prevent circular imports. + + get_examples (Callable[[], Iterable[Example]]): A function that produces a batch of examples. + method (str): The method name to show in error messages. + """ + if get_examples is None or not hasattr(get_examples, "__call__"): + err = Errors.E930.format(method=method, obj=type(get_examples)) + raise TypeError(err) + examples = get_examples() + if not examples: + err = Errors.E930.format(method=method, obj=examples) + raise TypeError(err) + validate_examples(examples, method) + + +cdef class Example: + def __init__(self, Doc predicted, Doc reference, *, alignment=None): + if predicted is None: + raise TypeError(Errors.E972.format(arg="predicted")) + if reference is None: + raise TypeError(Errors.E972.format(arg="reference")) + self.predicted = predicted + self.reference = reference + self._cached_alignment = alignment + + def __len__(self): + return len(self.predicted) + + @property + def predicted(self): + return self.x + + @predicted.setter + def predicted(self, doc): + self.x = doc + self._cached_alignment = None + self._cached_words_x = [t.text for t in doc] + + @property + def reference(self): + return self.y + + @reference.setter + def reference(self, doc): + self.y = doc + self._cached_alignment = None + self._cached_words_y = [t.text for t in doc] + + def copy(self): + return Example( + self.x.copy(), + self.y.copy() + ) + + @classmethod + def from_dict(cls, Doc predicted, dict example_dict): + if predicted is None: + raise ValueError(Errors.E976.format(n="first", type="Doc")) + if example_dict is None: + raise ValueError(Errors.E976.format(n="second", type="dict")) + example_dict = _fix_legacy_dict_data(example_dict) + tok_dict, doc_dict = _parse_example_dict_data(example_dict) + if "ORTH" not in tok_dict: + tok_dict["ORTH"] = [tok.text for tok in predicted] + tok_dict["SPACY"] = [tok.whitespace_ for tok in predicted] + return Example( + predicted, + annotations_to_doc(predicted.vocab, tok_dict, doc_dict) + ) + + @property + def alignment(self): + x_sig = hash64(self.x.c, sizeof(self.x.c[0]) * self.x.length, 0) + y_sig = hash64(self.y.c, sizeof(self.y.c[0]) * self.y.length, 0) + if self._cached_alignment is None: + words_x = [token.text for token in self.x] + words_y = [token.text for token in self.y] + self._x_sig = x_sig + self._y_sig = y_sig + self._cached_words_x = words_x + self._cached_words_y = words_y + self._cached_alignment = Alignment.from_strings(words_x, words_y) + return self._cached_alignment + elif self._x_sig == x_sig and self._y_sig == y_sig: + # If we have a cached alignment, check whether the cache is invalid + # due to retokenization. To make this check fast in loops, we first + # check a hash of the TokenC arrays. + return self._cached_alignment + else: + words_x = [token.text for token in self.x] + words_y = [token.text for token in self.y] + if words_x == self._cached_words_x and words_y == self._cached_words_y: + self._x_sig = x_sig + self._y_sig = y_sig + return self._cached_alignment + else: + self._cached_alignment = Alignment.from_strings(words_x, words_y) + self._cached_words_x = words_x + self._cached_words_y = words_y + self._x_sig = x_sig + self._y_sig = y_sig + return self._cached_alignment + + def _get_aligned_vectorized(self, align, gold_values): + # Fast path for Doc attributes/fields that are predominantly a single value, + # i.e., TAG, POS, MORPH. + x2y_single_toks = [] + x2y_single_toks_i = [] + + x2y_multiple_toks = [] + x2y_multiple_toks_i = [] + + # Gather indices of gold tokens aligned to the candidate tokens into two buckets. + # Bucket 1: All tokens that have a one-to-one alignment. + # Bucket 2: All tokens that have a one-to-many alignment. + for idx, token in enumerate(self.predicted): + aligned_gold_i = align[token.i] + aligned_gold_len = len(aligned_gold_i) + + if aligned_gold_len == 1: + x2y_single_toks.append(aligned_gold_i.item()) + x2y_single_toks_i.append(idx) + elif aligned_gold_len > 1: + x2y_multiple_toks.append(aligned_gold_i) + x2y_multiple_toks_i.append(idx) + + # Map elements of the first bucket directly to the output array. + output = numpy.full(len(self.predicted), None) + output[x2y_single_toks_i] = gold_values[x2y_single_toks].squeeze() + + # Collapse many-to-one alignments into one-to-one alignments if they + # share the same value. Map to None in all other cases. + for i in range(len(x2y_multiple_toks)): + aligned_gold_values = gold_values[x2y_multiple_toks[i]] + + # If all aligned tokens have the same value, use it. + if all_equal(aligned_gold_values): + x2y_multiple_toks[i] = aligned_gold_values[0].item() + else: + x2y_multiple_toks[i] = None + + output[x2y_multiple_toks_i] = x2y_multiple_toks + + return output.tolist() + + def _get_aligned_non_vectorized(self, align, gold_values): + # Slower path for fields that return multiple values (resulting + # in ragged arrays that cannot be vectorized trivially). + output = [None] * len(self.predicted) + + for token in self.predicted: + aligned_gold_i = align[token.i] + values = gold_values[aligned_gold_i].ravel() + if len(values) == 1: + output[token.i] = values.item() + elif all_equal(values): + # If all aligned tokens have the same value, use it. + output[token.i] = values[0].item() + + return output + + def get_aligned(self, field, as_string=False): + """Return an aligned array for a token attribute.""" + align = self.alignment.x2y + gold_values = self.reference.to_array([field]) + + if len(gold_values.shape) == 1: + output = self._get_aligned_vectorized(align, gold_values) + else: + output = self._get_aligned_non_vectorized(align, gold_values) + + vocab = self.reference.vocab + if as_string and field not in ["ENT_IOB", "SENT_START"]: + output = [vocab.strings[o] if o is not None else o for o in output] + + return output + + def get_aligned_parse(self, projectivize=True): + cand_to_gold = self.alignment.x2y + gold_to_cand = self.alignment.y2x + heads = [token.head.i for token in self.y] + deps = [token.dep_ for token in self.y] + + if projectivize: + proj_heads, proj_deps = nonproj.projectivize(heads, deps) + has_deps = [token.has_dep() for token in self.y] + has_heads = [token.has_head() for token in self.y] + + # ensure that missing data remains missing + heads = [h if has_heads[i] else heads[i] for i, h in enumerate(proj_heads)] + deps = [d if has_deps[i] else deps[i] for i, d in enumerate(proj_deps)] + + # Select all candidate tokens that are aligned to a single gold token. + c2g_single_toks = numpy.where(cand_to_gold.lengths == 1)[0] + + # Fetch all aligned gold token incides. + if c2g_single_toks.shape == cand_to_gold.lengths.shape: + # This the most likely case. + gold_i = cand_to_gold[:] + else: + gold_i = numpy.vectorize(lambda x: cand_to_gold[int(x)][0], otypes='i')(c2g_single_toks) + + # Fetch indices of all gold heads for the aligned gold tokens. + heads = numpy.asarray(heads, dtype='i') + gold_head_i = heads[gold_i] + + # Select all gold tokens that are heads of the previously selected + # gold tokens (and are aligned to a single candidate token). + g2c_len_heads = gold_to_cand.lengths[gold_head_i] + g2c_len_heads = numpy.where(g2c_len_heads == 1)[0] + g2c_i = numpy.vectorize(lambda x: gold_to_cand[int(x)][0], otypes='i')(gold_head_i[g2c_len_heads]).squeeze() + + # Update head/dep alignments with the above. + aligned_heads = numpy.full((self.x.length), None) + aligned_heads[c2g_single_toks[g2c_len_heads]] = g2c_i + + deps = numpy.asarray(deps) + aligned_deps = numpy.full((self.x.length), None) + aligned_deps[c2g_single_toks] = deps[gold_i] + + return aligned_heads.tolist(), aligned_deps.tolist() + + def get_aligned_sent_starts(self): + """Get list of SENT_START attributes aligned to the predicted tokenization. + If the reference does not have sentence starts, return a list of None values. + """ + if self.y.has_annotation("SENT_START"): + align = self.alignment.y2x + sent_starts = [False] * len(self.x) + for y_sent in self.y.sents: + x_start = int(align[y_sent.start][0]) + sent_starts[x_start] = True + return sent_starts + else: + return [None] * len(self.x) + + def get_aligned_spans_x2y(self, x_spans, allow_overlap=False): + return self._get_aligned_spans(self.y, x_spans, self.alignment.x2y, allow_overlap) + + def get_aligned_spans_y2x(self, y_spans, allow_overlap=False): + return self._get_aligned_spans(self.x, y_spans, self.alignment.y2x, allow_overlap) + + def _get_aligned_spans(self, doc, spans, align, allow_overlap): + seen = set() + output = [] + for span in spans: + indices = align[span.start : span.end] + if not allow_overlap: + indices = [idx for idx in indices if idx not in seen] + if len(indices) >= 1: + aligned_span = Span(doc, indices[0], indices[-1] + 1, label=span.label) + target_text = span.text.lower().strip().replace(" ", "") + our_text = aligned_span.text.lower().strip().replace(" ", "") + if our_text == target_text: + output.append(aligned_span) + seen.update(indices) + return output + + def get_aligned_ents_and_ner(self): + if not self.y.has_annotation("ENT_IOB"): + return [], [None] * len(self.x) + x_ents = self.get_aligned_spans_y2x(self.y.ents, allow_overlap=False) + # Default to 'None' for missing values + x_tags = offsets_to_biluo_tags( + self.x, + [(e.start_char, e.end_char, e.label_) for e in x_ents], + missing=None + ) + # Now fill the tokens we can align to O. + O = 2 # I=1, O=2, B=3 # no-cython-lint: E741 + for i, ent_iob in enumerate(self.get_aligned("ENT_IOB")): + if x_tags[i] is None: + if ent_iob == O: + x_tags[i] = "O" + elif self.x[i].is_space: + x_tags[i] = "O" + return x_ents, x_tags + + def get_aligned_ner(self): + _x_ents, x_tags = self.get_aligned_ents_and_ner() + return x_tags + + def get_matching_ents(self, check_label=True): + """Return entities that are shared between predicted and reference docs. + + If `check_label` is True, entities must have matching labels to be + kept. Otherwise only the character indices need to match. + """ + gold = {} + for ent in self.reference.ents: + gold[(ent.start_char, ent.end_char)] = ent.label + + keep = [] + for ent in self.predicted.ents: + key = (ent.start_char, ent.end_char) + if key not in gold: + continue + + if check_label and ent.label != gold[key]: + continue + + keep.append(ent) + + return keep + + def to_dict(self): + return { + "doc_annotation": { + "cats": dict(self.reference.cats), + "entities": doc_to_biluo_tags(self.reference), + "spans": self._spans_to_dict(), + "links": self._links_to_dict() + }, + "token_annotation": { + "ORTH": [t.text for t in self.reference], + "SPACY": [bool(t.whitespace_) for t in self.reference], + "TAG": [t.tag_ for t in self.reference], + "LEMMA": [t.lemma_ for t in self.reference], + "POS": [t.pos_ for t in self.reference], + "MORPH": [str(t.morph) for t in self.reference], + "HEAD": [t.head.i for t in self.reference], + "DEP": [t.dep_ for t in self.reference], + "SENT_START": [int(bool(t.is_sent_start)) for t in self.reference] + } + } + + def _spans_to_dict(self): + span_dict = {} + for key in self.reference.spans: + span_tuples = [] + for span in self.reference.spans[key]: + span_tuple = (span.start_char, span.end_char, span.label_, span.kb_id_) + span_tuples.append(span_tuple) + span_dict[key] = span_tuples + + return span_dict + + def _links_to_dict(self): + links = {} + for ent in self.reference.ents: + if ent.kb_id_: + links[(ent.start_char, ent.end_char)] = {ent.kb_id_: 1.0} + return links + + def split_sents(self): + """ Split the token annotations into multiple Examples based on + sent_starts and return a list of the new Examples""" + if not self.reference.has_annotation("SENT_START"): + return [self] + + align = self.alignment.y2x + seen_indices = set() + output = [] + for y_sent in self.reference.sents: + indices = align[y_sent.start : y_sent.end] + indices = [idx for idx in indices if idx not in seen_indices] + if indices: + x_sent = self.predicted[indices[0] : indices[-1] + 1] + output.append(Example(x_sent.as_doc(), y_sent.as_doc())) + seen_indices.update(indices) + return output + + @property + def text(self): + return self.x.text + + def __str__(self): + return str(self.to_dict()) + + def __repr__(self): + return str(self.to_dict()) + + +def _annot2array(vocab, tok_annot, doc_annot): + attrs = [] + values = [] + + for key, value in doc_annot.items(): + if value: + if key in ["entities", "cats", "spans"]: + pass + elif key == "links": + ent_kb_ids = _parse_links(vocab, tok_annot["ORTH"], tok_annot["SPACY"], value) + tok_annot["ENT_KB_ID"] = ent_kb_ids + else: + raise ValueError(Errors.E974.format(obj="doc", key=key)) + + for key, value in tok_annot.items(): + if key not in IDS: + raise ValueError(Errors.E974.format(obj="token", key=key)) + elif key in ["ORTH", "SPACY"]: + continue + elif key == "HEAD": + attrs.append(key) + row = [h-i if h is not None else 0 for i, h in enumerate(value)] + elif key == "DEP": + attrs.append(key) + row = [vocab.strings.add(h) if h is not None else MISSING_DEP for h in value] + elif key == "SENT_START": + attrs.append(key) + row = [to_ternary_int(v) for v in value] + elif key == "MORPH": + attrs.append(key) + row = [vocab.morphology.add(v) for v in value] + else: + attrs.append(key) + if not all(isinstance(v, str) for v in value): + types = set([type(v) for v in value]) + raise TypeError(Errors.E969.format(field=key, types=types)) from None + row = [vocab.strings.add(v) for v in value] + values.append([numpy.array(v, dtype=numpy.int32).astype(numpy.uint64) if v < 0 else v for v in row]) + array = numpy.array(values, dtype=numpy.uint64) + return attrs, array.T + + +def _add_spans_to_doc(doc, spans_data): + if not isinstance(spans_data, dict): + raise ValueError(Errors.E879) + for key, span_list in spans_data.items(): + spans = [] + if not isinstance(span_list, list): + raise ValueError(Errors.E879) + for span_tuple in span_list: + if not isinstance(span_tuple, (list, tuple)) or len(span_tuple) < 2: + raise ValueError(Errors.E879) + start_char = span_tuple[0] + end_char = span_tuple[1] + label = 0 + kb_id = 0 + if len(span_tuple) > 2: + label = span_tuple[2] + if len(span_tuple) > 3: + kb_id = span_tuple[3] + span = doc.char_span(start_char, end_char, label=label, kb_id=kb_id) + spans.append(span) + doc.spans[key] = spans + + +def _add_entities_to_doc(doc, ner_data): + if ner_data is None: + return + elif ner_data == []: + doc.ents = [] + elif not isinstance(ner_data, (list, tuple)): + raise ValueError(Errors.E973) + elif isinstance(ner_data[0], (list, tuple)): + return _add_entities_to_doc( + doc, + offsets_to_biluo_tags(doc, ner_data) + ) + elif isinstance(ner_data[0], str) or ner_data[0] is None: + return _add_entities_to_doc( + doc, + biluo_tags_to_spans(doc, ner_data) + ) + elif isinstance(ner_data[0], Span): + entities = [] + missing = [] + for span in ner_data: + if span.label: + entities.append(span) + else: + missing.append(span) + doc.set_ents(entities, missing=missing) + else: + raise ValueError(Errors.E973) + + +def _parse_example_dict_data(example_dict): + return ( + example_dict["token_annotation"], + example_dict["doc_annotation"] + ) + + +def _fix_legacy_dict_data(example_dict): + token_dict = example_dict.get("token_annotation", {}) + doc_dict = example_dict.get("doc_annotation", {}) + for key, value in example_dict.items(): + if value is not None: + if key in ("token_annotation", "doc_annotation"): + pass + elif key == "ids": + pass + elif key in ("cats", "links", "spans"): + doc_dict[key] = value + elif key in ("ner", "entities"): + doc_dict["entities"] = value + else: + token_dict[key] = value + # Remap keys + remapping = { + "words": "ORTH", + "tags": "TAG", + "pos": "POS", + "lemmas": "LEMMA", + "deps": "DEP", + "heads": "HEAD", + "sent_starts": "SENT_START", + "morphs": "MORPH", + "spaces": "SPACY", + } + old_token_dict = token_dict + token_dict = {} + for key, value in old_token_dict.items(): + if key in ("text", "ids", "brackets"): + pass + elif key in remapping.values(): + token_dict[key] = value + elif key.lower() in remapping: + token_dict[remapping[key.lower()]] = value + else: + all_keys = set(remapping.values()) + all_keys.update(remapping.keys()) + raise KeyError(Errors.E983.format(key=key, dict="token_annotation", keys=all_keys)) + text = example_dict.get("text", example_dict.get("raw")) + if _has_field(token_dict, "ORTH") and not _has_field(token_dict, "SPACY"): + token_dict["SPACY"] = _guess_spaces(text, token_dict["ORTH"]) + if "HEAD" in token_dict and "SENT_START" in token_dict: + # If heads are set, we don't also redundantly specify SENT_START. + token_dict.pop("SENT_START") + logger.debug(Warnings.W092) + return { + "token_annotation": token_dict, + "doc_annotation": doc_dict + } + + +def _has_field(annot, field): + if field not in annot: + return False + elif annot[field] is None: + return False + elif len(annot[field]) == 0: + return False + elif all([value is None for value in annot[field]]): + return False + else: + return True + + +def _parse_ner_tags(biluo_or_offsets, vocab, words, spaces): + if isinstance(biluo_or_offsets[0], (list, tuple)): + # Convert to biluo if necessary + # This is annoying but to convert the offsets we need a Doc + # that has the target tokenization. + reference = Doc(vocab, words=words, spaces=spaces) + biluo = offsets_to_biluo_tags(reference, biluo_or_offsets) + else: + biluo = biluo_or_offsets + ent_iobs = [] + ent_types = [] + for iob_tag in biluo_to_iob(biluo): + if iob_tag in (None, "-"): + ent_iobs.append("") + ent_types.append("") + else: + ent_iobs.append(iob_tag.split("-")[0]) + if iob_tag.startswith("I") or iob_tag.startswith("B"): + ent_types.append(remove_bilu_prefix(iob_tag)) + else: + ent_types.append("") + return ent_iobs, ent_types + + +def _parse_links(vocab, words, spaces, links): + reference = Doc(vocab, words=words, spaces=spaces) + starts = {token.idx: token.i for token in reference} + ends = {token.idx + len(token): token.i for token in reference} + ent_kb_ids = ["" for _ in reference] + + for index, annot_dict in links.items(): + true_kb_ids = [] + for key, value in annot_dict.items(): + if value == 1.0: + true_kb_ids.append(key) + if len(true_kb_ids) > 1: + raise ValueError(Errors.E980) + + if len(true_kb_ids) == 1: + start_char, end_char = index + start_token = starts.get(start_char) + end_token = ends.get(end_char) + if start_token is None or end_token is None: + raise ValueError(Errors.E981) + for i in range(start_token, end_token+1): + ent_kb_ids[i] = true_kb_ids[0] + + return ent_kb_ids + + +def _guess_spaces(text, words): + if text is None: + return None + spaces = [] + text_pos = 0 + # align words with text + for word in words: + try: + word_start = text[text_pos:].index(word) + except ValueError: + spaces.append(True) + continue + text_pos += word_start + len(word) + if text_pos < len(text) and text[text_pos] == " ": + spaces.append(True) + else: + spaces.append(False) + return spaces diff --git a/spacy/training/gold_io.pyx b/spacy/training/gold_io.pyx new file mode 100644 index 0000000..afbdf46 --- /dev/null +++ b/spacy/training/gold_io.pyx @@ -0,0 +1,226 @@ +# cython: profile=False +import warnings + +import srsly + +from .. import util +from ..errors import Warnings +from ..tokens import Doc +from .iob_utils import offsets_to_biluo_tags + + +def docs_to_json(docs, doc_id=0, ner_missing_tag="O"): + """Convert a list of Doc objects into the JSON-serializable format used by + the spacy train command. + + docs (iterable / Doc): The Doc object(s) to convert. + doc_id (int): Id for the JSON. + RETURNS (dict): The data in spaCy's JSON format + - each input doc will be treated as a paragraph in the output doc + """ + if isinstance(docs, Doc): + docs = [docs] + json_doc = {"id": doc_id, "paragraphs": []} + for i, doc in enumerate(docs): + raw = None if doc.has_unknown_spaces else doc.text + json_para = { + 'raw': raw, + "sentences": [], + "cats": [], + "entities": [], + "links": [] + } + for cat, val in doc.cats.items(): + json_cat = {"label": cat, "value": val} + json_para["cats"].append(json_cat) + # warning: entities information is currently duplicated as + # doc-level "entities" and token-level "ner" + for ent in doc.ents: + ent_tuple = (ent.start_char, ent.end_char, ent.label_) + json_para["entities"].append(ent_tuple) + if ent.kb_id_: + link_dict = {(ent.start_char, ent.end_char): {ent.kb_id_: 1.0}} + json_para["links"].append(link_dict) + biluo_tags = offsets_to_biluo_tags( + doc, json_para["entities"], missing=ner_missing_tag + ) + attrs = ("TAG", "POS", "MORPH", "LEMMA", "DEP", "ENT_IOB") + include_annotation = {attr: doc.has_annotation(attr) for attr in attrs} + for j, sent in enumerate(doc.sents): + json_sent = {"tokens": [], "brackets": []} + for token in sent: + json_token = { + "id": token.i, "orth": token.text, "space": token.whitespace_ + } + if include_annotation["TAG"]: + json_token["tag"] = token.tag_ + if include_annotation["POS"]: + json_token["pos"] = token.pos_ + if include_annotation["MORPH"]: + json_token["morph"] = str(token.morph) + if include_annotation["LEMMA"]: + json_token["lemma"] = token.lemma_ + if include_annotation["DEP"]: + json_token["head"] = token.head.i-token.i + json_token["dep"] = token.dep_ + if include_annotation["ENT_IOB"]: + json_token["ner"] = biluo_tags[token.i] + json_sent["tokens"].append(json_token) + json_para["sentences"].append(json_sent) + json_doc["paragraphs"].append(json_para) + return json_doc + + +def read_json_file(loc, docs_filter=None, limit=None): + """Read Example dictionaries from a json file or directory.""" + loc = util.ensure_path(loc) + if loc.is_dir(): + for filename in sorted(loc.iterdir()): + yield from read_json_file(loc / filename, limit=limit) + else: + with loc.open("rb") as file_: + utf8_str = file_.read() + for json_doc in json_iterate(utf8_str): + if docs_filter is not None and not docs_filter(json_doc): + continue + for json_paragraph in json_to_annotations(json_doc): + yield json_paragraph + + +def json_to_annotations(doc): + """Convert an item in the JSON-formatted training data to the format + used by Example. + + doc (dict): One entry in the training data. + YIELDS (tuple): The reformatted data - one training example per paragraph + """ + for paragraph in doc["paragraphs"]: + example = {"text": paragraph.get("raw", None)} + words = [] + spaces = [] + ids = [] + tags = [] + ner_tags = [] + pos = [] + morphs = [] + lemmas = [] + heads = [] + labels = [] + sent_starts = [] + brackets = [] + for sent in paragraph["sentences"]: + sent_start_i = len(words) + for i, token in enumerate(sent["tokens"]): + words.append(token["orth"]) + spaces.append(token.get("space", None)) + ids.append(token.get('id', sent_start_i + i)) + tags.append(token.get("tag", None)) + pos.append(token.get("pos", None)) + morphs.append(token.get("morph", None)) + lemmas.append(token.get("lemma", None)) + if "head" in token: + heads.append(token["head"] + sent_start_i + i) + else: + heads.append(None) + if "dep" in token: + labels.append(token["dep"]) + # Ensure ROOT label is case-insensitive + if labels[-1].lower() == "root": + labels[-1] = "ROOT" + else: + labels.append(None) + ner_tags.append(token.get("ner", None)) + if i == 0: + sent_starts.append(1) + else: + sent_starts.append(-1) + if "brackets" in sent: + brackets.extend( + ( + b["first"] + sent_start_i, + b["last"] + sent_start_i, + b["label"] + ) + for b in sent["brackets"] + ) + + example["token_annotation"] = dict( + ids=ids, + words=words, + spaces=spaces, + sent_starts=sent_starts, + brackets=brackets + ) + # avoid including dummy values that looks like gold info was present + if any(tags): + example["token_annotation"]["tags"] = tags + if any(pos): + example["token_annotation"]["pos"] = pos + if any(morphs): + example["token_annotation"]["morphs"] = morphs + if any(lemmas): + example["token_annotation"]["lemmas"] = lemmas + if any(head is not None for head in heads): + example["token_annotation"]["heads"] = heads + if any(labels): + example["token_annotation"]["deps"] = labels + + cats = {} + for cat in paragraph.get("cats", {}): + cats[cat["label"]] = cat["value"] + example["doc_annotation"] = dict( + cats=cats, + entities=ner_tags, + links=paragraph.get("links", []) + ) + yield example + + +def json_iterate(bytes utf8_str): + # We should've made these files jsonl...But since we didn't, parse out + # the docs one-by-one to reduce memory usage. + # It's okay to read in the whole file -- just don't parse it into JSON. + cdef long file_length = len(utf8_str) + if file_length > 2 ** 30: + warnings.warn(Warnings.W027.format(size=file_length)) + + raw = utf8_str + cdef int square_depth = 0 + cdef int curly_depth = 0 + cdef int inside_string = 0 + cdef int escape = 0 + cdef long start = -1 + cdef char c + cdef char quote = ord('"') + cdef char backslash = ord("\\") + cdef char open_square = ord("[") + cdef char close_square = ord("]") + cdef char open_curly = ord("{") + cdef char close_curly = ord("}") + for i in range(file_length): + c = raw[i] + if escape: + escape = False + continue + if c == backslash: + escape = True + continue + if c == quote: + inside_string = not inside_string + continue + if inside_string: + continue + if c == open_square: + square_depth += 1 + elif c == close_square: + square_depth -= 1 + elif c == open_curly: + if square_depth == 1 and curly_depth == 0: + start = i + curly_depth += 1 + elif c == close_curly: + curly_depth -= 1 + if square_depth == 1 and curly_depth == 0: + substr = utf8_str[start : i + 1].decode("utf8") + yield srsly.json_loads(substr) + start = -1 diff --git a/spacy/training/initialize.py b/spacy/training/initialize.py new file mode 100644 index 0000000..164a086 --- /dev/null +++ b/spacy/training/initialize.py @@ -0,0 +1,361 @@ +import gzip +import tarfile +import warnings +import zipfile +from itertools import islice +from pathlib import Path +from typing import IO, TYPE_CHECKING, Any, Dict, Optional, Union + +import numpy +import srsly +import tqdm +from thinc.api import Config, ConfigValidationError, fix_random_seed, set_gpu_allocator + +from ..errors import Errors, Warnings +from ..lookups import Lookups +from ..schemas import ConfigSchemaTraining +from ..util import ( + DEFAULT_OOV_PROB, + OOV_RANK, + ensure_path, + get_sourced_components, + load_model, + load_model_from_config, + logger, + registry, + resolve_dot_names, +) +from ..vectors import Mode as VectorsMode, Vectors +from .pretrain import get_tok2vec_ref + +if TYPE_CHECKING: + from ..language import Language # noqa: F401 + + +def init_nlp(config: Config, *, use_gpu: int = -1) -> "Language": + raw_config = config + config = raw_config.interpolate() + if "seed" not in config["training"]: + raise ValueError(Errors.E1015.format(value="[training] seed")) + if "gpu_allocator" not in config["training"]: + raise ValueError(Errors.E1015.format(value="[training] gpu_allocator")) + if config["training"]["seed"] is not None: + fix_random_seed(config["training"]["seed"]) + allocator = config["training"]["gpu_allocator"] + if use_gpu >= 0 and allocator: + set_gpu_allocator(allocator) + # Use original config here before it's resolved to functions + sourced = get_sourced_components(config) + nlp = load_model_from_config(raw_config, auto_fill=True) + logger.info("Set up nlp object from config") + config = nlp.config.interpolate() + # Resolve all training-relevant sections using the filled nlp config + T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type] + dot_names = [T["train_corpus"], T["dev_corpus"]] + if not isinstance(T["train_corpus"], str): + raise ConfigValidationError( + desc=Errors.E897.format( + field="training.train_corpus", type=type(T["train_corpus"]) + ) + ) + if not isinstance(T["dev_corpus"], str): + raise ConfigValidationError( + desc=Errors.E897.format( + field="training.dev_corpus", type=type(T["dev_corpus"]) + ) + ) + train_corpus, dev_corpus = resolve_dot_names(config, dot_names) + optimizer = T["optimizer"] + # Components that shouldn't be updated during training + frozen_components = T["frozen_components"] + # Sourced components that require resume_training + resume_components = [p for p in sourced if p not in frozen_components] + logger.info("Pipeline: %s", nlp.pipe_names) + if resume_components: + with nlp.select_pipes(enable=resume_components): + logger.info("Resuming training for: %s", resume_components) + nlp.resume_training(sgd=optimizer) + # Make sure that internal component names are synced and listeners are + # defined before initializing further + nlp._link_components() + with nlp.select_pipes(disable=[*frozen_components, *resume_components]): + if T["max_epochs"] == -1: + sample_size = 100 + logger.debug( + "Due to streamed train corpus, using only first %s examples for initialization. " + "If necessary, provide all labels in [initialize]. " + "More info: https://spacy.io/api/cli#init_labels", + sample_size, + ) + nlp.initialize( + lambda: islice(train_corpus(nlp), sample_size), sgd=optimizer + ) + else: + nlp.initialize(lambda: train_corpus(nlp), sgd=optimizer) + logger.info("Initialized pipeline components: %s", nlp.pipe_names) + # Detect components with listeners that are not frozen consistently + for name, proc in nlp.pipeline: + for listener in getattr( + proc, "listening_components", [] + ): # e.g. tok2vec/transformer + # Don't warn about components not in the pipeline + if listener not in nlp.pipe_names: + continue + if listener in frozen_components and name not in frozen_components: + logger.warning(Warnings.W087.format(name=name, listener=listener)) + # We always check this regardless, in case user freezes tok2vec + if listener not in frozen_components and name in frozen_components: + if name not in T["annotating_components"]: + logger.warning(Warnings.W086.format(name=name, listener=listener)) + return nlp + + +def init_vocab( + nlp: "Language", + *, + data: Optional[Path] = None, + lookups: Optional[Lookups] = None, + vectors: Optional[str] = None, +) -> None: + if lookups: + nlp.vocab.lookups = lookups + logger.info("Added vocab lookups: %s", ", ".join(lookups.tables)) + data_path = ensure_path(data) + if data_path is not None: + lex_attrs = srsly.read_jsonl(data_path) + for lexeme in nlp.vocab: + lexeme.rank = OOV_RANK + for attrs in lex_attrs: + if "settings" in attrs: + continue + lexeme = nlp.vocab[attrs["orth"]] + lexeme.set_attrs(**attrs) + if len(nlp.vocab): + oov_prob = min(lex.prob for lex in nlp.vocab) - 1 + else: + oov_prob = DEFAULT_OOV_PROB + nlp.vocab.cfg.update({"oov_prob": oov_prob}) + logger.info("Added %d lexical entries to the vocab", len(nlp.vocab)) + logger.info("Created vocabulary") + if vectors is not None: + load_vectors_into_model(nlp, vectors) + logger.info("Added vectors: %s", vectors) + # warn if source model vectors are not identical + sourced_vectors_hashes = nlp.meta.pop("_sourced_vectors_hashes", {}) + if len(sourced_vectors_hashes) > 0: + vectors_hash = hash(nlp.vocab.vectors.to_bytes(exclude=["strings"])) + for sourced_component, sourced_vectors_hash in sourced_vectors_hashes.items(): + if vectors_hash != sourced_vectors_hash: + warnings.warn(Warnings.W113.format(name=sourced_component)) + logger.info("Finished initializing nlp object") + + +def load_vectors_into_model( + nlp: "Language", name: Union[str, Path], *, add_strings: bool = True +) -> None: + """Load word vectors from an installed model or path into a model instance.""" + try: + # Load with the same vocab, which automatically adds the vectors to + # the current nlp object. Exclude lookups so they are not modified. + exclude = ["lookups"] + if not add_strings: + exclude.append("strings") + vectors_nlp = load_model(name, vocab=nlp.vocab, exclude=exclude) + except ConfigValidationError as e: + title = f"Config validation error for vectors {name}" + desc = ( + "This typically means that there's a problem in the config.cfg included " + "with the packaged vectors. Make sure that the vectors package you're " + "loading is compatible with the current version of spaCy." + ) + err = ConfigValidationError.from_error(e, title=title, desc=desc) + raise err from None + + if ( + len(vectors_nlp.vocab.vectors.keys()) == 0 + and vectors_nlp.vocab.vectors.mode != VectorsMode.floret + ) or ( + vectors_nlp.vocab.vectors.shape[0] == 0 + and vectors_nlp.vocab.vectors.mode == VectorsMode.floret + ): + logger.warning(Warnings.W112.format(name=name)) + + for lex in nlp.vocab: + lex.rank = nlp.vocab.vectors.key2row.get(lex.orth, OOV_RANK) # type: ignore[attr-defined] + + +def init_tok2vec( + nlp: "Language", pretrain_config: Dict[str, Any], init_config: Dict[str, Any] +) -> bool: + # Load pretrained tok2vec weights - cf. CLI command 'pretrain' + P = pretrain_config + I = init_config + weights_data = None + init_tok2vec = ensure_path(I["init_tok2vec"]) + if init_tok2vec is not None: + if not init_tok2vec.exists(): + err = f"can't find pretrained tok2vec: {init_tok2vec}" + errors = [{"loc": ["initialize", "init_tok2vec"], "msg": err}] + raise ConfigValidationError(config=nlp.config, errors=errors) + with init_tok2vec.open("rb") as file_: + weights_data = file_.read() + if weights_data is not None: + layer = get_tok2vec_ref(nlp, P) + layer.from_bytes(weights_data) + logger.info("Loaded pretrained weights from %s", init_tok2vec) + return True + return False + + +def convert_vectors( + nlp: "Language", + vectors_loc: Optional[Path], + *, + truncate: int, + prune: int, + name: Optional[str] = None, + mode: str = VectorsMode.default, + attr: str = "ORTH", +) -> None: + vectors_loc = ensure_path(vectors_loc) + if vectors_loc and vectors_loc.parts[-1].endswith(".npz"): + if attr != "ORTH": + raise ValueError( + "ORTH is the only attribute supported for vectors in .npz format." + ) + nlp.vocab.vectors = Vectors( + strings=nlp.vocab.strings, data=numpy.load(vectors_loc.open("rb")) + ) + for lex in nlp.vocab: + if lex.rank and lex.rank != OOV_RANK: + nlp.vocab.vectors.add(lex.orth, row=lex.rank) # type: ignore[attr-defined] + nlp.vocab.deduplicate_vectors() + else: + if vectors_loc: + logger.info("Reading vectors from %s", vectors_loc) + vectors_data, vector_keys, floret_settings = read_vectors( + vectors_loc, + truncate, + mode=mode, + ) + logger.info("Loaded vectors from %s", vectors_loc) + else: + vectors_data, vector_keys = (None, None) + if vector_keys is not None and mode != VectorsMode.floret: + for word in vector_keys: + if word not in nlp.vocab: + nlp.vocab[word] + if vectors_data is not None: + if mode == VectorsMode.floret: + nlp.vocab.vectors = Vectors( + strings=nlp.vocab.strings, + data=vectors_data, + attr=attr, + **floret_settings, + ) + else: + nlp.vocab.vectors = Vectors( + strings=nlp.vocab.strings, + data=vectors_data, + keys=vector_keys, + attr=attr, + ) + nlp.vocab.deduplicate_vectors() + if name is None: + # TODO: Is this correct? Does this matter? + nlp.vocab.vectors.name = f"{nlp.meta['lang']}_{nlp.meta['name']}.vectors" + else: + nlp.vocab.vectors.name = name + nlp.meta["vectors"]["name"] = nlp.vocab.vectors.name + if prune >= 1 and mode != VectorsMode.floret: + nlp.vocab.prune_vectors(prune) + + +def read_vectors( + vectors_loc: Path, truncate_vectors: int, *, mode: str = VectorsMode.default +): + f = ensure_shape(vectors_loc) + header_parts = next(f).split() + shape = tuple(int(size) for size in header_parts[:2]) + floret_settings = {} + if mode == VectorsMode.floret: + if len(header_parts) != 8: + raise ValueError( + "Invalid header for floret vectors. " + "Expected: bucket dim minn maxn hash_count hash_seed BOW EOW" + ) + floret_settings = { + "mode": "floret", + "minn": int(header_parts[2]), + "maxn": int(header_parts[3]), + "hash_count": int(header_parts[4]), + "hash_seed": int(header_parts[5]), + "bow": header_parts[6], + "eow": header_parts[7], + } + if truncate_vectors >= 1: + raise ValueError(Errors.E860) + else: + assert len(header_parts) == 2 + if truncate_vectors >= 1: + shape = (truncate_vectors, shape[1]) + vectors_data = numpy.zeros(shape=shape, dtype="f") + vectors_keys = [] + for i, line in enumerate(tqdm.tqdm(f, disable=None)): + line = line.rstrip() + pieces = line.rsplit(" ", vectors_data.shape[1]) + word = pieces.pop(0) + if len(pieces) != vectors_data.shape[1]: + raise ValueError(Errors.E094.format(line_num=i, loc=vectors_loc)) + vectors_data[i] = numpy.asarray(pieces, dtype="f") + vectors_keys.append(word) + if i == truncate_vectors - 1: + break + return vectors_data, vectors_keys, floret_settings + + +def open_file(loc: Union[str, Path]) -> IO: + """Handle .gz, .tar.gz or unzipped files""" + loc = ensure_path(loc) + if tarfile.is_tarfile(str(loc)): + return tarfile.open(str(loc), "r:gz") # type: ignore[return-value] + elif loc.parts[-1].endswith("gz"): + return (line.decode("utf8") for line in gzip.open(str(loc), "r")) # type: ignore[return-value] + elif loc.parts[-1].endswith("zip"): + zip_file = zipfile.ZipFile(str(loc)) + names = zip_file.namelist() + file_ = zip_file.open(names[0]) + return (line.decode("utf8") for line in file_) # type: ignore[return-value] + else: + return loc.open("r", encoding="utf8") + + +def ensure_shape(vectors_loc): + """Ensure that the first line of the data is the vectors shape. + If it's not, we read in the data and output the shape as the first result, + so that the reader doesn't have to deal with the problem. + """ + lines = open_file(vectors_loc) + first_line = next(lines) + try: + shape = tuple(int(size) for size in first_line.split()[:2]) + except ValueError: + shape = None + if shape is not None: + # All good, give the data + yield first_line + yield from lines + else: + # Figure out the shape, make it the first value, and then give the + # rest of the data. + width = len(first_line.split()) - 1 + length = 1 + for _ in lines: + length += 1 + yield f"{length} {width}" + # Reading the lines in again from file. This to avoid having to + # store all the results in a list in memory + lines2 = open_file(vectors_loc) + yield from lines2 + lines2.close() + lines.close() diff --git a/spacy/training/iob_utils.py b/spacy/training/iob_utils.py new file mode 100644 index 0000000..64d02a1 --- /dev/null +++ b/spacy/training/iob_utils.py @@ -0,0 +1,240 @@ +import warnings +from typing import Dict, Iterable, Iterator, List, Tuple, Union, cast + +from ..errors import Errors, Warnings +from ..tokens import Doc, Span + + +def iob_to_biluo(tags: Iterable[str]) -> List[str]: + out: List[str] = [] + tags = list(tags) + while tags: + out.extend(_consume_os(tags)) + out.extend(_consume_ent(tags)) + return out + + +def biluo_to_iob(tags: Iterable[str]) -> List[str]: + out = [] + for tag in tags: + if tag is None: + out.append(tag) + else: + tag = tag.replace("U-", "B-", 1).replace("L-", "I-", 1) + out.append(tag) + return out + + +def _consume_os(tags: List[str]) -> Iterator[str]: + while tags and tags[0] == "O": + yield tags.pop(0) + + +def _consume_ent(tags: List[str]) -> List[str]: + if not tags: + return [] + tag = tags.pop(0) + target_in = "I" + tag[1:] + target_last = "L" + tag[1:] + length = 1 + while tags and tags[0] in {target_in, target_last}: + length += 1 + tags.pop(0) + label = tag[2:] + if length == 1: + if len(label) == 0: + raise ValueError(Errors.E177.format(tag=tag)) + return ["U-" + label] + else: + start = "B-" + label + end = "L-" + label + middle = [f"I-{label}" for _ in range(1, length - 1)] + return [start] + middle + [end] + + +def doc_to_biluo_tags(doc: Doc, missing: str = "O"): + return offsets_to_biluo_tags( + doc, + [(ent.start_char, ent.end_char, ent.label_) for ent in doc.ents], + missing=missing, + ) + + +def _doc_to_biluo_tags_with_partial(doc: Doc) -> List[str]: + ents = doc_to_biluo_tags(doc, missing="-") + for i, token in enumerate(doc): + if token.ent_iob == 2: + ents[i] = "O" + return ents + + +def offsets_to_biluo_tags( + doc: Doc, entities: Iterable[Tuple[int, int, Union[str, int]]], missing: str = "O" +) -> List[str]: + """Encode labelled spans into per-token tags, using the + Begin/In/Last/Unit/Out scheme (BILUO). + + doc (Doc): The document that the entity offsets refer to. The output tags + will refer to the token boundaries within the document. + entities (iterable): A sequence of `(start, end, label)` triples. `start` + and `end` should be character-offset integers denoting the slice into + the original string. + missing (str): The label used for missing values, e.g. if tokenization + doesn’t align with the entity offsets. Defaults to "O". + RETURNS (list): A list of unicode strings, describing the tags. Each tag + string will be of the form either "", "O" or "{action}-{label}", where + action is one of "B", "I", "L", "U". The missing label is used where the + entity offsets don't align with the tokenization in the `Doc` object. + The training algorithm will view these as missing values. "O" denotes a + non-entity token. "B" denotes the beginning of a multi-token entity, + "I" the inside of an entity of three or more tokens, and "L" the end + of an entity of two or more tokens. "U" denotes a single-token entity. + + EXAMPLE: + >>> text = 'I like London.' + >>> entities = [(len('I like '), len('I like London'), 'LOC')] + >>> doc = nlp.tokenizer(text) + >>> tags = offsets_to_biluo_tags(doc, entities) + >>> assert tags == ["O", "O", 'U-LOC', "O"] + """ + # Ensure no overlapping entity labels exist + tokens_in_ents: Dict[int, Tuple[int, int, Union[str, int]]] = {} + starts = {token.idx: token.i for token in doc} + ends = {token.idx + len(token): token.i for token in doc} + biluo = ["-" for _ in doc] + # Handle entity cases + for start_char, end_char, label in entities: + if not label: + for s in starts: # account for many-to-one + if s >= start_char and s < end_char: + biluo[starts[s]] = "O" + else: + for token_index in range(start_char, end_char): + if token_index in tokens_in_ents.keys(): + raise ValueError( + Errors.E103.format( + span1=( + tokens_in_ents[token_index][0], + tokens_in_ents[token_index][1], + tokens_in_ents[token_index][2], + ), + span2=(start_char, end_char, label), + ) + ) + tokens_in_ents[token_index] = (start_char, end_char, label) + start_token = starts.get(start_char) + end_token = ends.get(end_char) + # Only interested if the tokenization is correct + if start_token is not None and end_token is not None: + if start_token == end_token: + biluo[start_token] = f"U-{label}" + else: + biluo[start_token] = f"B-{label}" + for i in range(start_token + 1, end_token): + biluo[i] = f"I-{label}" + biluo[end_token] = f"L-{label}" + # Now distinguish the O cases from ones where we miss the tokenization + entity_chars = set() + for start_char, end_char, label in entities: + for i in range(start_char, end_char): + entity_chars.add(i) + for token in doc: + for i in range(token.idx, token.idx + len(token)): + if i in entity_chars: + break + else: + biluo[token.i] = missing + if "-" in biluo and missing != "-": + ent_str = str(entities) + warnings.warn( + Warnings.W030.format( + text=doc.text[:50] + "..." if len(doc.text) > 50 else doc.text, + entities=ent_str[:50] + "..." if len(ent_str) > 50 else ent_str, + ) + ) + return biluo + + +def biluo_tags_to_spans(doc: Doc, tags: Iterable[str]) -> List[Span]: + """Encode per-token tags following the BILUO scheme into Span object, e.g. + to overwrite the doc.ents. + + doc (Doc): The document that the BILUO tags refer to. + tags (iterable): A sequence of BILUO tags with each tag describing one + token. Each tag string will be of the form of either "", "O" or + "{action}-{label}", where action is one of "B", "I", "L", "U". + RETURNS (list): A sequence of Span objects. Each token with a missing IOB + tag is returned as a Span with an empty label. + """ + token_offsets = tags_to_entities(tags) + spans = [] + for label, start_idx, end_idx in token_offsets: + span = Span(doc, start_idx, end_idx + 1, label=label) + spans.append(span) + return spans + + +def biluo_tags_to_offsets( + doc: Doc, tags: Iterable[str] +) -> List[Tuple[int, int, Union[str, int]]]: + """Encode per-token tags following the BILUO scheme into entity offsets. + + doc (Doc): The document that the BILUO tags refer to. + tags (iterable): A sequence of BILUO tags with each tag describing one + token. Each tags string will be of the form of either "", "O" or + "{action}-{label}", where action is one of "B", "I", "L", "U". + RETURNS (list): A sequence of `(start, end, label)` triples. `start` and + `end` will be character-offset integers denoting the slice into the + original string. + """ + spans = biluo_tags_to_spans(doc, tags) + return [(span.start_char, span.end_char, span.label_) for span in spans] + + +def tags_to_entities(tags: Iterable[str]) -> List[Tuple[str, int, int]]: + """Note that the end index returned by this function is inclusive. + To use it for Span creation, increment the end by 1.""" + entities = [] + start = None + for i, tag in enumerate(tags): + if tag is None or tag.startswith("-"): + # TODO: We shouldn't be getting these malformed inputs. Fix this. + if start is not None: + start = None + else: + entities.append(("", i, i)) + elif tag.startswith("O"): + pass + elif tag.startswith("I"): + if start is None: + raise ValueError( + Errors.E067.format(start="I", tags=list(tags)[: i + 1]) + ) + elif tag.startswith("U"): + entities.append((tag[2:], i, i)) + elif tag.startswith("B"): + start = i + elif tag.startswith("L"): + if start is None: + raise ValueError( + Errors.E067.format(start="L", tags=list(tags)[: i + 1]) + ) + entities.append((tag[2:], start, i)) + start = None + else: + raise ValueError(Errors.E068.format(tag=tag)) + return entities + + +def split_bilu_label(label: str) -> Tuple[str, str]: + return cast(Tuple[str, str], label.split("-", 1)) + + +def remove_bilu_prefix(label: str) -> str: + return label.split("-", 1)[1] + + +# Fallbacks to make backwards-compat easier +offsets_from_biluo_tags = biluo_tags_to_offsets +spans_from_biluo_tags = biluo_tags_to_spans +biluo_tags_from_offsets = offsets_to_biluo_tags diff --git a/spacy/training/loggers.py b/spacy/training/loggers.py new file mode 100644 index 0000000..7f20054 --- /dev/null +++ b/spacy/training/loggers.py @@ -0,0 +1,196 @@ +import sys +from pathlib import Path +from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union + +import srsly +import tqdm +from wasabi import Printer + +from .. import util +from ..errors import Errors + +if TYPE_CHECKING: + from ..language import Language # noqa: F401 + + +def setup_table( + *, cols: List[str], widths: List[int], max_width: int = 13 +) -> Tuple[List[str], List[int], List[str]]: + final_cols = [] + final_widths = [] + for col, width in zip(cols, widths): + if len(col) > max_width: + col = col[: max_width - 3] + "..." # shorten column if too long + final_cols.append(col.upper()) + final_widths.append(max(len(col), width)) + return final_cols, final_widths, ["r" for _ in final_widths] + + +# We cannot rename this method as it's directly imported +# and used by external packages such as spacy-loggers. +def console_logger( + progress_bar: bool = False, + console_output: bool = True, + output_file: Optional[Union[str, Path]] = None, +): + """The ConsoleLogger.v2 prints out training logs in the console and/or saves them to a jsonl file. + progress_bar (bool): Whether the logger should print a progress bar tracking the steps till the next evaluation pass. + console_output (bool): Whether the logger should print the logs on the console. + output_file (Optional[Union[str, Path]]): The file to save the training logs to. + """ + return console_logger_v3( + progress_bar=None if progress_bar is False else "eval", + console_output=console_output, + output_file=output_file, + ) + + +def console_logger_v3( + progress_bar: Optional[str] = None, + console_output: bool = True, + output_file: Optional[Union[str, Path]] = None, +): + """The ConsoleLogger.v3 prints out training logs in the console and/or saves them to a jsonl file. + progress_bar (Optional[str]): Type of progress bar to show in the console. Allowed values: + train - Tracks the number of steps from the beginning of training until the full training run is complete (training.max_steps is reached). + eval - Tracks the number of steps between the previous and next evaluation (training.eval_frequency is reached). + console_output (bool): Whether the logger should print the logs on the console. + output_file (Optional[Union[str, Path]]): The file to save the training logs to. + """ + _log_exist = False + if output_file: + output_file = util.ensure_path(output_file) # type: ignore + if output_file.exists(): # type: ignore + _log_exist = True + if not output_file.parents[0].exists(): # type: ignore + output_file.parents[0].mkdir(parents=True) # type: ignore + + def setup_printer( + nlp: "Language", stdout: IO = sys.stdout, stderr: IO = sys.stderr + ) -> Tuple[Callable[[Optional[Dict[str, Any]]], None], Callable[[], None]]: + write = lambda text: print(text, file=stdout, flush=True) + msg = Printer(no_print=True) + + nonlocal output_file + output_stream = None + if _log_exist: + write( + msg.warn( + f"Saving logs is disabled because {output_file} already exists." + ) + ) + output_file = None + elif output_file: + write(msg.info(f"Saving results to {output_file}")) + output_stream = open(output_file, "w", encoding="utf-8") + + # ensure that only trainable components are logged + logged_pipes = [ + name + for name, proc in nlp.pipeline + if hasattr(proc, "is_trainable") and proc.is_trainable + ] + max_steps = nlp.config["training"]["max_steps"] + eval_frequency = nlp.config["training"]["eval_frequency"] + score_weights = nlp.config["training"]["score_weights"] + score_cols = [col for col, value in score_weights.items() if value is not None] + loss_cols = [f"Loss {pipe}" for pipe in logged_pipes] + + if console_output: + spacing = 2 + table_header, table_widths, table_aligns = setup_table( + cols=["E", "#"] + loss_cols + score_cols + ["Score"], + widths=[3, 6] + [8 for _ in loss_cols] + [6 for _ in score_cols] + [6], + ) + write(msg.row(table_header, widths=table_widths, spacing=spacing)) + write(msg.row(["-" * width for width in table_widths], spacing=spacing)) + progress = None + expected_progress_types = ("train", "eval") + if progress_bar is not None and progress_bar not in expected_progress_types: + raise ValueError( + Errors.E1048.format( + unexpected=progress_bar, expected=expected_progress_types + ) + ) + + def log_step(info: Optional[Dict[str, Any]]) -> None: + nonlocal progress + + if info is None: + # If we don't have a new checkpoint, just return. + if progress is not None: + progress.update(1) + return + + losses = [] + log_losses = {} + for pipe_name in logged_pipes: + losses.append("{0:.2f}".format(float(info["losses"][pipe_name]))) + log_losses[pipe_name] = float(info["losses"][pipe_name]) + + scores = [] + log_scores = {} + for col in score_cols: + score = info["other_scores"].get(col, 0.0) + try: + score = float(score) + except TypeError: + err = Errors.E916.format(name=col, score_type=type(score)) + raise ValueError(err) from None + if col != "speed": + score *= 100 + scores.append("{0:.2f}".format(score)) + log_scores[str(col)] = score + + data = ( + [info["epoch"], info["step"]] + + losses + + scores + + ["{0:.2f}".format(float(info["score"]))] + ) + + if output_stream: + # Write to log file per log_step + log_data = { + "epoch": info["epoch"], + "step": info["step"], + "losses": log_losses, + "scores": log_scores, + "score": float(info["score"]), + } + output_stream.write(srsly.json_dumps(log_data) + "\n") + + if progress is not None: + progress.close() + if console_output: + write( + msg.row( + data, widths=table_widths, aligns=table_aligns, spacing=spacing + ) + ) + if progress_bar: + if progress_bar == "train": + total = max_steps + desc = f"Last Eval Epoch: {info['epoch']}" + initial = info["step"] + else: + total = eval_frequency + desc = f"Epoch {info['epoch'] + 1}" + initial = 0 + # Set disable=None, so that it disables on non-TTY + progress = tqdm.tqdm( + total=total, + disable=None, + leave=False, + file=stderr, + initial=initial, + ) + progress.set_description(desc) + + def finalize() -> None: + if output_stream: + output_stream.close() + + return log_step, finalize + + return setup_printer diff --git a/spacy/training/loop.py b/spacy/training/loop.py new file mode 100644 index 0000000..d6f1ad7 --- /dev/null +++ b/spacy/training/loop.py @@ -0,0 +1,386 @@ +import random +import shutil +import sys +from pathlib import Path +from timeit import default_timer as timer +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, +) + +from thinc.api import Config, Optimizer, constant, fix_random_seed, set_gpu_allocator +from wasabi import Printer + +from ..errors import Errors +from ..schemas import ConfigSchemaTraining +from ..util import logger, registry, resolve_dot_names +from .example import Example + +if TYPE_CHECKING: + from ..language import Language # noqa: F401 + + +DIR_MODEL_BEST = "model-best" +DIR_MODEL_LAST = "model-last" + + +def train( + nlp: "Language", + output_path: Optional[Path] = None, + *, + use_gpu: int = -1, + stdout: IO = sys.stdout, + stderr: IO = sys.stderr, +) -> Tuple["Language", Optional[Path]]: + """Train a pipeline. + + nlp (Language): The initialized nlp object with the full config. + output_path (Optional[Path]): Optional output path to save trained model to. + use_gpu (int): Whether to train on GPU. Make sure to call require_gpu + before calling this function. + stdout (file): A file-like object to write output messages. To disable + printing, set to io.StringIO. + stderr (file): A second file-like object to write output messages. To disable + printing, set to io.StringIO. + + RETURNS (tuple): The final nlp object and the path to the exported model. + """ + # We use no_print here so we can respect the stdout/stderr options. + msg = Printer(no_print=True) + # Create iterator, which yields out info after each optimization step. + config = nlp.config.interpolate() + if config["training"]["seed"] is not None: + fix_random_seed(config["training"]["seed"]) + allocator = config["training"]["gpu_allocator"] + if use_gpu >= 0 and allocator: + set_gpu_allocator(allocator) + T = registry.resolve(config["training"], schema=ConfigSchemaTraining) # type: ignore[arg-type] + dot_names = [T["train_corpus"], T["dev_corpus"]] + train_corpus, dev_corpus = resolve_dot_names(config, dot_names) + optimizer = T["optimizer"] + score_weights = T["score_weights"] + batcher = T["batcher"] + train_logger = T["logger"] + before_to_disk = create_before_to_disk_callback(T["before_to_disk"]) + before_update = T["before_update"] + + # Helper function to save checkpoints. This is a closure for convenience, + # to avoid passing in all the args all the time. + def save_checkpoint(is_best): + with nlp.use_params(optimizer.averages): + before_to_disk(nlp).to_disk(output_path / DIR_MODEL_LAST) + if is_best: + # Avoid saving twice (saving will be more expensive than + # the dir copy) + if (output_path / DIR_MODEL_BEST).exists(): + shutil.rmtree(output_path / DIR_MODEL_BEST) + shutil.copytree(output_path / DIR_MODEL_LAST, output_path / DIR_MODEL_BEST) + + # Components that shouldn't be updated during training + frozen_components = T["frozen_components"] + # Components that should set annotations on update + annotating_components = T["annotating_components"] + # Create iterator, which yields out info after each optimization step. + training_step_iterator = train_while_improving( + nlp, + optimizer, + create_train_batches(nlp, train_corpus, batcher, T["max_epochs"]), + create_evaluation_callback(nlp, dev_corpus, score_weights), + dropout=T["dropout"], + accumulate_gradient=T["accumulate_gradient"], + patience=T["patience"], + max_steps=T["max_steps"], + eval_frequency=T["eval_frequency"], + exclude=frozen_components, + annotating_components=annotating_components, + before_update=before_update, + ) + clean_output_dir(output_path) + stdout.write(msg.info(f"Pipeline: {nlp.pipe_names}") + "\n") + if frozen_components: + stdout.write(msg.info(f"Frozen components: {frozen_components}") + "\n") + if annotating_components: + stdout.write( + msg.info(f"Set annotations on update for: {annotating_components}") + "\n" + ) + stdout.write(msg.info(f"Initial learn rate: {optimizer.learn_rate}") + "\n") + with nlp.select_pipes(disable=frozen_components): + log_step, finalize_logger = train_logger(nlp, stdout, stderr) + try: + for batch, info, is_best_checkpoint in training_step_iterator: + if is_best_checkpoint is not None: + with nlp.select_pipes(disable=frozen_components): + update_meta(T, nlp, info) + if output_path is not None: + save_checkpoint(is_best_checkpoint) + info["output_path"] = str(output_path / DIR_MODEL_LAST) + log_step(info if is_best_checkpoint is not None else None) + except Exception as e: + if output_path is not None: + stdout.write( + msg.warn( + f"Aborting and saving the final best model. " + f"Encountered exception: {repr(e)}" + ) + + "\n" + ) + raise e + finally: + finalize_logger() + if output_path is not None: + save_checkpoint(False) + # This will only run if we did't hit an error + if optimizer.averages: + nlp.use_params(optimizer.averages) + if output_path is not None: + stdout.write( + msg.good("Saved pipeline to output directory", output_path / DIR_MODEL_LAST) + + "\n" + ) + return (nlp, output_path / DIR_MODEL_LAST) + else: + return (nlp, None) + + +def train_while_improving( + nlp: "Language", + optimizer: Optimizer, + train_data, + evaluate, + *, + dropout: float, + eval_frequency: int, + accumulate_gradient: int, + patience: int, + max_steps: int, + exclude: List[str], + annotating_components: List[str], + before_update: Optional[Callable[["Language", Dict[str, Any]], None]], +): + """Train until an evaluation stops improving. Works as a generator, + with each iteration yielding a tuple `(batch, info, is_best_checkpoint)`, + where info is a dict, and is_best_checkpoint is in [True, False, None] -- + None indicating that the iteration was not evaluated as a checkpoint. + The evaluation is conducted by calling the evaluate callback. + + Positional arguments: + nlp: The spaCy pipeline to evaluate. + optimizer: The optimizer callable. + train_data (Iterable[Batch]): A generator of batches, with the training + data. Each batch should be a Sized[Tuple[Input, Annot]]. The training + data iterable needs to take care of iterating over the epochs and + shuffling. + evaluate (Callable[[], Tuple[float, Any]]): A callback to perform evaluation. + The callback should take no arguments and return a tuple + `(main_score, other_scores)`. The main_score should be a float where + higher is better. other_scores can be any object. + + Every iteration, the function yields out a tuple with: + + * batch: A list of Example objects. + * info: A dict with various information about the last update (see below). + * is_best_checkpoint: A value in None, False, True, indicating whether this + was the best evaluation so far. You should use this to save the model + checkpoints during training. If None, evaluation was not conducted on + that iteration. False means evaluation was conducted, but a previous + evaluation was better. + + The info dict provides the following information: + + epoch (int): How many passes over the data have been completed. + step (int): How many steps have been completed. + score (float): The main score from the last evaluation. + other_scores: : The other scores from the last evaluation. + losses: The accumulated losses throughout training. + checkpoints: A list of previous results, where each result is a + (score, step, epoch) tuple. + """ + if isinstance(dropout, float): + dropouts = constant(dropout) + else: + dropouts = dropout + results = [] + losses: Dict[str, float] = {} + words_seen = 0 + start_time = timer() + for step, (epoch, batch) in enumerate(train_data): + if before_update: + before_update_args = {"step": step, "epoch": epoch} + before_update(nlp, before_update_args) + dropout = next(dropouts) # type: ignore + for subbatch in subdivide_batch(batch, accumulate_gradient): + nlp.update( + subbatch, + drop=dropout, + losses=losses, + sgd=False, # type: ignore[arg-type] + exclude=exclude, + annotates=annotating_components, + ) + # TODO: refactor this so we don't have to run it separately in here + for name, proc in nlp.pipeline: + if ( + name not in exclude + and hasattr(proc, "is_trainable") + and proc.is_trainable + and proc.model not in (True, False, None) # type: ignore[attr-defined] + ): + proc.finish_update(optimizer) # type: ignore[attr-defined] + optimizer.step_schedules() + if not (step % eval_frequency): + if optimizer.averages: + with nlp.use_params(optimizer.averages): + score, other_scores = evaluate() + else: + score, other_scores = evaluate() + results.append((score, step)) + is_best_checkpoint = score == max(results)[0] + else: + score, other_scores = (None, None) + is_best_checkpoint = None + words_seen += sum(len(eg) for eg in batch) + info = { + "epoch": epoch, + "step": step, + "score": score, + "other_scores": other_scores, + "losses": losses, + "checkpoints": results, + "seconds": int(timer() - start_time), + "words": words_seen, + } + yield batch, info, is_best_checkpoint + if is_best_checkpoint is not None: + losses = {} + # Stop if no improvement in `patience` updates (if specified) + # Negate step value so that the earliest best step is chosen for the + # same score, i.e. (1.0, 100) is chosen over (1.0, 200) + best_result = max((r_score, -r_step) for r_score, r_step in results) + best_step = -best_result[1] + if patience and (step - best_step) >= patience: + break + # Stop if we've exhausted our max steps (if specified) + if max_steps and step >= max_steps: + break + + +def subdivide_batch(batch, accumulate_gradient): + batch = list(batch) + batch.sort(key=lambda eg: len(eg.predicted)) + sub_len = len(batch) // accumulate_gradient + start = 0 + for i in range(accumulate_gradient): + subbatch = batch[start : start + sub_len] + if subbatch: + yield subbatch + start += len(subbatch) + subbatch = batch[start:] + if subbatch: + yield subbatch + + +def create_evaluation_callback( + nlp: "Language", dev_corpus: Callable, weights: Dict[str, float] +) -> Callable[[], Tuple[float, Dict[str, float]]]: + weights = {key: value for key, value in weights.items() if value is not None} + + def evaluate() -> Tuple[float, Dict[str, float]]: + nonlocal weights + try: + scores = nlp.evaluate(dev_corpus(nlp)) + except KeyError as e: + raise KeyError(Errors.E900.format(pipeline=nlp.pipe_names)) from e + # Calculate a weighted sum based on score_weights for the main score. + # We can only consider scores that are ints/floats, not dicts like + # entity scores per type etc. + scores = {key: value for key, value in scores.items() if value is not None} + weights = {key: value for key, value in weights.items() if key in scores} + for key, value in scores.items(): + if key in weights and not isinstance(value, (int, float)): + raise ValueError(Errors.E915.format(name=key, score_type=type(value))) + try: + weighted_score = sum( + scores.get(s, 0.0) * weights.get(s, 0.0) for s in weights + ) + except KeyError as e: + keys = list(scores.keys()) + err = Errors.E983.format(dict="score_weights", key=str(e), keys=keys) + raise KeyError(err) from None + return weighted_score, scores + + return evaluate + + +def create_train_batches( + nlp: "Language", + corpus: Callable[["Language"], Iterable[Example]], + batcher: Callable[[Iterable[Example]], Iterable[Example]], + max_epochs: int, +): + epoch = 0 + if max_epochs >= 0: + examples = list(corpus(nlp)) # type: Iterable[Example] + if not examples: + # Raise error if no data + raise ValueError(Errors.E986) + while max_epochs < 1 or epoch != max_epochs: + if max_epochs >= 0: + random.shuffle(examples) # type: ignore + else: + examples = corpus(nlp) + for batch in batcher(examples): + yield epoch, batch + epoch += 1 + + +def update_meta( + training: Union[Dict[str, Any], Config], nlp: "Language", info: Dict[str, Any] +) -> None: + nlp.meta["performance"] = {} + for metric in training["score_weights"]: + if metric is not None: + nlp.meta["performance"][metric] = info["other_scores"].get(metric, 0.0) + for pipe_name in nlp.pipe_names: + if pipe_name in info["losses"]: + nlp.meta["performance"][f"{pipe_name}_loss"] = info["losses"][pipe_name] + + +def create_before_to_disk_callback( + callback: Optional[Callable[["Language"], "Language"]], +) -> Callable[["Language"], "Language"]: + from ..language import Language # noqa: F811 + + def before_to_disk(nlp: Language) -> Language: + if not callback: + return nlp + modified_nlp = callback(nlp) + if not isinstance(modified_nlp, Language): + err = Errors.E914.format(name="before_to_disk", value=type(modified_nlp)) + raise ValueError(err) + return modified_nlp + + return before_to_disk + + +def clean_output_dir(path: Optional[Path]) -> None: + """Remove an existing output directory. Typically used to ensure that that + a directory like model-best and its contents aren't just being overwritten + by nlp.to_disk, which could preserve existing subdirectories (e.g. + components that don't exist anymore). + """ + if path is not None and path.exists(): + for subdir in [path / DIR_MODEL_BEST, path / DIR_MODEL_LAST]: + if subdir.exists(): + try: + shutil.rmtree(str(subdir)) + logger.debug("Removed existing output directory: %s", subdir) + except Exception as e: + raise IOError(Errors.E901.format(path=path)) from e diff --git a/spacy/training/pretrain.py b/spacy/training/pretrain.py new file mode 100644 index 0000000..32eada4 --- /dev/null +++ b/spacy/training/pretrain.py @@ -0,0 +1,260 @@ +import re +import time +from collections import Counter +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Union + +import srsly +from thinc.api import ( + Config, + Model, + Optimizer, + fix_random_seed, + set_dropout_rate, + set_gpu_allocator, +) +from thinc.config import ConfigValidationError +from wasabi import Printer + +from ..errors import Errors +from ..schemas import ConfigSchemaPretrain +from ..tokens import Doc +from ..util import dot_to_object, load_model_from_config, registry +from .example import Example + + +def pretrain( + config: Config, + output_dir: Path, + resume_path: Optional[Path] = None, + epoch_resume: Optional[int] = None, + use_gpu: int = -1, + silent: bool = True, + skip_last: bool = False, +): + msg = Printer(no_print=silent) + if config["training"]["seed"] is not None: + fix_random_seed(config["training"]["seed"]) + allocator = config["training"]["gpu_allocator"] + if use_gpu >= 0 and allocator: + set_gpu_allocator(allocator) + # ignore in pretraining because we're creating it now + config["initialize"]["init_tok2vec"] = None + nlp = load_model_from_config(config) + _config = nlp.config.interpolate() + P = registry.resolve(_config["pretraining"], schema=ConfigSchemaPretrain) # type: ignore[arg-type] + corpus = dot_to_object(_config, P["corpus"]) + corpus = registry.resolve({"corpus": corpus})["corpus"] + batcher = P["batcher"] + model = create_pretraining_model(nlp, P) + optimizer = P["optimizer"] + # Load in pretrained weights to resume from + if resume_path is not None: + epoch_resume = _resume_model(model, resume_path, epoch_resume, silent=silent) + else: + # Without '--resume-path' the '--epoch-resume' argument is ignored + epoch_resume = 0 + + objective = model.attrs["loss"] + # TODO: move this to logger function? + tracker = ProgressTracker(frequency=10000) + if P["n_save_epoch"]: + msg.divider( + f"Pre-training tok2vec layer - starting at epoch {epoch_resume} - saving every {P['n_save_epoch']} epoch" + ) + else: + msg.divider(f"Pre-training tok2vec layer - starting at epoch {epoch_resume}") + row_settings = {"widths": (3, 10, 10, 6, 4), "aligns": ("r", "r", "r", "r", "r")} + msg.row(("#", "# Words", "Total Loss", "Loss", "w/s"), **row_settings) + + def _save_model(epoch, is_temp=False, is_last=False): + is_temp_str = ".temp" if is_temp else "" + with model.use_params(optimizer.averages): + if is_last: + save_path = output_dir / f"model-last.bin" + else: + save_path = output_dir / f"model{epoch}{is_temp_str}.bin" + with (save_path).open("wb") as file_: + file_.write(model.get_ref("tok2vec").to_bytes()) + log = { + "nr_word": tracker.nr_word, + "loss": tracker.loss, + "epoch_loss": tracker.epoch_loss, + "epoch": epoch, + } + with (output_dir / "log.jsonl").open("a") as file_: + file_.write(srsly.json_dumps(log) + "\n") + + # TODO: I think we probably want this to look more like the + # 'create_train_batches' function? + try: + for epoch in range(epoch_resume, P["max_epochs"]): + for batch_id, batch in enumerate(batcher(corpus(nlp))): + docs = ensure_docs(batch) + loss = make_update(model, docs, optimizer, objective) + progress = tracker.update(epoch, loss, docs) + if progress: + msg.row(progress, **row_settings) + if P["n_save_every"] and (batch_id % P["n_save_every"] == 0): + _save_model(epoch, is_temp=True) + + if P["n_save_epoch"]: + if epoch % P["n_save_epoch"] == 0 or epoch == P["max_epochs"] - 1: + _save_model(epoch) + else: + _save_model(epoch) + tracker.epoch_loss = 0.0 + finally: + if not skip_last: + _save_model(P["max_epochs"], is_last=True) + + +def ensure_docs(examples_or_docs: Iterable[Union[Doc, Example]]) -> List[Doc]: + docs = [] + for eg_or_doc in examples_or_docs: + if isinstance(eg_or_doc, Doc): + docs.append(eg_or_doc) + else: + docs.append(eg_or_doc.reference) + return docs + + +def _resume_model( + model: Model, resume_path: Path, epoch_resume: Optional[int], silent: bool = True +) -> int: + msg = Printer(no_print=silent) + msg.info(f"Resume training tok2vec from: {resume_path}") + with resume_path.open("rb") as file_: + weights_data = file_.read() + model.get_ref("tok2vec").from_bytes(weights_data) + + if epoch_resume is None: + # Parse the epoch number from the given weight file + model_name = re.search(r"model\d+\.bin", str(resume_path)) + if model_name: + # Default weight file name so read epoch_start from it by cutting off 'model' and '.bin' + epoch_resume = int(model_name.group(0)[5:][:-4]) + 1 + else: + # No epoch given and couldn't infer it + raise ValueError(Errors.E1020) + + msg.info(f"Resuming from epoch: {epoch_resume}") + return epoch_resume + + +def make_update( + model: Model, docs: Iterable[Doc], optimizer: Optimizer, objective_func: Callable +) -> float: + """Perform an update over a single batch of documents. + + docs (iterable): A batch of `Doc` objects. + optimizer (callable): An optimizer. + RETURNS loss: A float for the loss. + """ + predictions, backprop = model.begin_update(docs) + loss, gradients = objective_func(model.ops, docs, predictions) + backprop(gradients) + model.finish_update(optimizer) + # Don't want to return a cupy object here + # The gradients are modified in-place by the BERT MLM, + # so we get an accurate loss + return float(loss) + + +def create_pretraining_model(nlp, pretrain_config): + """Define a network for the pretraining. We simply add an output layer onto + the tok2vec input model. The tok2vec input model needs to be a model that + takes a batch of Doc objects (as a list), and returns a list of arrays. + Each array in the output needs to have one row per token in the doc. + The actual tok2vec layer is stored as a reference, and only this bit will be + serialized to file and read back in when calling the 'train' command. + """ + with nlp.select_pipes(enable=[]): + nlp.initialize() + tok2vec = get_tok2vec_ref(nlp, pretrain_config) + # If the config referred to a Tok2VecListener, grab the original model instead + if type(tok2vec).__name__ == "Tok2VecListener": + original_tok2vec = ( + tok2vec.upstream_name if tok2vec.upstream_name != "*" else "tok2vec" + ) + tok2vec = nlp.get_pipe(original_tok2vec).model + try: + tok2vec.initialize(X=[nlp.make_doc("Give it a doc to infer shapes")]) + except ValueError: + component = pretrain_config["component"] + layer = pretrain_config["layer"] + raise ValueError(Errors.E874.format(component=component, layer=layer)) + + create_function = pretrain_config["objective"] + model = create_function(nlp.vocab, tok2vec) + model.initialize(X=[nlp.make_doc("Give it a doc to infer shapes")]) + set_dropout_rate(model, pretrain_config["dropout"]) + return model + + +def get_tok2vec_ref(nlp, pretrain_config): + tok2vec_component = pretrain_config["component"] + if tok2vec_component is None: + desc = ( + f"To use pretrained tok2vec weights, [pretraining.component] " + f"needs to specify the component that should load them." + ) + err = "component can't be null" + errors = [{"loc": ["pretraining", "component"], "msg": err}] + raise ConfigValidationError( + config=nlp.config["pretraining"], errors=errors, desc=desc + ) + layer = nlp.get_pipe(tok2vec_component).model + if pretrain_config["layer"]: + layer = layer.get_ref(pretrain_config["layer"]) + return layer + + +class ProgressTracker: + def __init__(self, frequency=1000000): + self.loss = 0.0 + self.prev_loss = 0.0 + self.nr_word = 0 + self.words_per_epoch = Counter() + self.frequency = frequency + self.last_time = time.time() + self.last_update = 0 + self.epoch_loss = 0.0 + + def update(self, epoch, loss, docs): + self.loss += loss + self.epoch_loss += loss + words_in_batch = sum(len(doc) for doc in docs) + self.words_per_epoch[epoch] += words_in_batch + self.nr_word += words_in_batch + words_since_update = self.nr_word - self.last_update + if words_since_update >= self.frequency: + wps = words_since_update / (time.time() - self.last_time) + self.last_update = self.nr_word + self.last_time = time.time() + loss_per_word = self.loss - self.prev_loss + status = ( + epoch, + self.nr_word, + _smart_round(self.loss, width=10), + _smart_round(loss_per_word, width=6), + int(wps), + ) + self.prev_loss = float(self.loss) + return status + else: + return None + + +def _smart_round( + figure: Union[float, int], width: int = 10, max_decimal: int = 4 +) -> str: + """Round large numbers as integers, smaller numbers as decimals.""" + n_digits = len(str(int(figure))) + n_decimal = width - (n_digits + 1) + if n_decimal <= 1: + return str(int(figure)) + else: + n_decimal = min(n_decimal, max_decimal) + format_str = "%." + str(n_decimal) + "f" + return format_str % figure diff --git a/spacy/ty.py b/spacy/ty.py new file mode 100644 index 0000000..c18ce28 --- /dev/null +++ b/spacy/ty.py @@ -0,0 +1,59 @@ +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, +) + +from thinc.api import Model, Optimizer + +from .compat import Protocol, runtime_checkable + +if TYPE_CHECKING: + from .language import Language + from .training import Example + + +@runtime_checkable +class TrainableComponent(Protocol): + model: Any + is_trainable: bool + + def update( + self, + examples: Iterable["Example"], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, + ) -> Dict[str, float]: ... + + def finish_update(self, sgd: Optimizer) -> None: ... + + +@runtime_checkable +class InitializableComponent(Protocol): + def initialize( + self, + get_examples: Callable[[], Iterable["Example"]], + nlp: "Language", + **kwargs: Any, + ): ... + + +@runtime_checkable +class ListenedToComponent(Protocol): + model: Any + listeners: Sequence[Model] + listener_map: Dict[str, Sequence[Model]] + listening_components: List[str] + + def add_listener(self, listener: Model, component_name: str) -> None: ... + + def remove_listener(self, listener: Model, component_name: str) -> bool: ... + + def find_listeners(self, component) -> None: ... diff --git a/spacy/typedefs.pxd b/spacy/typedefs.pxd new file mode 100644 index 0000000..72d4d99 --- /dev/null +++ b/spacy/typedefs.pxd @@ -0,0 +1,9 @@ +from libc.stdint cimport int32_t, uint8_t, uint16_t, uint32_t, uint64_t, uintptr_t + +ctypedef float weight_t +ctypedef uint64_t hash_t +ctypedef uint64_t class_t +ctypedef uint64_t attr_t +ctypedef uint64_t flags_t +ctypedef uint16_t len_t +ctypedef uint16_t tag_t diff --git a/spacy/typedefs.pyx b/spacy/typedefs.pyx new file mode 100644 index 0000000..61bf620 --- /dev/null +++ b/spacy/typedefs.pyx @@ -0,0 +1 @@ +# cython: profile=False diff --git a/spacy/util.py b/spacy/util.py new file mode 100644 index 0000000..14d7b53 --- /dev/null +++ b/spacy/util.py @@ -0,0 +1,1896 @@ +import functools +import importlib +import importlib.util +import inspect +import itertools +import logging +import os +import re +import shlex +import shutil +import socket +import stat +import subprocess +import sys +import tempfile +import warnings +from collections import defaultdict +from contextlib import contextmanager +from pathlib import Path +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Mapping, + NoReturn, + Optional, + Pattern, + Set, + Tuple, + Type, + Union, + cast, +) + +import catalogue +import numpy +import srsly +import thinc +from catalogue import Registry, RegistryError +from packaging.requirements import Requirement +from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.version import InvalidVersion, Version +from thinc.api import ( + Adam, + Config, + ConfigValidationError, + Model, + NumpyOps, + Optimizer, + get_current_ops, +) + +try: + import cupy.random +except ImportError: + cupy = None + +# These are functions that were previously (v2.x) available from spacy.util +# and have since moved to Thinc. We're importing them here so people's code +# doesn't break, but they should always be imported from Thinc from now on, +# not from spacy.util. +from thinc.api import compounding, decaying, fix_random_seed # noqa: F401 + +from . import about +from .compat import CudaStream, cupy, importlib_metadata, is_windows +from .errors import OLD_MODEL_SHORTCUTS, Errors, Warnings +from .symbols import ORTH + +if TYPE_CHECKING: + # This lets us add type hints for mypy etc. without causing circular imports + from .language import Language, PipeCallable # noqa: F401 + from .tokens import Doc, Span # noqa: F401 + from .vocab import Vocab # noqa: F401 + + +# fmt: off +OOV_RANK = numpy.iinfo(numpy.uint64).max +DEFAULT_OOV_PROB = -20 +LEXEME_NORM_LANGS = ["cs", "da", "de", "el", "en", "grc", "id", "lb", "mk", "pt", "ru", "sr", "ta", "th"] + +# Default order of sections in the config file. Not all sections needs to exist, +# and additional sections are added at the end, in alphabetical order. +CONFIG_SECTION_ORDER = ["paths", "variables", "system", "nlp", "components", "corpora", "training", "pretraining", "initialize"] + +LANG_ALIASES = { + "af": ["afr"], + "am": ["amh"], + "ar": ["ara"], + "az": ["aze"], + "bg": ["bul"], + "bn": ["ben"], + "bo": ["bod", "tib"], + "ca": ["cat"], + "cs": ["ces", "cze"], + "da": ["dan"], + "de": ["deu", "ger"], + "el": ["ell", "gre"], + "en": ["eng"], + "es": ["spa"], + "et": ["est"], + "eu": ["eus", "baq"], + "fa": ["fas", "per"], + "fi": ["fin"], + "fo": ["fao"], + "fr": ["fra", "fre"], + "ga": ["gle"], + "gd": ["gla"], + "gu": ["guj"], + "he": ["heb", "iw"], # "iw" is the obsolete ISO 639-1 code for Hebrew + "hi": ["hin"], + "hr": ["hrv", "scr"], # "scr" is the deprecated ISO 639-2/B for Croatian + "hu": ["hun"], + "hy": ["hye"], + "id": ["ind", "in"], # "in" is the obsolete ISO 639-1 code for Hebrew + "is": ["isl", "ice"], + "it": ["ita"], + "ja": ["jpn"], + "kn": ["kan"], + "ko": ["kor"], + "ky": ["kir"], + "la": ["lat"], + "lb": ["ltz"], + "lg": ["lug"], + "lt": ["lit"], + "lv": ["lav"], + "mk": ["mkd", "mac"], + "ml": ["mal"], + "mr": ["mar"], + "ms": ["msa", "may"], + "nb": ["nob"], + "ne": ["nep"], + "nl": ["nld", "dut"], + "nn": ["nno"], + "pl": ["pol"], + "pt": ["por"], + "ro": ["ron", "rom", "mo", "mol"], # "mo" and "mol" are deprecated codes for Moldavian + "ru": ["rus"], + "sa": ["san"], + "si": ["sin"], + "sk": ["slk", "slo"], + "sl": ["slv"], + "sq": ["sqi", "alb"], + "sr": ["srp", "scc"], # "scc" is the deprecated ISO 639-2/B code for Serbian + "sv": ["swe"], + "ta": ["tam"], + "te": ["tel"], + "th": ["tha"], + "ti": ["tir"], + "tl": ["tgl"], + "tn": ["tsn"], + "tr": ["tur"], + "tt": ["tat"], + "uk": ["ukr"], + "ur": ["urd"], + "vi": ["viw"], + "yo": ["yor"], + "zh": ["zho", "chi"], + + "xx": ["mul"], +} +# fmt: on + +logger = logging.getLogger("spacy") +logger_stream_handler = logging.StreamHandler() +logger_stream_handler.setFormatter( + logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s") +) +logger.addHandler(logger_stream_handler) + + +class ENV_VARS: + CONFIG_OVERRIDES = "SPACY_CONFIG_OVERRIDES" + + +class registry(thinc.registry): + languages = catalogue.create("spacy", "languages", entry_points=True) + architectures = catalogue.create("spacy", "architectures", entry_points=True) + tokenizers = catalogue.create("spacy", "tokenizers", entry_points=True) + lemmatizers = catalogue.create("spacy", "lemmatizers", entry_points=True) + lookups = catalogue.create("spacy", "lookups", entry_points=True) + displacy_colors = catalogue.create("spacy", "displacy_colors", entry_points=True) + misc = catalogue.create("spacy", "misc", entry_points=True) + # Callback functions used to manipulate nlp object etc. + callbacks = catalogue.create("spacy", "callbacks", entry_points=True) + batchers = catalogue.create("spacy", "batchers", entry_points=True) + readers = catalogue.create("spacy", "readers", entry_points=True) + augmenters = catalogue.create("spacy", "augmenters", entry_points=True) + loggers = catalogue.create("spacy", "loggers", entry_points=True) + scorers = catalogue.create("spacy", "scorers", entry_points=True) + vectors = catalogue.create("spacy", "vectors", entry_points=True) + # These are factories registered via third-party packages and the + # spacy_factories entry point. This registry only exists so we can easily + # load them via the entry points. The "true" factories are added via the + # Language.factory decorator (in the spaCy code base and user code) and those + # are the factories used to initialize components via registry.resolve. + _entry_point_factories = catalogue.create("spacy", "factories", entry_points=True) + factories = catalogue.create("spacy", "internal_factories") + # This is mostly used to get a list of all installed models in the current + # environment. spaCy models packaged with `spacy package` will "advertise" + # themselves via entry points. + models = catalogue.create("spacy", "models", entry_points=True) + cli = catalogue.create("spacy", "cli", entry_points=True) + + @classmethod + def ensure_populated(cls) -> None: + """Ensure the registry is populated with all necessary components.""" + from .registrations import REGISTRY_POPULATED, populate_registry + + if not REGISTRY_POPULATED: + populate_registry() + + @classmethod + def get_registry_names(cls) -> List[str]: + """List all available registries.""" + cls.ensure_populated() + names = [] + for name, value in inspect.getmembers(cls): + if not name.startswith("_") and isinstance(value, Registry): + names.append(name) + return sorted(names) + + @classmethod + def get(cls, registry_name: str, func_name: str) -> Callable: + """Get a registered function from the registry.""" + cls.ensure_populated() + # We're overwriting this classmethod so we're able to provide more + # specific error messages and implement a fallback to spacy-legacy. + if not hasattr(cls, registry_name): + names = ", ".join(cls.get_registry_names()) or "none" + raise RegistryError(Errors.E892.format(name=registry_name, available=names)) + reg = getattr(cls, registry_name) + try: + func = reg.get(func_name) + except RegistryError: + if func_name.startswith("spacy."): + legacy_name = func_name.replace("spacy.", "spacy-legacy.") + try: + return reg.get(legacy_name) + except catalogue.RegistryError: + pass + available = ", ".join(sorted(reg.get_all().keys())) or "none" + raise RegistryError( + Errors.E893.format( + name=func_name, reg_name=registry_name, available=available + ) + ) from None + return func + + @classmethod + def find( + cls, registry_name: str, func_name: str + ) -> Dict[str, Optional[Union[str, int]]]: + """Find information about a registered function, including the + module and path to the file it's defined in, the line number and the + docstring, if available. + + registry_name (str): Name of the catalogue registry. + func_name (str): Name of the registered function. + RETURNS (Dict[str, Optional[Union[str, int]]]): The function info. + """ + cls.ensure_populated() + # We're overwriting this classmethod so we're able to provide more + # specific error messages and implement a fallback to spacy-legacy. + if not hasattr(cls, registry_name): + names = ", ".join(cls.get_registry_names()) or "none" + raise RegistryError(Errors.E892.format(name=registry_name, available=names)) + reg = getattr(cls, registry_name) + try: + func_info = reg.find(func_name) + except RegistryError: + if func_name.startswith("spacy."): + legacy_name = func_name.replace("spacy.", "spacy-legacy.") + try: + return reg.find(legacy_name) + except catalogue.RegistryError: + pass + available = ", ".join(sorted(reg.get_all().keys())) or "none" + raise RegistryError( + Errors.E893.format( + name=func_name, reg_name=registry_name, available=available + ) + ) from None + return func_info + + @classmethod + def has(cls, registry_name: str, func_name: str) -> bool: + """Check whether a function is available in a registry.""" + cls.ensure_populated() + if not hasattr(cls, registry_name): + return False + reg = getattr(cls, registry_name) + if func_name.startswith("spacy."): + legacy_name = func_name.replace("spacy.", "spacy-legacy.") + return func_name in reg or legacy_name in reg + return func_name in reg + + +class SimpleFrozenDict(dict): + """Simplified implementation of a frozen dict, mainly used as default + function or method argument (for arguments that should default to empty + dictionary). Will raise an error if user or spaCy attempts to add to dict. + """ + + def __init__(self, *args, error: str = Errors.E095, **kwargs) -> None: + """Initialize the frozen dict. Can be initialized with pre-defined + values. + + error (str): The error message when user tries to assign to dict. + """ + super().__init__(*args, **kwargs) + self.error = error + + def __setitem__(self, key, value): + raise NotImplementedError(self.error) + + def pop(self, key, default=None): + raise NotImplementedError(self.error) + + def update(self, other): + raise NotImplementedError(self.error) + + +class SimpleFrozenList(list): + """Wrapper class around a list that lets us raise custom errors if certain + attributes/methods are accessed. Mostly used for properties like + Language.pipeline that return an immutable list (and that we don't want to + convert to a tuple to not break too much backwards compatibility). If a user + accidentally calls nlp.pipeline.append(), we can raise a more helpful error. + """ + + def __init__(self, *args, error: str = Errors.E927) -> None: + """Initialize the frozen list. + + error (str): The error message when user tries to mutate the list. + """ + self.error = error + super().__init__(*args) + + def append(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def clear(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def extend(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def insert(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def pop(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def remove(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def reverse(self, *args, **kwargs): + raise NotImplementedError(self.error) + + def sort(self, *args, **kwargs): + raise NotImplementedError(self.error) + + +def lang_class_is_loaded(lang: str) -> bool: + """Check whether a Language class is already loaded. Language classes are + loaded lazily, to avoid expensive setup code associated with the language + data. + + lang (str): Two-letter language code, e.g. 'en'. + RETURNS (bool): Whether a Language class has been loaded. + """ + return lang in registry.languages + + +def find_matching_language(lang: str) -> Optional[str]: + """ + Given a two-letter ISO 639-1 or three-letter ISO 639-3 language code, + find a supported spaCy language. + + Returns the language code if a matching language is available, or None + if there is no matching language. + + >>> find_matching_language('fra') # ISO 639-3 code for French + 'fr' + >>> find_matching_language('fre') # ISO 639-2/B code for French + 'fr' + >>> find_matching_language('iw') # Obsolete ISO 639-1 code for Hebrew + 'he' + >>> find_matching_language('mo') # Deprecated code for Moldavian + 'ro' + >>> find_matching_language('scc') # Deprecated ISO 639-2/B code for Serbian + 'sr' + >>> find_matching_language('zxx') + None + """ + import spacy.lang # noqa: F401 + + # Check aliases + for lang_code, aliases in LANG_ALIASES.items(): + if lang in aliases: + return lang_code + + return None + + +def get_lang_class(lang: str) -> Type["Language"]: + """Import and load a Language class. + + lang (str): Two-letter ISO 639-1 or three-letter ISO 639-3 language code, such as 'en' and 'eng'. + RETURNS (Language): Language class. + """ + # Check if language is registered / entry point is available + if lang in registry.languages: + return registry.languages.get(lang) + else: + # Find the language in the spacy.lang subpackage + try: + module = importlib.import_module(f".lang.{lang}", "spacy") + except ImportError as err: + # Find a matching language. For example, if the language 'eng' is + # requested, we can use language-matching to load `spacy.lang.en`. + match = find_matching_language(lang) + + if match: + lang = match + module = importlib.import_module(f".lang.{lang}", "spacy") + else: + raise ImportError(Errors.E048.format(lang=lang, err=err)) from err + set_lang_class(lang, getattr(module, module.__all__[0])) # type: ignore[attr-defined] + return registry.languages.get(lang) + + +def set_lang_class(name: str, cls: Type["Language"]) -> None: + """Set a custom Language class name that can be loaded via get_lang_class. + + name (str): Name of Language class. + cls (Language): Language class. + """ + registry.languages.register(name, func=cls) + + +def ensure_path(path: Any) -> Any: + """Ensure string is converted to a Path. + + path (Any): Anything. If string, it's converted to Path. + RETURNS: Path or original argument. + """ + if isinstance(path, str): + return Path(path) + else: + return path + + +def load_language_data(path: Union[str, Path]) -> Union[dict, list]: + """Load JSON language data using the given path as a base. If the provided + path isn't present, will attempt to load a gzipped version before giving up. + + path (str / Path): The data to load. + RETURNS: The loaded data. + """ + path = ensure_path(path) + if path.exists(): + return srsly.read_json(path) + path = path.with_suffix(path.suffix + ".gz") + if path.exists(): + return srsly.read_gzip_json(path) + raise ValueError(Errors.E160.format(path=path)) + + +def get_module_path(module: ModuleType) -> Path: + """Get the path of a Python module. + + module (ModuleType): The Python module. + RETURNS (Path): The path. + """ + if not hasattr(module, "__module__"): + raise ValueError(Errors.E169.format(module=repr(module))) + file_path = Path(cast(os.PathLike, sys.modules[module.__module__].__file__)) + return file_path.parent + + +# Default value for passed enable/disable values. +_DEFAULT_EMPTY_PIPES = SimpleFrozenList() + + +def load_model( + name: Union[str, Path], + *, + vocab: Union["Vocab", bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), +) -> "Language": + """Load a model from a package or data path. + + name (str): Package name or model path. + vocab (Vocab / True): Optional vocab to pass in on initialization. If True, + a new Vocab object will be created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All others will be disabled. + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. + config (Dict[str, Any] / Config): Config overrides as nested dict or dict + keyed by section values in dot notation. + RETURNS (Language): The loaded nlp object. + """ + kwargs = { + "vocab": vocab, + "disable": disable, + "enable": enable, + "exclude": exclude, + "config": config, + } + if isinstance(name, str): # name or string path + if name.startswith("blank:"): # shortcut for blank model + return get_lang_class(name.replace("blank:", ""))() + if is_package(name): # installed as package + return load_model_from_package(name, **kwargs) # type: ignore[arg-type] + if Path(name).exists(): # path to model data directory + return load_model_from_path(Path(name), **kwargs) # type: ignore[arg-type] + elif hasattr(name, "exists"): # Path or Path-like to model data + return load_model_from_path(name, **kwargs) # type: ignore[arg-type] + if name in OLD_MODEL_SHORTCUTS: + raise IOError(Errors.E941.format(name=name, full=OLD_MODEL_SHORTCUTS[name])) # type: ignore[index] + raise IOError(Errors.E050.format(name=name)) + + +def load_model_from_package( + name: str, + *, + vocab: Union["Vocab", bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), +) -> "Language": + """Load a model from an installed package. + + name (str): The package name. + vocab (Vocab / True): Optional vocab to pass in on initialization. If True, + a new Vocab object will be created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled + pipes will be loaded but they won't be run unless you explicitly + enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded + components won't be loaded. + config (Dict[str, Any] / Config): Config overrides as nested dict or dict + keyed by section values in dot notation. + RETURNS (Language): The loaded nlp object. + """ + cls = importlib.import_module(name) + return cls.load( + vocab=vocab, disable=disable, enable=enable, exclude=exclude, config=config + ) # type: ignore[attr-defined] + + +def load_model_from_path( + model_path: Path, + *, + meta: Optional[Dict[str, Any]] = None, + vocab: Union["Vocab", bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), +) -> "Language": + """Load a model from a data directory path. Creates Language class with + pipeline from config.cfg and then calls from_disk() with path. + + model_path (Path): Model path. + meta (Dict[str, Any]): Optional model meta. + vocab (Vocab / True): Optional vocab to pass in on initialization. If True, + a new Vocab object will be created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled + pipes will be loaded but they won't be run unless you explicitly + enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded + components won't be loaded. + config (Dict[str, Any] / Config): Config overrides as nested dict or dict + keyed by section values in dot notation. + RETURNS (Language): The loaded nlp object. + """ + if not model_path.exists(): + raise IOError(Errors.E052.format(path=model_path)) + if not meta: + meta = get_model_meta(model_path) + config_path = model_path / "config.cfg" + overrides = dict_to_dot(config, for_overrides=True) + config = load_config(config_path, overrides=overrides) + nlp = load_model_from_config( + config, + vocab=vocab, + disable=disable, + enable=enable, + exclude=exclude, + meta=meta, + ) + return nlp.from_disk(model_path, exclude=exclude, overrides=overrides) + + +def load_model_from_config( + config: Union[Dict[str, Any], Config], + *, + meta: Dict[str, Any] = SimpleFrozenDict(), + vocab: Union["Vocab", bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + auto_fill: bool = False, + validate: bool = True, +) -> "Language": + """Create an nlp object from a config. Expects the full config file including + a section "nlp" containing the settings for the nlp object. + + name (str): Package name or model path. + meta (Dict[str, Any]): Optional model meta. + vocab (Vocab / True): Optional vocab to pass in on initialization. If True, + a new Vocab object will be created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled + pipes will be loaded but they won't be run unless you explicitly + enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded + components won't be loaded. + auto_fill (bool): Whether to auto-fill config with missing defaults. + validate (bool): Whether to show config validation errors. + RETURNS (Language): The loaded nlp object. + """ + if "nlp" not in config: + raise ValueError(Errors.E985.format(config=config)) + nlp_config = config["nlp"] + if "lang" not in nlp_config or nlp_config["lang"] is None: + raise ValueError(Errors.E993.format(config=nlp_config)) + # This will automatically handle all codes registered via the languages + # registry, including custom subclasses provided via entry points + lang_cls = get_lang_class(nlp_config["lang"]) + nlp = lang_cls.from_config( + config, + vocab=vocab, + disable=disable, + enable=enable, + exclude=exclude, + auto_fill=auto_fill, + validate=validate, + meta=meta, + ) + return nlp + + +def get_sourced_components( + config: Union[Dict[str, Any], Config], +) -> Dict[str, Dict[str, Any]]: + """RETURNS (List[str]): All sourced components in the original config, + e.g. {"source": "en_core_web_sm"}. If the config contains a key + "factory", we assume it refers to a component factory. + """ + return { + name: cfg + for name, cfg in config.get("components", {}).items() + if "factory" not in cfg and "source" in cfg + } + + +def resolve_dot_names( + config: Config, dot_names: List[Optional[str]] +) -> Tuple[Any, ...]: + """Resolve one or more "dot notation" names, e.g. corpora.train. + The paths could point anywhere into the config, so we don't know which + top-level section we'll be looking within. + + We resolve the whole top-level section, although we could resolve less -- + we could find the lowest part of the tree. + """ + # TODO: include schema? + resolved = {} + output: List[Any] = [] + errors = [] + for name in dot_names: + if name is None: + output.append(name) + else: + section = name.split(".")[0] + # We want to avoid resolving the same thing twice + if section not in resolved: + if registry.is_promise(config[section]): + # Otherwise we can't resolve [corpus] if it's a promise + result = registry.resolve({"config": config[section]})["config"] + else: + result = registry.resolve(config[section]) + resolved[section] = result + try: + output.append(dot_to_object(resolved, name)) # type: ignore[arg-type] + except KeyError: + msg = f"not a valid section reference: {name}" + errors.append({"loc": name.split("."), "msg": msg}) + if errors: + raise ConfigValidationError(config=config, errors=errors) + return tuple(output) + + +def load_model_from_init_py( + init_file: Union[Path, str], + *, + vocab: Union["Vocab", bool] = True, + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), +) -> "Language": + """Helper function to use in the `load()` method of a model package's + __init__.py. + + vocab (Vocab / True): Optional vocab to pass in on initialization. If True, + a new Vocab object will be created. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled + pipes will be loaded but they won't be run unless you explicitly + enable them by calling nlp.enable_pipe. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other + pipes will be disabled (and can be enabled using `nlp.enable_pipe`). + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded + components won't be loaded. + config (Dict[str, Any] / Config): Config overrides as nested dict or dict + keyed by section values in dot notation. + RETURNS (Language): The loaded nlp object. + """ + model_path = Path(init_file).parent + meta = get_model_meta(model_path) + data_dir = f"{meta['lang']}_{meta['name']}-{meta['version']}" + data_path = model_path / data_dir + if not model_path.exists(): + raise IOError(Errors.E052.format(path=data_path)) + return load_model_from_path( + data_path, + vocab=vocab, + meta=meta, + disable=disable, + enable=enable, + exclude=exclude, + config=config, + ) + + +def load_config( + path: Union[str, Path], + overrides: Dict[str, Any] = SimpleFrozenDict(), + interpolate: bool = False, +) -> Config: + """Load a config file. Takes care of path validation and section order. + + path (Union[str, Path]): Path to the config file or "-" to read from stdin. + overrides: (Dict[str, Any]): Config overrides as nested dict or + dict keyed by section values in dot notation. + interpolate (bool): Whether to interpolate and resolve variables. + RETURNS (Config): The loaded config. + """ + config_path = ensure_path(path) + config = Config(section_order=CONFIG_SECTION_ORDER) + if str(config_path) == "-": # read from standard input + return config.from_str( + sys.stdin.read(), overrides=overrides, interpolate=interpolate + ) + else: + if not config_path or not config_path.is_file(): + raise IOError(Errors.E053.format(path=config_path, name="config file")) + return config.from_disk( + config_path, overrides=overrides, interpolate=interpolate + ) + + +def load_config_from_str( + text: str, overrides: Dict[str, Any] = SimpleFrozenDict(), interpolate: bool = False +): + """Load a full config from a string. Wrapper around Thinc's Config.from_str. + + text (str): The string config to load. + interpolate (bool): Whether to interpolate and resolve variables. + RETURNS (Config): The loaded config. + """ + return Config(section_order=CONFIG_SECTION_ORDER).from_str( + text, overrides=overrides, interpolate=interpolate + ) + + +def get_installed_models() -> List[str]: + """List all model packages currently installed in the environment. + + RETURNS (List[str]): The string names of the models. + """ + return list(registry.models.get_all().keys()) + + +def get_package_version(name: str) -> Optional[str]: + """Get the version of an installed package. Typically used to get model + package versions. + + name (str): The name of the installed Python package. + RETURNS (str / None): The version or None if package not installed. + """ + try: + return importlib_metadata.version(name) # type: ignore[attr-defined] + except importlib_metadata.PackageNotFoundError: # type: ignore[attr-defined] + return None + + +def is_compatible_version( + version: str, constraint: str, prereleases: bool = True +) -> Optional[bool]: + """Check if a version (e.g. "2.0.0") is compatible given a version + constraint (e.g. ">=1.9.0,<2.2.1"). If the constraint is a specific version, + it's interpreted as =={version}. + + version (str): The version to check. + constraint (str): The constraint string. + prereleases (bool): Whether to allow prereleases. If set to False, + prerelease versions will be considered incompatible. + RETURNS (bool / None): Whether the version is compatible, or None if the + version or constraint are invalid. + """ + # Handle cases where exact version is provided as constraint + if constraint[0].isdigit(): + constraint = f"=={constraint}" + try: + spec = SpecifierSet(constraint) + version = Version(version) # type: ignore[assignment] + except (InvalidSpecifier, InvalidVersion): + return None + spec.prereleases = prereleases + return version in spec + + +def is_unconstrained_version( + constraint: str, prereleases: bool = True +) -> Optional[bool]: + # We have an exact version, this is the ultimate constrained version + if constraint[0].isdigit(): + return False + try: + spec = SpecifierSet(constraint) + except InvalidSpecifier: + return None + spec.prereleases = prereleases + specs = [sp for sp in spec] + # We only have one version spec and it defines > or >= + if len(specs) == 1 and specs[0].operator in (">", ">="): + return True + # One specifier is exact version + if any(sp.operator in ("==") for sp in specs): + return False + has_upper = any(sp.operator in ("<", "<=") for sp in specs) + has_lower = any(sp.operator in (">", ">=") for sp in specs) + # We have a version spec that defines an upper and lower bound + if has_upper and has_lower: + return False + # Everything else, like only an upper version, only a lower version etc. + return True + + +def split_requirement(requirement: str) -> Tuple[str, str]: + """Split a requirement like spacy>=1.2.3 into ("spacy", ">=1.2.3").""" + req = Requirement(requirement) + return (req.name, str(req.specifier)) + + +def get_minor_version_range(version: str) -> str: + """Generate a version range like >=1.2.3,<1.3.0 based on a given version + (e.g. of spaCy). + """ + release = Version(version).release + return f">={version},<{release[0]}.{release[1] + 1}.0" + + +def get_model_lower_version(constraint: str) -> Optional[str]: + """From a version range like >=1.2.3,<1.3.0 return the lower pin.""" + try: + specset = SpecifierSet(constraint) + for spec in specset: + if spec.operator in (">=", "==", "~="): + return spec.version + except Exception: + pass + return None + + +def is_prerelease_version(version: str) -> bool: + """Check whether a version is a prerelease version. + + version (str): The version, e.g. "3.0.0.dev1". + RETURNS (bool): Whether the version is a prerelease version. + """ + return Version(version).is_prerelease + + +def get_base_version(version: str) -> str: + """Generate the base version without any prerelease identifiers. + + version (str): The version, e.g. "3.0.0.dev1". + RETURNS (str): The base version, e.g. "3.0.0". + """ + return Version(version).base_version + + +def get_minor_version(version: str) -> Optional[str]: + """Get the major + minor version (without patch or prerelease identifiers). + + version (str): The version. + RETURNS (str): The major + minor version or None if version is invalid. + """ + try: + v = Version(version) + except (TypeError, InvalidVersion): + return None + return f"{v.major}.{v.minor}" + + +def is_minor_version_match(version_a: str, version_b: str) -> bool: + """Compare two versions and check if they match in major and minor, without + patch or prerelease identifiers. Used internally for compatibility checks + that should be insensitive to patch releases. + + version_a (str): The first version + version_b (str): The second version. + RETURNS (bool): Whether the versions match. + """ + a = get_minor_version(version_a) + b = get_minor_version(version_b) + return a is not None and b is not None and a == b + + +def load_meta(path: Union[str, Path]) -> Dict[str, Any]: + """Load a model meta.json from a path and validate its contents. + + path (Union[str, Path]): Path to meta.json. + RETURNS (Dict[str, Any]): The loaded meta. + """ + path = ensure_path(path) + if not path.parent.exists(): + raise IOError(Errors.E052.format(path=path.parent)) + if not path.exists() or not path.is_file(): + raise IOError(Errors.E053.format(path=path.parent, name="meta.json")) + meta = srsly.read_json(path) + for setting in ["lang", "name", "version"]: + if setting not in meta or not meta[setting]: + raise ValueError(Errors.E054.format(setting=setting)) + if "spacy_version" in meta: + if not is_compatible_version(about.__version__, meta["spacy_version"]): + lower_version = get_model_lower_version(meta["spacy_version"]) + lower_version = get_base_version(lower_version) # type: ignore[arg-type] + if lower_version is not None: + lower_version = "v" + lower_version + elif "spacy_git_version" in meta: + lower_version = "git commit " + meta["spacy_git_version"] + else: + lower_version = "version unknown" + warn_msg = Warnings.W095.format( + model=f"{meta['lang']}_{meta['name']}", + model_version=meta["version"], + version=lower_version, + current=about.__version__, + ) + warnings.warn(warn_msg) + if is_unconstrained_version(meta["spacy_version"]): + warn_msg = Warnings.W094.format( + model=f"{meta['lang']}_{meta['name']}", + model_version=meta["version"], + version=meta["spacy_version"], + example=get_minor_version_range(about.__version__), + ) + warnings.warn(warn_msg) + return meta + + +def get_model_meta(path: Union[str, Path]) -> Dict[str, Any]: + """Get model meta.json from a directory path and validate its contents. + + path (str / Path): Path to model directory. + RETURNS (Dict[str, Any]): The model's meta data. + """ + model_path = ensure_path(path) + return load_meta(model_path / "meta.json") + + +def is_package(name: str) -> bool: + """Check if string maps to a package installed via pip. + + name (str): Name of package. + RETURNS (bool): True if installed package, False if not. + """ + try: + importlib_metadata.distribution(name) # type: ignore[attr-defined] + return True + except: # noqa: E722 + return False + + +def get_package_path(name: str) -> Path: + """Get the path to an installed package. + + name (str): Package name. + RETURNS (Path): Path to installed package. + """ + # Here we're importing the module just to find it. This is worryingly + # indirect, but it's otherwise very difficult to find the package. + pkg = importlib.import_module(name) + return Path(cast(Union[str, os.PathLike], pkg.__file__)).parent + + +def replace_model_node(model: Model, target: Model, replacement: Model) -> None: + """Replace a node within a model with a new one, updating refs. + + model (Model): The parent model. + target (Model): The target node. + replacement (Model): The node to replace the target with. + """ + # Place the node into the sublayers + for node in model.walk(): + if target in node.layers: + node.layers[node.layers.index(target)] = replacement + # Now fix any node references + for node in model.walk(): + for ref_name in node.ref_names: + if node.maybe_get_ref(ref_name) is target: + node.set_ref(ref_name, replacement) + + +def split_command(command: str) -> List[str]: + """Split a string command using shlex. Handles platform compatibility. + command (str) : The command to split + RETURNS (List[str]): The split command. + """ + return shlex.split(command, posix=not is_windows) + + +def run_command( + command: Union[str, List[str]], + *, + stdin: Optional[Any] = None, + capture: bool = False, +) -> subprocess.CompletedProcess: + """Run a command on the command line as a subprocess. If the subprocess + returns a non-zero exit code, a system exit is performed. + command (str / List[str]): The command. If provided as a string, the + string will be split using shlex.split. + stdin (Optional[Any]): stdin to read from or None. + capture (bool): Whether to capture the output and errors. If False, + the stdout and stderr will not be redirected, and if there's an error, + sys.exit will be called with the return code. You should use capture=False + when you want to turn over execution to the command, and capture=True + when you want to run the command more like a function. + RETURNS (Optional[CompletedProcess]): The process object. + """ + if isinstance(command, str): + cmd_list = split_command(command) + cmd_str = command + else: + cmd_list = command + cmd_str = " ".join(command) + try: + ret = subprocess.run( + cmd_list, + env=os.environ.copy(), + input=stdin, + encoding="utf8", + check=False, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + ) + except FileNotFoundError: + # Indicates the *command* wasn't found, it's an error before the command + # is run. + raise FileNotFoundError( + Errors.E970.format(str_command=cmd_str, tool=cmd_list[0]) + ) from None + if ret.returncode != 0 and capture: + message = f"Error running command:\n\n{cmd_str}\n\n" + message += f"Subprocess exited with status {ret.returncode}" + if ret.stdout is not None: + message += f"\n\nProcess log (stdout and stderr):\n\n" + message += ret.stdout + error = subprocess.SubprocessError(message) + error.ret = ret # type: ignore[attr-defined] + error.command = cmd_str # type: ignore[attr-defined] + raise error + elif ret.returncode != 0: + sys.exit(ret.returncode) + return ret + + +@contextmanager +def working_dir(path: Union[str, Path]) -> Iterator[Path]: + """Change current working directory and returns to previous on exit. + path (str / Path): The directory to navigate to. + YIELDS (Path): The absolute path to the current working directory. This + should be used if the block needs to perform actions within the working + directory, to prevent mismatches with relative paths. + """ + prev_cwd = Path.cwd() + current = Path(path).resolve() + os.chdir(str(current)) + try: + yield current + finally: + os.chdir(str(prev_cwd)) + + +@contextmanager +def make_tempdir() -> Generator[Path, None, None]: + """Execute a block in a temporary directory and remove the directory and + its contents at the end of the with block. + YIELDS (Path): The path of the temp directory. + """ + d = Path(tempfile.mkdtemp()) + yield d + + # On Windows, git clones use read-only files, which cause permission errors + # when being deleted. This forcibly fixes permissions. + def force_remove(rmfunc, path, ex): + os.chmod(path, stat.S_IWRITE) + rmfunc(path) + + try: + if sys.version_info >= (3, 12): + shutil.rmtree(str(d), onexc=force_remove) + else: + shutil.rmtree(str(d), onerror=force_remove) + except PermissionError as e: + warnings.warn(Warnings.W091.format(dir=d, msg=e)) + + +def is_in_jupyter() -> bool: + """Check if user is running spaCy from a Jupyter or Colab notebook by + detecting the IPython kernel. Mainly used for the displaCy visualizer. + RETURNS (bool): True if in Jupyter/Colab, False if not. + """ + # https://stackoverflow.com/a/39662359/6400719 + # https://stackoverflow.com/questions/15411967 + try: + if get_ipython().__class__.__name__ == "ZMQInteractiveShell": # type: ignore[name-defined] + return True # Jupyter notebook or qtconsole + if get_ipython().__class__.__module__ == "google.colab._shell": # type: ignore[name-defined] + return True # Colab notebook + except NameError: + pass # Probably standard Python interpreter + # additional check for Colab + try: + import google.colab + + return True # Colab notebook + except ImportError: + pass + return False + + +def is_in_interactive() -> bool: + """Check if user is running spaCy from an interactive Python + shell. Will return True in Jupyter notebooks too. + RETURNS (bool): True if in interactive mode, False if not. + """ + # https://stackoverflow.com/questions/2356399/tell-if-python-is-in-interactive-mode + return hasattr(sys, "ps1") or hasattr(sys, "ps2") + + +def get_object_name(obj: Any) -> str: + """Get a human-readable name of a Python object, e.g. a pipeline component. + + obj (Any): The Python object, typically a function or class. + RETURNS (str): A human-readable name. + """ + if hasattr(obj, "name") and obj.name is not None: + return obj.name + if hasattr(obj, "__name__"): + return obj.__name__ + if hasattr(obj, "__class__") and hasattr(obj.__class__, "__name__"): + return obj.__class__.__name__ + return repr(obj) + + +def is_same_func(func1: Callable, func2: Callable) -> bool: + """Approximately decide whether two functions are the same, even if their + identity is different (e.g. after they have been live reloaded). Mostly + used in the @Language.component and @Language.factory decorators to decide + whether to raise if a factory already exists. Allows decorator to run + multiple times with the same function. + + func1 (Callable): The first function. + func2 (Callable): The second function. + RETURNS (bool): Whether it's the same function (most likely). + """ + if not callable(func1) or not callable(func2): + return False + if not hasattr(func1, "__qualname__") or not hasattr(func2, "__qualname__"): + return False + same_name = func1.__qualname__ == func2.__qualname__ + same_file = inspect.getfile(func1) == inspect.getfile(func2) + same_code = inspect.getsourcelines(func1) == inspect.getsourcelines(func2) + return same_name and same_file and same_code + + +def get_cuda_stream( + require: bool = False, non_blocking: bool = True +) -> Optional[CudaStream]: + ops = get_current_ops() + if CudaStream is None: + return None + elif isinstance(ops, NumpyOps): + return None + else: + return CudaStream(non_blocking=non_blocking) + + +def get_async(stream, numpy_array): + if cupy is None: + return numpy_array + else: + array = cupy.ndarray(numpy_array.shape, order="C", dtype=numpy_array.dtype) + array.set(numpy_array, stream=stream) + return array + + +def read_regex(path: Union[str, Path]) -> Pattern: + path = ensure_path(path) + with path.open(encoding="utf8") as file_: + entries = file_.read().split("\n") + expression = "|".join( + ["^" + re.escape(piece) for piece in entries if piece.strip()] + ) + return re.compile(expression) + + +def compile_prefix_regex(entries: Iterable[Union[str, Pattern]]) -> Pattern: + """Compile a sequence of prefix rules into a regex object. + + entries (Iterable[Union[str, Pattern]]): The prefix rules, e.g. + spacy.lang.punctuation.TOKENIZER_PREFIXES. + RETURNS (Pattern): The regex object. to be used for Tokenizer.prefix_search. + """ + expression = "|".join(["^" + piece for piece in entries if piece.strip()]) # type: ignore[operator, union-attr] + return re.compile(expression) + + +def compile_suffix_regex(entries: Iterable[Union[str, Pattern]]) -> Pattern: + """Compile a sequence of suffix rules into a regex object. + + entries (Iterable[Union[str, Pattern]]): The suffix rules, e.g. + spacy.lang.punctuation.TOKENIZER_SUFFIXES. + RETURNS (Pattern): The regex object. to be used for Tokenizer.suffix_search. + """ + expression = "|".join([piece + "$" for piece in entries if piece.strip()]) # type: ignore[operator, union-attr] + return re.compile(expression) + + +def compile_infix_regex(entries: Iterable[Union[str, Pattern]]) -> Pattern: + """Compile a sequence of infix rules into a regex object. + + entries (Iterable[Union[str, Pattern]]): The infix rules, e.g. + spacy.lang.punctuation.TOKENIZER_INFIXES. + RETURNS (regex object): The regex object. to be used for Tokenizer.infix_finditer. + """ + expression = "|".join([piece for piece in entries if piece.strip()]) # type: ignore[misc, union-attr] + return re.compile(expression) + + +def add_lookups(default_func: Callable[[str], Any], *lookups) -> Callable[[str], Any]: + """Extend an attribute function with special cases. If a word is in the + lookups, the value is returned. Otherwise the previous function is used. + + default_func (callable): The default function to execute. + *lookups (dict): Lookup dictionary mapping string to attribute value. + RETURNS (callable): Lexical attribute getter. + """ + # This is implemented as functools.partial instead of a closure, to allow + # pickle to work. + return functools.partial(_get_attr_unless_lookup, default_func, lookups) + + +def _get_attr_unless_lookup( + default_func: Callable[[str], Any], lookups: Dict[str, Any], string: str +) -> Any: + for lookup in lookups: + if string in lookup: + return lookup[string] # type: ignore[index] + return default_func(string) + + +def update_exc( + base_exceptions: Dict[str, List[dict]], *addition_dicts +) -> Dict[str, List[dict]]: + """Update and validate tokenizer exceptions. Will overwrite exceptions. + + base_exceptions (Dict[str, List[dict]]): Base exceptions. + *addition_dicts (Dict[str, List[dict]]): Exceptions to add to the base dict, in order. + RETURNS (Dict[str, List[dict]]): Combined tokenizer exceptions. + """ + exc = dict(base_exceptions) + for additions in addition_dicts: + for orth, token_attrs in additions.items(): + if not all(isinstance(attr[ORTH], str) for attr in token_attrs): + raise ValueError(Errors.E055.format(key=orth, orths=token_attrs)) + described_orth = "".join(attr[ORTH] for attr in token_attrs) + if orth != described_orth: + raise ValueError(Errors.E056.format(key=orth, orths=described_orth)) + exc.update(additions) + exc = expand_exc(exc, "'", "’") + return exc + + +def expand_exc( + excs: Dict[str, List[dict]], search: str, replace: str +) -> Dict[str, List[dict]]: + """Find string in tokenizer exceptions, duplicate entry and replace string. + For example, to add additional versions with typographic apostrophes. + + excs (Dict[str, List[dict]]): Tokenizer exceptions. + search (str): String to find and replace. + replace (str): Replacement. + RETURNS (Dict[str, List[dict]]): Combined tokenizer exceptions. + """ + + def _fix_token(token, search, replace): + fixed = dict(token) + fixed[ORTH] = fixed[ORTH].replace(search, replace) + return fixed + + new_excs = dict(excs) + for token_string, tokens in excs.items(): + if search in token_string: + new_key = token_string.replace(search, replace) + new_value = [_fix_token(t, search, replace) for t in tokens] + new_excs[new_key] = new_value + return new_excs + + +def normalize_slice( + length: int, start: int, stop: int, step: Optional[int] = None +) -> Tuple[int, int]: + if not (step is None or step == 1): + raise ValueError(Errors.E057) + if start is None: + start = 0 + elif start < 0: + start += length + start = min(length, max(0, start)) + if stop is None: + stop = length + elif stop < 0: + stop += length + stop = min(length, max(start, stop)) + return start, stop + + +def filter_spans(spans: Iterable["Span"]) -> List["Span"]: + """Filter a sequence of spans and remove duplicates or overlaps. Useful for + creating named entities (where one token can only be part of one entity) or + when merging spans with `Retokenizer.merge`. When spans overlap, the (first) + longest span is preferred over shorter spans. + + spans (Iterable[Span]): The spans to filter. + RETURNS (List[Span]): The filtered spans. + """ + get_sort_key = lambda span: (span.end - span.start, -span.start) + sorted_spans = sorted(spans, key=get_sort_key, reverse=True) + result = [] + seen_tokens: Set[int] = set() + for span in sorted_spans: + # Check for end - 1 here because boundaries are inclusive + if span.start not in seen_tokens and span.end - 1 not in seen_tokens: + result.append(span) + seen_tokens.update(range(span.start, span.end)) + result = sorted(result, key=lambda span: span.start) + return result + + +def filter_chain_spans(*spans: Iterable["Span"]) -> List["Span"]: + return filter_spans(itertools.chain(*spans)) + + +def make_first_longest_spans_filter(): + return filter_chain_spans + + +def to_bytes(getters: Dict[str, Callable[[], bytes]], exclude: Iterable[str]) -> bytes: + return srsly.msgpack_dumps(to_dict(getters, exclude)) + + +def from_bytes( + bytes_data: bytes, + setters: Dict[str, Callable[[bytes], Any]], + exclude: Iterable[str], +) -> None: + return from_dict(srsly.msgpack_loads(bytes_data), setters, exclude) # type: ignore[return-value] + + +def to_dict( + getters: Dict[str, Callable[[], Any]], exclude: Iterable[str] +) -> Dict[str, Any]: + serialized = {} + for key, getter in getters.items(): + # Split to support file names like meta.json + if key.split(".")[0] not in exclude: + serialized[key] = getter() + return serialized + + +def from_dict( + msg: Dict[str, Any], + setters: Dict[str, Callable[[Any], Any]], + exclude: Iterable[str], +) -> Dict[str, Any]: + for key, setter in setters.items(): + # Split to support file names like meta.json + if key.split(".")[0] not in exclude and key in msg: + setter(msg[key]) + return msg + + +def to_disk( + path: Union[str, Path], + writers: Dict[str, Callable[[Path], None]], + exclude: Iterable[str], +) -> Path: + path = ensure_path(path) + if not path.exists(): + path.mkdir() + for key, writer in writers.items(): + # Split to support file names like meta.json + if key.split(".")[0] not in exclude: + writer(path / key) + return path + + +def from_disk( + path: Union[str, Path], + readers: Dict[str, Callable[[Path], None]], + exclude: Iterable[str], +) -> Path: + path = ensure_path(path) + for key, reader in readers.items(): + # Split to support file names like meta.json + if key.split(".")[0] not in exclude: + reader(path / key) + return path + + +def import_file(name: str, loc: Union[str, Path]) -> ModuleType: + """Import module from a file. Used to load models from a directory. + + name (str): Name of module to load. + loc (str / Path): Path to the file. + RETURNS: The loaded module. + """ + spec = importlib.util.spec_from_file_location(name, str(loc)) + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def minify_html(html: str) -> str: + """Perform a template-specific, rudimentary HTML minification for displaCy. + Disclaimer: NOT a general-purpose solution, only removes indentation and + newlines. + + html (str): Markup to minify. + RETURNS (str): "Minified" HTML. + """ + return html.strip().replace(" ", "").replace("\n", "") + + +def escape_html(text: str) -> str: + """Replace <, >, &, " with their HTML encoded representation. Intended to + prevent HTML errors in rendered displaCy markup. + + text (str): The original text. + RETURNS (str): Equivalent text to be safely used within HTML. + """ + text = text.replace("&", "&") + text = text.replace("<", "<") + text = text.replace(">", ">") + text = text.replace('"', """) + return text + + +def get_words_and_spaces( + words: Iterable[str], text: str +) -> Tuple[List[str], List[bool]]: + """Given a list of words and a text, reconstruct the original tokens and + return a list of words and spaces that can be used to create a Doc. This + can help recover destructive tokenization that didn't preserve any + whitespace information. + + words (Iterable[str]): The words. + text (str): The original text. + RETURNS (Tuple[List[str], List[bool]]): The words and spaces. + """ + if "".join("".join(words).split()) != "".join(text.split()): + raise ValueError(Errors.E194.format(text=text, words=words)) + text_words = [] + text_spaces = [] + text_pos = 0 + # normalize words to remove all whitespace tokens + norm_words = [word for word in words if not word.isspace()] + # align words with text + for word in norm_words: + try: + word_start = text[text_pos:].index(word) + except ValueError: + raise ValueError(Errors.E194.format(text=text, words=words)) from None + if word_start > 0: + text_words.append(text[text_pos : text_pos + word_start]) + text_spaces.append(False) + text_pos += word_start + text_words.append(word) + text_spaces.append(False) + text_pos += len(word) + if text_pos < len(text) and text[text_pos] == " ": + text_spaces[-1] = True + text_pos += 1 + if text_pos < len(text): + text_words.append(text[text_pos:]) + text_spaces.append(False) + return (text_words, text_spaces) + + +def copy_config(config: Union[Dict[str, Any], Config]) -> Config: + """Deep copy a Config. Will raise an error if the config contents are not + JSON-serializable. + + config (Config): The config to copy. + RETURNS (Config): The copied config. + """ + try: + return Config(config).copy() + except ValueError: + raise ValueError(Errors.E961.format(config=config)) from None + + +def dot_to_dict(values: Dict[str, Any]) -> Dict[str, dict]: + """Convert dot notation to a dict. For example: {"token.pos": True, + "token._.xyz": True} becomes {"token": {"pos": True, "_": {"xyz": True }}}. + + values (Dict[str, Any]): The key/value pairs to convert. + RETURNS (Dict[str, dict]): The converted values. + """ + result: Dict[str, dict] = {} + for key, value in values.items(): + path = result + parts = key.lower().split(".") + for i, item in enumerate(parts): + is_last = i == len(parts) - 1 + path = path.setdefault(item, value if is_last else {}) + return result + + +def dict_to_dot(obj: Dict[str, dict], *, for_overrides: bool = False) -> Dict[str, Any]: + """Convert dot notation to a dict. For example: {"token": {"pos": True, + "_": {"xyz": True }}} becomes {"token.pos": True, "token._.xyz": True}. + + obj (Dict[str, dict]): The dict to convert. + for_overrides (bool): Whether to enable special handling for registered + functions in overrides. + RETURNS (Dict[str, Any]): The key/value pairs. + """ + return { + ".".join(key): value + for key, value in walk_dict(obj, for_overrides=for_overrides) + } + + +def dot_to_object(config: Config, section: str): + """Convert dot notation of a "section" to a specific part of the Config. + e.g. "training.optimizer" would return the Optimizer object. + Throws an error if the section is not defined in this config. + + config (Config): The config. + section (str): The dot notation of the section in the config. + RETURNS: The object denoted by the section + """ + component = config + parts = section.split(".") + for item in parts: + try: + component = component[item] + except (KeyError, TypeError): + raise KeyError(Errors.E952.format(name=section)) from None + return component + + +def set_dot_to_object(config: Config, section: str, value: Any) -> None: + """Update a config at a given position from a dot notation. + + config (Config): The config. + section (str): The dot notation of the section in the config. + value (Any): The value to set in the config. + """ + component = config + parts = section.split(".") + for i, item in enumerate(parts): + try: + if i == len(parts) - 1: + component[item] = value + else: + component = component[item] + except (KeyError, TypeError): + raise KeyError(Errors.E952.format(name=section)) from None + + +def walk_dict( + node: Dict[str, Any], parent: List[str] = [], *, for_overrides: bool = False +) -> Iterator[Tuple[List[str], Any]]: + """Walk a dict and yield the path and values of the leaves. + + for_overrides (bool): Whether to treat registered functions that start with + @ as final values rather than dicts to traverse. + """ + for key, value in node.items(): + key_parent = [*parent, key] + if isinstance(value, dict) and ( + not for_overrides + or not any(value_key.startswith("@") for value_key in value) + ): + yield from walk_dict(value, key_parent, for_overrides=for_overrides) + else: + yield (key_parent, value) + + +def get_arg_names(func: Callable) -> List[str]: + """Get a list of all named arguments of a function (regular, + keyword-only). + + func (Callable): The function + RETURNS (List[str]): The argument names. + """ + argspec = inspect.getfullargspec(func) + return list(dict.fromkeys([*argspec.args, *argspec.kwonlyargs])) + + +def combine_score_weights( + weights: List[Dict[str, Optional[float]]], + overrides: Dict[str, Optional[float]] = SimpleFrozenDict(), +) -> Dict[str, Optional[float]]: + """Combine and normalize score weights defined by components, e.g. + {"ents_r": 0.2, "ents_p": 0.3, "ents_f": 0.5} and {"some_other_score": 1.0}. + + weights (List[dict]): The weights defined by the components. + overrides (Dict[str, Optional[Union[float, int]]]): Existing scores that + should be preserved. + RETURNS (Dict[str, float]): The combined and normalized weights. + """ + # We divide each weight by the total weight sum. + # We first need to extract all None/null values for score weights that + # shouldn't be shown in the table *or* be weighted + result: Dict[str, Optional[float]] = { + key: value for w_dict in weights for (key, value) in w_dict.items() + } + result.update(overrides) + weight_sum = sum([v if v else 0.0 for v in result.values()]) + for key, value in result.items(): + if value and weight_sum > 0: + result[key] = round(value / weight_sum, 2) + return result + + +class DummyTokenizer: + def __call__(self, text): + raise NotImplementedError + + def pipe(self, texts, **kwargs): + for text in texts: + yield self(text) + + # add dummy methods for to_bytes, from_bytes, to_disk and from_disk to + # allow serialization (see #1557) + def to_bytes(self, **kwargs): + return b"" + + def from_bytes(self, data: bytes, **kwargs) -> "DummyTokenizer": + return self + + def to_disk(self, path: Union[str, Path], **kwargs) -> None: + return None + + def from_disk(self, path: Union[str, Path], **kwargs) -> "DummyTokenizer": + return self + + +def create_default_optimizer() -> Optimizer: + return Adam() + + +def minibatch(items, size): + """Iterate over batches of items. `size` may be an iterator, + so that batch-size can vary on each step. + """ + if isinstance(size, int): + size_ = itertools.repeat(size) + else: + size_ = size + items = iter(items) + while True: + batch_size = next(size_) + batch = list(itertools.islice(items, int(batch_size))) + if len(batch) == 0: + break + yield list(batch) + + +def is_cython_func(func: Callable) -> bool: + """Slightly hacky check for whether a callable is implemented in Cython. + Can be used to implement slightly different behaviors, especially around + inspecting and parameter annotations. Note that this will only return True + for actual cdef functions and methods, not regular Python functions defined + in Python modules. + + func (Callable): The callable to check. + RETURNS (bool): Whether the callable is Cython (probably). + """ + attr = "__pyx_vtable__" + if hasattr(func, attr): # function or class instance + return True + # https://stackoverflow.com/a/55767059 + if ( + hasattr(func, "__qualname__") + and hasattr(func, "__module__") + and func.__module__ in sys.modules + ): # method + cls_func = vars(sys.modules[func.__module__])[func.__qualname__.split(".")[0]] + return hasattr(cls_func, attr) + return False + + +def check_bool_env_var(env_var: str) -> bool: + """Convert the value of an environment variable to a boolean. Add special + check for "0" (falsy) and consider everything else truthy, except unset. + + env_var (str): The name of the environment variable to check. + RETURNS (bool): Its boolean value. + """ + value = os.environ.get(env_var, False) + if value == "0": + return False + return bool(value) + + +def _pipe( + docs: Iterable["Doc"], + proc: "PipeCallable", + name: str, + default_error_handler: Callable[ + [str, "PipeCallable", List["Doc"], Exception], NoReturn + ], + kwargs: Mapping[str, Any], +) -> Iterator["Doc"]: + if hasattr(proc, "pipe"): + yield from proc.pipe(docs, **kwargs) + else: + # We added some args for pipe that __call__ doesn't expect. + kwargs = dict(kwargs) + error_handler = default_error_handler + if hasattr(proc, "get_error_handler"): + error_handler = proc.get_error_handler() + for arg in ["batch_size"]: + if arg in kwargs: + kwargs.pop(arg) + for doc in docs: + try: + doc = proc(doc, **kwargs) # type: ignore[call-arg] + yield doc + except Exception as e: + error_handler(name, proc, [doc], e) + + +def raise_error(proc_name, proc, docs, e): + raise e + + +def ignore_error(proc_name, proc, docs, e): + pass + + +def warn_if_jupyter_cupy(): + """Warn about require_gpu if a jupyter notebook + cupy + mismatched + contextvars vs. thread ops are detected + """ + if is_in_jupyter(): + from thinc.backends.cupy_ops import CupyOps + + if CupyOps.xp is not None: + from thinc.backends import contextvars_eq_thread_ops + + if not contextvars_eq_thread_ops(): + warnings.warn(Warnings.W111) + + +def check_lexeme_norms(vocab, component_name): + lexeme_norms = vocab.lookups.get_table("lexeme_norm", {}) + if len(lexeme_norms) == 0 and vocab.lang in LEXEME_NORM_LANGS: + langs = ", ".join(LEXEME_NORM_LANGS) + logger.debug(Warnings.W033.format(model=component_name, langs=langs)) + + +def to_ternary_int(val) -> int: + """Convert a value to the ternary 1/0/-1 int used for True/None/False in + attributes such as SENT_START: True/1/1.0 is 1 (True), None/0/0.0 is 0 + (None), any other values are -1 (False). + """ + if val is True: + return 1 + elif val is None: + return 0 + elif val is False: + return -1 + elif val == 1: + return 1 + elif val == 0: + return 0 + else: + return -1 + + +# The following implementation of packages_distributions() is adapted from +# importlib_metadata, which is distributed under the Apache 2.0 License. +# Copyright (c) 2017-2019 Jason R. Coombs, Barry Warsaw +# See licenses/3rd_party_licenses.txt +def packages_distributions() -> Dict[str, List[str]]: + """Return a mapping of top-level packages to their distributions. We're + inlining this helper from the importlib_metadata "backport" here, since + it's not available in the builtin importlib.metadata. + """ + pkg_to_dist = defaultdict(list) + for dist in importlib_metadata.distributions(): + for pkg in (dist.read_text("top_level.txt") or "").split(): + pkg_to_dist[pkg].append(dist.metadata["Name"]) + return dict(pkg_to_dist) + + +def all_equal(iterable): + """Return True if all the elements are equal to each other + (or if the input is an empty sequence), False otherwise.""" + g = itertools.groupby(iterable) + return next(g, True) and not next(g, False) + + +def _is_port_in_use(port: int, host: str = "localhost") -> bool: + """Check if 'host:port' is in use. Return True if it is, False otherwise. + + port (int): the port to check + host (str): the host to check (default "localhost") + RETURNS (bool): Whether 'host:port' is in use. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((host, port)) + return False + except socket.error: + return True + finally: + s.close() + + +def find_available_port(start: int, host: str, auto_select: bool = False) -> int: + """Given a starting port and a host, handle finding a port. + + If `auto_select` is False, a busy port will raise an error. + + If `auto_select` is True, the next free higher port will be used. + + start (int): the port to start looking from + host (str): the host to find a port on + auto_select (bool): whether to automatically select a new port if the given port is busy (default False) + RETURNS (int): The port to use. + """ + if not _is_port_in_use(start, host): + return start + + port = start + if not auto_select: + raise ValueError(Errors.E1050.format(port=port)) + + while _is_port_in_use(port, host) and port < 65535: + port += 1 + + if port == 65535 and _is_port_in_use(port, host): + raise ValueError(Errors.E1049.format(host=host)) + + # if we get here, the port changed + warnings.warn(Warnings.W124.format(host=host, port=start, serve_port=port)) + return port diff --git a/spacy/vectors.pyx b/spacy/vectors.pyx new file mode 100644 index 0000000..d1fb9a7 --- /dev/null +++ b/spacy/vectors.pyx @@ -0,0 +1,785 @@ +# cython: infer_types=True, binding=True +from typing import Callable + +from cython.operator cimport dereference as deref +from libc.stdint cimport uint32_t, uint64_t +from libcpp.set cimport set as cppset +from murmurhash.mrmr cimport hash128_x64 + +import warnings +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Union, cast + +import numpy +import srsly +from thinc.api import Ops, get_array_module, get_current_ops +from thinc.backends import get_array_ops +from thinc.types import Floats2d + +from .attrs cimport ORTH, attr_id_t +from .strings cimport StringStore + +from . import util +from .attrs import IDS +from .errors import Errors, Warnings +from .strings import get_string_id + +if TYPE_CHECKING: + from .vocab import Vocab # noqa: F401 # no-cython-lint + + +def unpickle_vectors(bytes_data): + return Vectors().from_bytes(bytes_data) + + +class Mode(str, Enum): + default = "default" + floret = "floret" + + @classmethod + def values(cls): + return list(cls.__members__.keys()) + + +cdef class BaseVectors: + def __init__(self, *, strings=None): + # Make sure abstract BaseVectors is not instantiated. + if self.__class__ == BaseVectors: + raise TypeError( + Errors.E1046.format(cls_name=self.__class__.__name__) + ) + + def __getitem__(self, key): + raise NotImplementedError + + def __contains__(self, key): + raise NotImplementedError + + def is_full(self): + raise NotImplementedError + + def get_batch(self, keys): + raise NotImplementedError + + @property + def shape(self): + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + @property + def vectors_length(self): + raise NotImplementedError + + @property + def size(self): + raise NotImplementedError + + def add(self, key, *, vector=None): + raise NotImplementedError + + def to_ops(self, ops: Ops): + pass + + # add dummy methods for to_bytes, from_bytes, to_disk and from_disk to + # allow serialization + def to_bytes(self, **kwargs): + return b"" + + def from_bytes(self, data: bytes, **kwargs): + return self + + def to_disk(self, path: Union[str, Path], **kwargs): + return None + + def from_disk(self, path: Union[str, Path], **kwargs): + return self + + +@util.registry.vectors("spacy.Vectors.v1") +def create_mode_vectors() -> Callable[["Vocab"], BaseVectors]: + def vectors_factory(vocab: "Vocab") -> BaseVectors: + return Vectors(strings=vocab.strings) + + return vectors_factory + + +cdef class Vectors(BaseVectors): + """Store, save and load word vectors. + + Vectors data is kept in the vectors.data attribute, which should be an + instance of numpy.ndarray (for CPU vectors) or cupy.ndarray + (for GPU vectors). + + In the default mode, `vectors.key2row` is a dictionary mapping word hashes + to rows in the vectors.data table. Multiple keys can be mapped to the same + vector, and not all of the rows in the table need to be assigned - so + len(list(vectors.keys())) may be greater or smaller than vectors.shape[0]. + + In floret mode, the floret settings (minn, maxn, etc.) are used to + calculate the vector from the rows corresponding to the key's ngrams. + + DOCS: https://spacy.io/api/vectors + """ + cdef public object strings + cdef public object name + cdef readonly object mode + cdef public object data + cdef public object key2row + cdef cppset[int] _unset + cdef readonly uint32_t minn + cdef readonly uint32_t maxn + cdef readonly uint32_t hash_count + cdef readonly uint32_t hash_seed + cdef readonly unicode bow + cdef readonly unicode eow + cdef readonly attr_id_t attr + + def __init__(self, *, strings=None, shape=None, data=None, keys=None, name=None, mode=Mode.default, minn=0, maxn=0, hash_count=1, hash_seed=0, bow="<", eow=">", attr="ORTH"): + """Create a new vector store. + + strings (StringStore): The string store. + shape (tuple): Size of the table, as (# entries, # columns) + data (numpy.ndarray or cupy.ndarray): The vector data. + keys (iterable): A sequence of keys, aligned with the data. + name (str): A name to identify the vectors table. + mode (str): Vectors mode: "default" or "floret" (default: "default"). + minn (int): The floret char ngram minn (default: 0). + maxn (int): The floret char ngram maxn (default: 0). + hash_count (int): The floret hash count (1-4, default: 1). + hash_seed (int): The floret hash seed (default: 0). + bow (str): The floret BOW string (default: "<"). + eow (str): The floret EOW string (default: ">"). + attr (Union[int, str]): The token attribute for the vector keys + (default: "ORTH"). + + DOCS: https://spacy.io/api/vectors#init + """ + self.strings = strings + if self.strings is None: + self.strings = StringStore() + self.name = name + if mode not in Mode.values(): + raise ValueError( + Errors.E202.format( + name="vectors", + mode=mode, + modes=str(Mode.values()) + ) + ) + self.mode = Mode(mode).value + self.key2row = {} + self.minn = minn + self.maxn = maxn + self.hash_count = hash_count + self.hash_seed = hash_seed + self.bow = bow + self.eow = eow + if isinstance(attr, int): + self.attr = attr + else: + attr = attr.upper() + if attr == "TEXT": + attr = "ORTH" + self.attr = IDS.get(attr, ORTH) + + if self.mode == Mode.default: + if data is None: + if shape is None: + shape = (0, 0) + ops = get_current_ops() + data = ops.xp.zeros(shape, dtype="f") + self._unset = cppset[int]({i for i in range(data.shape[0])}) + else: + self._unset = cppset[int]() + self.data = data + if keys is not None: + for i, key in enumerate(keys): + self.add(key, row=i) + elif self.mode == Mode.floret: + if maxn < minn: + raise ValueError(Errors.E863) + if hash_count < 1 or hash_count >= 5: + raise ValueError(Errors.E862) + if data is None: + raise ValueError(Errors.E864) + if keys is not None: + raise ValueError(Errors.E861) + self.data = data + self._unset = cppset[int]() + + @property + def shape(self): + """Get `(rows, dims)` tuples of number of rows and number of dimensions + in the vector table. + + RETURNS (tuple): A `(rows, dims)` pair. + + DOCS: https://spacy.io/api/vectors#shape + """ + return self.data.shape + + @property + def size(self): + """The vector size i,e. rows * dims. + + RETURNS (int): The vector size. + + DOCS: https://spacy.io/api/vectors#size + """ + return self.data.size + + @property + def is_full(self): + """Whether the vectors table is full. + + RETURNS (bool): `True` if no slots are available for new keys. + + DOCS: https://spacy.io/api/vectors#is_full + """ + if self.mode == Mode.floret: + return True + return self._unset.size() == 0 + + @property + def n_keys(self): + """Get the number of keys in the table. Note that this is the number + of all keys, not just unique vectors. + + RETURNS (int): The number of keys in the table for default vectors. + For floret vectors, return -1. + + DOCS: https://spacy.io/api/vectors#n_keys + """ + if self.mode == Mode.floret: + return -1 + return len(self.key2row) + + def __reduce__(self): + return (unpickle_vectors, (self.to_bytes(),)) + + def __getitem__(self, key): + """Get a vector by key. If the key is not found, a KeyError is raised. + + key (str/int): The key to get the vector for. + RETURNS (ndarray): The vector for the key. + + DOCS: https://spacy.io/api/vectors#getitem + """ + if self.mode == Mode.default: + i = self.key2row.get(get_string_id(key), None) + if i is None: + raise KeyError(Errors.E058.format(key=key)) + else: + return self.data[i] + elif self.mode == Mode.floret: + return self.get_batch([key])[0] + raise KeyError(Errors.E058.format(key=key)) + + def __setitem__(self, key, vector): + """Set a vector for the given key. + + key (str/int): The key to set the vector for. + vector (ndarray): The vector to set. + + DOCS: https://spacy.io/api/vectors#setitem + """ + if self.mode == Mode.floret: + warnings.warn(Warnings.W115.format(method="Vectors.__setitem__")) + return + key = get_string_id(key) + i = self.key2row[key] + self.data[i] = vector + if self._unset.count(i): + self._unset.erase(self._unset.find(i)) + + def __iter__(self): + """Iterate over the keys in the table. + + YIELDS (int): A key in the table. + + DOCS: https://spacy.io/api/vectors#iter + """ + yield from self.key2row + + def __len__(self): + """Return the number of vectors in the table. + + RETURNS (int): The number of vectors in the data. + + DOCS: https://spacy.io/api/vectors#len + """ + return self.data.shape[0] + + def __contains__(self, key): + """Check whether a key has been mapped to a vector entry in the table. + + key (int): The key to check. + RETURNS (bool): Whether the key has a vector entry. + + DOCS: https://spacy.io/api/vectors#contains + """ + if self.mode == Mode.floret: + return True + else: + return key in self.key2row + + def __eq__(self, other): + # Check for equality, with faster checks first + return ( + self.shape == other.shape + and self.key2row == other.key2row + and self.to_bytes(exclude=["strings"]) == other.to_bytes(exclude=["strings"]) + ) + + def resize(self, shape, inplace=False): + """Resize the underlying vectors array. If inplace=True, the memory + is reallocated. This may cause other references to the data to become + invalid, so only use inplace=True if you're sure that's what you want. + + If the number of vectors is reduced, keys mapped to rows that have been + deleted are removed. These removed items are returned as a list of + `(key, row)` tuples. + + shape (tuple): A `(rows, dims)` tuple. + inplace (bool): Reallocate the memory. + RETURNS (list): The removed items as a list of `(key, row)` tuples. + + DOCS: https://spacy.io/api/vectors#resize + """ + if self.mode == Mode.floret: + warnings.warn(Warnings.W115.format(method="Vectors.resize")) + return -1 + xp = get_array_module(self.data) + if inplace: + if shape[1] != self.data.shape[1]: + raise ValueError(Errors.E193.format(new_dim=shape[1], curr_dim=self.data.shape[1])) + if xp == numpy: + self.data.resize(shape, refcheck=False) + else: + raise ValueError(Errors.E192) + else: + resized_array = xp.zeros(shape, dtype=self.data.dtype) + copy_shape = (min(shape[0], self.data.shape[0]), min(shape[1], self.data.shape[1])) + resized_array[:copy_shape[0], :copy_shape[1]] = self.data[:copy_shape[0], :copy_shape[1]] + self.data = resized_array + self._sync_unset() + removed_items = [] + for key, row in self.key2row.copy().items(): + if row >= shape[0]: + self.key2row.pop(key) + removed_items.append((key, row)) + return removed_items + + def keys(self): + """RETURNS (iterable): A sequence of keys in the table.""" + return self.key2row.keys() + + def values(self): + """Iterate over vectors that have been assigned to at least one key. + + Note that some vectors may be unassigned, so the number of vectors + returned may be less than the length of the vectors table. + + YIELDS (ndarray): A vector in the table. + + DOCS: https://spacy.io/api/vectors#values + """ + for row, vector in enumerate(range(self.data.shape[0])): + if not self._unset.count(row): + yield vector + + def items(self): + """Iterate over `(key, vector)` pairs. + + YIELDS (tuple): A key/vector pair. + + DOCS: https://spacy.io/api/vectors#items + """ + for key, row in self.key2row.items(): + yield key, self.data[row] + + def find(self, *, key=None, keys=None, row=None, rows=None): + """Look up one or more keys by row, or vice versa. + + key (Union[int, str]): Find the row that the given key points to. + Returns int, -1 if missing. + keys (Iterable[Union[int, str]]): Find rows that the keys point to. + Returns ndarray. + row (int): Find the first key that points to the row. + Returns int. + rows (Iterable[int]): Find the keys that point to the rows. + Returns ndarray. + RETURNS: The requested key, keys, row or rows. + """ + if self.mode == Mode.floret: + raise ValueError( + Errors.E858.format( + mode=self.mode, + alternative="Use Vectors[key] instead.", + ) + ) + if sum(arg is None for arg in (key, keys, row, rows)) != 3: + bad_kwargs = {"key": key, "keys": keys, "row": row, "rows": rows} + raise ValueError(Errors.E059.format(kwargs=bad_kwargs)) + xp = get_array_module(self.data) + if key is not None: + key = get_string_id(key) + return self.key2row.get(int(key), -1) + elif keys is not None: + keys = [get_string_id(key) for key in keys] + rows = [self.key2row.get(int(key), -1) for key in keys] + return xp.asarray(rows, dtype="i") + else: + row2key = {row: key for key, row in self.key2row.items()} + if row is not None: + return row2key[row] + else: + results = [row2key[row] for row in rows] + return xp.asarray(results, dtype="uint64") + + def _get_ngram_hashes(self, unicode s): + """Calculate up to 4 32-bit hash values with MurmurHash3_x64_128 using + the floret hash settings. + key (str): The string key. + RETURNS: A list of the integer hashes. + """ + # MurmurHash3_x64_128 returns an array of 2 uint64_t values. + cdef uint64_t[2] out + chars = s.encode("utf8") + cdef char* utf8_string = chars + hash128_x64(utf8_string, len(chars), self.hash_seed, &out) + rows = [ + out[0] & 0xffffffffu, + out[0] >> 32, + out[1] & 0xffffffffu, + out[1] >> 32, + ] + return rows[:min(self.hash_count, 4)] + + def _get_ngrams(self, unicode key): + """Get all padded ngram strings using the ngram settings. + key (str): The string key. + RETURNS: A list of the ngram strings for the padded key. + """ + key = self.bow + key + self.eow + ngrams = [key] + [ + key[start:start+ngram_size] + for ngram_size in range(self.minn, self.maxn + 1) + for start in range(0, len(key) - ngram_size + 1) + ] + return ngrams + + def get_batch(self, keys): + """Get the vectors for the provided keys efficiently as a batch. + keys (Iterable[Union[int, str]]): The keys. + RETURNS: The requested vectors from the vector table. + """ + ops = get_array_ops(self.data) + if self.mode == Mode.default: + rows = self.find(keys=keys) + vecs = self.data[rows] + elif self.mode == Mode.floret: + keys = [self.strings.as_string(key) for key in keys] + if sum(len(key) for key in keys) == 0: + return ops.xp.zeros((len(keys), self.data.shape[1])) + unique_keys = tuple(set(keys)) + row_index = {key: i for i, key in enumerate(unique_keys)} + rows = [row_index[key] for key in keys] + indices = [] + lengths = [] + for key in unique_keys: + if key == "": + ngram_rows = [] + else: + ngram_rows = [ + h % self.data.shape[0] + for ngram in self._get_ngrams(key) + for h in self._get_ngram_hashes(ngram) + ] + indices.extend(ngram_rows) + lengths.append(len(ngram_rows)) + indices = ops.asarray(indices, dtype="int32") + lengths = ops.asarray(lengths, dtype="int32") + vecs = ops.reduce_mean(cast(Floats2d, self.data[indices]), lengths) + vecs = vecs[rows] + return ops.as_contig(vecs) + + def add(self, key, *, vector=None, row=None): + """Add a key to the table. Keys can be mapped to an existing vector + by setting `row`, or a new vector can be added. + + key (int): The key to add. + vector (ndarray / None): A vector to add for the key. + row (int / None): The row number of a vector to map the key to. + RETURNS (int): The row the vector was added to. + + DOCS: https://spacy.io/api/vectors#add + """ + if self.mode == Mode.floret: + warnings.warn(Warnings.W115.format(method="Vectors.add")) + return -1 + # use int for all keys and rows in key2row for more efficient access + # and serialization + key = int(get_string_id(key)) + if row is not None: + row = int(row) + if row is None and key in self.key2row: + row = self.key2row[key] + elif row is None: + if self.is_full: + raise ValueError(Errors.E060.format(rows=self.data.shape[0], + cols=self.data.shape[1])) + row = deref(self._unset.begin()) + if row < self.data.shape[0]: + self.key2row[key] = row + else: + raise ValueError(Errors.E197.format(row=row, key=key)) + if vector is not None: + xp = get_array_module(self.data) + vector = xp.asarray(vector) + self.data[row] = vector + if self._unset.count(row): + self._unset.erase(self._unset.find(row)) + return row + + def most_similar(self, queries, *, batch_size=1024, n=1, sort=True): + """For each of the given vectors, find the n most similar entries + to it, by cosine. + + Queries are by vector. Results are returned as a `(keys, best_rows, + scores)` tuple. If `queries` is large, the calculations are performed in + chunks, to avoid consuming too much memory. You can set the `batch_size` + to control the size/space trade-off during the calculations. + + queries (ndarray): An array with one or more vectors. + batch_size (int): The batch size to use. + n (int): The number of entries to return for each query. + sort (bool): Whether to sort the n entries returned by score. + RETURNS (tuple): The most similar entries as a `(keys, best_rows, scores)` + tuple. + """ + if self.mode == Mode.floret: + raise ValueError(Errors.E858.format( + mode=self.mode, + alternative="", + )) + xp = get_array_module(self.data) + filled = sorted(list({row for row in self.key2row.values()})) + if len(filled) < n: + raise ValueError(Errors.E198.format(n=n, n_rows=len(filled))) + filled = xp.asarray(filled) + + norms = xp.linalg.norm(self.data[filled], axis=1, keepdims=True) + norms[norms == 0] = 1 + vectors = self.data[filled] / norms + + best_rows = xp.zeros((queries.shape[0], n), dtype='i') + scores = xp.zeros((queries.shape[0], n), dtype='f') + # Work in batches, to avoid memory problems. + for i in range(0, queries.shape[0], batch_size): + batch = queries[i : i+batch_size] + batch_norms = xp.linalg.norm(batch, axis=1, keepdims=True) + batch_norms[batch_norms == 0] = 1 + batch /= batch_norms + # batch e.g. (1024, 300) + # vectors e.g. (10000, 300) + # sims e.g. (1024, 10000) + sims = xp.dot(batch, vectors.T) + best_rows[i:i+batch_size] = xp.argpartition(sims, -n, axis=1)[:, -n:] + scores[i:i+batch_size] = xp.partition(sims, -n, axis=1)[:, -n:] + + if sort and n >= 2: + sorted_index = xp.arange(scores.shape[0])[:, None][i:i+batch_size], \ + xp.argsort(scores[i:i+batch_size], axis=1)[:, ::-1] + scores[i:i+batch_size] = scores[sorted_index] + best_rows[i:i+batch_size] = best_rows[sorted_index] + + for i, j in numpy.ndindex(best_rows.shape): + best_rows[i, j] = filled[best_rows[i, j]] + # Round values really close to 1 or -1 + scores = xp.around(scores, decimals=4, out=scores) + # Account for numerical error we want to return in range -1, 1 + scores = xp.clip(scores, a_min=-1, a_max=1, out=scores) + row2key = {row: key for key, row in self.key2row.items()} + + numpy_rows = get_current_ops().to_numpy(best_rows) + keys = xp.asarray( + [ + [row2key[row] for row in numpy_rows[i] if row in row2key] + for i in range(len(queries)) + ], + dtype="uint64" + ) + return (keys, best_rows, scores) + + def to_ops(self, ops: Ops): + self.data = ops.asarray(self.data) + + def _get_cfg(self): + if self.mode == Mode.default: + return { + "mode": Mode(self.mode).value, + } + elif self.mode == Mode.floret: + return { + "mode": Mode(self.mode).value, + "minn": self.minn, + "maxn": self.maxn, + "hash_count": self.hash_count, + "hash_seed": self.hash_seed, + "bow": self.bow, + "eow": self.eow, + "attr": self.attr, + } + + def _set_cfg(self, cfg): + self.mode = Mode(cfg.get("mode", Mode.default)).value + self.minn = cfg.get("minn", 0) + self.maxn = cfg.get("maxn", 0) + self.hash_count = cfg.get("hash_count", 0) + self.hash_seed = cfg.get("hash_seed", 0) + self.bow = cfg.get("bow", "<") + self.eow = cfg.get("eow", ">") + self.attr = cfg.get("attr", ORTH) + + def to_disk(self, path, *, exclude=tuple()): + """Save the current state to a directory. + + path (str / Path): A path to a directory, which will be created if + it doesn't exists. + + DOCS: https://spacy.io/api/vectors#to_disk + """ + xp = get_array_module(self.data) + if xp is numpy: + save_array = lambda arr, file_: xp.save(file_, arr, allow_pickle=False) # no-cython-lint + else: + save_array = lambda arr, file_: xp.save(file_, arr) # no-cython-lint + + def save_vectors(path): + # the source of numpy.save indicates that the file object is closed after use. + # but it seems that somehow this does not happen, as ResourceWarnings are raised here. + # in order to not rely on this, wrap in context manager. + ops = get_current_ops() + with path.open("wb") as _file: + save_array(ops.to_numpy(self.data, byte_order="<"), _file) + + serializers = { + "strings": lambda p: self.strings.to_disk(p.with_suffix(".json")), + "vectors": lambda p: save_vectors(p), + "key2row": lambda p: srsly.write_msgpack(p, self.key2row), + "vectors.cfg": lambda p: srsly.write_json(p, self._get_cfg()), + } + return util.to_disk(path, serializers, exclude) + + def from_disk(self, path, *, exclude=tuple()): + """Loads state from a directory. Modifies the object in place and + returns it. + + path (str / Path): Directory path, string or Path-like object. + RETURNS (Vectors): The modified object. + + DOCS: https://spacy.io/api/vectors#from_disk + """ + def load_key2row(path): + if path.exists(): + self.key2row = srsly.read_msgpack(path) + for key, row in self.key2row.items(): + if self._unset.count(row): + self._unset.erase(self._unset.find(row)) + + def load_keys(path): + if path.exists(): + keys = numpy.load(str(path)) + for i, key in enumerate(keys): + self.add(key, row=i) + + def load_vectors(path): + ops = get_current_ops() + if path.exists(): + self.data = ops.xp.load(str(path)) + self.to_ops(ops) + + def load_settings(path): + if path.exists(): + self._set_cfg(srsly.read_json(path)) + + serializers = { + "strings": lambda p: self.strings.from_disk(p.with_suffix(".json")), + "vectors": load_vectors, + "keys": load_keys, + "key2row": load_key2row, + "vectors.cfg": load_settings, + } + + util.from_disk(path, serializers, exclude) + self._sync_unset() + return self + + def to_bytes(self, *, exclude=tuple()): + """Serialize the current state to a binary string. + + exclude (list): String names of serialization fields to exclude. + RETURNS (bytes): The serialized form of the `Vectors` object. + + DOCS: https://spacy.io/api/vectors#to_bytes + """ + def serialize_weights(): + if hasattr(self.data, "to_bytes"): + return self.data.to_bytes() + else: + ops = get_current_ops() + return srsly.msgpack_dumps(ops.to_numpy(self.data, byte_order="<")) + + serializers = { + "strings": lambda: self.strings.to_bytes(), + "key2row": lambda: srsly.msgpack_dumps(self.key2row), + "vectors": serialize_weights, + "vectors.cfg": lambda: srsly.json_dumps(self._get_cfg()), + } + return util.to_bytes(serializers, exclude) + + def from_bytes(self, data, *, exclude=tuple()): + """Load state from a binary string. + + data (bytes): The data to load from. + exclude (list): String names of serialization fields to exclude. + RETURNS (Vectors): The `Vectors` object. + + DOCS: https://spacy.io/api/vectors#from_bytes + """ + def deserialize_weights(b): + if hasattr(self.data, "from_bytes"): + self.data.from_bytes() + else: + xp = get_array_module(self.data) + self.data = xp.asarray(srsly.msgpack_loads(b)) + ops = get_current_ops() + self.to_ops(ops) + + deserializers = { + "strings": lambda b: self.strings.from_bytes(b), + "key2row": lambda b: self.key2row.update(srsly.msgpack_loads(b)), + "vectors": deserialize_weights, + "vectors.cfg": lambda b: self._set_cfg(srsly.json_loads(b)) + } + util.from_bytes(data, deserializers, exclude) + self._sync_unset() + return self + + def clear(self): + """Clear all entries in the vector table. + + DOCS: https://spacy.io/api/vectors#clear + """ + if self.mode == Mode.floret: + raise ValueError(Errors.E859) + self.key2row = {} + self._sync_unset() + + def _sync_unset(self): + filled = {row for row in self.key2row.values()} + self._unset = cppset[int]({row for row in range(self.data.shape[0]) if row not in filled}) diff --git a/spacy/vocab.pxd b/spacy/vocab.pxd new file mode 100644 index 0000000..c2bfe12 --- /dev/null +++ b/spacy/vocab.pxd @@ -0,0 +1,49 @@ +from cymem.cymem cimport Pool +from libcpp.vector cimport vector +from murmurhash.mrmr cimport hash64 +from preshed.maps cimport PreshMap + +from .morphology cimport Morphology +from .strings cimport StringStore +from .structs cimport LexemeC, TokenC +from .typedefs cimport attr_t, hash_t + + +cdef LexemeC EMPTY_LEXEME + + +cdef union LexemesOrTokens: + const LexemeC* const* lexemes + const TokenC* tokens + + +cdef struct _Cached: + LexemesOrTokens data + bint is_lex + int length + + +cdef class Vocab: + cdef Pool mem + cdef readonly StringStore strings + cdef public Morphology morphology + cdef public object _vectors + cdef public object _lookups + cdef public object writing_system + cdef public object get_noun_chunks + cdef readonly int length + cdef public object _unused_object # TODO remove in v4, see #9150 + cdef public object lex_attr_getters + cdef public object cfg + + cdef const LexemeC* get(self, Pool mem, str string) except NULL + cdef const LexemeC* get_by_orth(self, Pool mem, attr_t orth) except NULL + cdef const TokenC* make_fused_token(self, substrings) except NULL + + cdef const LexemeC* _new_lexeme(self, Pool mem, str string) except NULL + cdef int _add_lex_to_vocab(self, hash_t key, const LexemeC* lex, bint is_transient) except -1 + cdef const LexemeC* _new_lexeme(self, Pool mem, str string) except NULL + + cdef PreshMap _by_orth + cdef Pool _non_temp_mem + cdef vector[attr_t] _transient_orths diff --git a/spacy/vocab.pyi b/spacy/vocab.pyi new file mode 100644 index 0000000..906a4c0 --- /dev/null +++ b/spacy/vocab.pyi @@ -0,0 +1,83 @@ +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Union + +from cymem.cymem import Pool +from thinc.types import Floats1d, FloatsXd + +from .lexeme import Lexeme +from .lookups import Lookups +from .morphology import Morphology +from .strings import StringStore +from .tokens import Doc, Span +from .vectors import Vectors + +def create_vocab( + lang: Optional[str], defaults: Any, vectors_name: Optional[str] = ... +) -> Vocab: ... + +class Vocab: + cfg: Dict[str, Any] + get_noun_chunks: Optional[Callable[[Union[Doc, Span]], Iterator[Span]]] + lookups: Lookups + morphology: Morphology + strings: StringStore + vectors: Vectors + writing_system: Dict[str, Any] + def __init__( + self, + lex_attr_getters: Optional[Dict[str, Callable[[str], Any]]] = ..., + strings: Optional[Union[List[str], StringStore]] = ..., + lookups: Optional[Lookups] = ..., + oov_prob: float = ..., + vectors_name: Optional[str] = ..., + writing_system: Dict[str, Any] = ..., + get_noun_chunks: Optional[Callable[[Union[Doc, Span]], Iterator[Span]]] = ..., + ) -> None: ... + @property + def lang(self) -> str: ... + def __len__(self) -> int: ... + def add_flag( + self, flag_getter: Callable[[str], bool], flag_id: int = ... + ) -> int: ... + def __contains__(self, key: str) -> bool: ... + def __iter__(self) -> Iterator[Lexeme]: ... + def __getitem__(self, id_or_string: Union[str, int]) -> Lexeme: ... + @property + def vectors_length(self) -> int: ... + def reset_vectors( + self, *, width: Optional[int] = ..., shape: Optional[int] = ... + ) -> None: ... + def deduplicate_vectors(self) -> None: ... + def prune_vectors(self, nr_row: int, batch_size: int = ...) -> Dict[str, float]: ... + def get_vector( + self, + orth: Union[int, str], + minn: Optional[int] = ..., + maxn: Optional[int] = ..., + ) -> FloatsXd: ... + def set_vector(self, orth: Union[int, str], vector: Floats1d) -> None: ... + def has_vector(self, orth: Union[int, str]) -> bool: ... + def to_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = ... + ) -> None: ... + def from_disk( + self, path: Union[str, Path], *, exclude: Iterable[str] = ... + ) -> Vocab: ... + def to_bytes(self, *, exclude: Iterable[str] = ...) -> bytes: ... + def from_bytes( + self, bytes_data: bytes, *, exclude: Iterable[str] = ... + ) -> Vocab: ... + @contextmanager + def memory_zone(self, mem: Optional[Pool] = None) -> Iterator[Pool]: ... + +def pickle_vocab(vocab: Vocab) -> Any: ... +def unpickle_vocab( + sstore: StringStore, + vectors: Any, + morphology: Any, + _unused_object: Any, + lex_attr_getters: Any, + lookups: Any, + get_noun_chunks: Any, +) -> Vocab: ... diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx new file mode 100644 index 0000000..4bf80c8 --- /dev/null +++ b/spacy/vocab.pyx @@ -0,0 +1,646 @@ +import functools +from contextlib import ExitStack, contextmanager +from typing import Iterator, Optional + +import numpy +import srsly +from thinc.api import get_array_module, get_current_ops +from preshed.maps cimport map_clear + +from .attrs cimport LANG, ORTH +from .lexeme cimport EMPTY_LEXEME, OOV_RANK, Lexeme +from .tokens.token cimport Token +from .typedefs cimport attr_t + +from . import util +from .attrs import IS_STOP, NORM, intify_attrs +from .compat import copy_reg +from .errors import Errors +from .lang.lex_attrs import LEX_ATTRS, get_lang, is_stop +from .lang.norm_exceptions import BASE_NORMS +from .lookups import Lookups +from .vectors import Mode as VectorsMode +from .vectors import Vectors + + +def create_vocab(lang, defaults, vectors_name=None): + # If the spacy-lookups-data package is installed, we pre-populate the lookups + # with lexeme data, if available + lex_attrs = {**LEX_ATTRS, **defaults.lex_attr_getters} + # This is messy, but it's the minimal working fix to Issue #639. + lex_attrs[IS_STOP] = functools.partial(is_stop, stops=defaults.stop_words) + # Ensure that getter can be pickled + lex_attrs[LANG] = functools.partial(get_lang, lang=lang) + lex_attrs[NORM] = util.add_lookups( + lex_attrs.get(NORM, LEX_ATTRS[NORM]), + BASE_NORMS, + ) + return Vocab( + lex_attr_getters=lex_attrs, + writing_system=defaults.writing_system, + get_noun_chunks=defaults.syntax_iterators.get("noun_chunks"), + vectors_name=vectors_name, + ) + + +cdef class Vocab: + """A look-up table that allows you to access `Lexeme` objects. The `Vocab` + instance also provides access to the `StringStore`, and owns underlying + C-data that is shared between `Doc` objects. + + DOCS: https://spacy.io/api/vocab + """ + def __init__( + self, + lex_attr_getters=None, + strings=tuple(), + lookups=None, + oov_prob=-20., + vectors_name=None, + writing_system={}, # no-cython-lint + get_noun_chunks=None, + **deprecated_kwargs + ): + """Create the vocabulary. + + lex_attr_getters (dict): A dictionary mapping attribute IDs to + functions to compute them. Defaults to `None`. + strings (StringStore): StringStore that maps strings to integers, and + vice versa. + lookups (Lookups): Container for large lookup tables and dictionaries. + oov_prob (float): Default OOV probability. + vectors_name (str): Optional name to identify the vectors table. + get_noun_chunks (Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]): + A function that yields base noun phrases used for Doc.noun_chunks. + """ + lex_attr_getters = lex_attr_getters if lex_attr_getters is not None else {} + if lookups in (None, True, False): + lookups = Lookups() + self.cfg = {'oov_prob': oov_prob} + self.mem = Pool() + self._by_orth = PreshMap() + self.strings = StringStore() + self.length = 0 + if strings: + for string in strings: + _ = self[string] + self.lex_attr_getters = lex_attr_getters + self.morphology = Morphology(self.strings) + self.vectors = Vectors(strings=self.strings, name=vectors_name) + self.lookups = lookups + self.writing_system = writing_system + self.get_noun_chunks = get_noun_chunks + # During a memory_zone we replace our mem object with one + # that's passed to us. We keep a reference to our non-temporary + # memory here, in case we need to make an allocation we want to + # guarantee is not temporary. This is also how we check whether + # we're in a memory zone: we check whether self.mem is self._non_temp_mem + self._non_temp_mem = self.mem + + @property + def vectors(self): + return self._vectors + + @vectors.setter + def vectors(self, vectors): + if hasattr(vectors, "strings"): + for s in vectors.strings: + self.strings.add(s, allow_transient=False) + self._vectors = vectors + self._vectors.strings = self.strings + + @property + def lang(self): + langfunc = None + if self.lex_attr_getters: + langfunc = self.lex_attr_getters.get(LANG, None) + return langfunc("_") if langfunc else "" + + @property + def in_memory_zone(self) -> bool: + return self.mem is not self._non_temp_mem + + def __len__(self): + """The current number of lexemes stored. + + RETURNS (int): The current number of lexemes stored. + """ + return self.length + + @contextmanager + def memory_zone(self, mem: Optional[Pool] = None) -> Iterator[Pool]: + """Begin a block where resources allocated during the block will + be freed at the end of it. If a resources was created within the + memory zone block, accessing it outside the block is invalid. + Behaviour of this invalid access is undefined. Memory zones should + not be nested. + + The memory zone is helpful for services that need to process large + volumes of text with a defined memory budget. + """ + if mem is None: + mem = Pool() + # The ExitStack allows programmatic nested context managers. + # We don't know how many we need, so it would be awkward to have + # them as nested blocks. + with ExitStack() as stack: + contexts = [stack.enter_context(self.strings.memory_zone(mem))] + if hasattr(self.morphology, "memory_zone"): + contexts.append(stack.enter_context(self.morphology.memory_zone(mem))) + if hasattr(self._vectors, "memory_zone"): + contexts.append(stack.enter_context(self._vectors.memory_zone(mem))) + self.mem = mem + try: + yield mem + finally: + self._clear_transient_orths() + self.mem = self._non_temp_mem + + def add_flag(self, flag_getter, int flag_id=-1): + """Set a new boolean flag to words in the vocabulary. + + The flag_getter function will be called over the words currently in the + vocab, and then applied to new words as they occur. You'll then be able + to access the flag value on each token using token.check_flag(flag_id). + See also: `Lexeme.set_flag`, `Lexeme.check_flag`, `Token.set_flag`, + `Token.check_flag`. + + flag_getter (callable): A function `f(str) -> bool`, to get the + flag value. + flag_id (int): An integer between 1 and 63 (inclusive), specifying + the bit at which the flag will be stored. If -1, the lowest + available bit will be chosen. + RETURNS (int): The integer ID by which the flag value can be checked. + + DOCS: https://spacy.io/api/vocab#add_flag + """ + if flag_id == -1: + for bit in range(1, 64): + if bit not in self.lex_attr_getters: + flag_id = bit + break + else: + raise ValueError(Errors.E062) + elif flag_id >= 64 or flag_id < 1: + raise ValueError(Errors.E063.format(value=flag_id)) + for lex in self: + lex.set_flag(flag_id, flag_getter(lex.orth_)) + self.lex_attr_getters[flag_id] = flag_getter + return flag_id + + cdef const LexemeC* get(self, Pool mem, str string) except NULL: + """Get a pointer to a `LexemeC` from the lexicon, creating a new + `Lexeme` if necessary. + """ + if string == "": + return &EMPTY_LEXEME + cdef LexemeC* lex + cdef hash_t key = self.strings[string] + lex = self._by_orth.get(key) + if lex != NULL: + assert lex.orth in self.strings + if lex.orth != key: + raise KeyError(Errors.E064.format(string=lex.orth, + orth=key, orth_id=string)) + return lex + else: + return self._new_lexeme(mem, string) + + cdef const LexemeC* get_by_orth(self, Pool mem, attr_t orth) except NULL: + """Get a pointer to a `LexemeC` from the lexicon, creating a new + `Lexeme` if necessary using memory acquired from the given pool. If the + pool is the lexicon's own memory, the lexeme is saved in the lexicon. + """ + if orth == 0: + return &EMPTY_LEXEME + cdef LexemeC* lex + lex = self._by_orth.get(orth) + if lex != NULL: + return lex + else: + return self._new_lexeme(mem, self.strings[orth]) + + cdef const LexemeC* _new_lexeme(self, Pool mem, str string) except NULL: + # The mem argument is deprecated, replaced by memory zones. Same with + # this size heuristic. + mem = self.mem + lex = mem.alloc(1, sizeof(LexemeC)) + lex.orth = self.strings.add(string, allow_transient=True) + lex.length = len(string) + if self.vectors is not None and hasattr(self.vectors, "key2row"): + lex.id = self.vectors.key2row.get(lex.orth, OOV_RANK) + else: + lex.id = OOV_RANK + if self.lex_attr_getters is not None: + for attr, func in self.lex_attr_getters.items(): + value = func(string) + if isinstance(value, str): + value = self.strings.add(value, allow_transient=True) + if value is not None: + Lexeme.set_struct_attr(lex, attr, value) + self._add_lex_to_vocab(lex.orth, lex, self.mem is not self._non_temp_mem) + if lex == NULL: + raise ValueError(Errors.E085.format(string=string)) + return lex + + cdef int _add_lex_to_vocab(self, hash_t key, const LexemeC* lex, bint is_transient) except -1: + self._by_orth.set(lex.orth, lex) + self.length += 1 + if is_transient and self.in_memory_zone: + self._transient_orths.push_back(lex.orth) + + def _clear_transient_orths(self): + """Remove transient lexemes from the index (generally at the end of the memory zone)""" + for orth in self._transient_orths: + map_clear(self._by_orth.c_map, orth) + self._transient_orths.clear() + + def __contains__(self, key): + """Check whether the string or int key has an entry in the vocabulary. + + string (str): The ID string. + RETURNS (bool) Whether the string has an entry in the vocabulary. + + DOCS: https://spacy.io/api/vocab#contains + """ + cdef hash_t int_key + if isinstance(key, bytes): + int_key = self.strings[key.decode("utf8")] + elif isinstance(key, str): + int_key = self.strings[key] + else: + int_key = key + lex = self._by_orth.get(int_key) + return lex is not NULL + + def __iter__(self): + """Iterate over the lexemes in the vocabulary. + + YIELDS (Lexeme): An entry in the vocabulary. + + DOCS: https://spacy.io/api/vocab#iter + """ + cdef attr_t key + cdef size_t addr + for key, addr in self._by_orth.items(): + lex = Lexeme(self, key) + yield lex + + def __getitem__(self, id_or_string): + """Retrieve a lexeme, given an int ID or a unicode string. If a + previously unseen unicode string is given, a new lexeme is created and + stored. + + id_or_string (int or str): The integer ID of a word, or its unicode + string. If `int >= Lexicon.size`, `IndexError` is raised. If + `id_or_string` is neither an int nor a unicode string, `ValueError` + is raised. + RETURNS (Lexeme): The lexeme indicated by the given ID. + + EXAMPLE: + >>> apple = nlp.vocab.strings["apple"] + >>> assert nlp.vocab[apple] == nlp.vocab[u"apple"] + + DOCS: https://spacy.io/api/vocab#getitem + """ + cdef attr_t orth + if isinstance(id_or_string, str): + orth = self.strings.add(id_or_string, allow_transient=True) + else: + orth = id_or_string + return Lexeme(self, orth) + + cdef const TokenC* make_fused_token(self, substrings) except NULL: + cdef int i + tokens = self.mem.alloc(len(substrings) + 1, sizeof(TokenC)) + for i, props in enumerate(substrings): + props = intify_attrs(props, strings_map=self.strings, + _do_deprecated=True) + token = &tokens[i] + # Set the special tokens up to have arbitrary attributes + lex = self.get_by_orth(self.mem, props[ORTH]) + token.lex = lex + for attr_id, value in props.items(): + Token.set_struct_attr(token, attr_id, value) + # NORM is the only one that overlaps between the two + # (which is maybe not great?) + if attr_id != NORM: + Lexeme.set_struct_attr(lex, attr_id, value) + return tokens + + @property + def vectors_length(self): + if hasattr(self.vectors, "shape"): + return self.vectors.shape[1] + else: + return -1 + + def reset_vectors(self, *, width=None, shape=None): + """Drop the current vector table. Because all vectors must be the same + width, you have to call this to change the size of the vectors. + """ + if not isinstance(self.vectors, Vectors): + raise ValueError(Errors.E849.format(method="reset_vectors", vectors_type=type(self.vectors))) + if width is not None and shape is not None: + raise ValueError(Errors.E065.format(width=width, shape=shape)) + elif shape is not None: + self.vectors = Vectors(strings=self.strings, shape=shape) + else: + width = width if width is not None else self.vectors.shape[1] + self.vectors = Vectors(strings=self.strings, shape=(self.vectors.shape[0], width)) + + def deduplicate_vectors(self): + if not isinstance(self.vectors, Vectors): + raise ValueError(Errors.E849.format(method="deduplicate_vectors", vectors_type=type(self.vectors))) + if self.vectors.mode != VectorsMode.default: + raise ValueError(Errors.E858.format( + mode=self.vectors.mode, + alternative="" + )) + ops = get_current_ops() + xp = get_array_module(self.vectors.data) + filled = xp.asarray( + sorted(list({row for row in self.vectors.key2row.values()})) + ) + # deduplicate data and remap keys + data = numpy.unique(ops.to_numpy(self.vectors.data[filled]), axis=0) + data = ops.asarray(data) + if data.shape == self.vectors.data.shape: + # nothing to deduplicate + return + row_by_bytes = {row.tobytes(): i for i, row in enumerate(data)} + key2row = { + key: row_by_bytes[self.vectors.data[row].tobytes()] + for key, row in self.vectors.key2row.items() + } + # replace vectors with deduplicated version + self.vectors = Vectors(strings=self.strings, data=data, name=self.vectors.name) + for key, row in key2row.items(): + self.vectors.add(key, row=row) + + def prune_vectors(self, nr_row, batch_size=1024): + """Reduce the current vector table to `nr_row` unique entries. Words + mapped to the discarded vectors will be remapped to the closest vector + among those remaining. + + For example, suppose the original table had vectors for the words: + ['sat', 'cat', 'feline', 'reclined']. If we prune the vector table to + two rows, we would discard the vectors for 'feline' and 'reclined'. + These words would then be remapped to the closest remaining vector + -- so "feline" would have the same vector as "cat", and "reclined" + would have the same vector as "sat". + + The similarities are judged by cosine. The original vectors may + be large, so the cosines are calculated in minibatches, to reduce + memory usage. + + nr_row (int): The number of rows to keep in the vector table. + batch_size (int): Batch of vectors for calculating the similarities. + Larger batch sizes might be faster, while temporarily requiring + more memory. + RETURNS (dict): A dictionary keyed by removed words mapped to + `(string, score)` tuples, where `string` is the entry the removed + word was mapped to, and `score` the similarity score between the + two words. + + DOCS: https://spacy.io/api/vocab#prune_vectors + """ + if not isinstance(self.vectors, Vectors): + raise ValueError(Errors.E849.format(method="prune_vectors", vectors_type=type(self.vectors))) + if self.vectors.mode != VectorsMode.default: + raise ValueError(Errors.E858.format( + mode=self.vectors.mode, + alternative="" + )) + ops = get_current_ops() + xp = get_array_module(self.vectors.data) + # Make sure all vectors are in the vocab + for orth in self.vectors: + self[orth] + # Make prob negative so it sorts by rank ascending + # (key2row contains the rank) + priority = [] + cdef Lexeme lex + cdef attr_t value + for lex in self: + value = Lexeme.get_struct_attr(lex.c, self.vectors.attr) + if value in self.vectors.key2row: + priority.append((-lex.prob, self.vectors.key2row[value], value)) + priority.sort() + indices = xp.asarray([i for (prob, i, key) in priority], dtype="uint64") + keys = xp.asarray([key for (prob, i, key) in priority], dtype="uint64") + keep = xp.ascontiguousarray(self.vectors.data[indices[:nr_row]]) + toss = xp.ascontiguousarray(self.vectors.data[indices[nr_row:]]) + self.vectors = Vectors(strings=self.strings, data=keep, keys=keys[:nr_row], name=self.vectors.name) + syn_keys, syn_rows, scores = self.vectors.most_similar(toss, batch_size=batch_size) + syn_keys = ops.to_numpy(syn_keys) + remap = {} + for i, key in enumerate(ops.to_numpy(keys[nr_row:])): + self.vectors.add(key, row=syn_rows[i][0]) + word = self.strings[key] + synonym = self.strings[syn_keys[i][0]] + score = scores[i][0] + remap[word] = (synonym, score) + return remap + + def get_vector(self, orth): + """Retrieve a vector for a word in the vocabulary. Words can be looked + up by string or int ID. If the current vectors do not contain an entry + for the word, a 0-vector with the same number of dimensions as the + current vectors is returned. + + orth (int / unicode): The hash value of a word, or its unicode string. + RETURNS (numpy.ndarray or cupy.ndarray): A word vector. Size + and shape determined by the `vocab.vectors` instance. Usually, a + numpy ndarray of shape (300,) and dtype float32. + + DOCS: https://spacy.io/api/vocab#get_vector + """ + if isinstance(orth, str): + orth = self.strings.add(orth, allow_transient=True) + cdef Lexeme lex = self[orth] + key = Lexeme.get_struct_attr(lex.c, self.vectors.attr) + if self.has_vector(key): + return self.vectors[key] + xp = get_array_module(self.vectors.data) + vectors = xp.zeros((self.vectors_length,), dtype="f") + return vectors + + def set_vector(self, orth, vector): + """Set a vector for a word in the vocabulary. Words can be referenced + by string or int ID. + + orth (int / str): The word. + vector (numpy.ndarray or cupy.nadarry[ndim=1, dtype='float32']): The vector to set. + + DOCS: https://spacy.io/api/vocab#set_vector + """ + if isinstance(orth, str): + orth = self.strings.add(orth, allow_transient=False) + cdef Lexeme lex = self[orth] + key = Lexeme.get_struct_attr(lex.c, self.vectors.attr) + if self.vectors.is_full and key not in self.vectors: + new_rows = max(100, int(self.vectors.shape[0]*1.3)) + if self.vectors.shape[1] == 0: + width = vector.size + else: + width = self.vectors.shape[1] + self.vectors.resize((new_rows, width)) + row = self.vectors.add(key, vector=vector) + if row >= 0: + lex.rank = row + + def has_vector(self, orth): + """Check whether a word has a vector. Returns False if no vectors have + been loaded. Words can be looked up by string or int ID. + + orth (int / str): The word. + RETURNS (bool): Whether the word has a vector. + + DOCS: https://spacy.io/api/vocab#has_vector + """ + if isinstance(orth, str): + orth = self.strings.add(orth, allow_transient=True) + cdef Lexeme lex = self[orth] + key = Lexeme.get_struct_attr(lex.c, self.vectors.attr) + return key in self.vectors + + @property + def lookups(self): + return self._lookups + + @lookups.setter + def lookups(self, lookups): + self._lookups = lookups + if lookups.has_table("lexeme_norm"): + self.lex_attr_getters[NORM] = util.add_lookups( + self.lex_attr_getters.get(NORM, LEX_ATTRS[NORM]), + self.lookups.get_table("lexeme_norm"), + ) + + def to_disk(self, path, *, exclude=tuple()): + """Save the current state to a directory. + + path (str or Path): A path to a directory, which will be created if + it doesn't exist. + exclude (Iterable[str]): String names of serialization fields to exclude. + + DOCS: https://spacy.io/api/vocab#to_disk + """ + path = util.ensure_path(path) + if not path.exists(): + path.mkdir() + if "strings" not in exclude: + self.strings.to_disk(path / "strings.json") + if "vectors" not in exclude: + self.vectors.to_disk(path, exclude=["strings"]) + if "lookups" not in exclude: + self.lookups.to_disk(path) + + def from_disk(self, path, *, exclude=tuple()): + """Loads state from a directory. Modifies the object in place and + returns it. + + path (str or Path): A path to a directory. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Vocab): The modified `Vocab` object. + + DOCS: https://spacy.io/api/vocab#to_disk + """ + path = util.ensure_path(path) + if "strings" not in exclude: + self.strings.from_disk(path / "strings.json") # TODO: add exclude? + if "vectors" not in exclude: + if self.vectors is not None: + self.vectors.from_disk(path, exclude=["strings"]) + if "lookups" not in exclude: + self.lookups.from_disk(path) + if "lexeme_norm" in self.lookups: + self.lex_attr_getters[NORM] = util.add_lookups( + self.lex_attr_getters.get(NORM, LEX_ATTRS[NORM]), self.lookups.get_table("lexeme_norm") + ) + self.length = 0 + self._by_orth = PreshMap() + return self + + def to_bytes(self, *, exclude=tuple()): + """Serialize the current state to a binary string. + + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (bytes): The serialized form of the `Vocab` object. + + DOCS: https://spacy.io/api/vocab#to_bytes + """ + def deserialize_vectors(): + if self.vectors is None: + return None + else: + return self.vectors.to_bytes(exclude=["strings"]) + + getters = { + "strings": lambda: self.strings.to_bytes(), + "vectors": deserialize_vectors, + "lookups": lambda: self.lookups.to_bytes(), + } + return util.to_bytes(getters, exclude) + + def from_bytes(self, bytes_data, *, exclude=tuple()): + """Load state from a binary string. + + bytes_data (bytes): The data to load from. + exclude (Iterable[str]): String names of serialization fields to exclude. + RETURNS (Vocab): The `Vocab` object. + + DOCS: https://spacy.io/api/vocab#from_bytes + """ + def serialize_vectors(b): + if self.vectors is None: + return None + else: + return self.vectors.from_bytes(b, exclude=["strings"]) + + setters = { + "strings": lambda b: self.strings.from_bytes(b), + "vectors": lambda b: serialize_vectors(b), + "lookups": lambda b: self.lookups.from_bytes(b), + } + util.from_bytes(bytes_data, setters, exclude) + if "lexeme_norm" in self.lookups: + self.lex_attr_getters[NORM] = util.add_lookups( + self.lex_attr_getters.get(NORM, LEX_ATTRS[NORM]), self.lookups.get_table("lexeme_norm") + ) + self.length = 0 + self._by_orth = PreshMap() + return self + + def _reset_cache(self, keys, strings): + # I'm not sure this made sense. Disable it for now. + raise NotImplementedError + + +def pickle_vocab(vocab): + sstore = vocab.strings + vectors = vocab.vectors + morph = vocab.morphology + _unused_object = vocab._unused_object + lex_attr_getters = srsly.pickle_dumps(vocab.lex_attr_getters) + lookups = vocab.lookups + get_noun_chunks = vocab.get_noun_chunks + return (unpickle_vocab, + (sstore, vectors, morph, _unused_object, lex_attr_getters, lookups, get_noun_chunks)) + + +def unpickle_vocab(sstore, vectors, morphology, _unused_object, + lex_attr_getters, lookups, get_noun_chunks): + cdef Vocab vocab = Vocab() + vocab.vectors = vectors + vocab.strings = sstore + vocab.morphology = morphology + vocab._unused_object = _unused_object + vocab.lex_attr_getters = srsly.pickle_loads(lex_attr_getters) + vocab.lookups = lookups + vocab.get_noun_chunks = get_noun_chunks + return vocab + + +copy_reg.pickle(Vocab, pickle_vocab, unpickle_vocab) diff --git a/website/.dockerignore b/website/.dockerignore new file mode 100644 index 0000000..e4a8855 --- /dev/null +++ b/website/.dockerignore @@ -0,0 +1,9 @@ +.cache/ +.next/ +public/ +node_modules +.npm +logs +*.log +npm-debug.log* +quickstart-training-generator.js diff --git a/website/.eslintrc b/website/.eslintrc new file mode 100644 index 0000000..98d2c16 --- /dev/null +++ b/website/.eslintrc @@ -0,0 +1,27 @@ +{ + "extends": ["standard", "prettier"], + "plugins": ["standard", "react", "react-hooks"], + "rules": { + "no-var": "error", + "no-unused-vars": 1, + "arrow-spacing": ["error", { "before": true, "after": true }], + "indent": ["error", 4], + "semi": ["error", "never"], + "arrow-parens": ["error", "as-needed"], + "standard/object-curly-even-spacing": ["error", "either"], + "standard/array-bracket-even-spacing": ["error", "either"], + "standard/computed-property-even-spacing": ["error", "even"], + "standard/no-callback-literal": ["error", ["cb", "callback"]], + "react/jsx-uses-react": "error", + "react/jsx-uses-vars": "error", + "react-hooks/rules-of-hooks": "error", + "react-hooks/exhaustive-deps": "warn" + }, + "parser": "babel-eslint", + "parserOptions": { + "ecmaVersion": 8 + }, + "env": { + "browser": true + } +} diff --git a/website/.eslintrc.json b/website/.eslintrc.json new file mode 100644 index 0000000..1c2aa65 --- /dev/null +++ b/website/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/website/.gitignore b/website/.gitignore new file mode 100644 index 0000000..599c095 --- /dev/null +++ b/website/.gitignore @@ -0,0 +1,46 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +quickstart-training-generator.js + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts + +!.vscode/extensions.json +!public + +public/robots.txt +public/sitemap* +public/sw.js* +public/workbox* diff --git a/website/.nvmrc b/website/.nvmrc new file mode 100644 index 0000000..3c03207 --- /dev/null +++ b/website/.nvmrc @@ -0,0 +1 @@ +18 diff --git a/website/.prettierignore b/website/.prettierignore new file mode 100644 index 0000000..d0d878e --- /dev/null +++ b/website/.prettierignore @@ -0,0 +1 @@ +.next \ No newline at end of file diff --git a/website/.prettierrc b/website/.prettierrc new file mode 100644 index 0000000..03904b1 --- /dev/null +++ b/website/.prettierrc @@ -0,0 +1,37 @@ +{ + "semi": false, + "singleQuote": true, + "trailingComma": "es5", + "tabWidth": 4, + "printWidth": 100, + "overrides": [ + { + "files": "*.sass", + "options": { + "printWidth": 999 + } + }, + { + "files": "*.mdx", + "options": { + "tabWidth": 2, + "printWidth": 80, + "proseWrap": "always" + } + }, + { + "files": ["package.json", "package-lock.json"], + "options": { + "tabWidth": 2, + "printWidth": 80, + "proseWrap": "always" + } + }, + { + "files": "*.html", + "options": { + "htmlWhitespaceSensitivity": "strict" + } + } + ] +} diff --git a/website/Dockerfile b/website/Dockerfile new file mode 100644 index 0000000..9b2f6ca --- /dev/null +++ b/website/Dockerfile @@ -0,0 +1,14 @@ +FROM node:18 + +USER node + +# This is so the installed node_modules will be up one directory +# from where a user mounts files, so that they don't accidentally mount +# their own node_modules from a different build +# https://nodejs.org/api/modules.html#modules_loading_from_node_modules_folders +WORKDIR /home/node +COPY --chown=node package.json . +COPY --chown=node package-lock.json . +RUN npm install + +WORKDIR /home/node/website/ diff --git a/website/README.md b/website/README.md new file mode 100644 index 0000000..a434efe --- /dev/null +++ b/website/README.md @@ -0,0 +1,100 @@ +# spacy.io website and docs + +![Netlify Status](https://api.netlify.com/api/v1/badges/d65fe97d-99ab-47f8-a339-1d8987251da0/deploy-status) + +The styleguide for the spaCy website is available at +[spacy.io/styleguide](https://spacy.io/styleguide). + +## Setup and installation + +```bash +# Clone the repository +git clone https://github.com/explosion/spaCy +cd spaCy/website + +# Switch to the correct Node version +# +# If you don't have NVM and don't want to use it, you can manually switch to the Node version +# stated in /.nvmrc and skip this step +nvm use + +# Install the dependencies +npm install + +# Start the development server +npm run dev +``` + +If you are planning on making edits to the site, you should also set up the +[Prettier](https://prettier.io/) code formatter. It takes care of formatting +Markdown and other files automatically. +[See here](https://prettier.io/docs/en/editors.html) for the available +extensions for your code editor. The +[`.prettierrc`](https://github.com/explosion/spaCy/tree/master/website/.prettierrc) +file in the root defines the settings used in this codebase. + +## Building & developing the site with Docker + +While it shouldn't be necessary and is not recommended you can run this site in a Docker container. + +If you'd like to do this, **be sure you do _not_ include your local +`node_modules` folder**, since there are some dependencies that need to be built +for the image system. Rename it before using. + +First build the Docker image. This only needs to be done on the first run +or when changes are made to `Dockerfile` or the website dependencies: + +```bash +docker build -t spacy-io . +``` + +You can then build and run the website with: + +```bash +docker run -it \ + --rm \ + -v $(pwd):/home/node/website \ + -p 3000:3000 \ + spacy-io \ + npm run dev -- -H 0.0.0.0 +``` + +This will allow you to access the built website at http://0.0.0.0:3000/ in your +browser, and still edit code in your editor while having the site reflect those +changes. + +## Project structure + +```yaml +├── docs # the actual markdown content +├── meta # JSON-formatted site metadata +| ├── dynamicMeta.js # At build time generated meta data +| ├── languages.json # supported languages and statistical models +| ├── sidebars.json # sidebar navigations for different sections +| ├── site.json # general site metadata +| ├── type-annotations.json # Type annotations +| └── universe.json # data for the spaCy universe section +├── pages # Next router pages +├── public # static images and other assets +├── setup # Jinja setup +├── src # source +| ├── components # React components +| ├── fonts # webfonts +| ├── images # images used in the layout +| ├── plugins # custom plugins to transform Markdown +| ├── styles # CSS modules and global styles +| ├── templates # page layouts +| | ├── docs.js # layout template for documentation pages +| | ├── index.js # global layout template +| | ├── models.js # layout template for model pages +| | └── universe.js # layout templates for universe +| └── widgets # non-reusable components with content, e.g. changelog +├── .eslintrc.json # ESLint config file +├── .nvmrc # NVM config file +| # (to support "nvm use" to switch to correct Node version) +| +├── .prettierrc # Prettier config file +├── next.config.mjs # Next config file +├── package.json # package settings and dependencies +└── tsconfig.json # TypeScript config file +``` diff --git a/website/UNIVERSE.md b/website/UNIVERSE.md new file mode 100644 index 0000000..ac4e2e6 --- /dev/null +++ b/website/UNIVERSE.md @@ -0,0 +1,104 @@ + + +# spaCy Universe + +The [spaCy Universe](https://spacy.io/universe) collects the many great +resources developed with or for spaCy. It includes standalone packages, plugins, +extensions, educational materials, operational utilities and bindings for other +languages. + +If you have a project that you want the spaCy community to make use of, you can +suggest it by submitting a pull request to this repository. The Universe +database is open-source and collected in a simple JSON file. + +Looking for inspiration for your own spaCy plugin or extension? Check out the +[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22) +discussion forum. + +## Checklist + +### Projects + +✅ Libraries and packages should be **open-source** (with a user-friendly +license) and at least somewhat **documented** (e.g. a simple `README` with usage +instructions). + +✅ We're happy to include work in progress and prereleases, but we'd like to +keep the emphasis on projects that should be useful to the community **right +away**. + +✅ Demos and visualizers should be available via a **public URL**. + +### Educational Materials + +✅ Books should be **available for purchase or download** (not just pre-order). +Ebooks and self-published books are fine, too, if they include enough +substantial content. + +✅ The `"url"` of book entries should either point to the publisher's website or +a reseller of your choice (ideally one that ships worldwide or as close as +possible). + +✅ If an online course is only available behind a paywall, it should at least +have a **free excerpt** or chapter available, so users know what to expect. + +## JSON format + +To add a project, fork this repository, edit the +[`universe.json`](meta/universe.json) and add an object of the following format +to the list of `"resources"`. Before you submit your pull request, make sure to +use a linter to verify that your markup is correct. + +```json +{ + "id": "unique-project-id", + "title": "Project title", + "slogan": "A short summary", + "description": "A longer description – *Markdown allowed!*", + "github": "user/repo", + "pip": "package-name", + "code_example": [ + "import spacy", + "import package_name", + "", + "nlp = spacy.load('en')", + "nlp.add_pipe(package_name)" + ], + "code_language": "python", + "url": "https://example.com", + "thumb": "https://example.com/thumb.jpg", + "image": "https://example.com/image.jpg", + "author": "Your Name", + "author_links": { + "twitter": "username", + "github": "username", + "website": "https://example.com" + }, + "category": ["pipeline", "standalone"], + "tags": ["some-tag", "etc"] +} +``` + +| Field | Type | Description | +| --------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | string | Unique ID of the project. | +| `title` | string | Project title. If not set, the `id` will be used as the display title. | +| `slogan` | string | A short description of the project. Displayed in the overview and under the title. | +| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. | +| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. | +| `pip` | string | Package name on pip. If available, the installation command will be displayed. | +| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. | +| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. | +| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). | +| `url` | string | Optional project link to display as button. | +| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. | +| `image` | string | Optional URL to project image to display with description. | +| `author` | string | Name(s) of project author(s). | +| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. | +| `category` | list | One or more categories to assign to project. Must be one of the available options. | +| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. | + +To separate them from the projects, educational materials also specify +`"type": "education`. Books can also set a `"cover"` field containing a URL to a +cover image. If available, it's used in the overview and displayed on the +individual book page. diff --git a/website/docs/api/architectures.mdx b/website/docs/api/architectures.mdx new file mode 100644 index 0000000..956234a --- /dev/null +++ b/website/docs/api/architectures.mdx @@ -0,0 +1,1352 @@ +--- +title: Model Architectures +teaser: Pre-defined model architectures included with the core library +source: spacy/ml/models +menu: + - ['Tok2Vec', 'tok2vec-arch'] + - ['Transformers', 'transformers'] + - ['Pretraining', 'pretrain'] + - ['Parser & NER', 'parser'] + - ['Tagging', 'tagger'] + - ['Text Classification', 'textcat'] + - ['Span Classification', 'spancat'] + - ['Entity Linking', 'entitylinker'] + - ['Coreference', 'coref-architectures'] +--- + +A **model architecture** is a function that wires up a +[`Model`](https://thinc.ai/docs/api-model) instance, which you can then use in a +pipeline component or as a layer of a larger network. This page documents +spaCy's built-in architectures that are used for different NLP tasks. All +trainable [built-in components](/api#architecture-pipeline) expect a `model` +argument defined in the config and document their the default architecture. +Custom architectures can be registered using the +[`@spacy.registry.architectures`](/api/top-level#registry) decorator and used as +part of the [training config](/usage/training#custom-functions). Also see the +usage documentation on +[layers and model architectures](/usage/layers-architectures). + +## Tok2Vec architectures {id="tok2vec-arch",source="spacy/ml/models/tok2vec.py"} + +### spacy.Tok2Vec.v2 {id="Tok2Vec"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.Tok2Vec.v2" +> +> [model.embed] +> @architectures = "spacy.CharacterEmbed.v2" +> # ... +> +> [model.encode] +> @architectures = "spacy.MaxoutWindowEncoder.v2" +> # ... +> ``` + +Construct a tok2vec model out of two subnetworks: one for embedding and one for +encoding. See the +["Embed, Encode, Attend, Predict"](https://explosion.ai/blog/deep-learning-formula-nlp) +blog post for background. + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embed` | Embed tokens into context-independent word vector representations. For example, [CharacterEmbed](/api/architectures#CharacterEmbed) or [MultiHashEmbed](/api/architectures#MultiHashEmbed). ~~Model[List[Doc], List[Floats2d]]~~ | +| `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder](/api/architectures#MaxoutWindowEncoder). ~~Model[List[Floats2d], List[Floats2d]]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.HashEmbedCNN.v2 {id="HashEmbedCNN"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.HashEmbedCNN.v2" +> pretrained_vectors = null +> width = 96 +> depth = 4 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +Build spaCy's "standard" tok2vec layer. This layer is defined by a +[MultiHashEmbed](/api/architectures#MultiHashEmbed) embedding layer that uses +subword features, and a +[MaxoutWindowEncoder](/api/architectures#MaxoutWindowEncoder) encoding layer +consisting of a CNN and a layer-normalized maxout activation function. + +| Name | Description | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The width of the input and output. These are required to be the same, so that residual connections can be used. Recommended values are `96`, `128` or `300`. ~~int~~ | +| `depth` | The number of convolutional layers to use. Recommended values are between `2` and `8`. ~~int~~ | +| `embed_size` | The number of rows in the hash embedding tables. This can be surprisingly small, due to the use of the hash embeddings. Recommended values are between `2000` and `10000`. ~~int~~ | +| `window_size` | The number of tokens on either side to concatenate during the convolutions. The receptive field of the CNN will be `depth * window_size * 2 + 1`, so a 4-layer network with a window size of `2` will be sensitive to 17 words at a time. Recommended value is `1`. ~~int~~ | +| `maxout_pieces` | The number of pieces to use in the maxout non-linearity. If `1`, the [`Mish`](https://thinc.ai/docs/api-layers#mish) non-linearity is used instead. Recommended values are `1`-`3`. ~~int~~ | +| `subword_features` | Whether to also embed subword features, specifically the prefix, suffix and word shape. This is recommended for alphabetic languages like English, but not if single-character tokens are used for a language such as Chinese. ~~bool~~ | +| `pretrained_vectors` | Whether to also use static vectors. ~~bool~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.Tok2VecListener.v1 {id="Tok2VecListener"} + +> #### Example config +> +> ```ini +> [components.tok2vec] +> factory = "tok2vec" +> +> [components.tok2vec.model] +> @architectures = "spacy.HashEmbedCNN.v2" +> width = 342 +> +> [components.tagger] +> factory = "tagger" +> +> [components.tagger.model] +> @architectures = "spacy.Tagger.v2" +> +> [components.tagger.model.tok2vec] +> @architectures = "spacy.Tok2VecListener.v1" +> width = ${components.tok2vec.model.width} +> ``` + +A listener is used as a sublayer within a component such as a +[`DependencyParser`](/api/dependencyparser), +[`EntityRecognizer`](/api/entityrecognizer)or +[`TextCategorizer`](/api/textcategorizer). Usually you'll have multiple +listeners connecting to a single upstream [`Tok2Vec`](/api/tok2vec) component +that's earlier in the pipeline. The listener layers act as **proxies**, passing +the predictions from the `Tok2Vec` component into downstream components, and +communicating gradients back upstream. + +Instead of defining its own `Tok2Vec` instance, a model architecture like +[Tagger](/api/architectures#tagger) can define a listener as its `tok2vec` +argument that connects to the shared `tok2vec` component in the pipeline. + +Listeners work by caching the `Tok2Vec` output for a given batch of `Doc`s. This +means that in order for a component to work with the listener, the batch of +`Doc`s passed to the listener must be the same as the batch of `Doc`s passed to +the `Tok2Vec`. As a result, any manipulation of the `Doc`s which would affect +`Tok2Vec` output, such as to create special contexts or remove `Doc`s for which +no prediction can be made, must happen inside the model, **after** the call to +the `Tok2Vec` component. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `width` | The width of the vectors produced by the "upstream" [`Tok2Vec`](/api/tok2vec) component. ~~int~~ | +| `upstream` | A string to identify the "upstream" `Tok2Vec` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Tok2Vec` component. You'll almost never have multiple upstream `Tok2Vec` components, so the wildcard string will almost always be fine. ~~str~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.MultiHashEmbed.v2 {id="MultiHashEmbed"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.MultiHashEmbed.v2" +> width = 64 +> attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] +> rows = [2000, 1000, 1000, 1000] +> include_static_vectors = true +> ``` + +Construct an embedding layer that separately embeds a number of lexical +attributes using hash embedding, concatenates the results, and passes it through +a feed-forward subnetwork to build a mixed representation. The features used can +be configured with the `attrs` argument. The suggested attributes are `NORM`, +`PREFIX`, `SUFFIX` and `SHAPE`. This lets the model take into account some +subword information, without construction a fully character-based +representation. If pretrained vectors are available, they can be included in the +representation as well, with the vectors table kept static (i.e. it's not +updated). + +| Name | Description | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The output width. Also used as the width of the embedding tables. Recommended values are between `64` and `300`. If static vectors are included, a learned linear layer is used to map the vectors to the specified width before concatenating it with the other embedding outputs. A single maxout layer is then used to reduce the concatenated vectors to the final width. ~~int~~ | +| `attrs` | The token attributes to embed. A separate embedding table will be constructed for each attribute. ~~List[Union[int, str]]~~ | +| `rows` | The number of rows for each embedding tables. Can be low, due to the hashing trick. Recommended values are between `1000` and `10000`. The layer needs surprisingly few rows, due to its use of the hashing trick. Generally between 2000 and 10000 rows is sufficient, even for very large vocabularies. A number of rows must be specified for each table, so the `rows` list must be of the same length as the `attrs` parameter. ~~List[int]~~ | +| `include_static_vectors` | Whether to also use static word vectors. Requires a vectors table to be loaded in the [`Doc`](/api/doc) objects' vocab. ~~bool~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.CharacterEmbed.v2 {id="CharacterEmbed"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.CharacterEmbed.v2" +> width = 128 +> rows = 7000 +> nM = 64 +> nC = 8 +> ``` + +Construct an embedded representation based on character embeddings, using a +feed-forward network. A fixed number of UTF-8 byte characters are used for each +word, taken from the beginning and end of the word equally. Padding is used in +the center for words that are too short. + +For instance, let's say `nC=4`, and the word is "jumping". The characters used +will be `"jung"` (two from the start, two from the end). If we had `nC=8`, the +characters would be `"jumpping"`: 4 from the start, 4 from the end. This ensures +that the final character is always in the last position, instead of being in an +arbitrary position depending on the word length. + +The characters are embedded in a embedding table with a given number of rows, +and the vectors concatenated. A hash-embedded vector of the `NORM` of the word +is also concatenated on, and the result is then passed through a feed-forward +network to construct a single vector to represent the information. + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The width of the output vector and the `NORM` hash embedding. ~~int~~ | +| `rows` | The number of rows in the `NORM` hash embedding table. ~~int~~ | +| `nM` | The dimensionality of the character embeddings. Recommended values are between `16` and `64`. ~~int~~ | +| `nC` | The number of UTF-8 bytes to embed per word. Recommended values are between `3` and `8`, although it may depend on the length of words in the language. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.MaxoutWindowEncoder.v2 {id="MaxoutWindowEncoder"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.MaxoutWindowEncoder.v2" +> width = 128 +> window_size = 1 +> maxout_pieces = 3 +> depth = 4 +> ``` + +Encode context using convolutions with maxout activation, layer normalization +and residual connections. + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between `64` and `300`. ~~int~~ | +| `window_size` | The number of words to concatenate around each token to construct the convolution. Recommended value is `1`. ~~int~~ | +| `maxout_pieces` | The number of maxout pieces to use. Recommended values are `2` or `3`. ~~int~~ | +| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | + +### spacy.MishWindowEncoder.v2 {id="MishWindowEncoder"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.MishWindowEncoder.v2" +> width = 64 +> window_size = 1 +> depth = 4 +> ``` + +Encode context using convolutions with +[`Mish`](https://thinc.ai/docs/api-layers#mish) activation, layer normalization +and residual connections. + +| Name | Description | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between `64` and `300`. ~~int~~ | +| `window_size` | The number of words to concatenate around each token to construct the convolution. Recommended value is `1`. ~~int~~ | +| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | + +### spacy.TorchBiLSTMEncoder.v1 {id="TorchBiLSTMEncoder"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.TorchBiLSTMEncoder.v1" +> width = 64 +> depth = 2 +> dropout = 0.0 +> ``` + +Encode context using bidirectional LSTM layers. Requires +[PyTorch](https://pytorch.org). + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between `64` and `300`. ~~int~~ | +| `depth` | The number of recurrent layers, for instance `depth=2` results in stacking two LSTMs together. ~~int~~ | +| `dropout` | Creates a Dropout layer on the outputs of each LSTM layer except the last layer. Set to 0.0 to disable this functionality. ~~float~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | + +### spacy.StaticVectors.v2 {id="StaticVectors"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.StaticVectors.v2" +> nO = null +> nM = null +> dropout = 0.2 +> key_attr = "ORTH" +> +> [model.init_W] +> @initializers = "glorot_uniform_init.v1" +> ``` + +Embed [`Doc`](/api/doc) objects with their vocab's vectors table, applying a +learned linear projection to control the dimensionality. Unknown tokens are +mapped to a zero vector. See the documentation on +[static vectors](/usage/embeddings-transformers#static-vectors) for details. + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nO` | The output width of the layer, after the linear projection. ~~Optional[int]~~ | +| `nM` | The width of the static vectors. ~~Optional[int]~~ | +| `dropout` | Optional dropout rate. If set, it's applied per dimension over the whole batch. Defaults to `None`. ~~Optional[float]~~ | +| `init_W` | The [initialization function](https://thinc.ai/docs/api-initializers). Defaults to [`glorot_uniform_init`](https://thinc.ai/docs/api-initializers#glorot_uniform_init). ~~Callable[[Ops, Tuple[int, ...]]], FloatsXd]~~ | +| `key_attr` | This setting is ignored in spaCy v3.6+. To set a custom key attribute for vectors, configure it through [`Vectors`](/api/vectors) or [`spacy init vectors`](/api/cli#init-vectors). Defaults to `"ORTH"`. ~~str~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Ragged]~~ | + +### spacy.FeatureExtractor.v1 {id="FeatureExtractor"} + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.FeatureExtractor.v1" +> columns = ["NORM", "PREFIX", "SUFFIX", "SHAPE", "ORTH"] +> ``` + +Extract arrays of input features from [`Doc`](/api/doc) objects. Expects a list +of feature names to extract, which should refer to token attributes. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------ | +| `columns` | The token attributes to extract. ~~List[Union[int, str]]~~ | +| **CREATES** | The created feature extraction layer. ~~Model[List[Doc], List[Ints2d]]~~ | + +## Transformer architectures {id="transformers",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"} + +The following architectures are provided by the package +[`spacy-transformers`](https://github.com/explosion/spacy-transformers). See the +[usage documentation](/usage/embeddings-transformers#transformers) for how to +integrate the architectures into your training config. + + + +Note that in order to use these architectures in your config, you need to +install the +[`spacy-transformers`](https://github.com/explosion/spacy-transformers). See the +[installation docs](/usage/embeddings-transformers#transformers-installation) +for details and system requirements. + + + +### spacy-transformers.TransformerModel.v3 {id="TransformerModel"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy-transformers.TransformerModel.v3" +> name = "roberta-base" +> tokenizer_config = {"use_fast": true} +> transformer_config = {} +> mixed_precision = true +> grad_scaler_config = {"init_scale": 32768} +> +> [model.get_spans] +> @span_getters = "spacy-transformers.strided_spans.v1" +> window = 128 +> stride = 96 +> ``` + +Load and wrap a transformer model from the +[HuggingFace `transformers`](https://huggingface.co/transformers) library. You +can use any transformer that has pretrained weights and a PyTorch +implementation. The `name` variable is passed through to the underlying library, +so it can be either a string or a path. If it's a string, the pretrained weights +will be downloaded via the transformers library if they are not already +available locally. + +In order to support longer documents, the +[TransformerModel](/api/architectures#TransformerModel) layer allows you to pass +in a `get_spans` function that will divide up the [`Doc`](/api/doc) objects +before passing them through the transformer. Your spans are allowed to overlap +or exclude tokens. This layer is usually used directly by the +[`Transformer`](/api/transformer) component, which allows you to share the +transformer weights across your pipeline. For a layer that's configured for use +in other components, see +[Tok2VecTransformer](/api/architectures#Tok2VecTransformer). + +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Any model name that can be loaded by [`transformers.AutoModel`](https://huggingface.co/transformers/model_doc/auto.html#transformers.AutoModel). ~~str~~ | +| `get_spans` | Function that takes a batch of [`Doc`](/api/doc) object and returns lists of [`Span`](/api) objects to process by the transformer. [See here](/api/transformer#span_getters) for built-in options and examples. ~~Callable[[List[Doc]], List[Span]]~~ | +| `tokenizer_config` | Tokenizer settings passed to [`transformers.AutoTokenizer`](https://huggingface.co/transformers/model_doc/auto.html#transformers.AutoTokenizer). ~~Dict[str, Any]~~ | +| `transformer_config` | Transformer settings passed to [`transformers.AutoConfig`](https://huggingface.co/transformers/model_doc/auto.html?highlight=autoconfig#transformers.AutoConfig) ~~Dict[str, Any]~~ | +| `mixed_precision` | Replace whitelisted ops by half-precision counterparts. Speeds up training and prediction on GPUs with [Tensor Cores](https://developer.nvidia.com/tensor-cores) and reduces GPU memory use. ~~bool~~ | +| `grad_scaler_config` | Configuration to pass to `thinc.api.PyTorchGradScaler` during training when `mixed_precision` is enabled. ~~Dict[str, Any]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], FullTransformerBatch]~~ | +| | | + + + Mixed-precision support is currently an experimental feature. + + + + +- The `transformer_config` argument was added in + `spacy-transformers.TransformerModel.v2`. +- The `mixed_precision` and `grad_scaler_config` arguments were added in + `spacy-transformers.TransformerModel.v3`. + +The other arguments are shared between all versions. + + + +### spacy-transformers.TransformerListener.v1 {id="TransformerListener"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy-transformers.TransformerListener.v1" +> grad_factor = 1.0 +> +> [model.pooling] +> @layers = "reduce_mean.v1" +> ``` + +Create a `TransformerListener` layer, which will connect to a +[`Transformer`](/api/transformer) component earlier in the pipeline. The layer +takes a list of [`Doc`](/api/doc) objects as input, and produces a list of +2-dimensional arrays as output, with each array having one row per token. Most +spaCy models expect a sublayer with this signature, making it easy to connect +them to a transformer model via this sublayer. Transformer models usually +operate over wordpieces, which usually don't align one-to-one against spaCy +tokens. The layer therefore requires a reduction operation in order to calculate +a single token vector given zero or more wordpiece vectors. + +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `pooling` | A reduction layer used to calculate the token vectors based on zero or more wordpiece vectors. If in doubt, mean pooling (see [`reduce_mean`](https://thinc.ai/docs/api-layers#reduce_mean)) is usually a good choice. ~~Model[Ragged, Floats2d]~~ | +| `grad_factor` | Reweight gradients from the component before passing them upstream. You can set this to `0` to "freeze" the transformer weights with respect to the component, or use it to make some components more significant than others. Leaving it at `1.0` is usually fine. ~~float~~ | +| `upstream` | A string to identify the "upstream" `Transformer` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Transformer` component. You'll almost never have multiple upstream `Transformer` components, so the wildcard string will almost always be fine. ~~str~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy-transformers.Tok2VecTransformer.v3 {id="Tok2VecTransformer"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy-transformers.Tok2VecTransformer.v3" +> name = "albert-base-v2" +> tokenizer_config = {"use_fast": false} +> transformer_config = {} +> grad_factor = 1.0 +> mixed_precision = true +> grad_scaler_config = {"init_scale": 32768} +> ``` + +Use a transformer as a [`Tok2Vec`](/api/tok2vec) layer directly. This does +**not** allow multiple components to share the transformer weights and does +**not** allow the transformer to set annotations into the [`Doc`](/api/doc) +object, but it's a **simpler solution** if you only need the transformer within +one component. + +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_spans` | Function that takes a batch of [`Doc`](/api/doc) object and returns lists of [`Span`](/api) objects to process by the transformer. [See here](/api/transformer#span_getters) for built-in options and examples. ~~Callable[[List[Doc]], List[Span]]~~ | +| `tokenizer_config` | Tokenizer settings passed to [`transformers.AutoTokenizer`](https://huggingface.co/transformers/model_doc/auto.html#transformers.AutoTokenizer). ~~Dict[str, Any]~~ | +| `transformer_config` | Settings to pass to the transformers forward pass. ~~Dict[str, Any]~~ | +| `pooling` | A reduction layer used to calculate the token vectors based on zero or more wordpiece vectors. If in doubt, mean pooling (see [`reduce_mean`](https://thinc.ai/docs/api-layers#reduce_mean)) is usually a good choice. ~~Model[Ragged, Floats2d]~~ | +| `grad_factor` | Reweight gradients from the component before passing them upstream. You can set this to `0` to "freeze" the transformer weights with respect to the component, or use it to make some components more significant than others. Leaving it at `1.0` is usually fine. ~~float~~ | +| `mixed_precision` | Replace whitelisted ops by half-precision counterparts. Speeds up training and prediction on GPUs with [Tensor Cores](https://developer.nvidia.com/tensor-cores) and reduces GPU memory use. ~~bool~~ | +| `grad_scaler_config` | Configuration to pass to `thinc.api.PyTorchGradScaler` during training when `mixed_precision` is enabled. ~~Dict[str, Any]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + + + Mixed-precision support is currently an experimental feature. + + + + +- The `transformer_config` argument was added in + `spacy-transformers.Tok2VecTransformer.v2`. +- The `mixed_precision` and `grad_scaler_config` arguments were added in + `spacy-transformers.Tok2VecTransformer.v3`. + +The other arguments are shared between all versions. + + + +## Curated Transformer architectures {id="curated-trf",source="https://github.com/explosion/spacy-curated-transformers/blob/main/spacy_curated_transformers/models/architectures.py"} + +The following architectures are provided by the package +[`spacy-curated-transformers`](https://github.com/explosion/spacy-curated-transformers). +See the [usage documentation](/usage/embeddings-transformers#transformers) for +how to integrate the architectures into your training config. + +When loading the model +[from the Hugging Face Hub](/api/curatedtransformer#hf_trfencoder_loader), the +model config's parameters must be same as the hyperparameters used by the +pre-trained model. The +[`init fill-curated-transformer`](/api/cli#init-fill-curated-transformer) CLI +command can be used to automatically fill in these values. + +### spacy-curated-transformers.AlbertTransformer.v1 + +Construct an ALBERT transformer model. + +| Name | Description | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| `vocab_size` | Vocabulary size. ~~int~~ | +| `with_spans` | Callback that constructs a span generator model. ~~Callable~~ | +| `piece_encoder` | The piece encoder to segment input tokens. ~~Model~~ | +| `attention_probs_dropout_prob` | Dropout probability of the self-attention layers. ~~float~~ | +| `embedding_width` | Width of the embedding representations. ~~int~~ | +| `hidden_act` | Activation used by the point-wise feed-forward layers. ~~str~~ | +| `hidden_dropout_prob` | Dropout probability of the point-wise feed-forward and embedding layers. ~~float~~ | +| `hidden_width` | Width of the final representations. ~~int~~ | +| `intermediate_width` | Width of the intermediate projection layer in the point-wise feed-forward layer. ~~int~~ | +| `layer_norm_eps` | Epsilon for layer normalization. ~~float~~ | +| `max_position_embeddings` | Maximum length of position embeddings. ~~int~~ | +| `model_max_length` | Maximum length of model inputs. ~~int~~ | +| `num_attention_heads` | Number of self-attention heads. ~~int~~ | +| `num_hidden_groups` | Number of layer groups whose constituents share parameters. ~~int~~ | +| `num_hidden_layers` | Number of hidden layers. ~~int~~ | +| `padding_idx` | Index of the padding meta-token. ~~int~~ | +| `type_vocab_size` | Type vocabulary size. ~~int~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model~~ | + +### spacy-curated-transformers.BertTransformer.v1 + +Construct a BERT transformer model. + +| Name | Description | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| `vocab_size` | Vocabulary size. ~~int~~ | +| `with_spans` | Callback that constructs a span generator model. ~~Callable~~ | +| `piece_encoder` | The piece encoder to segment input tokens. ~~Model~~ | +| `attention_probs_dropout_prob` | Dropout probability of the self-attention layers. ~~float~~ | +| `hidden_act` | Activation used by the point-wise feed-forward layers. ~~str~~ | +| `hidden_dropout_prob` | Dropout probability of the point-wise feed-forward and embedding layers. ~~float~~ | +| `hidden_width` | Width of the final representations. ~~int~~ | +| `intermediate_width` | Width of the intermediate projection layer in the point-wise feed-forward layer. ~~int~~ | +| `layer_norm_eps` | Epsilon for layer normalization. ~~float~~ | +| `max_position_embeddings` | Maximum length of position embeddings. ~~int~~ | +| `model_max_length` | Maximum length of model inputs. ~~int~~ | +| `num_attention_heads` | Number of self-attention heads. ~~int~~ | +| `num_hidden_layers` | Number of hidden layers. ~~int~~ | +| `padding_idx` | Index of the padding meta-token. ~~int~~ | +| `type_vocab_size` | Type vocabulary size. ~~int~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model~~ | + +### spacy-curated-transformers.CamembertTransformer.v1 + +Construct a CamemBERT transformer model. + +| Name | Description | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| `vocab_size` | Vocabulary size. ~~int~~ | +| `with_spans` | Callback that constructs a span generator model. ~~Callable~~ | +| `piece_encoder` | The piece encoder to segment input tokens. ~~Model~~ | +| `attention_probs_dropout_prob` | Dropout probability of the self-attention layers. ~~float~~ | +| `hidden_act` | Activation used by the point-wise feed-forward layers. ~~str~~ | +| `hidden_dropout_prob` | Dropout probability of the point-wise feed-forward and embedding layers. ~~float~~ | +| `hidden_width` | Width of the final representations. ~~int~~ | +| `intermediate_width` | Width of the intermediate projection layer in the point-wise feed-forward layer. ~~int~~ | +| `layer_norm_eps` | Epsilon for layer normalization. ~~float~~ | +| `max_position_embeddings` | Maximum length of position embeddings. ~~int~~ | +| `model_max_length` | Maximum length of model inputs. ~~int~~ | +| `num_attention_heads` | Number of self-attention heads. ~~int~~ | +| `num_hidden_layers` | Number of hidden layers. ~~int~~ | +| `padding_idx` | Index of the padding meta-token. ~~int~~ | +| `type_vocab_size` | Type vocabulary size. ~~int~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model~~ | + +### spacy-curated-transformers.RobertaTransformer.v1 + +Construct a RoBERTa transformer model. + +| Name | Description | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| `vocab_size` | Vocabulary size. ~~int~~ | +| `with_spans` | Callback that constructs a span generator model. ~~Callable~~ | +| `piece_encoder` | The piece encoder to segment input tokens. ~~Model~~ | +| `attention_probs_dropout_prob` | Dropout probability of the self-attention layers. ~~float~~ | +| `hidden_act` | Activation used by the point-wise feed-forward layers. ~~str~~ | +| `hidden_dropout_prob` | Dropout probability of the point-wise feed-forward and embedding layers. ~~float~~ | +| `hidden_width` | Width of the final representations. ~~int~~ | +| `intermediate_width` | Width of the intermediate projection layer in the point-wise feed-forward layer. ~~int~~ | +| `layer_norm_eps` | Epsilon for layer normalization. ~~float~~ | +| `max_position_embeddings` | Maximum length of position embeddings. ~~int~~ | +| `model_max_length` | Maximum length of model inputs. ~~int~~ | +| `num_attention_heads` | Number of self-attention heads. ~~int~~ | +| `num_hidden_layers` | Number of hidden layers. ~~int~~ | +| `padding_idx` | Index of the padding meta-token. ~~int~~ | +| `type_vocab_size` | Type vocabulary size. ~~int~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model~~ | + +### spacy-curated-transformers.XlmrTransformer.v1 + +Construct a XLM-RoBERTa transformer model. + +| Name | Description | +| ------------------------------ | ---------------------------------------------------------------------------------------- | +| `vocab_size` | Vocabulary size. ~~int~~ | +| `with_spans` | Callback that constructs a span generator model. ~~Callable~~ | +| `piece_encoder` | The piece encoder to segment input tokens. ~~Model~~ | +| `attention_probs_dropout_prob` | Dropout probability of the self-attention layers. ~~float~~ | +| `hidden_act` | Activation used by the point-wise feed-forward layers. ~~str~~ | +| `hidden_dropout_prob` | Dropout probability of the point-wise feed-forward and embedding layers. ~~float~~ | +| `hidden_width` | Width of the final representations. ~~int~~ | +| `intermediate_width` | Width of the intermediate projection layer in the point-wise feed-forward layer. ~~int~~ | +| `layer_norm_eps` | Epsilon for layer normalization. ~~float~~ | +| `max_position_embeddings` | Maximum length of position embeddings. ~~int~~ | +| `model_max_length` | Maximum length of model inputs. ~~int~~ | +| `num_attention_heads` | Number of self-attention heads. ~~int~~ | +| `num_hidden_layers` | Number of hidden layers. ~~int~~ | +| `padding_idx` | Index of the padding meta-token. ~~int~~ | +| `type_vocab_size` | Type vocabulary size. ~~int~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model~~ | + +### spacy-curated-transformers.ScalarWeight.v1 + +Construct a model that accepts a list of transformer layer outputs and returns a +weighted representation of the same. + +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------- | +| `num_layers` | Number of transformer hidden layers. ~~int~~ | +| `dropout_prob` | Dropout probability. ~~float~~ | +| `mixed_precision` | Use mixed-precision training. ~~bool~~ | +| `grad_scaler_config` | Configuration passed to the PyTorch gradient scaler. ~~dict~~ | +| **CREATES** | The model using the architecture ~~Model[ScalarWeightInT, ScalarWeightOutT]~~ | + +### spacy-curated-transformers.TransformerLayersListener.v1 + +Construct a listener layer that communicates with one or more upstream +Transformer components. This layer extracts the output of the last transformer +layer and performs pooling over the individual pieces of each `Doc` token, +returning their corresponding representations. The upstream name should either +be the wildcard string '\*', or the name of the Transformer component. + +In almost all cases, the wildcard string will suffice as there'll only be one +upstream Transformer component. But in certain situations, e.g: you have +disjoint datasets for certain tasks, or you'd like to use a pre-trained pipeline +but a downstream task requires its own token representations, you could end up +with more than one Transformer component in the pipeline. + +| Name | Description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `layers` | The number of layers produced by the upstream transformer component, excluding the embedding layer. ~~int~~ | +| `width` | The width of the vectors produced by the upstream transformer component. ~~int~~ | +| `pooling` | Model that is used to perform pooling over the piece representations. ~~Model~~ | +| `upstream_name` | A string to identify the 'upstream' Transformer component to communicate with. ~~str~~ | +| `grad_factor` | Factor to multiply gradients with. ~~float~~ | +| **CREATES** | A model that returns the relevant vectors from an upstream transformer component. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy-curated-transformers.LastTransformerLayerListener.v1 + +Construct a listener layer that communicates with one or more upstream +Transformer components. This layer extracts the output of the last transformer +layer and performs pooling over the individual pieces of each Doc token, +returning their corresponding representations. The upstream name should either +be the wildcard string '\*', or the name of the Transformer component. + +In almost all cases, the wildcard string will suffice as there'll only be one +upstream Transformer component. But in certain situations, e.g: you have +disjoint datasets for certain tasks, or you'd like to use a pre-trained pipeline +but a downstream task requires its own token representations, you could end up +with more than one Transformer component in the pipeline. + +| Name | Description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `width` | The width of the vectors produced by the upstream transformer component. ~~int~~ | +| `pooling` | Model that is used to perform pooling over the piece representations. ~~Model~~ | +| `upstream_name` | A string to identify the 'upstream' Transformer component to communicate with. ~~str~~ | +| `grad_factor` | Factor to multiply gradients with. ~~float~~ | +| **CREATES** | A model that returns the relevant vectors from an upstream transformer component. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy-curated-transformers.ScalarWeightingListener.v1 + +Construct a listener layer that communicates with one or more upstream +Transformer components. This layer calculates a weighted representation of all +transformer layer outputs and performs pooling over the individual pieces of +each Doc token, returning their corresponding representations. + +Requires its upstream Transformer components to return all layer outputs from +their models. The upstream name should either be the wildcard string '\*', or +the name of the Transformer component. + +In almost all cases, the wildcard string will suffice as there'll only be one +upstream Transformer component. But in certain situations, e.g: you have +disjoint datasets for certain tasks, or you'd like to use a pre-trained pipeline +but a downstream task requires its own token representations, you could end up +with more than one Transformer component in the pipeline. + +| Name | Description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `width` | The width of the vectors produced by the upstream transformer component. ~~int~~ | +| `weighting` | Model that is used to perform the weighting of the different layer outputs. ~~Model~~ | +| `pooling` | Model that is used to perform pooling over the piece representations. ~~Model~~ | +| `upstream_name` | A string to identify the 'upstream' Transformer component to communicate with. ~~str~~ | +| `grad_factor` | Factor to multiply gradients with. ~~float~~ | +| **CREATES** | A model that returns the relevant vectors from an upstream transformer component. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy-curated-transformers.BertWordpieceEncoder.v1 + +Construct a WordPiece piece encoder model that accepts a list of token sequences +or documents and returns a corresponding list of piece identifiers. This encoder +also splits each token on punctuation characters, as expected by most BERT +models. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.ByteBpeEncoder.v1 + +Construct a Byte-BPE piece encoder model that accepts a list of token sequences +or documents and returns a corresponding list of piece identifiers. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.CamembertSentencepieceEncoder.v1 + +Construct a SentencePiece piece encoder model that accepts a list of token +sequences or documents and returns a corresponding list of piece identifiers +with CamemBERT post-processing applied. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.CharEncoder.v1 + +Construct a character piece encoder model that accepts a list of token sequences +or documents and returns a corresponding list of piece identifiers. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.SentencepieceEncoder.v1 + +Construct a SentencePiece piece encoder model that accepts a list of token +sequences or documents and returns a corresponding list of piece identifiers. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.WordpieceEncoder.v1 + +Construct a WordPiece piece encoder model that accepts a list of token sequences +or documents and returns a corresponding list of piece identifiers. This encoder +also splits each token on punctuation characters, as expected by most BERT +models. + +This model must be separately initialized using an appropriate loader. + +### spacy-curated-transformers.XlmrSentencepieceEncoder.v1 + +Construct a SentencePiece piece encoder model that accepts a list of token +sequences or documents and returns a corresponding list of piece identifiers +with XLM-RoBERTa post-processing applied. + +This model must be separately initialized using an appropriate loader. + +## Pretraining architectures {id="pretrain",source="spacy/ml/models/multi_task.py"} + +The spacy `pretrain` command lets you initialize a `Tok2Vec` layer in your +pipeline with information from raw text. To this end, additional layers are +added to build a network for a temporary task that forces the `Tok2Vec` layer to +learn something about sentence structure and word cooccurrence statistics. Two +pretraining objectives are available, both of which are variants of the cloze +task [Devlin et al. (2018)](https://arxiv.org/abs/1810.04805) introduced for +BERT. + +For more information, see the section on +[pretraining](/usage/embeddings-transformers#pretraining). + +### spacy.PretrainVectors.v1 {id="pretrain_vectors"} + +> #### Example config +> +> ```ini +> [pretraining] +> component = "tok2vec" +> +> [initialize] +> vectors = "en_core_web_lg" +> ... +> +> [pretraining.objective] +> @architectures = "spacy.PretrainVectors.v1" +> maxout_pieces = 3 +> hidden_size = 300 +> loss = "cosine" +> ``` + +Predict the word's vector from a static embeddings table as pretraining +objective for a Tok2Vec layer. To use this objective, make sure that the +`initialize.vectors` section in the config refers to a model with static +vectors. + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `maxout_pieces` | The number of maxout pieces to use. Recommended values are `2` or `3`. ~~int~~ | +| `hidden_size` | Size of the hidden layer of the model. ~~int~~ | +| `loss` | The loss function can be either "cosine" or "L2". We typically recommend to use "cosine". ~~~str~~ | +| **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | + +### spacy.PretrainCharacters.v1 {id="pretrain_chars"} + +> #### Example config +> +> ```ini +> [pretraining] +> component = "tok2vec" +> ... +> +> [pretraining.objective] +> @architectures = "spacy.PretrainCharacters.v1" +> maxout_pieces = 3 +> hidden_size = 300 +> n_characters = 4 +> ``` + +Predict some number of leading and trailing UTF-8 bytes as pretraining objective +for a Tok2Vec layer. + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `maxout_pieces` | The number of maxout pieces to use. Recommended values are `2` or `3`. ~~int~~ | +| `hidden_size` | Size of the hidden layer of the model. ~~int~~ | +| `n_characters` | The window of characters - e.g. if `n_characters = 2`, the model will try to predict the first two and last two characters of the word. ~~int~~ | +| **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | + +## Parser & NER architectures {id="parser"} + +### spacy.TransitionBasedParser.v2 {id="TransitionBasedParser",source="spacy/ml/models/parser.py"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TransitionBasedParser.v2" +> state_type = "ner" +> extra_state_tokens = false +> hidden_width = 64 +> maxout_pieces = 2 +> use_upper = true +> +> [model.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v2" +> pretrained_vectors = null +> width = 96 +> depth = 4 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +Build a transition-based parser model. Can apply to NER or dependency parsing. +Transition-based parsing is an approach to structured prediction where the task +of predicting the structure is mapped to a series of state transitions. You +might find [this tutorial](https://explosion.ai/blog/parsing-english-in-python) +helpful for background information. The neural network state prediction model +consists of either two or three subnetworks: + +- **tok2vec**: Map each token into a vector representation. This subnetwork is + run once for each batch. +- **lower**: Construct a feature-specific vector for each `(token, feature)` + pair. This is also run once for each batch. Constructing the state + representation is then a matter of summing the component features and applying + the non-linearity. +- **upper** (optional): A feed-forward network that predicts scores from the + state representation. If not present, the output from the lower model is used + as action scores directly. + +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | Subnetwork to map tokens into vector representations. ~~Model[List[Doc], List[Floats2d]]~~ | +| `state_type` | Which task to extract features for. Possible values are "ner" and "parser". ~~str~~ | +| `extra_state_tokens` | Whether to use an expanded feature set when extracting the state tokens. Slightly slower, but sometimes improves accuracy slightly. Defaults to `False`. ~~bool~~ | +| `hidden_width` | The width of the hidden layer. ~~int~~ | +| `maxout_pieces` | How many pieces to use in the state prediction layer. Recommended values are `1`, `2` or `3`. If `1`, the maxout non-linearity is replaced with a [`Relu`](https://thinc.ai/docs/api-layers#relu) non-linearity if `use_upper` is `True`, and no non-linearity if `False`. ~~int~~ | +| `use_upper` | Whether to use an additional hidden layer after the state vector in order to predict the action scores. It is recommended to set this to `False` for large pretrained models such as transformers, and `True` for smaller networks. The upper layer is computed on CPU, which becomes a bottleneck on larger GPU-based models, where it's also less necessary. ~~bool~~ | +| `nO` | The number of actions the model will predict between. Usually inferred from data at the beginning of training, or loaded from disk. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Docs], List[List[Floats2d]]]~~ | + + + +[TransitionBasedParser.v1](/api/legacy#TransitionBasedParser_v1) had the exact +same signature, but the `use_upper` argument was `True` by default. + + + +## Tagging architectures {id="tagger",source="spacy/ml/models/tagger.py"} + +### spacy.Tagger.v2 {id="Tagger"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.Tagger.v2" +> nO = null +> normalize = false +> +> [model.tok2vec] +> # ... +> ``` + +Build a tagger model, using a provided token-to-vector component. The tagger +model adds a linear layer with softmax activation to predict scores given the +token vectors. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------ | +| `tok2vec` | Subnetwork to map tokens into vector representations. ~~Model[List[Doc], List[Floats2d]]~~ | +| `nO` | The number of tags to output. Inferred from the data if `None`. ~~Optional[int]~~ | +| `normalize` | Normalize probabilities during inference. Defaults to `False`. ~~bool~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + + + +- The `normalize` argument was added in `spacy.Tagger.v2`. `spacy.Tagger.v1` + always normalizes probabilities during inference. + +The other arguments are shared between all versions. + + + +## Text classification architectures {id="textcat",source="spacy/ml/models/textcat.py"} + +A text classification architecture needs to take a [`Doc`](/api/doc) as input, +and produce a score for each potential label class. Textcat challenges can be +binary (e.g. sentiment analysis) or involve multiple possible labels. +Multi-label challenges can either have mutually exclusive labels (each example +has exactly one label), or multiple labels may be applicable at the same time. + +As the properties of text classification problems can vary widely, we provide +several different built-in architectures. It is recommended to experiment with +different architectures and settings to determine what works best on your +specific data and challenge. + + + +When the architecture for a text classification challenge contains a setting for +`exclusive_classes`, it is important to use the correct value for the correct +pipeline component. The `textcat` component should always be used for +single-label use-cases where `exclusive_classes = true`, while the +`textcat_multilabel` should be used for multi-label settings with +`exclusive_classes = false`. + + + +### spacy.TextCatEnsemble.v2 {id="TextCatEnsemble"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatEnsemble.v2" +> nO = null +> +> [model.linear_model] +> @architectures = "spacy.TextCatBOW.v3" +> exclusive_classes = true +> length = 262144 +> ngram_size = 1 +> no_output_layer = false +> +> [model.tok2vec] +> @architectures = "spacy.Tok2Vec.v2" +> +> [model.tok2vec.embed] +> @architectures = "spacy.MultiHashEmbed.v2" +> width = 64 +> rows = [2000, 2000, 1000, 1000, 1000, 1000] +> attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] +> include_static_vectors = false +> +> [model.tok2vec.encode] +> @architectures = "spacy.MaxoutWindowEncoder.v2" +> width = ${model.tok2vec.embed.width} +> window_size = 1 +> maxout_pieces = 3 +> depth = 2 +> ``` + +Stacked ensemble of a linear bag-of-words model and a neural network model. The +neural network is built upon a Tok2Vec layer and uses attention. The setting for +whether or not this model should cater for multi-label classification, is taken +from the linear model, where it is stored in `model.attrs["multi_label"]`. + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `linear_model` | The linear bag-of-words model. ~~Model[List[Doc], Floats2d]~~ | +| `tok2vec` | The `tok2vec` layer to build the neural network upon. ~~Model[List[Doc], List[Floats2d]]~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + + + +[TextCatEnsemble.v1](/api/legacy#TextCatEnsemble_v1) was functionally similar, +but used an internal `tok2vec` instead of taking it as argument: + +| Name | Description | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `pretrained_vectors` | Whether or not pretrained vectors will be used in addition to the feature vectors. ~~bool~~ | +| `width` | Output dimension of the feature encoding step. ~~int~~ | +| `embed_size` | Input dimension of the feature encoding step. ~~int~~ | +| `conv_depth` | Depth of the tok2vec layer. ~~int~~ | +| `window_size` | The number of contextual vectors to [concatenate](https://thinc.ai/docs/api-layers#expand_window) from the left and from the right. ~~int~~ | +| `ngram_size` | Determines the maximum length of the n-grams in the BOW model. For instance, `ngram_size=3`would give unigram, trigram and bigram features. ~~int~~ | +| `dropout` | The dropout rate. ~~float~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + + + +### spacy.TextCatBOW.v3 {id="TextCatBOW"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatBOW.v3" +> exclusive_classes = false +> length = 262144 +> ngram_size = 1 +> no_output_layer = false +> nO = null +> ``` + +An n-gram "bag-of-words" model. This architecture should run much faster than +the others, but may not be as accurate, especially if texts are short. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `ngram_size` | Determines the maximum length of the n-grams in the BOW model. For instance, `ngram_size=3` would give unigram, trigram and bigram features. ~~int~~ | +| `no_output_layer` | Whether or not to add an output layer to the model (`Softmax` activation if `exclusive_classes` is `True`, else `Logistic`). ~~bool~~ | +| `length` | The size of the weights vector. The length will be rounded up to the next power of two if it is not a power of two. Defaults to `262144`. ~~int~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + + + +- [TextCatBOW.v1](/api/legacy#TextCatBOW_v1) was not yet resizable. Since v2, + new labels can be added to this component, even after training. +- [TextCatBOW.v1](/api/legacy#TextCatBOW_v1) and + [TextCatBOW.v2](/api/legacy#TextCatBOW_v2) used an erroneous sparse linear + layer that only used a small number of the allocated parameters. +- [TextCatBOW.v1](/api/legacy#TextCatBOW_v1) and + [TextCatBOW.v2](/api/legacy#TextCatBOW_v2) did not have the `length` argument. + + + +### spacy.TextCatParametricAttention.v1 {id="TextCatParametricAttention"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatParametricAttention.v1" +> exclusive_classes = true +> nO = null +> +> [model.tok2vec] +> @architectures = "spacy.Tok2Vec.v2" +> +> [model.tok2vec.embed] +> @architectures = "spacy.MultiHashEmbed.v2" +> width = 64 +> rows = [2000, 2000, 1000, 1000, 1000, 1000] +> attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] +> include_static_vectors = false +> +> [model.tok2vec.encode] +> @architectures = "spacy.MaxoutWindowEncoder.v2" +> width = ${model.tok2vec.embed.width} +> window_size = 1 +> maxout_pieces = 3 +> depth = 2 +> ``` + +A neural network model that is built upon Tok2Vec and uses parametric attention +to attend to tokens that are relevant to text classification. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The `tok2vec` layer to build the neural network upon. ~~Model[List[Doc], List[Floats2d]]~~ | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.TextCatReduce.v1 {id="TextCatReduce"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatReduce.v1" +> exclusive_classes = false +> use_reduce_first = false +> use_reduce_last = false +> use_reduce_max = false +> use_reduce_mean = true +> nO = null +> +> [model.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v2" +> pretrained_vectors = null +> width = 96 +> depth = 4 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +A classifier that pools token hidden representations of each `Doc` using first, +max or mean reduction and then applies a classification layer. Reductions are +concatenated when multiple reductions are used. + + + +`TextCatReduce` is a generalization of the older +[`TextCatCNN`](/api/legacy#TextCatCNN_v2) model. `TextCatCNN` always uses a mean +reduction, whereas `TextCatReduce` also supports first/max reductions. + + + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `use_reduce_first` | Pool by using the hidden representation of the first token of a `Doc`. ~~bool~~ | +| `use_reduce_last` | Pool by using the hidden representation of the last token of a `Doc`. ~~bool~~ | +| `use_reduce_max` | Pool by taking the maximum values of the hidden representations of a `Doc`. ~~bool~~ | +| `use_reduce_mean` | Pool by taking the mean of all hidden representations of a `Doc`. ~~bool~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +## Span classification architectures {id="spancat",source="spacy/ml/models/spancat.py"} + +### spacy.SpanCategorizer.v1 {id="SpanCategorizer"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.SpanCategorizer.v1" +> scorer = {"@layers": "spacy.LinearLogistic.v1"} +> +> [model.reducer] +> @layers = spacy.mean_max_reducer.v1" +> hidden_size = 128 +> +> [model.tok2vec] +> @architectures = "spacy.Tok2Vec.v1" +> +> [model.tok2vec.embed] +> @architectures = "spacy.MultiHashEmbed.v1" +> # ... +> +> [model.tok2vec.encode] +> @architectures = "spacy.MaxoutWindowEncoder.v1" +> # ... +> ``` + +Build a span categorizer model to power a +[`SpanCategorizer`](/api/spancategorizer) component, given a token-to-vector +model, a reducer model to map the sequence of vectors for each span down to a +single vector, and a scorer model to map the vectors to probabilities. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------- | +| `tok2vec` | The token-to-vector model. ~~Model[List[Doc], List[Floats2d]]~~ | +| `reducer` | The reducer model. ~~Model[Ragged, Floats2d]~~ | +| `scorer` | The scorer model. ~~Model[Floats2d, Floats2d]~~ | +| **CREATES** | The model using the architecture. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | + +### spacy.mean_max_reducer.v1 {id="mean_max_reducer"} + +Reduce sequences by concatenating their mean and max pooled vectors, and then +combine the concatenated vectors with a hidden layer. + +| Name | Description | +| ------------- | ------------------------------------- | +| `hidden_size` | The size of the hidden layer. ~~int~~ | + +## Entity linking architectures {id="entitylinker",source="spacy/ml/models/entity_linker.py"} + +An [`EntityLinker`](/api/entitylinker) component disambiguates textual mentions +(tagged as named entities) to unique identifiers, grounding the named entities +into the "real world". This requires 3 main components: + +- A [`KnowledgeBase`](/api/kb) (KB) holding the unique identifiers, potential + synonyms and prior probabilities. +- A candidate generation step to produce a set of likely identifiers, given a + certain textual mention. +- A machine learning [`Model`](https://thinc.ai/docs/api-model) that picks the + most plausible ID from the set of candidates. + +### spacy.EntityLinker.v2 {id="EntityLinker"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.EntityLinker.v2" +> nO = null +> +> [model.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v2" +> pretrained_vectors = null +> width = 96 +> depth = 2 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +The `EntityLinker` model architecture is a Thinc `Model` with a +[`Linear`](https://thinc.ai/api-layers#linear) output layer. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `nO` | Output dimension, determined by the length of the vectors encoding each entity in the KB. If the `nO` dimension is not set, the entity linking component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.EmptyKB.v1 {id="EmptyKB.v1"} + +A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab) +instance. + +| Name | Description | +| ---------------------- | ----------------------------------------------------------------------------------- | +| `entity_vector_length` | The length of the vectors encoding each entity in the KB. Defaults to `64`. ~~int~~ | + +### spacy.EmptyKB.v2 {id="EmptyKB"} + +A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab) +instance. This is the default when a new entity linker component is created. It +returns a `Callable[[Vocab, int], InMemoryLookupKB]`. + +### spacy.KBFromFile.v1 {id="KBFromFile"} + +A function that reads an existing `KnowledgeBase` from file. + +| Name | Description | +| --------- | -------------------------------------------------------- | +| `kb_path` | The location of the KB that was stored to file. ~~Path~~ | + +### spacy.CandidateGenerator.v1 {id="CandidateGenerator"} + +A function that takes as input a [`KnowledgeBase`](/api/kb) and a +[`Span`](/api/span) object denoting a named entity, and returns a list of +plausible [`Candidate`](/api/kb/#candidate) objects. The default +`CandidateGenerator` uses the text of a mention to find its potential aliases in +the `KnowledgeBase`. Note that this function is case-dependent. + +### spacy.CandidateBatchGenerator.v1 {id="CandidateBatchGenerator"} + +A function that takes as input a [`KnowledgeBase`](/api/kb) and an `Iterable` of +[`Span`](/api/span) objects denoting named entities, and returns a list of +plausible [`Candidate`](/api/kb/#candidate) objects per specified +[`Span`](/api/span). The default `CandidateBatchGenerator` uses the text of a +mention to find its potential aliases in the `KnowledgeBase`. Note that this +function is case-dependent. + +## Coreference {id="coref-architectures",tag="experimental"} + +A [`CoreferenceResolver`](/api/coref) component identifies tokens that refer to +the same entity. A [`SpanResolver`](/api/span-resolver) component infers spans +from single tokens. Together these components can be used to reproduce +traditional coreference models. You can also omit the `SpanResolver` if working +with only token-level clusters is acceptable. + +### spacy-experimental.Coref.v1 {id="Coref",tag="experimental"} + +> #### Example Config +> +> ```ini +> +> [model] +> @architectures = "spacy-experimental.Coref.v1" +> distance_embedding_size = 20 +> dropout = 0.3 +> hidden_size = 1024 +> depth = 2 +> antecedent_limit = 50 +> antecedent_batch_size = 512 +> +> [model.tok2vec] +> @architectures = "spacy-transformers.TransformerListener.v1" +> grad_factor = 1.0 +> upstream = "transformer" +> pooling = {"@layers":"reduce_mean.v1"} +> ``` + +The `Coref` model architecture is a Thinc `Model`. + +| Name | Description | +| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `distance_embedding_size` | A representation of the distance between candidates. ~~int~~ | +| `dropout` | The dropout to use internally. Unlike some Thinc models, this has separate dropout for the internal PyTorch layers. ~~float~~ | +| `hidden_size` | Size of the main internal layers. ~~int~~ | +| `depth` | Depth of the internal network. ~~int~~ | +| `antecedent_limit` | How many candidate antecedents to keep after rough scoring. This has a significant effect on memory usage. Typical values would be 50 to 200, or higher for very long documents. ~~int~~ | +| `antecedent_batch_size` | Internal batch size. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy-experimental.SpanResolver.v1 {id="SpanResolver",tag="experimental"} + +> #### Example Config +> +> ```ini +> +> [model] +> @architectures = "spacy-experimental.SpanResolver.v1" +> hidden_size = 1024 +> distance_embedding_size = 64 +> conv_channels = 4 +> window_size = 1 +> max_distance = 128 +> prefix = "coref_head_clusters" +> +> [model.tok2vec] +> @architectures = "spacy-transformers.TransformerListener.v1" +> grad_factor = 1.0 +> upstream = "transformer" +> pooling = {"@layers":"reduce_mean.v1"} +> ``` + +The `SpanResolver` model architecture is a Thinc `Model`. Note that +`MentionClusters` is `List[List[Tuple[int, int]]]`. + +| Name | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `hidden_size` | Size of the main internal layers. ~~int~~ | +| `distance_embedding_size` | A representation of the distance between two candidates. ~~int~~ | +| `conv_channels` | The number of channels in the internal CNN. ~~int~~ | +| `window_size` | The number of neighboring tokens to consider in the internal CNN. `1` means consider one token on each side. ~~int~~ | +| `max_distance` | The longest possible length of a predicted span. ~~int~~ | +| `prefix` | The prefix that indicates spans to use for input data. ~~string~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[MentionClusters]]~~ | diff --git a/website/docs/api/attributeruler.mdx b/website/docs/api/attributeruler.mdx new file mode 100644 index 0000000..c183191 --- /dev/null +++ b/website/docs/api/attributeruler.mdx @@ -0,0 +1,267 @@ +--- +title: AttributeRuler +tag: class +source: spacy/pipeline/attributeruler.py +version: 3 +teaser: 'Pipeline component for rule-based token attribute assignment' +api_string_name: attribute_ruler +api_trainable: false +--- + +The attribute ruler lets you set token attributes for tokens identified by +[`Matcher` patterns](/usage/rule-based-matching#matcher). The attribute ruler is +typically used to handle exceptions for token attributes and to map values +between attributes such as mapping fine-grained POS tags to coarse-grained POS +tags. See the [usage guide](/usage/linguistic-features/#mappings-exceptions) for +examples. + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). + +> #### Example +> +> ```python +> config = {"validate": True} +> nlp.add_pipe("attribute_ruler", config=config) +> ``` + +| Setting | Description | +| ---------- | --------------------------------------------------------------------------------------------- | +| `validate` | Whether patterns should be validated (passed to the `Matcher`). Defaults to `False`. ~~bool~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/attributeruler.py +``` + +## AttributeRuler.\_\_init\_\_ {id="init",tag="method"} + +Initialize the attribute ruler. + +> #### Example +> +> ```python +> # Construction via add_pipe +> ruler = nlp.add_pipe("attribute_ruler") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `vocab` | The shared vocabulary to pass to the matcher. ~~Vocab~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. ~~str~~ | +| _keyword-only_ | | +| `validate` | Whether patterns should be validated (passed to the [`Matcher`](/api/matcher#init)). Defaults to `False`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"tag`", `"pos"`, `"morph"` and `"lemma"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | + +## AttributeRuler.\_\_call\_\_ {id="call",tag="method"} + +Apply the attribute ruler to a `Doc`, setting token attributes for tokens +matched by the provided patterns. + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## AttributeRuler.add {id="add",tag="method"} + +Add patterns to the attribute ruler. The patterns are a list of `Matcher` +patterns and the attributes are a dict of attributes to set on the matched +token. If the pattern matches a span of more than one token, the `index` can be +used to set the attributes for the token at that index in the span. The `index` +may be negative to index from the end of the span. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> patterns = [[{"TAG": "VB"}]] +> attrs = {"POS": "VERB"} +> ruler.add(patterns=patterns, attrs=attrs) +> ``` + +| Name | Description | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `patterns` | The `Matcher` patterns to add. ~~Iterable[List[Dict[Union[int, str], Any]]]~~ | +| `attrs` | The attributes to assign to the target token in the matched span. ~~Dict[str, Any]~~ | +| `index` | The index of the token in the matched span to modify. May be negative to index from the end of the span. Defaults to `0`. ~~int~~ | + +## AttributeRuler.add_patterns {id="add_patterns",tag="method"} + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> patterns = [ +> { +> "patterns": [[{"TAG": "VB"}]], "attrs": {"POS": "VERB"} +> }, +> { +> "patterns": [[{"LOWER": "two"}, {"LOWER": "apples"}]], +> "attrs": {"LEMMA": "apple"}, +> "index": -1 +> }, +> ] +> ruler.add_patterns(patterns) +> ``` + +Add patterns from a list of pattern dicts. Each pattern dict can specify the +keys `"patterns"`, `"attrs"` and `"index"`, which match the arguments of +[`AttributeRuler.add`](/api/attributeruler#add). + +| Name | Description | +| ---------- | -------------------------------------------------------------------------- | +| `patterns` | The patterns to add. ~~Iterable[Dict[str, Union[List[dict], dict, int]]]~~ | + +## AttributeRuler.patterns {id="patterns",tag="property"} + +Get all patterns that have been added to the attribute ruler in the +`patterns_dict` format accepted by +[`AttributeRuler.add_patterns`](/api/attributeruler#add_patterns). + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| **RETURNS** | The patterns added to the attribute ruler. ~~List[Dict[str, Union[List[dict], dict, int]]]~~ | + +## AttributeRuler.initialize {id="initialize",tag="method"} + +Initialize the component with data and used before training to load in rules +from a file. This method is typically called by +[`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> ruler.initialize(lambda: [], nlp=nlp, patterns=patterns) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.attribute_ruler] +> +> [initialize.components.attribute_ruler.patterns] +> @readers = "srsly.read_json.v1" +> path = "corpus/attribute_ruler_patterns.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects (the training data). Not used by this component. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `patterns` | A list of pattern dicts with the keys as the arguments to [`AttributeRuler.add`](/api/attributeruler#add) (`patterns`/`attrs`/`index`) to add as patterns. Defaults to `None`. ~~Optional[Iterable[Dict[str, Union[List[dict], dict, int]]]]~~ | +| `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | +| `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]]~~ | + +## AttributeRuler.load_from_tag_map {id="load_from_tag_map",tag="method"} + +Load attribute ruler patterns from a tag map. + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. ~~Dict[str, Dict[Union[int, str], Union[int, str]]]~~ | + +## AttributeRuler.load_from_morph_rules {id="load_from_morph_rules",tag="method"} + +Load attribute ruler patterns from morph rules. + +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. ~~Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | + +## AttributeRuler.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> ruler.to_disk("/path/to/attribute_ruler") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## AttributeRuler.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> ruler.from_disk("/path/to/attribute_ruler") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `AttributeRuler` object. ~~AttributeRuler~~ | + +## AttributeRuler.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("attribute_ruler") +> ruler = ruler.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `AttributeRuler` object. ~~bytes~~ | + +## AttributeRuler.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler_bytes = ruler.to_bytes() +> ruler = nlp.add_pipe("attribute_ruler") +> ruler.from_bytes(ruler_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `AttributeRuler` object. ~~AttributeRuler~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = ruler.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ---------- | --------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `patterns` | The `Matcher` patterns. You usually don't want to exclude this. | +| `attrs` | The attributes to set. You usually don't want to exclude this. | +| `indices` | The token indices. You usually don't want to exclude this. | diff --git a/website/docs/api/attributes.mdx b/website/docs/api/attributes.mdx new file mode 100644 index 0000000..9cb76ac --- /dev/null +++ b/website/docs/api/attributes.mdx @@ -0,0 +1,77 @@ +--- +title: Attributes +teaser: Token attributes +source: spacy/attrs.pyx +--- + +[Token](/api/token) attributes are specified using internal IDs in many places +including: + +- [`Matcher` patterns](/api/matcher#patterns), +- [`Doc.to_array`](/api/doc#to_array) and + [`Doc.from_array`](/api/doc#from_array) +- [`Doc.has_annotation`](/api/doc#has_annotation) +- [`MultiHashEmbed`](/api/architectures#MultiHashEmbed) Tok2Vec architecture + `attrs` + +> ```python +> import spacy +> from spacy.attrs import DEP +> +> nlp = spacy.blank("en") +> doc = nlp("There are many attributes.") +> +> # DEP always has the same internal value +> assert DEP == 76 +> +> # "DEP" is automatically converted to DEP +> assert DEP == nlp.vocab.strings["DEP"] +> assert doc.has_annotation(DEP) == doc.has_annotation("DEP") +> +> # look up IDs in spacy.attrs.IDS +> from spacy.attrs import IDS +> assert IDS["DEP"] == DEP +> ``` + +All methods automatically convert between the string version of an ID (`"DEP"`) +and the internal integer symbols (`DEP`). The internal IDs can be imported from +`spacy.attrs` or retrieved from the [`StringStore`](/api/stringstore). A map +from string attribute names to internal attribute IDs is stored in +`spacy.attrs.IDS`. + +The corresponding [`Token` object attributes](/api/token#attributes) can be +accessed using the same names in lowercase, e.g. `token.orth` or `token.length`. +For attributes that represent string values, the internal integer ID is accessed +as `Token.attr`, e.g. `token.dep`, while the string value can be retrieved by +appending `_` as in `token.dep_`. + +| Attribute | Description | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `DEP` | The token's dependency label. ~~str~~ | +| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | +| `ENT_IOB` | The IOB part of the token's entity tag. Uses custom integer values rather than the string store: unset is `0`, `I` is `1`, `O` is `2`, and `B` is `3`. ~~str~~ | +| `ENT_KB_ID` | The token's entity knowledge base ID. ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `IS_ALPHA` | Token text consists of alphabetic characters. ~~bool~~ | +| `IS_ASCII` | Token text consists of ASCII characters. ~~bool~~ | +| `IS_DIGIT` | Token text consists of digits. ~~bool~~ | +| `IS_LOWER` | Token text is in lowercase. ~~bool~~ | +| `IS_PUNCT` | Token is punctuation. ~~bool~~ | +| `IS_SPACE` | Token is whitespace. ~~bool~~ | +| `IS_STOP` | Token is a stop word. ~~bool~~ | +| `IS_TITLE` | Token text is in titlecase. ~~bool~~ | +| `IS_UPPER` | Token text is in uppercase. ~~bool~~ | +| `LEMMA` | The token's lemma. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `LIKE_EMAIL` | Token text resembles an email address. ~~bool~~ | +| `LIKE_NUM` | Token text resembles a number. ~~bool~~ | +| `LIKE_URL` | Token text resembles a URL. ~~bool~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `MORPH` | The token's morphological analysis. ~~MorphAnalysis~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `POS` | The token's universal part of speech (UPOS). ~~str~~ | +| `SENT_START` | Token is start of sentence. ~~bool~~ | +| `SHAPE` | The token's shape. ~~str~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `TAG` | The token's fine-grained part of speech. ~~str~~ | diff --git a/website/docs/api/basevectors.mdx b/website/docs/api/basevectors.mdx new file mode 100644 index 0000000..993b9a3 --- /dev/null +++ b/website/docs/api/basevectors.mdx @@ -0,0 +1,143 @@ +--- +title: BaseVectors +teaser: Abstract class for word vectors +tag: class +source: spacy/vectors.pyx +version: 3.7 +--- + +`BaseVectors` is an abstract class to support the development of custom vectors +implementations. + +For use in training with [`StaticVectors`](/api/architectures#staticvectors), +`get_batch` must be implemented. For improved performance, use efficient +batching in `get_batch` and implement `to_ops` to copy the vector data to the +current device. See an example custom implementation for +[BPEmb subword embeddings](/usage/embeddings-transformers#custom-vectors). + +## BaseVectors.\_\_init\_\_ {id="init",tag="method"} + +Create a new vector store. + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `strings` | The string store. A new string store is created if one is not provided. Defaults to `None`. ~~Optional[StringStore]~~ | + +## BaseVectors.\_\_getitem\_\_ {id="getitem",tag="method"} + +Get a vector by key. If the key is not found in the table, a `KeyError` should +be raised. + +| Name | Description | +| ----------- | ---------------------------------------------------------------- | +| `key` | The key to get the vector for. ~~Union[int, str]~~ | +| **RETURNS** | The vector for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## BaseVectors.\_\_len\_\_ {id="len",tag="method"} + +Return the number of vectors in the table. + +| Name | Description | +| ----------- | ------------------------------------------- | +| **RETURNS** | The number of vectors in the table. ~~int~~ | + +## BaseVectors.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether there is a vector entry for the given key. + +| Name | Description | +| ----------- | -------------------------------------------- | +| `key` | The key to check. ~~int~~ | +| **RETURNS** | Whether the key has a vector entry. ~~bool~~ | + +## BaseVectors.add {id="add",tag="method"} + +Add a key to the table, if possible. If no keys can be added, return `-1`. + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------- | +| `key` | The key to add. ~~Union[str, int]~~ | +| **RETURNS** | The row the vector was added to, or `-1` if the operation is not supported. ~~int~~ | + +## BaseVectors.shape {id="shape",tag="property"} + +Get `(rows, dims)` tuples of number of rows and number of dimensions in the +vector table. + +| Name | Description | +| ----------- | ------------------------------------------ | +| **RETURNS** | A `(rows, dims)` pair. ~~Tuple[int, int]~~ | + +## BaseVectors.size {id="size",tag="property"} + +The vector size, i.e. `rows * dims`. + +| Name | Description | +| ----------- | ------------------------ | +| **RETURNS** | The vector size. ~~int~~ | + +## BaseVectors.is_full {id="is_full",tag="property"} + +Whether the vectors table is full and no slots are available for new keys. + +| Name | Description | +| ----------- | ------------------------------------------- | +| **RETURNS** | Whether the vectors table is full. ~~bool~~ | + +## BaseVectors.get_batch {id="get_batch",tag="method",version="3.2"} + +Get the vectors for the provided keys efficiently as a batch. Required to use +the vectors with [`StaticVectors`](/api/architectures#StaticVectors) for +training. + +| Name | Description | +| ------ | --------------------------------------- | +| `keys` | The keys. ~~Iterable[Union[int, str]]~~ | + +## BaseVectors.to_ops {id="to_ops",tag="method"} + +Dummy method. Implement this to change the embedding matrix to use different +Thinc ops. + +| Name | Description | +| ----- | -------------------------------------------------------- | +| `ops` | The Thinc ops to switch the embedding matrix to. ~~Ops~~ | + +## BaseVectors.to_disk {id="to_disk",tag="method"} + +Dummy method to allow serialization. Implement to save vector data with the +pipeline. + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## BaseVectors.from_disk {id="from_disk",tag="method"} + +Dummy method to allow serialization. Implement to load vector data from a saved +pipeline. + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified vectors object. ~~BaseVectors~~ | + +## BaseVectors.to_bytes {id="to_bytes",tag="method"} + +Dummy method to allow serialization. Implement to serialize vector data to a +binary string. + +| Name | Description | +| ----------- | ---------------------------------------------------- | +| **RETURNS** | The serialized form of the vectors object. ~~bytes~~ | + +## BaseVectors.from_bytes {id="from_bytes",tag="method"} + +Dummy method to allow serialization. Implement to load vector data from a binary +string. + +| Name | Description | +| ----------- | ----------------------------------- | +| `data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The vectors object. ~~BaseVectors~~ | diff --git a/website/docs/api/cli.mdx b/website/docs/api/cli.mdx new file mode 100644 index 0000000..add6b14 --- /dev/null +++ b/website/docs/api/cli.mdx @@ -0,0 +1,1747 @@ +--- +title: Command Line Interface +teaser: Download, train and package pipelines, and debug spaCy +source: spacy/cli +menu: + - ['download', 'download'] + - ['info', 'info'] + - ['validate', 'validate'] + - ['init', 'init'] + - ['find-function', 'find-function'] + - ['convert', 'convert'] + - ['debug', 'debug'] + - ['train', 'train'] + - ['pretrain', 'pretrain'] + - ['evaluate', 'evaluate'] + - ['benchmark', 'benchmark'] + - ['apply', 'apply'] + - ['find-threshold', 'find-threshold'] + - ['assemble', 'assemble'] + - ['package', 'package'] + - ['project', 'project'] + - ['huggingface-hub', 'huggingface-hub'] +--- + +spaCy's CLI provides a range of helpful commands for downloading and training +pipelines, converting data and debugging your config, data and installation. For +a list of available commands, you can type `python -m spacy --help`. You can +also add the `--help` flag to any command or subcommand to see the description, +available arguments and usage. + +## download {id="download",tag="command"} + +Download [trained pipelines](/usage/models) for spaCy. The downloader finds the +best-matching compatible version and uses `pip install` to download the Python +package. Direct downloads don't perform any compatibility checks and require the +pipeline name to be specified with its version (e.g. `en_core_web_sm-3.0.0`). + +> #### Downloading best practices +> +> The `download` command is mostly intended as a convenient, interactive wrapper +> – it performs compatibility checks and prints detailed messages in case things +> go wrong. It's **not recommended** to use this command as part of an automated +> process. If you know which package your project needs, you should consider a +> [direct download via pip](/usage/models#download-pip), or uploading the +> package to a local PyPi installation and fetching it straight from there. This +> will also allow you to add it as a versioned package dependency to your +> project. + +```bash +$ python -m spacy download [model] [--direct] [--sdist] [pip_args] [--url url] +``` + +| Name | Description | +| ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | Pipeline package name, e.g. [`en_core_web_sm`](/models/en#en_core_web_sm). ~~str (positional)~~ | +| `--direct`, `-D` | Force direct download of exact package version. ~~bool (flag)~~ | +| `--sdist`, `-S` 3 | Download the source package (`.tar.gz` archive) instead of the default pre-built binary wheel. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| pip args | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ | +| **CREATES** | The installed pipeline package in your `site-packages` directory. | +| `--url`, `-U` | Download from a mirror repository at the given url | + +## info {id="info",tag="command"} + +Print information about your spaCy installation, trained pipelines and local +setup, and generate [Markdown](https://en.wikipedia.org/wiki/Markdown)-formatted +markup to copy-paste into +[GitHub issues](https://github.com/explosion/spaCy/issues). + +```bash +$ python -m spacy info [--markdown] [--silent] [--exclude] +``` + +> #### Example +> +> ```bash +> $ python -m spacy info en_core_web_lg --markdown +> ``` + +```bash +$ python -m spacy info [model] [--markdown] [--silent] [--exclude] +``` + +| Name | Description | +| -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ | +| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ | +| `--silent`, `-s` | Don't print anything, just return the values. ~~bool (flag)~~ | +| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ | +| `--url`, `-u` 3.5.0 | Print the URL to download the most recent compatible version of the pipeline. Requires a pipeline name. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Information about your spaCy installation. | + +## validate {id="validate",version="2",tag="command"} + +Find all trained pipeline packages installed in the current environment and +check whether they are compatible with the currently installed version of spaCy. +Should be run after upgrading spaCy via `pip install -U spacy` to ensure that +all installed packages can be used with the new version. It will show a list of +packages and their installed versions. If any package is out of date, the latest +compatible versions and command for updating are shown. + +> #### Automated validation +> +> You can also use the `validate` command as part of your build process or test +> suite, to ensure all packages are up to date before proceeding. If +> incompatible packages are found, it will return `1`. + +```bash +$ python -m spacy validate +``` + +| Name | Description | +| ---------- | -------------------------------------------------------------------- | +| **PRINTS** | Details about the compatibility of your installed pipeline packages. | + +## init {id="init",version="3"} + +The `spacy init` CLI includes helpful commands for initializing training config +files and pipeline directories. + +### init config {id="init-config",version="3",tag="command"} + +Initialize and save a [`config.cfg` file](/usage/training#config) using the +**recommended settings** for your use case. It works just like the +[quickstart widget](/usage/training#quickstart), only that it also auto-fills +all default values and exports a [training](/usage/training#config)-ready +config. The settings you specify will impact the suggested model architectures +and pipeline setup, as well as the hyperparameters. You can also adjust and +customize those settings in your config file later. + +> #### Example +> +> ```bash +> $ python -m spacy init config config.cfg --lang en --pipeline ner,textcat --optimize accuracy +> ``` + +```bash +$ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [--gpu] [--pretraining] [--force] +``` + +| Name | Description | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `output_file` | Path to output `.cfg` file or `-` to write the config to stdout (so you can pipe it forward to a file or to the `train` command). Note that if you're writing to stdout, no additional logging info is printed. ~~Path (positional)~~ | +| `--lang`, `-l` | Optional code of the [language](/usage/models#languages) to use. Defaults to `"en"`. ~~str (option)~~ | +| `--pipeline`, `-p` | Comma-separated list of trainable [pipeline components](/usage/processing-pipelines#built-in) to include. Defaults to `"tagger,parser,ner"`. ~~str (option)~~ | +| `--optimize`, `-o` | `"efficiency"` or `"accuracy"`. Whether to optimize for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). This will impact the choice of architecture, pretrained weights and related hyperparameters. Defaults to `"efficiency"`. ~~str (option)~~ | +| `--gpu`, `-G` | Whether the model can run on GPU. This will impact the choice of architecture, pretrained weights and related hyperparameters. ~~bool (flag)~~ | +| `--pretraining`, `-pt` | Include config for pretraining (with [`spacy pretrain`](/api/cli#pretrain)). Defaults to `False`. ~~bool (flag)~~ | +| `--force`, `-f` | Force overwriting the output file if it already exists. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | The config file for training. | + +### init fill-config {id="init-fill-config",version="3"} + +Auto-fill a partial [.cfg file](/usage/training#config) with **all default +values**, e.g. a config generated with the +[quickstart widget](/usage/training#quickstart). Config files used for training +should always be complete and not contain any hidden defaults or missing values, +so this command helps you create your final training config. In order to find +the available settings and defaults, all functions referenced in the config will +be created, and their signatures are used to find the defaults. If your config +contains a problem that can't be resolved automatically, spaCy will show you a +validation error with more details. + +> #### Example +> +> ```bash +> $ python -m spacy init fill-config base.cfg config.cfg --diff +> ``` +> +> #### Example diff +> +> ![Screenshot of visual diff in terminal](/images/cli_init_fill-config_diff.jpg) + +```bash +$ python -m spacy init fill-config [base_path] [output_file] [--diff] +``` + +| Name | Description | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `base_path` | Path to base config to fill, e.g. generated by the [quickstart widget](/usage/training#quickstart). ~~Path (positional)~~ | +| `output_file` | Path to output `.cfg` file or "-" to write to stdout so you can pipe it to a file. Defaults to "-" (stdout). ~~Path (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--pretraining`, `-pt` | Include config for pretraining (with [`spacy pretrain`](/api/cli#pretrain)). Defaults to `False`. ~~bool (flag)~~ | +| `--diff`, `-D` | Print a visual diff highlighting the changes. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Complete and auto-filled config file for training. | + +### init fill-curated-transformer {id="init-fill-curated-transformer",version="3.7",tag="command"} + +Auto-fill the Hugging Face model hyperpameters and loader parameters of a +[Curated Transformer](/api/curatedtransformer) pipeline component in a +[.cfg file](/usage/training#config). The name and revision of the +[Hugging Face model](https://huggingface.co/models) can either be passed as +command-line arguments or read from the +`initialize.components.transformer.encoder_loader` config section. + +```bash +$ python -m spacy init fill-curated-transformer [base_path] [output_file] [--model-name] [--model-revision] [--pipe-name] [--code] +``` + +| Name | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `base_path` | Path to base config to fill, e.g. generated by the [quickstart widget](/usage/training#quickstart). ~~Path (positional)~~ | +| `output_file` | Path to output `.cfg` file or "-" to write to stdout so you can pipe it to a file. Defaults to "-" (stdout). ~~Path (positional)~~ | +| `--model-name`, `-m` | Name of the Hugging Face model. Defaults to the model name from the encoder loader config. ~~Optional[str] (option)~~ | +| `--model-revision`, `-r` | Revision of the Hugging Face model. Defaults to `main`. ~~Optional[str] (option)~~ | +| `--pipe-name`, `-n` | Name of the Curated Transformer pipe whose config is to be filled. Defaults to the first transformer pipe. ~~Optional[str] (option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| **CREATES** | Complete and auto-filled config file for training. | + +### init vectors {id="init-vectors",version="3",tag="command"} + +Convert [word vectors](/usage/linguistic-features#vectors-similarity) for use +with spaCy. Will export an `nlp` object that you can use in the +[`[initialize]`](/api/data-formats#config-initialize) block of your config to +initialize a model with vectors. See the usage guide on +[static vectors](/usage/embeddings-transformers#static-vectors) for details on +how to use vectors in your model. + + + +This functionality was previously available as part of the command `init-model`. + + + +```bash +$ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--truncate] [--name] [--verbose] +``` + +| Name | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lang` | Pipeline language. Two-letter [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) or three-letter [ISO 639-3 code](https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes), such as `en` and `eng`. ~~str (positional)~~ | +| `vectors_loc` | Location of vectors. Should be a file where the first row contains the dimensions of the vectors, followed by a space-separated Word2Vec table. File can be provided in `.txt` format or as a zipped text file in `.zip` or `.tar.gz` format. ~~Path (positional)~~ | +| `output_dir` | Pipeline output directory. Will be created if it doesn't exist. ~~Path (positional)~~ | +| `--truncate`, `-t` | Number of vectors to truncate to when reading in vectors file. Defaults to `0` for no truncation. ~~int (option)~~ | +| `--prune`, `-p` | Number of vectors to prune the vocabulary to. Defaults to `-1` for no pruning. ~~int (option)~~ | +| `--mode`, `-m` | Vectors mode: `default` or [`floret`](https://github.com/explosion/floret). Defaults to `default`. ~~str \(option)~~ | +| `--attr`, `-a` | Token attribute to use for vectors, e.g. `LOWER` or `NORM`) Defaults to `ORTH`. ~~str \(option)~~ | +| `--name`, `-n` | Name to assign to the word vectors in the `meta.json`, e.g. `en_core_web_md.vectors`. ~~Optional[str] \(option)~~ | +| `--verbose`, `-V` | Print additional information and explanations. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A spaCy pipeline directory containing the vocab and vectors. | + +### init labels {id="init-labels",version="3",tag="command"} + +Generate JSON files for the labels in the data. This helps speed up the training +process, since spaCy won't have to preprocess the data to extract the labels. +After generating the labels, you can provide them to components that accept a +`labels` argument on initialization via the +[`[initialize]`](/api/data-formats#config-initialize) block of your config. + +> #### Example config +> +> ```ini +> [initialize.components.ner] +> +> [initialize.components.ner.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/ner.json +> ``` + +```bash +$ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [--gpu-id] [overrides] +``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `output_path` | Output directory for the label files. Will create one JSON file per component. ~~Path (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--verbose`, `-V` | Show more detailed messages for debugging purposes. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **CREATES** | The label files. | + +## find-function {id="find-function",version="3.7",tag="command"} + +Find the module, path and line number to the file for a given registered +function. This functionality is helpful to understand where registered +functions, as used in the config file, are defined. + +```bash +$ python -m spacy find-function [func_name] [--registry] +``` + +> #### Example +> +> ```bash +> $ python -m spacy find-function spacy.TextCatBOW.v1 +> ``` + +| Name | Description | +| ------------------ | ----------------------------------------------------- | +| `func_name` | Name of the registered function. ~~str (positional)~~ | +| `--registry`, `-r` | Name of the catalogue registry. ~~str (option)~~ | + +## convert {id="convert",tag="command"} + +Convert files into spaCy's +[binary training data format](/api/data-formats#binary-training), a serialized +[`DocBin`](/api/docbin), for use with the `train` command and other experiment +management functions. The converter can be specified on the command line, or +chosen based on the file extension of the input file. + +```bash +$ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] [--n-sents] [--seg-sents] [--base] [--morphology] [--merge-subtokens] [--ner-map] [--lang] +``` + +| Name | Description | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `input_path` | Input file or directory. ~~Path (positional)~~ | +| `output_dir` | Output directory for converted file. Defaults to `"-"`, meaning data will be written to `stdout`. ~~Optional[Path] \(option)~~ | +| `--converter`, `-c` | Name of converter to use (see below). ~~str (option)~~ | +| `--file-type`, `-t` | Type of file to create. Either `spacy` (default) for binary [`DocBin`](/api/docbin) data or `json` for v2.x JSON format. ~~str (option)~~ | +| `--n-sents`, `-n` | Number of sentences per document. Supported for: `conll`, `conllu`, `iob`, `ner` ~~int (option)~~ | +| `--seg-sents`, `-s` | Segment sentences. Supported for: `conll`, `ner` ~~bool (flag)~~ | +| `--base`, `-b`, `--model` | Trained spaCy pipeline for sentence segmentation to use as base (for `--seg-sents`). ~~Optional[str] (option)~~ | +| `--morphology`, `-m` | Enable appending morphology to tags. Supported for: `conllu` ~~bool (flag)~~ | +| `--merge-subtokens`, `-T` | Merge CoNLL-U subtokens ~~bool (flag)~~ | +| `--ner-map`, `-nm` | NER tag mapping (as JSON-encoded dict of entity types). Supported for: `conllu` ~~Optional[Path] (option)~~ | +| `--lang`, `-l` | Language code (if tokenizer required). ~~Optional[str] \(option)~~ | +| `--concatenate`, `-C` | Concatenate output to a single file ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). | + +### Converters {id="converters"} + +| ID | Description | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `auto` | Automatically pick converter based on file extension and file content (default). | +| `json` | JSON-formatted training data used in spaCy v2.x. | +| `conllu` | Universal Dependencies `.conllu` format. | +| `ner` / `conll` | NER with IOB/IOB2/BILUO tags, one token per line with columns separated by whitespace. The first column is the token and the final column is the NER tag. Sentences are separated by blank lines and documents are separated by the line `-DOCSTART- -X- O O`. Supports CoNLL 2003 NER format. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | +| `iob` | NER with IOB/IOB2/BILUO tags, one sentence per line with tokens separated by whitespace and annotation separated by `\|`, either `word\|B-ENT`or`word\|POS\|B-ENT`. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | + +## debug {id="debug",version="3"} + +The `spacy debug` CLI includes helpful commands for debugging and profiling your +configs, data and implementations. + +### debug config {id="debug-config",version="3",tag="command"} + +Debug a [`config.cfg` file](/usage/training#config) and show validation errors. +The command will create all objects in the tree and validate them. Note that +some config validation errors are blocking and will prevent the rest of the +config from being resolved. This means that you may not see all validation +errors at once and some issues are only shown once previous errors have been +fixed. To auto-fill a partial config and save the result, you can use the +[`init fill-config`](/api/cli#init-fill-config) command. + +```bash +$ python -m spacy debug config [config_path] [--code] [--show-functions] [--show-variables] [overrides] +``` + +> #### Example +> +> ```bash +> $ python -m spacy debug config config.cfg +> ``` + + + +``` +✘ Config validation error +dropout field required +optimizer field required +optimize extra fields not permitted + +{'seed': 0, 'accumulate_gradient': 1, 'dev_corpus': 'corpora.dev', 'train_corpus': 'corpora.train', 'gpu_allocator': None, 'patience': 1600, 'max_epochs': 0, 'max_steps': 20000, 'eval_frequency': 200, 'frozen_components': [], 'optimize': None, 'before_to_disk': None, 'batcher': {'@batchers': 'spacy.batch_by_words.v1', 'discard_oversize': False, 'tolerance': 0.2, 'get_length': None, 'size': {'@schedules': 'compounding.v1', 'start': 100, 'stop': 1000, 'compound': 1.001, 't': 0.0}}, 'logger': {'@loggers': 'spacy.ConsoleLogger.v1', 'progress_bar': False}, 'score_weights': {'tag_acc': 0.5, 'dep_uas': 0.25, 'dep_las': 0.25, 'sents_f': 0.0}} + +If your config contains missing values, you can run the 'init fill-config' +command to fill in all the defaults, if possible: + +python -m spacy init fill-config tmp/starter-config_invalid.cfg tmp/starter-config_invalid.cfg +``` + + + + + +```bash +$ python -m spacy debug config ./config.cfg --show-functions --show-variables +``` + +``` +============================= Config validation ============================= +✔ Config is valid + +=============================== Variables (6) =============================== + +Variable Value +----------------------------------------- ---------------------------------- +${components.tok2vec.model.encode.width} 96 +${paths.dev} 'hello' +${paths.init_tok2vec} None +${paths.raw} None +${paths.train} '' +${system.seed} 0 + + +========================= Registered functions (17) ========================= +ℹ [nlp.tokenizer] +Registry @tokenizers +Name spacy.Tokenizer.v1 +Module spacy.language +File /path/to/spacy/language.py (line 64) +ℹ [components.ner.model] +Registry @architectures +Name spacy.TransitionBasedParser.v1 +Module spacy.ml.models.parser +File /path/to/spacy/ml/models/parser.py (line 11) +ℹ [components.ner.model.tok2vec] +Registry @architectures +Name spacy.Tok2VecListener.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 16) +ℹ [components.parser.model] +Registry @architectures +Name spacy.TransitionBasedParser.v1 +Module spacy.ml.models.parser +File /path/to/spacy/ml/models/parser.py (line 11) +ℹ [components.parser.model.tok2vec] +Registry @architectures +Name spacy.Tok2VecListener.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 16) +ℹ [components.tagger.model] +Registry @architectures +Name spacy.Tagger.v1 +Module spacy.ml.models.tagger +File /path/to/spacy/ml/models/tagger.py (line 9) +ℹ [components.tagger.model.tok2vec] +Registry @architectures +Name spacy.Tok2VecListener.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 16) +ℹ [components.tok2vec.model] +Registry @architectures +Name spacy.Tok2Vec.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 72) +ℹ [components.tok2vec.model.embed] +Registry @architectures +Name spacy.MultiHashEmbed.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 93) +ℹ [components.tok2vec.model.encode] +Registry @architectures +Name spacy.MaxoutWindowEncoder.v1 +Module spacy.ml.models.tok2vec +File /path/to/spacy/ml/models/tok2vec.py (line 207) +ℹ [corpora.dev] +Registry @readers +Name spacy.Corpus.v1 +Module spacy.training.corpus +File /path/to/spacy/training/corpus.py (line 18) +ℹ [corpora.train] +Registry @readers +Name spacy.Corpus.v1 +Module spacy.training.corpus +File /path/to/spacy/training/corpus.py (line 18) +ℹ [training.logger] +Registry @loggers +Name spacy.ConsoleLogger.v1 +Module spacy.training.loggers +File /path/to/spacy/training/loggers.py (line 8) +ℹ [training.batcher] +Registry @batchers +Name spacy.batch_by_words.v1 +Module spacy.training.batchers +File /path/to/spacy/training/batchers.py (line 49) +ℹ [training.batcher.size] +Registry @schedules +Name compounding.v1 +Module thinc.schedules +File /path/to/thinc/thinc/schedules.py (line 43) +ℹ [training.optimizer] +Registry @optimizers +Name Adam.v1 +Module thinc.optimizers +File /path/to/thinc/thinc/optimizers.py (line 58) +ℹ [training.optimizer.learn_rate] +Registry @schedules +Name warmup_linear.v1 +Module thinc.schedules +File /path/to/thinc/thinc/schedules.py (line 91) +``` + + + +| Name | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--show-functions`, `-F` | Show an overview of all registered function blocks used in the config and where those functions come from, including the module name, Python file and line number. ~~bool (flag)~~ | +| `--show-variables`, `-V` | Show an overview of all variables referenced in the config, e.g. `${paths.train}` and their values that will be used. This also reflects any config overrides provided on the CLI, e.g. `--paths.train /path`. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **PRINTS** | Config validation errors, if available. | + +### debug data {id="debug-data",tag="command"} + +Analyze, debug and validate your training and development data. Get useful +stats, and find problems like invalid entity annotations, cyclic dependencies, +low data labels and more. + + + +The `debug data` command is now available as a subcommand of `spacy debug`. It +takes the same arguments as `train` and reads settings off the +[`config.cfg` file](/usage/training#config) and optional +[overrides](/usage/training#config-overrides) on the CLI. + + + + + +If your pipeline contains a `spancat` component, then this command will also +report span characteristics such as the average span length and the span (or +span boundary) distinctiveness. The distinctiveness measure shows how different +the tokens are with respect to the rest of the corpus using the KL-divergence of +the token distributions. To learn more, you can check out Papay et al.'s work on +[_Dissecting Span Identification Tasks with Performance Prediction_ (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). + + + +```bash +$ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides] +``` + +> #### Example +> +> ```bash +> $ python -m spacy debug data ./config.cfg +> ``` + + + +``` +=========================== Data format validation =========================== +✔ Corpus is loadable +✔ Pipeline can be initialized with data + +=============================== Training stats =============================== +Training pipeline: tagger, parser, ner +Starting with blank model 'en' +18127 training docs +2939 evaluation docs +⚠ 34 training examples also in evaluation data + +============================== Vocab & Vectors ============================== +ℹ 2083156 total words in the data (56962 unique) +⚠ 13020 misaligned tokens in the training data +⚠ 2423 misaligned tokens in the dev data +10 most common words: 'the' (98429), ',' (91756), '.' (87073), 'to' (50058), +'of' (49559), 'and' (44416), 'a' (34010), 'in' (31424), 'that' (22792), 'is' +(18952) +ℹ No word vectors present in the model + +========================== Named Entity Recognition ========================== +ℹ 18 new labels, 0 existing labels +528978 missing values (tokens with '-' label) +New: 'ORG' (23860), 'PERSON' (21395), 'GPE' (21193), 'DATE' (18080), 'CARDINAL' +(10490), 'NORP' (9033), 'MONEY' (5164), 'PERCENT' (3761), 'ORDINAL' (2122), +'LOC' (2113), 'TIME' (1616), 'WORK_OF_ART' (1229), 'QUANTITY' (1150), 'FAC' +(1134), 'EVENT' (974), 'PRODUCT' (935), 'LAW' (444), 'LANGUAGE' (338) +✔ Good amount of examples for all labels +✔ Examples without occurrences available for all labels +✔ No entities consisting of or starting/ending with whitespace + +=========================== Part-of-speech Tagging =========================== +ℹ 49 labels in data +'NN' (266331), 'IN' (227365), 'DT' (185600), 'NNP' (164404), 'JJ' (119830), +'NNS' (110957), '.' (101482), ',' (92476), 'RB' (90090), 'PRP' (90081), 'VB' +(74538), 'VBD' (68199), 'CC' (62862), 'VBZ' (50712), 'VBP' (43420), 'VBN' +(42193), 'CD' (40326), 'VBG' (34764), 'TO' (31085), 'MD' (25863), 'PRP$' +(23335), 'HYPH' (13833), 'POS' (13427), 'UH' (13322), 'WP' (10423), 'WDT' +(9850), 'RP' (8230), 'WRB' (8201), ':' (8168), '''' (7392), '``' (6984), 'NNPS' +(5817), 'JJR' (5689), '$' (3710), 'EX' (3465), 'JJS' (3118), 'RBR' (2872), +'-RRB-' (2825), '-LRB-' (2788), 'PDT' (2078), 'XX' (1316), 'RBS' (1142), 'FW' +(794), 'NFP' (557), 'SYM' (440), 'WP$' (294), 'LS' (293), 'ADD' (191), 'AFX' +(24) + +============================= Dependency Parsing ============================= +ℹ Found 111703 sentences with an average length of 18.6 words. +ℹ Found 2251 nonprojective train sentences +ℹ Found 303 nonprojective dev sentences +ℹ 47 labels in train data +ℹ 211 labels in projectivized train data +'punct' (236796), 'prep' (188853), 'pobj' (182533), 'det' (172674), 'nsubj' +(169481), 'compound' (116142), 'ROOT' (111697), 'amod' (107945), 'dobj' (93540), +'aux' (86802), 'advmod' (86197), 'cc' (62679), 'conj' (59575), 'poss' (36449), +'ccomp' (36343), 'advcl' (29017), 'mark' (27990), 'nummod' (24582), 'relcl' +(21359), 'xcomp' (21081), 'attr' (18347), 'npadvmod' (17740), 'acomp' (17204), +'auxpass' (15639), 'appos' (15368), 'neg' (15266), 'nsubjpass' (13922), 'case' +(13408), 'acl' (12574), 'pcomp' (10340), 'nmod' (9736), 'intj' (9285), 'prt' +(8196), 'quantmod' (7403), 'dep' (4300), 'dative' (4091), 'agent' (3908), 'expl' +(3456), 'parataxis' (3099), 'oprd' (2326), 'predet' (1946), 'csubj' (1494), +'subtok' (1147), 'preconj' (692), 'meta' (469), 'csubjpass' (64), 'iobj' (1) +⚠ Low number of examples for label 'iobj' (1) +⚠ Low number of examples for 130 labels in the projectivized dependency +trees used for training. You may want to projectivize labels such as punct +before training in order to improve parser performance. +⚠ Projectivized labels with low numbers of examples: appos||attr: 12 +advmod||dobj: 13 prep||ccomp: 12 nsubjpass||ccomp: 15 pcomp||prep: 14 +amod||dobj: 9 attr||xcomp: 14 nmod||nsubj: 17 prep||advcl: 2 prep||prep: 5 +nsubj||conj: 12 advcl||advmod: 18 ccomp||advmod: 11 ccomp||pcomp: 5 acl||pobj: +10 npadvmod||acomp: 7 dobj||pcomp: 14 nsubjpass||pcomp: 1 nmod||pobj: 8 +amod||attr: 6 nmod||dobj: 12 aux||conj: 1 neg||conj: 1 dative||xcomp: 11 +pobj||dative: 3 xcomp||acomp: 19 advcl||pobj: 2 nsubj||advcl: 2 csubj||ccomp: 1 +advcl||acl: 1 relcl||nmod: 2 dobj||advcl: 10 advmod||advcl: 3 nmod||nsubjpass: 6 +amod||pobj: 5 cc||neg: 1 attr||ccomp: 16 advcl||xcomp: 3 nmod||attr: 4 +advcl||nsubjpass: 5 advcl||ccomp: 4 ccomp||conj: 1 punct||acl: 1 meta||acl: 1 +parataxis||acl: 1 prep||acl: 1 amod||nsubj: 7 ccomp||ccomp: 3 acomp||xcomp: 5 +dobj||acl: 5 prep||oprd: 6 advmod||acl: 2 dative||advcl: 1 pobj||agent: 5 +xcomp||amod: 1 dep||advcl: 1 prep||amod: 8 relcl||compound: 1 advcl||csubj: 3 +npadvmod||conj: 2 npadvmod||xcomp: 4 advmod||nsubj: 3 ccomp||amod: 7 +advcl||conj: 1 nmod||conj: 2 advmod||nsubjpass: 2 dep||xcomp: 2 appos||ccomp: 1 +advmod||dep: 1 advmod||advmod: 5 aux||xcomp: 8 dep||advmod: 1 dative||ccomp: 2 +prep||dep: 1 conj||conj: 1 dep||ccomp: 4 cc||ROOT: 1 prep||ROOT: 1 nsubj||pcomp: +3 advmod||prep: 2 relcl||dative: 1 acl||conj: 1 advcl||attr: 4 prep||npadvmod: 1 +nsubjpass||xcomp: 1 neg||advmod: 1 xcomp||oprd: 1 advcl||advcl: 1 dobj||dep: 3 +nsubjpass||parataxis: 1 attr||pcomp: 1 ccomp||parataxis: 1 advmod||attr: 1 +nmod||oprd: 1 appos||nmod: 2 advmod||relcl: 1 appos||npadvmod: 1 appos||conj: 1 +prep||expl: 1 nsubjpass||conj: 1 punct||pobj: 1 cc||pobj: 1 conj||pobj: 1 +punct||conj: 1 ccomp||dep: 1 oprd||xcomp: 3 ccomp||xcomp: 1 ccomp||nsubj: 1 +nmod||dep: 1 xcomp||ccomp: 1 acomp||advcl: 1 intj||advmod: 1 advmod||acomp: 2 +relcl||oprd: 1 advmod||prt: 1 advmod||pobj: 1 appos||nummod: 1 relcl||npadvmod: +3 mark||advcl: 1 aux||ccomp: 1 amod||nsubjpass: 1 npadvmod||advmod: 1 conj||dep: +1 nummod||pobj: 1 amod||npadvmod: 1 intj||pobj: 1 nummod||npadvmod: 1 +xcomp||xcomp: 1 aux||dep: 1 advcl||relcl: 1 +⚠ The following labels were found only in the train data: xcomp||amod, +advcl||relcl, prep||nsubjpass, acl||nsubj, nsubjpass||conj, xcomp||oprd, +advmod||conj, advmod||advmod, iobj, advmod||nsubjpass, dobj||conj, ccomp||amod, +meta||acl, xcomp||xcomp, prep||attr, prep||ccomp, advcl||acomp, acl||dobj, +advcl||advcl, pobj||agent, prep||advcl, nsubjpass||xcomp, prep||dep, +acomp||xcomp, aux||ccomp, ccomp||dep, conj||dep, relcl||compound, +nsubjpass||ccomp, nmod||dobj, advmod||advcl, advmod||acl, dobj||advcl, +dative||xcomp, prep||nsubj, ccomp||ccomp, nsubj||ccomp, xcomp||acomp, +prep||acomp, dep||advmod, acl||pobj, appos||dobj, npadvmod||acomp, cc||ROOT, +relcl||nsubj, nmod||pobj, acl||nsubjpass, ccomp||advmod, pcomp||prep, +amod||dobj, advmod||attr, advcl||csubj, appos||attr, dobj||pcomp, prep||ROOT, +relcl||pobj, advmod||pobj, amod||nsubj, ccomp||xcomp, prep||oprd, +npadvmod||advmod, appos||nummod, advcl||pobj, neg||advmod, acl||attr, +appos||nsubjpass, csubj||ccomp, amod||nsubjpass, intj||pobj, dep||advcl, +cc||neg, xcomp||ccomp, dative||ccomp, nmod||oprd, pobj||dative, prep||dobj, +dep||ccomp, relcl||attr, ccomp||nsubj, advcl||xcomp, nmod||dep, advcl||advmod, +ccomp||conj, pobj||prep, advmod||acomp, advmod||relcl, attr||pcomp, +ccomp||parataxis, oprd||xcomp, intj||advmod, nmod||nsubjpass, prep||npadvmod, +parataxis||acl, prep||pobj, advcl||dobj, amod||pobj, prep||acl, conj||pobj, +advmod||dep, punct||pobj, ccomp||acomp, acomp||advcl, nummod||npadvmod, +dobj||dep, npadvmod||xcomp, advcl||conj, relcl||npadvmod, punct||acl, +relcl||dobj, dobj||xcomp, nsubjpass||parataxis, dative||advcl, relcl||nmod, +advcl||ccomp, appos||npadvmod, ccomp||pcomp, prep||amod, mark||advcl, +prep||advmod, prep||xcomp, appos||nsubj, attr||ccomp, advmod||prt, dobj||ccomp, +aux||conj, advcl||nsubj, conj||conj, advmod||ccomp, advcl||nsubjpass, +attr||xcomp, nmod||conj, npadvmod||conj, relcl||dative, prep||expl, +nsubjpass||pcomp, advmod||xcomp, advmod||dobj, appos||pobj, nsubj||conj, +relcl||nsubjpass, advcl||attr, appos||ccomp, advmod||prep, prep||conj, +nmod||attr, punct||conj, neg||conj, dep||xcomp, aux||xcomp, dobj||acl, +nummod||pobj, amod||npadvmod, nsubj||pcomp, advcl||acl, appos||nmod, +relcl||oprd, prep||prep, cc||pobj, nmod||nsubj, amod||attr, aux||dep, +appos||conj, advmod||nsubj, nsubj||advcl, acl||conj +To train a parser, your data should include at least 20 instances of each label. +⚠ Multiple root labels (ROOT, nsubj, aux, npadvmod, prep) found in +training data. spaCy's parser uses a single root label ROOT so this distinction +will not be available. + +================================== Summary ================================== +✔ 5 checks passed +⚠ 8 warnings +``` + + + +| Name | Description | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--ignore-warnings`, `-IW` | Ignore warnings, only show stats and errors. ~~bool (flag)~~ | +| `--verbose`, `-V` | Print additional information and explanations. ~~bool (flag)~~ | +| `--no-format`, `-NF` | Don't pretty-print the results. Use this if you want to write to a file. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **PRINTS** | Debugging information. | + +### debug diff-config {id="debug-diff",tag="command"} + +Show a diff of a config file with respect to spaCy's defaults or another config +file. If additional settings were used in the creation of the config file, then +you must supply these as extra parameters to the command when comparing to the +default settings. The generated diff can also be used when posting to the +discussion forum to provide more information for the maintainers. + +```bash +$ python -m spacy debug diff-config [config_path] [--compare-to] [--optimize] [--gpu] [--pretraining] [--markdown] +``` + +> #### Example +> +> ```bash +> $ python -m spacy debug diff-config ./config.cfg +> ``` + + + +``` +ℹ Found user-defined language: 'en' +ℹ Found user-defined pipelines: ['tok2vec', 'tagger', 'parser', +'ner'] +[paths] ++ train = "./data/train.spacy" ++ dev = "./data/dev.spacy" +- train = null +- dev = null +vectors = null +init_tok2vec = null + +[system] +gpu_allocator = null ++ seed = 42 +- seed = 0 + +[nlp] +lang = "en" +pipeline = ["tok2vec","tagger","parser","ner"] +batch_size = 1000 +disabled = [] +before_creation = null +after_creation = null +after_pipeline_creation = null +tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"} + +[components] + +[components.ner] +factory = "ner" +incorrect_spans_key = null +moves = null +scorer = {"@scorers":"spacy.ner_scorer.v1"} +update_with_oracle_cut_size = 100 + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "ner" +extra_state_tokens = false +- hidden_width = 64 ++ hidden_width = 36 +maxout_pieces = 2 +use_upper = true +nO = null + +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +upstream = "*" + +[components.parser] +factory = "parser" +learn_tokens = false +min_action_freq = 30 +moves = null +scorer = {"@scorers":"spacy.parser_scorer.v1"} +update_with_oracle_cut_size = 100 + +[components.parser.model] +@architectures = "spacy.TransitionBasedParser.v2" +state_type = "parser" +extra_state_tokens = false +hidden_width = 128 +maxout_pieces = 3 +use_upper = true +nO = null + +[components.parser.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +upstream = "*" + +[components.tagger] +factory = "tagger" +neg_prefix = "!" +overwrite = false +scorer = {"@scorers":"spacy.tagger_scorer.v1"} + +[components.tagger.model] +@architectures = "spacy.Tagger.v1" +nO = null + +[components.tagger.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +width = ${components.tok2vec.model.encode.width} +upstream = "*" + +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v2" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = ${components.tok2vec.model.encode.width} +attrs = ["NORM","PREFIX","SUFFIX","SHAPE"] +rows = [5000,2500,2500,2500] +include_static_vectors = false + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = 96 +depth = 4 +window_size = 1 +maxout_pieces = 3 + +[corpora] + +[corpora.dev] +@readers = "spacy.Corpus.v1" +path = ${paths.dev} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +max_length = 0 +gold_preproc = false +limit = 0 +augmenter = null + +[training] +dev_corpus = "corpora.dev" +train_corpus = "corpora.train" +seed = ${system.seed} +gpu_allocator = ${system.gpu_allocator} +dropout = 0.1 +accumulate_gradient = 1 +patience = 1600 +max_epochs = 0 +max_steps = 20000 +eval_frequency = 200 +frozen_components = [] +annotating_components = [] +before_to_disk = null + +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +discard_oversize = false +tolerance = 0.2 +get_length = null + +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +t = 0.0 + +[training.logger] +@loggers = "spacy.ConsoleLogger.v1" +progress_bar = false + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 0.00000001 +learn_rate = 0.001 + +[training.score_weights] +tag_acc = 0.33 +dep_uas = 0.17 +dep_las = 0.17 +dep_las_per_type = null +sents_p = null +sents_r = null +sents_f = 0.0 +ents_f = 0.33 +ents_p = 0.0 +ents_r = 0.0 +ents_per_type = null + +[pretraining] + +[initialize] +vectors = ${paths.vectors} +init_tok2vec = ${paths.init_tok2vec} +vocab_data = null +lookups = null +before_init = null +after_init = null + +[initialize.components] + +[initialize.tokenizer] +``` + + + +| Name | Description | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. ~~Union[Path, str] \(positional)~~ | +| `compare_to` | Path to another config file to diff against, or `None` to compare against default settings. ~~Optional[Union[Path, str] \(option)~~ | +| `optimize`, `-o` | `"efficiency"` or `"accuracy"`. Whether the config was optimized for efficiency (faster inference, smaller model, lower memory consumption) or higher accuracy (potentially larger and slower model). Only relevant when comparing against a default config. Defaults to `"efficiency"`. ~~str (option)~~ | +| `gpu`, `-G` | Whether the config was made to run on a GPU. Only relevant when comparing against a default config. ~~bool (flag)~~ | +| `pretraining`, `-pt` | Include config for pretraining (with [`spacy pretrain`](/api/cli#pretrain)). Only relevant when comparing against a default config. Defaults to `False`. ~~bool (flag)~~ | +| `markdown`, `-md` | Generate Markdown for Github issues. Defaults to `False`. ~~bool (flag)~~ | +| **PRINTS** | Diff between the two config files. | + +### debug profile {id="debug-profile",tag="command"} + +Profile which functions take the most time in a spaCy pipeline. Input should be +formatted as one JSON object per line with a key `"text"`. It can either be +provided as a JSONL file, or be read from `sys.sytdin`. If no input file is +specified, the IMDB dataset is loaded via +[`ml_datasets`](https://github.com/explosion/ml_datasets). + + + +The `profile` command is now available as a subcommand of `spacy debug`. + + + +```bash +$ python -m spacy debug profile [model] [inputs] [--n-texts] +``` + +| Name | Description | +| ----------------- | ---------------------------------------------------------------------------------- | +| `model` | A loadable spaCy pipeline (package name or path). ~~str (positional)~~ | +| `inputs` | Path to input file, or `-` for standard input. ~~Path (positional)~~ | +| `--n-texts`, `-n` | Maximum number of texts to use if available. Defaults to `10000`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Profiling information for the pipeline. | + +### debug model {id="debug-model",version="3",tag="command"} + +Debug a Thinc [`Model`](https://thinc.ai/docs/api-model) by running it on a +sample text and checking how it updates its internal weights and parameters. + +```bash +$ python -m spacy debug model [config_path] [component] [--layers] [--dimensions] [--parameters] [--gradients] [--attributes] [--print-step0] [--print-step1] [--print-step2] [--print-step3] [--gpu-id] +``` + + + +In this example log, we just print the name of each layer after creation of the +model ("Step 0"), which helps us to understand the internal structure of the +Neural Network, and to focus on specific layers that we want to inspect further +(see next example). + +```bash +$ python -m spacy debug model ./config.cfg tagger -P0 +``` + +``` +ℹ Using CPU +ℹ Fixing random seed: 0 +ℹ Analysing model with ID 62 + +========================== STEP 0 - before training ========================== +ℹ Layer 0: model ID 62: +'extract_features>>list2ragged>>with_array-ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed>>with_array-maxout>>layernorm>>dropout>>ragged2list>>with_array-residual>>residual>>residual>>residual>>with_array-softmax' +ℹ Layer 1: model ID 59: +'extract_features>>list2ragged>>with_array-ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed>>with_array-maxout>>layernorm>>dropout>>ragged2list>>with_array-residual>>residual>>residual>>residual' +ℹ Layer 2: model ID 61: 'with_array-softmax' +ℹ Layer 3: model ID 24: +'extract_features>>list2ragged>>with_array-ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed>>with_array-maxout>>layernorm>>dropout>>ragged2list' +ℹ Layer 4: model ID 58: 'with_array-residual>>residual>>residual>>residual' +ℹ Layer 5: model ID 60: 'softmax' +ℹ Layer 6: model ID 13: 'extract_features' +ℹ Layer 7: model ID 14: 'list2ragged' +ℹ Layer 8: model ID 16: +'with_array-ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed' +ℹ Layer 9: model ID 22: 'with_array-maxout>>layernorm>>dropout' +ℹ Layer 10: model ID 23: 'ragged2list' +ℹ Layer 11: model ID 57: 'residual>>residual>>residual>>residual' +ℹ Layer 12: model ID 15: +'ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed|ints-getitem>>hashembed' +ℹ Layer 13: model ID 21: 'maxout>>layernorm>>dropout' +ℹ Layer 14: model ID 32: 'residual' +ℹ Layer 15: model ID 40: 'residual' +ℹ Layer 16: model ID 48: 'residual' +ℹ Layer 17: model ID 56: 'residual' +ℹ Layer 18: model ID 3: 'ints-getitem>>hashembed' +ℹ Layer 19: model ID 6: 'ints-getitem>>hashembed' +ℹ Layer 20: model ID 9: 'ints-getitem>>hashembed' +... +``` + +In this example log, we see how initialization of the model (Step 1) propagates +the correct values for the `nI` (input) and `nO` (output) dimensions of the +various layers. In the `softmax` layer, this step also defines the `W` matrix as +an all-zero matrix determined by the `nO` and `nI` dimensions. After a first +training step (Step 2), this matrix has clearly updated its values through the +training feedback loop. + +```bash +$ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P2 +``` + +``` +ℹ Using CPU +ℹ Fixing random seed: 0 +ℹ Analysing model with ID 62 + +========================= STEP 0 - before training ========================= +ℹ Layer 5: model ID 60: 'softmax' +ℹ - dim nO: None +ℹ - dim nI: 96 +ℹ - param W: None +ℹ - param b: None +ℹ Layer 15: model ID 40: 'residual' +ℹ - dim nO: None +ℹ - dim nI: None + +======================= STEP 1 - after initialization ======================= +ℹ Layer 5: model ID 60: 'softmax' +ℹ - dim nO: 4 +ℹ - dim nI: 96 +ℹ - param W: (4, 96) - sample: [0. 0. 0. 0. 0.] +ℹ - param b: (4,) - sample: [0. 0. 0. 0.] +ℹ Layer 15: model ID 40: 'residual' +ℹ - dim nO: 96 +ℹ - dim nI: None + +========================== STEP 2 - after training ========================== +ℹ Layer 5: model ID 60: 'softmax' +ℹ - dim nO: 4 +ℹ - dim nI: 96 +ℹ - param W: (4, 96) - sample: [ 0.00283958 -0.00294119 0.00268396 -0.00296219 +-0.00297141] +ℹ - param b: (4,) - sample: [0.00300002 0.00300002 0.00300002 0.00300002] +ℹ Layer 15: model ID 40: 'residual' +ℹ - dim nO: 96 +ℹ - dim nI: None +``` + + + +| Name | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `component` | Name of the pipeline component of which the model should be analyzed. ~~str (positional)~~ | +| `--layers`, `-l` | Comma-separated names of layer IDs to print. ~~str (option)~~ | +| `--dimensions`, `-DIM` | Show dimensions of each layer. ~~bool (flag)~~ | +| `--parameters`, `-PAR` | Show parameters of each layer. ~~bool (flag)~~ | +| `--gradients`, `-GRAD` | Show gradients of each layer. ~~bool (flag)~~ | +| `--attributes`, `-ATTR` | Show attributes of each layer. ~~bool (flag)~~ | +| `--print-step0`, `-P0` | Print model before training. ~~bool (flag)~~ | +| `--print-step1`, `-P1` | Print model after initialization. ~~bool (flag)~~ | +| `--print-step2`, `-P2` | Print model after training. ~~bool (flag)~~ | +| `--print-step3`, `-P3` | Print final predictions. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **PRINTS** | Debugging information. | + +### debug pieces {id="debug-pieces",version="3.7",tag="command"} + +Analyze word- or sentencepiece stats. + +```bash +$ python -m spacy debug pieces [config_path] [--code] [--name] [overrides] +``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to config file. ~~Union[Path, str] (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--name`, `-n` | Name of the Curated Transformer pipe whose config is to be filled. Defaults to the first transformer pipe. ~~Optional[str] (option)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **PRINTS** | Debugging information. | + + + +```bash +$ python -m spacy debug pieces ./config.cfg +``` + +``` +========================= Training corpus statistics ========================= +Median token length: 1.0 +Mean token length: 1.54 +Token length range: [1, 13] + +======================= Development corpus statistics ======================= +Median token length: 1.0 +Mean token length: 1.44 +Token length range: [1, 8] +``` + + + +## train {id="train",tag="command"} + +Train a pipeline. Expects data in spaCy's +[binary format](/api/data-formats#training) and a +[config file](/api/data-formats#config) with all settings and hyperparameters. +Will save out the best model from all epochs, as well as the final pipeline. The +`--code` argument can be used to provide a Python file that's imported before +the training process starts. This lets you register +[custom functions](/usage/training#custom-functions) and architectures and refer +to them in your config, all while still using spaCy's built-in `train` workflow. +If you need to manage complex multi-step training workflows, check out the new +[spaCy projects](/usage/projects). + + + +The `train` command doesn't take a long list of command-line arguments anymore +and instead expects a single [`config.cfg` file](/usage/training#config) +containing all settings for the pipeline, training process and hyperparameters. +Config values can be [overwritten](/usage/training#config-overrides) on the CLI +if needed. For example, `--paths.train ./train.spacy` sets the variable `train` +in the section `[paths]`. + + + +> #### Example +> +> ```bash +> $ python -m spacy train config.cfg --output ./output --paths.train ./train --paths.dev ./dev +> ``` + +```bash +$ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] [overrides] +``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `--output`, `-o` | Directory to store trained pipeline in. Will be created if it doesn't exist. ~~Optional[Path] \(option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--verbose`, `-V` | Show more detailed messages during training. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | +| **CREATES** | The final trained pipeline and the best trained pipeline. | + +### Calling the training function from Python {id="train-function",version="3.2"} + +The training CLI exposes a `train` helper function that lets you run the +training just like `spacy train`. Usually it's easier to use the command line +directly, but if you need to kick off training from code this is how to do it. + +> #### Example +> +> ```python +> from spacy.cli.train import train +> +> train("./config.cfg", overrides={"paths.train": "./train.spacy", "paths.dev": "./dev.spacy"}) +> +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `config_path` | Path to the config to use for training. ~~Union[str, Path]~~ | +| `output_path` | Optional name of directory to save output model in. If not provided a model will not be saved. ~~Optional[Union[str, Path]]~~ | +| _keyword-only_ | | +| `use_gpu` | Which GPU to use. Defaults to -1 for no GPU. ~~int~~ | +| `overrides` | Values to override config settings. ~~Dict[str, Any]~~ | + +## pretrain {id="pretrain",version="2.1",tag="command,experimental"} + +Pretrain the "token to vector" ([`Tok2vec`](/api/tok2vec)) layer of pipeline +components on raw text, using an approximate language-modeling objective. +Specifically, we load pretrained vectors, and train a component like a CNN, +BiLSTM, etc to predict vectors which match the pretrained ones. The weights are +saved to a directory after each epoch. You can then include a **path to one of +these pretrained weights files** in your +[training config](/usage/training#config) as the `init_tok2vec` setting when you +train your pipeline. This technique may be especially helpful if you have little +labelled data. See the usage docs on +[pretraining](/usage/embeddings-transformers#pretraining) for more info. To read +the raw text, a [`JsonlCorpus`](/api/top-level#jsonlcorpus) is typically used. + + + +As of spaCy v3.0, the `pretrain` command takes the same +[config file](/usage/training#config) as the `train` command. This ensures that +settings are consistent between pretraining and training. Settings for +pretraining can be defined in the `[pretraining]` block of the config file and +auto-generated by setting `--pretraining` on +[`init fill-config`](/api/cli#init-fill-config). Also see the +[data format](/api/data-formats#config) for details. + + + +> #### Example +> +> ```bash +> $ python -m spacy pretrain config.cfg ./output_pretrain --paths.raw_text ./data.jsonl +> ``` + +```bash +$ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [--epoch-resume] [--gpu-id] [overrides] +``` + +| Name | Description | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `output_dir` | Directory to save binary weights to on each epoch. ~~Path (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--resume-path`, `-r` | Path to pretrained weights from which to resume pretraining. ~~Optional[Path] \(option)~~ | +| `--epoch-resume`, `-er` | The epoch to resume counting from when using `--resume-path`. Prevents unintended overwriting of existing weight files. ~~Optional[int] \(option)~~ | +| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | +| `--skip-last`, `-L` 3.5.2 | Skip saving `model-last.bin`. Defaults to `False`. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--training.dropout 0.2`. ~~Any (option/flag)~~ | +| **CREATES** | The pretrained weights that can be used to initialize `spacy train`. | + +## evaluate {id="evaluate",version="2",tag="command"} + +The `evaluate` subcommand is superseded by +[`spacy benchmark accuracy`](#benchmark-accuracy). `evaluate` is provided as an +alias to `benchmark accuracy` for compatibility. + +## benchmark {id="benchmark", version="3.5"} + +The `spacy benchmark` CLI includes commands for benchmarking the accuracy and +speed of your spaCy pipelines. + +### accuracy {id="benchmark-accuracy", version="3.5", tag="command"} + +Evaluate the accuracy of a trained pipeline. Expects a loadable spaCy pipeline +(package name or path) and evaluation data in the +[binary `.spacy` format](/api/data-formats#binary-training). The +`--gold-preproc` option sets up the evaluation examples with gold-standard +sentences and tokens for the predictions. Gold preprocessing helps the +annotations align to the tokenization, and may result in sequences of more +consistent length. However, it may reduce runtime accuracy due to train/test +skew. To render a sample of dependency parses in a HTML file using the +[displaCy visualizations](/usage/visualizers), set as output directory as the +`--displacy-path` argument. + +```bash +$ python -m spacy benchmark accuracy [model] [data_path] [--output] [--code] [--gold-preproc] [--gpu-id] [--displacy-path] [--displacy-limit] [--per-component] [--spans-key] +``` + +| Name | Description | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of evaluation data in spaCy's [binary format](/api/data-formats#training). ~~Path (positional)~~ | +| `--output`, `-o` | Output JSON file for metrics. If not set, no metrics will be exported. ~~Optional[Path] \(option)~~ | +| `--code`, `-c` 3 | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--gold-preproc`, `-G` | Use gold preprocessing. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--displacy-path`, `-dp` | Directory to output rendered parses as HTML. If not set, no visualizations will be generated. ~~Optional[Path] \(option)~~ | +| `--displacy-limit`, `-dl` | Number of parses to generate per file. Defaults to `25`. Keep in mind that a significantly higher number might cause the `.html` files to render slowly. ~~int (option)~~ | +| `--per-component`, `-P` 3.6 | Whether to return the scores keyed by component name. Defaults to `False`. ~~bool (flag)~~ | +| `--spans-key`, `-sk` 3.6.2 | Spans key to use when evaluating `Doc.spans`. Defaults to `sc`. ~~str (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Training results and optional metrics and visualizations. | + +### speed {id="benchmark-speed", version="3.5", tag="command"} + +Benchmark the speed of a trained pipeline with a 95% confidence interval. +Expects a loadable spaCy pipeline (package name or path) and benchmark data in +the [binary `.spacy` format](/api/data-formats#binary-training). The pipeline is +warmed up before any measurements are taken. + +```cli +$ python -m spacy benchmark speed [model] [data_path] [--code] [--batch_size] [--no-shuffle] [--gpu-id] [--batches] [--warmup] +``` + +| Name | Description | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to benchmark the speed of. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of benchmark data in spaCy's [binary format](/api/data-formats#training). ~~Path (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--batch-size`, `-b` | Set the batch size. If not set, the pipeline's batch size is used. ~~Optional[int] \(option)~~ | +| `--no-shuffle` | Do not shuffle documents in the benchmark data. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--batches` | Number of batches to benchmark on. Defaults to `50`. ~~Optional[int] \(option)~~ | +| `--warmup`, `-w` | Iterations over the benchmark data for warmup. Defaults to `3` ~~Optional[int] \(option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Pipeline speed in words per second with a 95% confidence interval. | + +## apply {id="apply", version="3.5", tag="command"} + +Applies a trained pipeline to data and stores the resulting annotated documents +in a `DocBin`. The input can be a single file or a directory. The recognized +input formats are: + +1. `.spacy` +2. `.jsonl` containing a user specified `text_key` +3. Files with any other extension are assumed to be plain text files containing + a single document. + +When a directory is provided it is traversed recursively to collect all files. + +When loading a .spacy file, any potential annotations stored on the `Doc` that are not overwritten by the pipeline will be preserved. +If you want to evaluate the pipeline on raw text only, make sure that the .spacy file does not contain any annotations. + +```bash +$ python -m spacy apply [model] [data-path] [output-file] [--code] [--text-key] [--force-overwrite] [--gpu-id] [--batch-size] [--n-process] +``` + +| Name | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to apply to the data. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of data to be evaluated in spaCy's [binary format](/api/data-formats#training), jsonl, or plain text. ~~Path (positional)~~ | +| `output-file` | Output `DocBin` path. ~~str (positional)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--text-key`, `-tk` | The key for `.jsonl` files to use to grab the texts from. Defaults to `text`. ~~Optional[str] \(option)~~ | +| `--force-overwrite`, `-F` | If the provided `output-file` already exists, then force `apply` to overwrite it. If this is `False` (default) then quits with a warning instead. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--batch-size`, `-b` | Batch size to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--n-process`, `-n` | Number of processes to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A `DocBin` with the annotations from the `model` for all the files found in `data-path`. | + +## find-threshold {id="find-threshold",version="3.5",tag="command"} + +Runs prediction trials for a trained model with varying thresholds to maximize +the specified metric. The search space for the threshold is traversed linearly +from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout` +(the corresponding API call to `spacy.cli.find_threshold.find_threshold()` +returns all results). + +This is applicable only for components whose predictions are influenced by +thresholds - e.g. `textcat_multilabel` and `spancat`, but not `textcat`. Note +that the full path to the corresponding threshold attribute in the config has to +be provided. + +> #### Examples +> +> ```bash +> # For textcat_multilabel: +> $ python -m spacy find-threshold my_nlp data.spacy textcat_multilabel threshold cats_macro_f +> ``` +> +> ```bash +> # For spancat: +> $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f +> ``` + +| Name | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Path to file with DocBin with docs to use for threshold search. ~~Path (positional)~~ | +| `pipe_name` | Name of pipe to examine thresholds for. ~~str (positional)~~ | +| `threshold_key` | Key of threshold attribute in component's configuration. ~~str (positional)~~ | +| `scores_key` | Name of score to metric to optimize. ~~str (positional)~~ | +| `--n_trials`, `-n` | Number of trials to determine optimal thresholds. ~~int (option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--gold-preproc`, `-G` | Use gold preprocessing. ~~bool (flag)~~ | +| `--verbose`, `-V`, `-VV` | Display more information for debugging purposes. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | + +## assemble {id="assemble",tag="command"} + +Assemble a pipeline from a config file without additional training. Expects a +[config file](/api/data-formats#config) with all settings and hyperparameters. +The `--code` argument can be used to import a Python file that lets you register +[custom functions](/usage/training#custom-functions) and refer to them in your +config. + +> #### Example +> +> ```bash +> $ python -m spacy assemble config.cfg ./output +> ``` + +```bash +$ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [overrides] +``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `config_path` | Path to the [config](/api/data-formats#config) file containing all settings and hyperparameters. If `-`, the data will be [read from stdin](/usage/training#config-stdin). ~~Union[Path, str] \(positional)~~ | +| `output_dir` | Directory to store the final pipeline in. Will be created if it doesn't exist. ~~Optional[Path] \(option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions). ~~Optional[Path] \(option)~~ | +| `--verbose`, `-V` | Show more detailed messages during processing. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.data ./data`. ~~Any (option/flag)~~ | +| **CREATES** | The final assembled pipeline. | + +## package {id="package",tag="command"} + +Generate an installable [Python package](/usage/training#models-generating) from +an existing pipeline data directory. All data files are copied over. If +additional code files are provided (e.g. Python files containing custom +registered functions like +[pipeline components](/usage/processing-pipelines#custom-components)), they are +copied into the package and imported in the `__init__.py`. If the path to a +[`meta.json`](/api/data-formats#meta) is supplied, or a `meta.json` is found in +the input directory, this file is used. Otherwise, the data can be entered +directly from the command line. spaCy will then create a build artifact that you +can distribute and install with `pip install`. As of v3.1, the `package` command +will also create a formatted `README.md` based on the pipeline information +defined in the `meta.json`. If a `README.md` is already present in the source +directory, it will be used instead. + + + +The `spacy package` command now also builds the `.tar.gz` archive automatically, +so you don't have to run `python setup.py sdist` separately anymore. To disable +this, you can set `--build none`. You can also choose to build a binary wheel +(which installs more efficiently) by setting `--build wheel`, or to build both +the sdist and wheel by setting `--build sdist,wheel`. + + + +```bash +$ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--create-meta] [--build] [--name] [--version] [--force] +``` + +> #### Example +> +> ```bash +> $ python -m spacy package /input /output +> $ cd /output/en_pipeline-0.0.0 +> $ pip install dist/en_pipeline-0.0.0.tar.gz +> ``` + +| Name | Description | +| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `input_dir` | Path to directory containing pipeline data. ~~Path (positional)~~ | +| `output_dir` | Directory to create package folder in. ~~Path (positional)~~ | +| `--code`, `-c` 3 | Comma-separated paths to Python files to be included in the package and imported in its `__init__.py`. This allows including [registering functions](/usage/training#custom-functions) and [custom components](/usage/processing-pipelines#custom-components). ~~str (option)~~ | +| `--meta-path`, `-m` | Path to [`meta.json`](/api/data-formats#meta) file (optional). ~~Optional[Path] \(option)~~ | +| `--create-meta`, `-C` | Create a `meta.json` file on the command line, even if one already exists in the directory. If an existing file is found, its entries will be shown as the defaults in the command line prompt. ~~bool (flag)~~ | +| `--build`, `-b` 3 | Comma-separated artifact formats to build. Can be `sdist` (for a `.tar.gz` archive) and/or `wheel` (for a binary `.whl` file), or `none` if you want to run this step manually. The generated artifacts can be installed by `pip install`. Defaults to `sdist`. ~~str (option)~~ | +| `--name`, `-n` 3 | Package name to override in meta. ~~Optional[str] \(option)~~ | +| `--version`, `-v` 3 | Package version to override in meta. Useful when training new versions, as it doesn't require editing the meta template. ~~Optional[str] \(option)~~ | +| `--force`, `-f` | Force overwriting of existing folder in output directory. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A Python package containing the spaCy pipeline. | + +## project {id="project",version="3"} + +The `spacy project` CLI includes subcommands for working with +[spaCy projects](/usage/projects), end-to-end workflows for building and +deploying custom spaCy pipelines. + +### project clone {id="project-clone",tag="command"} + +Clone a project template from a Git repository. Calls into `git` under the hood +and can use the sparse checkout feature if available, so you're only downloading +what you need. By default, spaCy's +[project templates repo](https://github.com/explosion/projects) is used, but you +can provide any other repo (public or private) that you have access to using the +`--repo` option. + +```bash +$ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project clone pipelines/ner_wikiner +> ``` +> +> Clone from custom repo: +> +> ```bash +> $ python -m spacy project clone template --repo https://github.com/your_org/your_repo +> ``` + +| Name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | The name of the template to clone, relative to the repo. Can be a top-level directory or a subdirectory like `dir/template`. ~~str (positional)~~ | +| `dest` | Where to clone the project. Defaults to current working directory. ~~Path (positional)~~ | +| `--repo`, `-r` | The repository to clone from. Can be any public or private Git repo you have access to. ~~str (option)~~ | +| `--branch`, `-b` | The branch to clone from. Defaults to `master`. ~~str (option)~~ | +| `--sparse`, `-S` | Enable [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) to only check out and download what's needed. Requires Git v22.2+. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | The cloned [project directory](/usage/projects#project-files). | + +### project assets {id="project-assets",tag="command"} + +Fetch project assets like datasets and pretrained weights. Assets are defined in +the `assets` section of the [`project.yml`](/usage/projects#project-yml). If a +`checksum` is provided, the file is only downloaded if no local file with the +same checksum exists and spaCy will show an error if the checksum of the +downloaded file doesn't match. If assets don't specify a `url` they're +considered "private" and you have to take care of putting them into the +destination directory yourself. If a local path is provided, the asset is copied +into the current project. + +```bash +$ python -m spacy project assets [project_dir] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project assets [--sparse] +> ``` + +| Name | Description | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--extra`, `-e` 3.3.1 | Download assets marked as "extra". Default false. ~~bool (flag)~~ | +| `--sparse`, `-S` | Enable [sparse checkout](https://git-scm.com/docs/git-sparse-checkout) to only check out and download what's needed. Requires Git v22.2+. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Downloaded or copied assets defined in the `project.yml`. | + +### project run {id="project-run",tag="command"} + +Run a named command or workflow defined in the +[`project.yml`](/usage/projects#project-yml). If a workflow name is specified, +all commands in the workflow are run, in order. If commands define +[dependencies or outputs](/usage/projects#deps-outputs), they will only be +re-run if state has changed. For example, if the input dataset changes, a +preprocessing command that depends on those files will be re-run. + +```bash +$ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project run train +> ``` + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------- | +| `subcommand` | Name of the command or workflow to run. ~~str (positional)~~ | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--force`, `-F` | Force re-running steps, even if nothing changed. ~~bool (flag)~~ | +| `--dry`, `-D` | Perform a dry run and don't execute scripts. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **EXECUTES** | The command defined in the `project.yml`. | + +### project push {id="project-push",tag="command"} + +Upload all available files or directories listed as in the `outputs` section of +commands to a remote storage. Outputs are archived and compressed prior to +upload, and addressed in the remote storage using the output's relative path +(URL encoded), a hash of its command string and dependencies, and a hash of its +file contents. This means `push` should **never overwrite** a file in your +remote. If all the hashes match, the contents are the same and nothing happens. +If the contents are different, the new version of the file is uploaded. Deleting +obsolete files is left up to you. + +Remotes can be defined in the `remotes` section of the +[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses +[`cloudpathlib`](https://cloudpathlib.drivendata.org) to communicate with the +remote storages, so you can use any protocol that `cloudpathlib` supports, +including [S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. + +```bash +$ python -m spacy project push [remote] [project_dir] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project push my_bucket +> ``` +> +> ```yaml +> ### project.yml +> remotes: +> my_bucket: 's3://my-spacy-bucket' +> ``` + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------- | +| `remote` | The name of the remote to upload to. Defaults to `"default"`. ~~str (positional)~~ | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **UPLOADS** | All project outputs that exist and are not already stored in the remote. | + +### project pull {id="project-pull",tag="command"} + +Download all files or directories listed as `outputs` for commands, unless they +are already present locally. When searching for files in the remote, `pull` +won't just look at the output path, but will also consider the **command +string** and the **hashes of the dependencies**. For instance, let's say you've +previously pushed a checkpoint to the remote, but now you've changed some +hyper-parameters. Because you've changed the inputs to the command, if you run +`pull`, you won't retrieve the stale result. If you train your pipeline and push +the outputs to the remote, the outputs will be saved alongside the prior +outputs, so if you change the config back, you'll be able to fetch back the +result. + +Remotes can be defined in the `remotes` section of the +[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses +[`Pathy`](https://github.com/justindujardin/pathy) to communicate with the +remote storages, so you can use any protocol that `Pathy` supports, including +[S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. + +```bash +$ python -m spacy project pull [remote] [project_dir] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project pull my_bucket +> ``` +> +> ```yaml +> ### project.yml +> remotes: +> my_bucket: 's3://my-spacy-bucket' +> ``` + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------- | +| `remote` | The name of the remote to download from. Defaults to `"default"`. ~~str (positional)~~ | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **DOWNLOADS** | All project outputs that do not exist locally and can be found in the remote. | + +### project document {id="project-document",tag="command"} + +Auto-generate a pretty Markdown-formatted `README` for your project, based on +its [`project.yml`](/usage/projects#project-yml). Will create sections that +document the available commands, workflows and assets. The auto-generated +content will be placed between two hidden markers, so you can add your own +custom content before or after the auto-generated documentation. When you re-run +the `project document` command, only the auto-generated part is replaced. + +```bash +$ python -m spacy project document [project_dir] [--output] [--no-emoji] +``` + +> #### Example +> +> ```bash +> $ python -m spacy project document --output README.md +> ``` + + + +For more examples, see the templates in our +[`projects`](https://github.com/explosion/projects) repo. + +![Screenshot of auto-generated Markdown Readme](/images/project_document.jpg) + + + +| Name | Description | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `--output`, `-o` | Path to output file or `-` for stdout (default). If a file is specified and it already exists and contains auto-generated docs, only the auto-generated docs section is replaced. ~~Path (positional)~~ | +| `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ | +| **CREATES** | The Markdown-formatted project documentation. | + +### project dvc {id="project-dvc",tag="command"} + +Auto-generate [Data Version Control](https://dvc.org) (DVC) config file. Calls +[`dvc run`](https://dvc.org/doc/command-reference/run) with `--no-exec` under +the hood to generate the `dvc.yaml`. A DVC project can only define one pipeline, +so you need to specify one workflow defined in the +[`project.yml`](/usage/projects#project-yml). If no workflow is specified, the +first defined workflow is used. The DVC config will only be updated if the +`project.yml` changed. For details, see the +[DVC integration](/usage/projects#dvc) docs. + + + +This command requires DVC to be installed and initialized in the project +directory, e.g. via [`dvc init`](https://dvc.org/doc/command-reference/init). +You'll also need to add the assets you want to track with +[`dvc add`](https://dvc.org/doc/command-reference/add). + + + +```bash +$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet] +``` + +> #### Example +> +> ```bash +> $ git init +> $ dvc init +> $ python -m spacy project dvc all +> ``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------- | +| `project_dir` | Path to project directory. Defaults to current working directory. ~~Path (positional)~~ | +| `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ | +| `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ | +| `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ | +| `--quiet`, `-q` | Print no output generated by DVC. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | + +## huggingface-hub {id="huggingface-hub",version="3.1"} + +The `spacy huggingface-cli` CLI includes commands for uploading your trained +spaCy pipelines to the [Hugging Face Hub](https://huggingface.co/). + +> #### Installation +> +> ```bash +> $ pip install spacy-huggingface-hub +> $ huggingface-cli login +> ``` + + + +To use this command, you need the +[`spacy-huggingface-hub`](https://github.com/explosion/spacy-huggingface-hub) +package installed. Installing the package will automatically add the +`huggingface-hub` command to the spaCy CLI. + + + +### huggingface-hub push {id="huggingface-hub-push",tag="command"} + +Push a spaCy pipeline to the Hugging Face Hub. Expects a `.whl` file packaged +with [`spacy package`](/api/cli#package) and `--build wheel`. For more details, +see the spaCy project [integration](/usage/projects#huggingface_hub). + +```bash +$ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--verbose] +``` + +> #### Example +> +> ```bash +> $ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl +> ``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------- | +| `whl_path` | The path to the `.whl` file packaged with [`spacy package`](https://spacy.io/api/cli#package). ~~Path(positional)~~ | +| `--org`, `-o` | Optional name of organization to which the pipeline should be uploaded. ~~str (option)~~ | +| `--msg`, `-m` | Commit message to use for update. Defaults to `"Update spaCy pipeline"`. ~~str (option)~~ | +| `--verbose`, `-V` | Output additional info for debugging, e.g. the full generated hub metadata. ~~bool (flag)~~ | +| **UPLOADS** | The pipeline to the hub. | diff --git a/website/docs/api/coref.mdx b/website/docs/api/coref.mdx new file mode 100644 index 0000000..0b9ebb8 --- /dev/null +++ b/website/docs/api/coref.mdx @@ -0,0 +1,353 @@ +--- +title: CoreferenceResolver +tag: class,experimental +source: spacy-experimental/coref/coref_component.py +teaser: 'Pipeline component for word-level coreference resolution' +api_base_class: /api/pipe +api_string_name: coref +api_trainable: true +--- + +> #### Installation +> +> ```bash +> $ pip install -U spacy-experimental +> ``` + + + +This component is not yet integrated into spaCy core, and is available via the +extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "experimental_coref"` in your +[training config](/usage/training#config) or +`nlp.add_pipe("experimental_coref")` will work out-of-the-box. + + + +A `CoreferenceResolver` component groups tokens into clusters that refer to the +same thing. Clusters are represented as SpanGroups that start with a prefix +(`coref_clusters` by default). + +A `CoreferenceResolver` component can be paired with a +[`SpanResolver`](/api/span-resolver) to expand single tokens to spans. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `Doc.spans` as a [`SpanGroup`](/api/spangroup). The +span key will be a prefix plus a serial number referring to the coreference +cluster, starting from zero. + +The span key prefix defaults to `"coref_clusters"`, but can be passed as a +parameter. + +| Location | Value | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------- | +| `Doc.spans[prefix + "_" + cluster_number]` | One coreference cluster, represented as single-token spans. Cluster numbers start from 1. ~~SpanGroup~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#coref-architectures) documentation for +details on the architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy_experimental.coref.coref_component import DEFAULT_COREF_MODEL +> from spacy_experimental.coref.coref_util import DEFAULT_CLUSTER_PREFIX +> config={ +> "model": DEFAULT_COREF_MODEL, +> "span_cluster_prefix": DEFAULT_CLUSTER_PREFIX, +> } +> nlp.add_pipe("experimental_coref", config=config) +> ``` + +| Setting | Description | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Coref](/api/architectures#Coref). ~~Model~~ | +| `span_cluster_prefix` | The prefix for the keys for clusters saved to `doc.spans`. Defaults to `coref_clusters`. ~~str~~ | + +## CoreferenceResolver.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> coref = nlp.add_pipe("experimental_coref") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_coref.v1"}} +> coref = nlp.add_pipe("experimental_coref", config=config) +> +> # Construction from class +> from spacy_experimental.coref.coref_component import CoreferenceResolver +> coref = CoreferenceResolver(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| --------------------- | --------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `span_cluster_prefix` | The prefix for the key for saving clusters of spans. ~~bool~~ | + +## CoreferenceResolver.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/coref#call) and [`pipe`](/api/coref#pipe) delegate to the +[`predict`](/api/coref#predict) and +[`set_annotations`](/api/coref#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> coref = nlp.add_pipe("experimental_coref") +> # This usually happens under the hood +> processed = coref(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## CoreferenceResolver.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/coref#call) and +[`pipe`](/api/coref#pipe) delegate to the [`predict`](/api/coref#predict) and +[`set_annotations`](/api/coref#set_annotations) methods. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> for doc in coref.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## CoreferenceResolver.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## CoreferenceResolver.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. Clusters are returned as a list of `MentionClusters`, one for +each input `Doc`. A `MentionClusters` instance is just a list of lists of pairs +of `int`s, where each item corresponds to a cluster, and the `int`s correspond +to token indices. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> clusters = coref.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | + +## CoreferenceResolver.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of documents, saving coreference clusters in `Doc.spans`. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> clusters = coref.predict([doc1, doc2]) +> coref.set_annotations([doc1, doc2], clusters) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `clusters` | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | + +## CoreferenceResolver.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects. Delegates to +[`predict`](/api/coref#predict). + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> optimizer = nlp.initialize() +> losses = coref.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## CoreferenceResolver.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> optimizer = coref.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## CoreferenceResolver.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> with coref.use_params(optimizer.averages): +> coref.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## CoreferenceResolver.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.to_disk("/path/to/coref") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## CoreferenceResolver.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.from_disk("/path/to/coref") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `CoreferenceResolver` object. ~~CoreferenceResolver~~ | + +## CoreferenceResolver.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref_bytes = coref.to_bytes() +> ``` + +Serialize the pipe to a bytestring, including the `KnowledgeBase`. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `CoreferenceResolver` object. ~~bytes~~ | + +## CoreferenceResolver.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> coref_bytes = coref.to_bytes() +> coref = nlp.add_pipe("experimental_coref") +> coref.from_bytes(coref_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `CoreferenceResolver` object. ~~CoreferenceResolver~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = coref.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/corpus.mdx b/website/docs/api/corpus.mdx new file mode 100644 index 0000000..75e8f5c --- /dev/null +++ b/website/docs/api/corpus.mdx @@ -0,0 +1,242 @@ +--- +title: Corpus +teaser: An annotated corpus +tag: class +source: spacy/training/corpus.py +version: 3 +--- + +This class manages annotated corpora and can be used for training and +development datasets in the [`DocBin`](/api/docbin) (`.spacy`) format. To +customize the data loading during training, you can register your own +[data readers and batchers](/usage/training#custom-code-readers-batchers). Also +see the usage guide on [data utilities](/usage/training#data) for more details +and examples. + +## Config and implementation {id="config"} + +`spacy.Corpus.v1` is a registered function that creates a `Corpus` of training +or evaluation data. It takes the same arguments as the `Corpus` class and +returns a callable that yields [`Example`](/api/example) objects. You can +replace it with your own registered function in the +[`@readers` registry](/api/top-level#registry) to customize the data loading and +streaming. + +> #### Example config +> +> ```ini +> [paths] +> train = "corpus/train.spacy" +> +> [corpora.train] +> @readers = "spacy.Corpus.v1" +> path = ${paths.train} +> gold_preproc = false +> max_length = 0 +> limit = 0 +> augmenter = null +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Path~~ | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/training/corpus.py +``` + +## Corpus.\_\_init\_\_ {id="init",tag="method"} + +Create a `Corpus` for iterating [Example](/api/example) objects from a file or +directory of [`.spacy` data files](/api/data-formats#binary-training). The +`gold_preproc` setting lets you specify whether to set up the `Example` object +with gold-standard sentences and tokens for the predictions. Gold preprocessing +helps the annotations align to the tokenization, and may result in sequences of +more consistent length. However, it may reduce runtime accuracy due to +train/test skew. + +> #### Example +> +> ```python +> from spacy.training import Corpus +> +> # With a single file +> corpus = Corpus("./data/train.spacy") +> +> # With a directory +> corpus = Corpus("./data", limit=10) +> ``` + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. Defaults to `False`. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ | +| `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ | + +## Corpus.\_\_call\_\_ {id="call",tag="method"} + +Yield examples from the data. + +> #### Example +> +> ```python +> from spacy.training import Corpus +> import spacy +> +> corpus = Corpus("./train.spacy") +> nlp = spacy.blank("en") +> train_data = corpus(nlp) +> ``` + +| Name | Description | +| ---------- | -------------------------------------- | +| `nlp` | The current `nlp` object. ~~Language~~ | +| **YIELDS** | The examples. ~~Example~~ | + +## JsonlCorpus {id="jsonlcorpus",tag="class"} + +Iterate Doc objects from a file or directory of JSONL (newline-delimited JSON) +formatted raw text files. Can be used to read the raw text corpus for language +model [pretraining](/usage/embeddings-transformers#pretraining) from a JSONL +file. + +> #### Tip: Writing JSONL +> +> Our utility library [`srsly`](https://github.com/explosion/srsly) provides a +> handy `write_jsonl` helper that takes a file path and list of dictionaries and +> writes out JSONL-formatted data. +> +> ```python +> import srsly +> data = [{"text": "Some text"}, {"text": "More..."}] +> srsly.write_jsonl("/path/to/text.jsonl", data) +> ``` + +```json {title="Example"} +{"text": "Can I ask where you work now and what you do, and if you enjoy it?"} +{"text": "They may just pull out of the Seattle market completely, at least until they have autonomous vehicles."} +{"text": "My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in."} +``` + +### JsonlCorpus.\_\_init\_\_ {id="jsonlcorpus",tag="method"} + +Initialize the reader. + +> #### Example +> +> ```python +> from spacy.training import JsonlCorpus +> +> corpus = JsonlCorpus("./data/texts.jsonl") +> ``` +> +> ```ini +> ### Example config +> [corpora.pretrain] +> @readers = "spacy.JsonlCorpus.v1" +> path = "corpus/raw_text.jsonl" +> min_length = 0 +> max_length = 0 +> limit = 0 +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects newline-delimited JSON with a key `"text"` for each record. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `min_length` | Minimum document length (in tokens). Shorter documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | + +### JsonlCorpus.\_\_call\_\_ {id="jsonlcorpus-call",tag="method"} + +Yield examples from the data. + +> #### Example +> +> ```python +> from spacy.training import JsonlCorpus +> import spacy +> +> corpus = JsonlCorpus("./texts.jsonl") +> nlp = spacy.blank("en") +> data = corpus(nlp) +> ``` + +| Name | Description | +| ---------- | -------------------------------------- | +| `nlp` | The current `nlp` object. ~~Language~~ | +| **YIELDS** | The examples. ~~Example~~ | + +## PlainTextCorpus {id="plaintextcorpus",tag="class",version="3.5.1"} + +Iterate over documents from a plain text file. Can be used to read the raw text +corpus for language model +[pretraining](/usage/embeddings-transformers#pretraining). The expected file +format is: + +- UTF-8 encoding +- One document per line +- Blank lines are ignored. + +```text {title="Example"} +Can I ask where you work now and what you do, and if you enjoy it? +They may just pull out of the Seattle market completely, at least until they have autonomous vehicles. +My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in. +``` + +### PlainTextCorpus.\_\_init\_\_ {id="plaintextcorpus-init",tag="method"} + +Initialize the reader. + +> #### Example +> +> ```python +> from spacy.training import PlainTextCorpus +> +> corpus = PlainTextCorpus("./data/docs.txt") +> ``` +> +> ```ini +> ### Example config +> [corpora.pretrain] +> @readers = "spacy.PlainTextCorpus.v1" +> path = "corpus/raw_text.txt" +> min_length = 0 +> max_length = 0 +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects newline-delimited documents in UTF8 format. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `min_length` | Minimum document length (in tokens). Shorter documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | + +### PlainTextCorpus.\_\_call\_\_ {id="plaintextcorpus-call",tag="method"} + +Yield examples from the data. + +> #### Example +> +> ```python +> from spacy.training import PlainTextCorpus +> import spacy +> +> corpus = PlainTextCorpus("./docs.txt") +> nlp = spacy.blank("en") +> data = corpus(nlp) +> ``` + +| Name | Description | +| ---------- | -------------------------------------- | +| `nlp` | The current `nlp` object. ~~Language~~ | +| **YIELDS** | The examples. ~~Example~~ | diff --git a/website/docs/api/curatedtransformer.mdx b/website/docs/api/curatedtransformer.mdx new file mode 100644 index 0000000..3e63ef7 --- /dev/null +++ b/website/docs/api/curatedtransformer.mdx @@ -0,0 +1,580 @@ +--- +title: CuratedTransformer +teaser: + Pipeline component for multi-task learning with Curated Transformer models +tag: class +source: github.com/explosion/spacy-curated-transformers/blob/main/spacy_curated_transformers/pipeline/transformer.py +version: 3.7 +api_base_class: /api/pipe +api_string_name: curated_transformer +--- + + + +This component is available via the extension package +[`spacy-curated-transformers`](https://github.com/explosion/spacy-curated-transformers). +It exposes the component via entry points, so if you have the package installed, +using `factory = "curated_transformer"` in your +[training config](/usage/training#config) will work out-of-the-box. + + + +This pipeline component lets you use a curated set of transformer models in your +pipeline. spaCy Curated Transformers currently supports the following model +types: + +- ALBERT +- BERT +- CamemBERT +- RoBERTa +- XLM-RoBERT + +If you want to use another type of model, use +[spacy-transformers](/api/spacy-transformers), which allows you to use all +Hugging Face transformer models with spaCy. + +You will usually connect downstream components to a shared Curated Transformer +pipe using one of the Curated Transformer listener layers. This works similarly +to spaCy's [Tok2Vec](/api/tok2vec), and the +[Tok2VecListener](/api/architectures/#Tok2VecListener) sublayer. The component +assigns the output of the transformer to the `Doc`'s extension attributes. To +access the values, you can use the custom +[`Doc._.trf_data`](#assigned-attributes) attribute. + +For more details, see the [usage documentation](/usage/embeddings-transformers). + +## Assigned Attributes {id="assigned-attributes"} + +The component sets the following +[custom extension attribute](/usage/processing-pipeline#custom-components-attributes): + +| Location | Value | +| ---------------- | -------------------------------------------------------------------------- | +| `Doc._.trf_data` | Curated Transformer outputs for the `Doc` object. ~~DocTransformerOutput~~ | + +## Config and Implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#curated-trf) documentation for details +on the curated transformer architectures and their arguments and +hyperparameters. + +> #### Example +> +> ```python +> from spacy_curated_transformers.pipeline.transformer import DEFAULT_CONFIG +> +> nlp.add_pipe("curated_transformer", config=DEFAULT_CONFIG) +> ``` + +| Setting | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. Defaults to [`XlmrTransformer`](/api/architectures#curated-trf). ~~Model~~ | +| `frozen` | If `True`, the model's weights are frozen and no backpropagation is performed. ~~bool~~ | +| `all_layer_outputs` | If `True`, the model returns the outputs of all the layers. Otherwise, only the output of the last layer is returned. This must be set to `True` if any of the pipe's downstream listeners require the outputs of all transformer layers. ~~bool~~ | + +```python +https://github.com/explosion/spacy-curated-transformers/blob/main/spacy_curated_transformers/pipeline/transformer.py +``` + +## CuratedTransformer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> trf = nlp.add_pipe("curated_transformer") +> +> # Construction via add_pipe with custom config +> config = { +> "model": { +> "@architectures": "spacy-curated-transformers.XlmrTransformer.v1", +> "vocab_size": 250002, +> "num_hidden_layers": 12, +> "hidden_width": 768, +> "piece_encoder": { +> "@architectures": "spacy-curated-transformers.XlmrSentencepieceEncoder.v1" +> } +> } +> } +> trf = nlp.add_pipe("curated_transformer", config=config) +> +> # Construction from class +> from spacy_curated_transformers import CuratedTransformer +> trf = CuratedTransformer(nlp.vocab, model) +> ``` + +Construct a `CuratedTransformer` component. One or more subsequent spaCy +components can use the transformer outputs as features in its model, with +gradients backpropagated to the single shared weights. The activations from the +transformer are saved in the [`Doc._.trf_data`](#assigned-attributes) extension +attribute. You can also provide a callback to set additional annotations. In +your application, you would normally use a shortcut for this and instantiate the +component using its string name and [`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | One of the supported pre-trained transformer models. ~~Model~~ | +| _keyword-only_ | | +| `name` | The component instance name. ~~str~~ | +| `frozen` | If `True`, the model's weights are frozen and no backpropagation is performed. ~~bool~~ | +| `all_layer_outputs` | If `True`, the model returns the outputs of all the layers. Otherwise, only the output of the last layer is returned. This must be set to `True` if any of the pipe's downstream listeners require the outputs of all transformer layers. ~~bool~~ | + +## CuratedTransformer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/curatedtransformer#call) and +[`pipe`](/api/curatedtransformer#pipe) delegate to the +[`predict`](/api/curatedtransformer#predict) and +[`set_annotations`](/api/curatedtransformer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> trf = nlp.add_pipe("curated_transformer") +> # This usually happens under the hood +> processed = trf(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## CuratedTransformer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/curatedtransformer#call) +and [`pipe`](/api/curatedtransformer#pipe) delegate to the +[`predict`](/api/curatedtransformer#predict) and +[`set_annotations`](/api/curatedtransformer#set_annotations) methods. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> for doc in trf.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## CuratedTransformer.initialize {id="initialize",tag="method"} + +Initialize the component for training and return an +[`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a +function that returns an iterable of [`Example`](/api/example) objects. **At +least one example should be supplied.** The data examples are used to +**initialize the model** of the component and can either be the full training +data or a representative sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> trf.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `encoder_loader` | Initialization callback for the transformer model. ~~Optional[Callable]~~ | +| `piece_loader` | Initialization callback for the input piece encoder. ~~Optional[Callable]~~ | + +## CuratedTransformer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> scores = trf.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## CuratedTransformer.set_annotations {id="set_annotations",tag="method"} + +Assign the extracted features to the `Doc` objects. By default, the +[`DocTransformerOutput`](/api/curatedtransformer#doctransformeroutput) object is +written to the [`Doc._.trf_data`](#assigned-attributes) attribute. Your +`set_extra_annotations` callback is then called, if provided. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> scores = trf.predict(docs) +> trf.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------------ | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `CuratedTransformer.predict`. | + +## CuratedTransformer.update {id="update",tag="method"} + +Prepare for an update to the transformer. + +Like the [`Tok2Vec`](api/tok2vec) component, the `CuratedTransformer` component +is unusual in that it does not receive "gold standard" annotations to calculate +a weight update. The optimal output of the transformer data is unknown; it's a +hidden layer inside the network that is updated by backpropagating from output +layers. + +The `CuratedTransformer` component therefore does not perform a weight update +during its own `update` method. Instead, it runs its transformer model and +communicates the output and the backpropagation callback to any downstream +components that have been connected to it via the transformer listener sublayer. +If there are multiple listeners, the last layer will actually backprop to the +transformer and call the optimizer, while the others simply increment the +gradients. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> optimizer = nlp.initialize() +> losses = trf.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | A batch of [`Example`](/api/example) objects. Only the [`Example.predicted`](/api/example#predicted) `Doc` object is used, the reference `Doc` is ignored. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## CuratedTransformer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> optimizer = trf.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## CuratedTransformer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> with trf.use_params(optimizer.averages): +> trf.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## CuratedTransformer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> trf.to_disk("/path/to/transformer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## CuratedTransformer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> trf.from_disk("/path/to/transformer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `CuratedTransformer` object. ~~CuratedTransformer~~ | + +## CuratedTransformer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> trf = nlp.add_pipe("curated_transformer") +> trf_bytes = trf.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `CuratedTransformer` object. ~~bytes~~ | + +## CuratedTransformer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> trf_bytes = trf.to_bytes() +> trf = nlp.add_pipe("curated_transformer") +> trf.from_bytes(trf_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `CuratedTransformer` object. ~~CuratedTransformer~~ | + +## Serialization Fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = trf.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | + +## DocTransformerOutput {id="doctransformeroutput",tag="dataclass"} + +Curated Transformer outputs for one `Doc` object. Stores the dense +representations generated by the transformer for each piece identifier. Piece +identifiers are grouped by token. Instances of this class are typically assigned +to the [`Doc._.trf_data`](/api/curatedtransformer#assigned-attributes) extension +attribute. + +> #### Example +> +> ```python +> # Get the last hidden layer output for "is" (token index 1) +> doc = nlp("This is a text.") +> tensors = doc._.trf_data.last_hidden_layer_state[1] +> ``` + +| Name | Description | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `all_outputs` | List of `Ragged` tensors that correspends to outputs of the different transformer layers. Each tensor element corresponds to a piece identifier's representation. ~~List[Ragged]~~ | +| `last_layer_only` | If only the last transformer layer's outputs are preserved. ~~bool~~ | + +### DocTransformerOutput.embedding_layer {id="doctransformeroutput-embeddinglayer",tag="property"} + +Return the output of the transformer's embedding layer or `None` if +`last_layer_only` is `True`. + +| Name | Description | +| ----------- | -------------------------------------------- | +| **RETURNS** | Embedding layer output. ~~Optional[Ragged]~~ | + +### DocTransformerOutput.last_hidden_layer_state {id="doctransformeroutput-lasthiddenlayerstate",tag="property"} + +Return the output of the transformer's last hidden layer. + +| Name | Description | +| ----------- | ------------------------------------ | +| **RETURNS** | Last hidden layer output. ~~Ragged~~ | + +### DocTransformerOutput.all_hidden_layer_states {id="doctransformeroutput-allhiddenlayerstates",tag="property"} + +Return the outputs of all transformer layers (excluding the embedding layer). + +| Name | Description | +| ----------- | -------------------------------------- | +| **RETURNS** | Hidden layer outputs. ~~List[Ragged]~~ | + +### DocTransformerOutput.num_outputs {id="doctransformeroutput-numoutputs",tag="property"} + +Return the number of layer outputs stored in the `DocTransformerOutput` instance +(including the embedding layer). + +| Name | Description | +| ----------- | -------------------------- | +| **RETURNS** | Numbef of outputs. ~~int~~ | + +## Span Getters {id="span_getters",source="github.com/explosion/spacy-transformers/blob/master/spacy_curated_transformers/span_getters.py"} + +Span getters are functions that take a batch of [`Doc`](/api/doc) objects and +return a lists of [`Span`](/api/span) objects for each doc to be processed by +the transformer. This is used to manage long documents by cutting them into +smaller sequences before running the transformer. The spans are allowed to +overlap, and you can also omit sections of the `Doc` if they are not relevant. +Span getters can be referenced in the +`[components.transformer.model.with_spans]` block of the config to customize the +sequences processed by the transformer. + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | A batch of `Doc` objects. ~~Iterable[Doc]~~ | +| **RETURNS** | The spans to process by the transformer. ~~List[List[Span]]~~ | + +### WithStridedSpans.v1 {id="strided_spans",tag="registered function"} + +> #### Example config +> +> ```ini +> [transformer.model.with_spans] +> @architectures = "spacy-curated-transformers.WithStridedSpans.v1" +> stride = 96 +> window = 128 +> ``` + +Create a span getter for strided spans. If you set the `window` and `stride` to +the same value, the spans will cover each token once. Setting `stride` lower +than `window` will allow for an overlap, so that some tokens are counted twice. +This can be desirable, because it allows all tokens to have both a left and +right context. + +| Name | Description | +| -------- | ------------------------ | +| `window` | The window size. ~~int~~ | +| `stride` | The stride size. ~~int~~ | + +## Model Loaders + +[Curated Transformer models](/api/architectures#curated-trf) are constructed +with default hyperparameters and randomized weights when the pipeline is +created. To load the weights of an existing pre-trained model into the pipeline, +one of the following loader callbacks can be used. The pre-trained model must +have the same hyperparameters as the model used by the pipeline. + +### HFTransformerEncoderLoader.v1 {id="hf_trfencoder_loader",tag="registered_function"} + +Construct a callback that initializes a supported transformer model with weights +from a corresponding HuggingFace model. + +| Name | Description | +| ---------- | ------------------------------------------ | +| `name` | Name of the HuggingFace model. ~~str~~ | +| `revision` | Name of the model revision/branch. ~~str~~ | + +### PyTorchCheckpointLoader.v1 {id="pytorch_checkpoint_loader",tag="registered_function"} + +Construct a callback that initializes a supported transformer model with weights +from a PyTorch checkpoint. + +| Name | Description | +| ------ | ---------------------------------------- | +| `path` | Path to the PyTorch checkpoint. ~~Path~~ | + +## Tokenizer Loaders + +[Curated Transformer models](/api/architectures#curated-trf) must be paired with +a matching tokenizer (piece encoder) model in a spaCy pipeline. As with the +transformer models, tokenizers are constructed with an empty vocabulary during +pipeline creation - They need to be initialized with an appropriate loader +before use in training/inference. + +### ByteBPELoader.v1 {id="bytebpe_loader",tag="registered_function"} + +Construct a callback that initializes a Byte-BPE piece encoder model. + +| Name | Description | +| ------------- | ------------------------------------- | +| `vocab_path` | Path to the vocabulary file. ~~Path~~ | +| `merges_path` | Path to the merges file. ~~Path~~ | + +### CharEncoderLoader.v1 {id="charencoder_loader",tag="registered_function"} + +Construct a callback that initializes a character piece encoder model. + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `path` | Path to the serialized character model. ~~Path~~ | +| `bos_piece` | Piece used as a beginning-of-sentence token. Defaults to `"[BOS]"`. ~~str~~ | +| `eos_piece` | Piece used as a end-of-sentence token. Defaults to `"[EOS]"`. ~~str~~ | +| `unk_piece` | Piece used as a stand-in for unknown tokens. Defaults to `"[UNK]"`. ~~str~~ | +| `normalize` | Unicode normalization form to use. Defaults to `"NFKC"`. ~~str~~ | + +### HFPieceEncoderLoader.v1 {id="hf_pieceencoder_loader",tag="registered_function"} + +Construct a callback that initializes a HuggingFace piece encoder model. Used in +conjunction with the HuggingFace model loader. + +| Name | Description | +| ---------- | ------------------------------------------ | +| `name` | Name of the HuggingFace model. ~~str~~ | +| `revision` | Name of the model revision/branch. ~~str~~ | + +### SentencepieceLoader.v1 {id="sentencepiece_loader",tag="registered_function"} + +Construct a callback that initializes a SentencePiece piece encoder model. + +| Name | Description | +| ------ | ---------------------------------------------------- | +| `path` | Path to the serialized SentencePiece model. ~~Path~~ | + +### WordpieceLoader.v1 {id="wordpiece_loader",tag="registered_function"} + +Construct a callback that initializes a WordPiece piece encoder model. + +| Name | Description | +| ------ | ------------------------------------------------ | +| `path` | Path to the serialized WordPiece model. ~~Path~~ | + +## Callbacks + +### gradual_transformer_unfreezing.v1 {id="gradual_transformer_unfreezing",tag="registered_function"} + +Construct a callback that can be used to gradually unfreeze the weights of one +or more Transformer components during training. This can be used to prevent +catastrophic forgetting during fine-tuning. + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `target_pipes` | A dictionary whose keys and values correspond to the names of Transformer components and the training step at which they should be unfrozen respectively. ~~Dict[str, int]~~ | diff --git a/website/docs/api/cython-classes.mdx b/website/docs/api/cython-classes.mdx new file mode 100644 index 0000000..ce7c039 --- /dev/null +++ b/website/docs/api/cython-classes.mdx @@ -0,0 +1,209 @@ +--- +title: Cython Classes +menu: + - ['Doc', 'doc'] + - ['Token', 'token'] + - ['Span', 'span'] + - ['Lexeme', 'lexeme'] + - ['Vocab', 'vocab'] + - ['StringStore', 'stringstore'] +--- + +## Doc {id="doc",tag="cdef class",source="spacy/tokens/doc.pxd"} + +The `Doc` object holds an array of [`TokenC`](/api/cython-structs#tokenc) +structs. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see [`Doc`](/api/doc). + + + +### Attributes {id="doc_attributes"} + +| Name | Description | +| ------------ | -------------------------------------------------------------------------------------------------------- | +| `mem` | A memory pool. Allocated memory will be freed once the `Doc` object is garbage collected. ~~cymem.Pool~~ | +| `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ | +| `c` | A pointer to a [`TokenC`](/api/cython-structs#tokenc) struct. ~~TokenC\*~~ | +| `length` | The number of tokens in the document. ~~int~~ | +| `max_length` | The underlying size of the `Doc.c` array. ~~int~~ | + +### Doc.push_back {id="doc_push_back",tag="method"} + +Append a token to the `Doc`. The token can be provided as a +[`LexemeC`](/api/cython-structs#lexemec) or +[`TokenC`](/api/cython-structs#tokenc) pointer, using Cython's +[fused types](http://cython.readthedocs.io/en/latest/src/userguide/fusedtypes.html). + +> #### Example +> +> ```python +> from spacy.tokens cimport Doc +> from spacy.vocab cimport Vocab +> +> doc = Doc(Vocab()) +> lexeme = doc.vocab.get("hello") +> doc.push_back(lexeme, True) +> assert doc.text == "hello " +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------- | +| `lex_or_tok` | The word to append to the `Doc`. ~~LexemeOrToken~~ | +| `has_space` | Whether the word has trailing whitespace. ~~bint~~ | + +## Token {id="token",tag="cdef class",source="spacy/tokens/token.pxd"} + +A Cython class providing access and methods for a +[`TokenC`](/api/cython-structs#tokenc) struct. Note that the `Token` object does +not own the struct. It only receives a pointer to it. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see [`Token`](/api/token). + + + +### Attributes {id="token_attributes"} + +| Name | Description | +| ------- | -------------------------------------------------------------------------- | +| `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ | +| `c` | A pointer to a [`TokenC`](/api/cython-structs#tokenc) struct. ~~TokenC\*~~ | +| `i` | The offset of the token within the document. ~~int~~ | +| `doc` | The parent document. ~~Doc~~ | + +### Token.cinit {id="token_cinit",tag="method"} + +Create a `Token` object from a `TokenC*` pointer. + +> #### Example +> +> ```python +> token = Token.cinit(&doc.c[3], doc, 3) +> ``` + +| Name | Description | +| -------- | -------------------------------------------------------------------------- | +| `vocab` | A reference to the shared `Vocab`. ~~Vocab~~ | +| `c` | A pointer to a [`TokenC`](/api/cython-structs#tokenc) struct. ~~TokenC\*~~ | +| `offset` | The offset of the token within the document. ~~int~~ | +| `doc` | The parent document. ~~int~~ | + +## Span {id="span",tag="cdef class",source="spacy/tokens/span.pxd"} + +A Cython class providing access and methods for a slice of a `Doc` object. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see [`Span`](/api/span). + + + +### Attributes {id="span_attributes"} + +| Name | Description | +| ------------ | ----------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `start` | The index of the first token of the span. ~~int~~ | +| `end` | The index of the first token after the span. ~~int~~ | +| `start_char` | The index of the first character of the span. ~~int~~ | +| `end_char` | The index of the last character of the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~attr_t (uint64_t)~~ | + +## Lexeme {id="lexeme",tag="cdef class",source="spacy/lexeme.pxd"} + +A Cython class providing access and methods for an entry in the vocabulary. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme). + + + +### Attributes {id="lexeme_attributes"} + +| Name | Description | +| ------- | ----------------------------------------------------------------------------- | +| `c` | A pointer to a [`LexemeC`](/api/cython-structs#lexemec) struct. ~~LexemeC\*~~ | +| `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ | +| `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | + +## Vocab {id="vocab",tag="cdef class",source="spacy/vocab.pxd"} + +A Cython class providing access and methods for a vocabulary and other data +shared across a language. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab). + + + +### Attributes {id="vocab_attributes"} + +| Name | Description | +| --------- | ---------------------------------------------------------------------------------------------------------- | +| `mem` | A memory pool. Allocated memory will be freed once the `Vocab` object is garbage collected. ~~cymem.Pool~~ | +| `strings` | A `StringStore` that maps string to hash values and vice versa. ~~StringStore~~ | +| `length` | The number of entries in the vocabulary. ~~int~~ | + +### Vocab.get {id="vocab_get",tag="method"} + +Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the +vocabulary. + +> #### Example +> +> ```python +> lexeme = vocab.get(vocab.mem, "hello") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------- | +| `mem` | A memory pool. Allocated memory will be freed once the `Vocab` object is garbage collected. ~~cymem.Pool~~ | +| `string` | The string of the word to look up. ~~str~~ | +| **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | + +### Vocab.get_by_orth {id="vocab_get_by_orth",tag="method"} + +Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the +vocabulary. + +> #### Example +> +> ```python +> lexeme = vocab.get_by_orth(doc[0].lex.norm) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------- | +| `mem` | A memory pool. Allocated memory will be freed once the `Vocab` object is garbage collected. ~~cymem.Pool~~ | +| `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | +| **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | + +## StringStore {id="stringstore",tag="cdef class",source="spacy/strings.pxd"} + +A lookup table to retrieve strings by 64-bit hashes. + + + +This section documents the extra C-level attributes and methods that can't be +accessed from Python. For the Python documentation, see +[`StringStore`](/api/stringstore). + + + +### Attributes {id="stringstore_attributes"} + +| Name | Description | +| ------ | ---------------------------------------------------------------------------------------------------------------- | +| `mem` | A memory pool. Allocated memory will be freed once the `StringStore` object is garbage collected. ~~cymem.Pool~~ | +| `keys` | A list of hash values in the `StringStore`. ~~vector[hash_t] \(vector[uint64_t])~~ | diff --git a/website/docs/api/cython-structs.mdx b/website/docs/api/cython-structs.mdx new file mode 100644 index 0000000..106a27e --- /dev/null +++ b/website/docs/api/cython-structs.mdx @@ -0,0 +1,253 @@ +--- +title: Cython Structs +teaser: C-language objects that let you group variables together +next: /api/cython-classes +menu: + - ['TokenC', 'tokenc'] + - ['LexemeC', 'lexemec'] +--- + +## TokenC {id="tokenc",tag="C struct",source="spacy/structs.pxd"} + +Cython data container for the `Token` object. + +> #### Example +> +> ```python +> token = &doc.c[3] +> token_ptr = &doc.c[3] +> ``` + +| Name | Description | +| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lex` | A pointer to the lexeme for the token. ~~const LexemeC\*~~ | +| `morph` | An ID allowing lookup of morphological attributes. ~~uint64_t~~ | +| `pos` | Coarse-grained part-of-speech tag. ~~univ_pos_t~~ | +| `spacy` | A binary value indicating whether the token has trailing whitespace. ~~bint~~ | +| `tag` | Fine-grained part-of-speech tag. ~~attr_t (uint64_t)~~ | +| `idx` | The character offset of the token within the parent document. ~~int~~ | +| `lemma` | Base form of the token, with no inflectional suffixes. ~~attr_t (uint64_t)~~ | +| `sense` | Space for storing a word sense ID, currently unused. ~~attr_t (uint64_t)~~ | +| `head` | Offset of the syntactic parent relative to the token. ~~int~~ | +| `dep` | Syntactic dependency relation. ~~attr_t (uint64_t)~~ | +| `l_kids` | Number of left children. ~~uint32_t~~ | +| `r_kids` | Number of right children. ~~uint32_t~~ | +| `l_edge` | Offset of the leftmost token of this token's syntactic descendants. ~~uint32_t~~ | +| `r_edge` | Offset of the rightmost token of this token's syntactic descendants. ~~uint32_t~~ | +| `sent_start` | Ternary value indicating whether the token is the first word of a sentence. `0` indicates a missing value, `-1` indicates `False` and `1` indicates `True`. The default value, 0, is interpreted as no sentence break. Sentence boundary detectors will usually set 0 for all tokens except tokens that follow a sentence boundary. ~~int~~ | +| `ent_iob` | IOB code of named entity tag. `0` indicates a missing value, `1` indicates `I`, `2` indicates `0` and `3` indicates `B`. ~~int~~ | +| `ent_type` | Named entity type. ~~attr_t (uint64_t)~~ | +| `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~attr_t (uint64_t)~~ | + +### Token.get_struct_attr {id="token_get_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} + +Get the value of an attribute from the `TokenC` struct by attribute ID. + +> #### Example +> +> ```python +> from spacy.attrs cimport IS_ALPHA +> from spacy.tokens cimport Token +> +> is_alpha = Token.get_struct_attr(&doc.c[3], IS_ALPHA) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------- | +| `token` | A pointer to a `TokenC` struct. ~~const TokenC\*~~ | +| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | + +### Token.set_struct_attr {id="token_set_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} + +Set the value of an attribute of the `TokenC` struct by attribute ID. + +> #### Example +> +> ```python +> from spacy.attrs cimport TAG +> from spacy.tokens cimport Token +> +> token = &doc.c[3] +> Token.set_struct_attr(token, TAG, 0) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------- | +| `token` | A pointer to a `TokenC` struct. ~~const TokenC\*~~ | +| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| `value` | The value to set. ~~attr_t (uint64_t)~~ | + +### token_by_start {id="token_by_start",tag="function",source="spacy/tokens/doc.pxd"} + +Find a token in a `TokenC*` array by the offset of its first character. + +> #### Example +> +> ```python +> from spacy.tokens.doc cimport Doc, token_by_start +> from spacy.vocab cimport Vocab +> +> doc = Doc(Vocab(), words=["hello", "world"]) +> assert token_by_start(doc.c, doc.length, 6) == 1 +> assert token_by_start(doc.c, doc.length, 4) == -1 +> ``` + +| Name | Description | +| ------------ | ----------------------------------------------------------------- | +| `tokens` | A `TokenC*` array. ~~const TokenC\*~~ | +| `length` | The number of tokens in the array. ~~int~~ | +| `start_char` | The start index to search for. ~~int~~ | +| **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | + +### token_by_end {id="token_by_end",tag="function",source="spacy/tokens/doc.pxd"} + +Find a token in a `TokenC*` array by the offset of its final character. + +> #### Example +> +> ```python +> from spacy.tokens.doc cimport Doc, token_by_end +> from spacy.vocab cimport Vocab +> +> doc = Doc(Vocab(), words=["hello", "world"]) +> assert token_by_end(doc.c, doc.length, 5) == 0 +> assert token_by_end(doc.c, doc.length, 1) == -1 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------- | +| `tokens` | A `TokenC*` array. ~~const TokenC\*~~ | +| `length` | The number of tokens in the array. ~~int~~ | +| `end_char` | The end index to search for. ~~int~~ | +| **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | + +### set_children_from_heads {id="set_children_from_heads",tag="function",source="spacy/tokens/doc.pxd"} + +Set attributes that allow lookup of syntactic children on a `TokenC*` array. +This function must be called after making changes to the `TokenC.head` +attribute, in order to make the parse tree navigation consistent. + +> #### Example +> +> ```python +> from spacy.tokens.doc cimport Doc, set_children_from_heads +> from spacy.vocab cimport Vocab +> +> doc = Doc(Vocab(), words=["Baileys", "from", "a", "shoe"]) +> doc.c[0].head = 0 +> doc.c[1].head = 0 +> doc.c[2].head = 3 +> doc.c[3].head = 1 +> set_children_from_heads(doc.c, doc.length) +> assert doc.c[3].l_kids == 1 +> ``` + +| Name | Description | +| -------- | ------------------------------------------ | +| `tokens` | A `TokenC*` array. ~~const TokenC\*~~ | +| `length` | The number of tokens in the array. ~~int~~ | + +## LexemeC {id="lexemec",tag="C struct",source="spacy/structs.pxd"} + +Struct holding information about a lexical type. `LexemeC` structs are usually +owned by the `Vocab`, and accessed through a read-only pointer on the `TokenC` +struct. + +> #### Example +> +> ```python +> lex = doc.c[3].lex +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `flags` | Bit-field for binary lexical flag values. ~~flags_t (uint64_t)~~ | +| `id` | Usually used to map lexemes to rows in a matrix, e.g. for word vectors. Does not need to be unique, so currently misnamed. ~~attr_t (uint64_t)~~ | +| `length` | Number of unicode characters in the lexeme. ~~attr_t (uint64_t)~~ | +| `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | +| `lower` | ID of the lowercase form of the lexeme. ~~attr_t (uint64_t)~~ | +| `norm` | ID of the lexeme's norm, i.e. a normalized form of the text. ~~attr_t (uint64_t)~~ | +| `shape` | Transform of the lexeme's string, to show orthographic features. ~~attr_t (uint64_t)~~ | +| `prefix` | Length-N substring from the start of the lexeme. Defaults to `N=1`. ~~attr_t (uint64_t)~~ | +| `suffix` | Length-N substring from the end of the lexeme. Defaults to `N=3`. ~~attr_t (uint64_t)~~ | + +### Lexeme.get_struct_attr {id="lexeme_get_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} + +Get the value of an attribute from the `LexemeC` struct by attribute ID. + +> #### Example +> +> ```python +> from spacy.attrs cimport IS_ALPHA +> from spacy.lexeme cimport Lexeme +> +> lexeme = doc.c[3].lex +> is_alpha = Lexeme.get_struct_attr(lexeme, IS_ALPHA) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------- | +| `lex` | A pointer to a `LexemeC` struct. ~~const LexemeC\*~~ | +| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | + +### Lexeme.set_struct_attr {id="lexeme_set_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} + +Set the value of an attribute of the `LexemeC` struct by attribute ID. + +> #### Example +> +> ```python +> from spacy.attrs cimport NORM +> from spacy.lexeme cimport Lexeme +> +> lexeme = doc.c[3].lex +> Lexeme.set_struct_attr(lexeme, NORM, lexeme.lower) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------- | +| `lex` | A pointer to a `LexemeC` struct. ~~const LexemeC\*~~ | +| `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| `value` | The value to set. ~~attr_t (uint64_t)~~ | + +### Lexeme.c_check_flag {id="lexeme_c_check_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} + +Check the value of a binary flag attribute. + +> #### Example +> +> ```python +> from spacy.attrs cimport IS_STOP +> from spacy.lexeme cimport Lexeme +> +> lexeme = doc.c[3].lex +> is_stop = Lexeme.c_check_flag(lexeme, IS_STOP) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------------- | +| `lexeme` | A pointer to a `LexemeC` struct. ~~const LexemeC\*~~ | +| `flag_id` | The ID of the flag to look up. The flag IDs are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| **RETURNS** | The boolean value of the flag. ~~bint~~ | + +### Lexeme.c_set_flag {id="lexeme_c_set_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} + +Set the value of a binary flag attribute. + +> #### Example +> +> ```python +> from spacy.attrs cimport IS_STOP +> from spacy.lexeme cimport Lexeme +> +> lexeme = doc.c[3].lex +> Lexeme.c_set_flag(lexeme, IS_STOP, 0) +> ``` + +| Name | Description | +| --------- | --------------------------------------------------------------------------------------------- | +| `lexeme` | A pointer to a `LexemeC` struct. ~~const LexemeC\*~~ | +| `flag_id` | The ID of the flag to look up. The flag IDs are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | +| `value` | The value to set. ~~bint~~ | diff --git a/website/docs/api/cython.mdx b/website/docs/api/cython.mdx new file mode 100644 index 0000000..5f744ee --- /dev/null +++ b/website/docs/api/cython.mdx @@ -0,0 +1,134 @@ +--- +title: Cython Architecture +next: /api/cython-structs +menu: + - ['Overview', 'overview'] + - ['Conventions', 'conventions'] +--- + +## Overview {id="overview",hidden="true"} + +> #### What's Cython? +> +> [Cython](http://cython.org/) is a language for writing C extensions for +> Python. Most Python code is also valid Cython, but you can add type +> declarations to get efficient memory-managed code just like C or C++. + +This section documents spaCy's C-level data structures and interfaces, intended +for use from Cython. Some of the attributes are primarily for internal use, and +all C-level functions and methods are designed for speed over safety – if you +make a mistake and access an array out-of-bounds, the program may crash +abruptly. + +With Cython there are four ways of declaring complex data types. Unfortunately +we use all four in different places, as they all have different utility: + +| Declaration | Description | Example | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| `class` | A normal Python class. | [`Language`](/api/language) | +| `cdef class` | A Python extension type. Differs from a normal Python class in that its attributes can be defined on the underlying struct. Can have C-level objects as attributes (notably structs and pointers), and can have methods which have C-level objects as arguments or return types. | [`Lexeme`](/api/cython-classes#lexeme) | +| `cdef struct` | A struct is just a collection of variables, sort of like a named tuple, except the memory is contiguous. Structs can't have methods, only attributes. | [`LexemeC`](/api/cython-structs#lexemec) | +| `cdef cppclass` | A C++ class. Like a struct, this can be allocated on the stack, but can have methods, a constructor and a destructor. Differs from `cdef class` in that it can be created and destroyed without acquiring the Python global interpreter lock. This style is the most obscure. | [`StateC`](%%GITHUB_SPACY/spacy/pipeline/_parser_internals/_state.pxd) | + +The most important classes in spaCy are defined as `cdef class` objects. The +underlying data for these objects is usually gathered into a struct, which is +usually named `c`. For instance, the [`Lexeme`](/api/cython-classes#lexeme) +class holds a [`LexemeC`](/api/cython-structs#lexemec) struct, at `Lexeme.c`. +This lets you shed the Python container, and pass a pointer to the underlying +data into C-level functions. + +## Conventions {id="conventions"} + +spaCy's core data structures are implemented as [Cython](http://cython.org/) +`cdef` classes. Memory is managed through the +[`cymem`](https://github.com/explosion/cymem) `cymem.Pool` class, which allows +you to allocate memory which will be freed when the `Pool` object is garbage +collected. This means you usually don't have to worry about freeing memory. You +just have to decide which Python object owns the memory, and make it own the +`Pool`. When that object goes out of scope, the memory will be freed. You do +have to take care that no pointers outlive the object that owns them — but this +is generally quite easy. + +All Cython modules should have the `# cython: infer_types=True` compiler +directive at the top of the file. This makes the code much cleaner, as it avoids +the need for many type declarations. If possible, you should prefer to declare +your functions `nogil`, even if you don't especially care about multi-threading. +The reason is that `nogil` functions help the Cython compiler reason about your +code quite a lot — you're telling the compiler that no Python dynamics are +possible. This lets many errors be raised, and ensures your function will run at +C speed. + +Cython gives you many choices of sequences: you could have a Python list, a +numpy array, a memory view, a C++ vector, or a pointer. Pointers are preferred, +because they are fastest, have the most explicit semantics, and let the compiler +check your code more strictly. C++ vectors are also great — but you should only +use them internally in functions. It's less friendly to accept a vector as an +argument, because that asks the user to do much more work. Here's how to get a +pointer from a numpy array, memory view or vector: + +```python +cdef void get_pointers(np.ndarray[int, mode='c'] numpy_array, vector[int] cpp_vector, int[::1] memory_view) nogil: +pointer1 = numpy_array.data +pointer2 = cpp_vector.data() +pointer3 = &memory_view[0] +``` + +Both C arrays and C++ vectors reassure the compiler that no Python operations +are possible on your variable. This is a big advantage: it lets the Cython +compiler raise many more errors for you. + +When getting a pointer from a numpy array or memoryview, take care that the data +is actually stored in C-contiguous order — otherwise you'll get a pointer to +nonsense. The type-declarations in the code above should generate runtime errors +if buffers with incorrect memory layouts are passed in. To iterate over the +array, the following style is preferred: + +```python +cdef int c_total(const int* int_array, int length) nogil: + total = 0 + for item in int_array[:length]: + total += item + return total +``` + +If this is confusing, consider that the compiler couldn't deal with +`for item in int_array:` — there's no length attached to a raw pointer, so how +could we figure out where to stop? The length is provided in the slice notation +as a solution to this. Note that we don't have to declare the type of `item` in +the code above — the compiler can easily infer it. This gives us tidy code that +looks quite like Python, but is exactly as fast as C — because we've made sure +the compilation to C is trivial. + +Your functions cannot be declared `nogil` if they need to create Python objects +or call Python functions. This is perfectly okay — you shouldn't torture your +code just to get `nogil` functions. However, if your function isn't `nogil`, you +should compile your module with `cython -a --cplus my_module.pyx` and open the +resulting `my_module.html` file in a browser. This will let you see how Cython +is compiling your code. Calls into the Python run-time will be in bright yellow. +This lets you easily see whether Cython is able to correctly type your code, or +whether there are unexpected problems. + +Working in Cython is very rewarding once you're over the initial learning curve. +As with C and C++, the first way you write something in Cython will often be the +performance-optimal approach. In contrast, Python optimization generally +requires a lot of experimentation. Is it faster to have an `if item in my_dict` +check, or to use `.get()`? What about `try`/`except`? Does this numpy operation +create a copy? There's no way to guess the answers to these questions, and +you'll usually be dissatisfied with your results — so there's no way to know +when to stop this process. In the worst case, you'll make a mess that invites +the next reader to try their luck too. This is like one of those +[volcanic gas-traps](http://www.wemjournal.org/article/S1080-6032%2809%2970088-2/abstract), +where the rescuers keep passing out from low oxygen, causing another rescuer to +follow — only to succumb themselves. In short, just say no to optimizing your +Python. If it's not fast enough the first time, just switch to Cython. + + + +- [Official Cython documentation](http://docs.cython.org/en/latest/) + (cython.org) +- [Writing C in Cython](https://explosion.ai/blog/writing-c-in-cython) + (explosion.ai) +- [Multi-threading spaCy’s parser and named entity recognizer](https://explosion.ai/blog/multithreading-with-cython) + (explosion.ai) + + diff --git a/website/docs/api/data-formats.mdx b/website/docs/api/data-formats.mdx new file mode 100644 index 0000000..c9d88f8 --- /dev/null +++ b/website/docs/api/data-formats.mdx @@ -0,0 +1,597 @@ +--- +title: Data formats +teaser: Details on spaCy's input and output data formats +menu: + - ['Training Config', 'config'] + - ['Training Data', 'training'] + - ['Vocabulary', 'vocab-jsonl'] + - ['Pipeline Meta', 'meta'] +--- + +This section documents input and output formats of data used by spaCy, including +the [training config](/usage/training#config), training data and lexical +vocabulary data. For an overview of label schemes used by the models, see the +[models directory](/models). Each trained pipeline documents the label schemes +used in its components, depending on the data it was trained on. + +## Training config {id="config",version="3"} + +Config files define the training process and pipeline and can be passed to +[`spacy train`](/api/cli#train). They use +[Thinc's configuration system](https://thinc.ai/docs/usage-config) under the +hood. For details on how to use training configs, see the +[usage documentation](/usage/training#config). To get started with the +recommended settings for your use case, check out the +[quickstart widget](/usage/training#quickstart) or run the +[`init config`](/api/cli#init-config) command. + +> #### What does the @ mean? +> +> The `@` syntax lets you refer to function names registered in the +> [function registry](/api/top-level#registry). For example, +> `@architectures = "spacy.HashEmbedCNN.v2"` refers to a registered function of +> the name [spacy.HashEmbedCNN.v2](/api/architectures#HashEmbedCNN) and all +> other values defined in its block will be passed into that function as +> arguments. Those arguments depend on the registered function. See the usage +> guide on [registered functions](/usage/training#config-functions) for details. + +```ini +%%GITHUB_SPACY/spacy/default_config.cfg +``` + + + +Under the hood, spaCy's configs are powered by our machine learning library +[Thinc's config system](https://thinc.ai/docs/usage-config), which uses +[`pydantic`](https://github.com/samuelcolvin/pydantic/) for data validation +based on type hints. See [`spacy/schemas.py`](%%GITHUB_SPACY/spacy/schemas.py) +for the schemas used to validate the default config. Arguments of registered +functions are validated against their type annotations, if available. To debug +your config and check that it's valid, you can run the +[`spacy debug config`](/api/cli#debug-config) command. + + + +### nlp {id="config-nlp",tag="section"} + +> #### Example +> +> ```ini +> [nlp] +> lang = "en" +> pipeline = ["tagger", "parser", "ner"] +> before_creation = null +> after_creation = null +> after_pipeline_creation = null +> batch_size = 1000 +> +> [nlp.tokenizer] +> @tokenizers = "spacy.Tokenizer.v1" +> ``` + +Defines the `nlp` object, its tokenizer and +[processing pipeline](/usage/processing-pipelines) component names. + +| Name | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lang` | Pipeline language [ISO code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). Defaults to `null`. ~~str~~ | +| `pipeline` | Names of pipeline components in order. Should correspond to sections in the `[components]` block, e.g. `[components.ner]`. See docs on [defining components](/usage/training#config-components). Defaults to `[]`. ~~List[str]~~ | +| `disabled` | Names of pipeline components that are loaded but disabled by default and not run as part of the pipeline. Should correspond to components listed in `pipeline`. After a pipeline is loaded, disabled components can be enabled using [`Language.enable_pipe`](/api/language#enable_pipe). ~~List[str]~~ | +| `before_creation` | Optional [callback](/usage/training#custom-code-nlp-callbacks) to modify `Language` subclass before it's initialized. Defaults to `null`. ~~Optional[Callable[[Type[Language]], Type[Language]]]~~ | +| `after_creation` | Optional [callback](/usage/training#custom-code-nlp-callbacks) to modify `nlp` object right after it's initialized. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | +| `after_pipeline_creation` | Optional [callback](/usage/training#custom-code-nlp-callbacks) to modify `nlp` object after the pipeline components have been added. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | +| `tokenizer` | The tokenizer to use. Defaults to [`Tokenizer`](/api/tokenizer). ~~Callable[[str], Doc]~~ | +| `batch_size` | Default batch size for [`Language.pipe`](/api/language#pipe) and [`Language.evaluate`](/api/language#evaluate). ~~int~~ | + +### components {id="config-components",tag="section"} + +> #### Example +> +> ```ini +> [components.textcat] +> factory = "textcat" +> +> [components.textcat.model] +> @architectures = "spacy.TextCatBOW.v2" +> exclusive_classes = true +> ngram_size = 1 +> no_output_layer = false +> ``` + +This section includes definitions of the +[pipeline components](/usage/processing-pipelines) and their models, if +available. Components in this section can be referenced in the `pipeline` of the +`[nlp]` block. Component blocks need to specify either a `factory` (named +function to use to create component) or a `source` (name of path of trained +pipeline to copy components from). See the docs on +[defining pipeline components](/usage/training#config-components) for details. + +### paths, system {id="config-variables",tag="variables"} + +These sections define variables that can be referenced across the other sections +as variables. For example `${paths.train}` uses the value of `train` defined in +the block `[paths]`. If your config includes custom registered functions that +need paths, you can define them here. All config values can also be +[overwritten](/usage/training#config-overrides) on the CLI when you run +[`spacy train`](/api/cli#train), which is especially relevant for data paths +that you don't want to hard-code in your config file. + +```bash +$ python -m spacy train config.cfg --paths.train ./corpus/train.spacy +``` + +### corpora {id="config-corpora",tag="section"} + +> #### Example +> +> ```ini +> [corpora] +> +> [corpora.train] +> @readers = "spacy.Corpus.v1" +> path = ${paths:train} +> +> [corpora.dev] +> @readers = "spacy.Corpus.v1" +> path = ${paths:dev} +> +> [corpora.pretrain] +> @readers = "spacy.JsonlCorpus.v1" +> path = ${paths.raw} +> +> [corpora.my_custom_data] +> @readers = "my_custom_reader.v1" +> ``` + +This section defines a **dictionary** mapping of string keys to functions. Each +function takes an `nlp` object and yields [`Example`](/api/example) objects. By +default, the two keys `train` and `dev` are specified and each refer to a +[`Corpus`](/api/top-level#Corpus). When pretraining, an additional `pretrain` +section is added that defaults to a [`JsonlCorpus`](/api/top-level#jsonlcorpus). +You can also register custom functions that return a callable. + +| Name | Description | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `train` | Training data corpus, typically used in `[training]` block. ~~Callable[[Language], Iterator[Example]]~~ | +| `dev` | Development data corpus, typically used in `[training]` block. ~~Callable[[Language], Iterator[Example]]~~ | +| `pretrain` | Raw text for [pretraining](/usage/embeddings-transformers#pretraining), typically used in `[pretraining]` block (if available). ~~Callable[[Language], Iterator[Example]]~~ | +| ... | Any custom or alternative corpora. ~~Callable[[Language], Iterator[Example]]~~ | + +Alternatively, the `[corpora]` block can refer to **one function** that returns +a dictionary keyed by the corpus names. This can be useful if you want to load a +single corpus once and then divide it up into `train` and `dev` partitions. + +> #### Example +> +> ```ini +> [corpora] +> @readers = "my_custom_reader.v1" +> train_path = ${paths:train} +> dev_path = ${paths:dev} +> shuffle = true +> +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `corpora` | A dictionary keyed by string names, mapped to corpus functions that receive the current `nlp` object and return an iterator of [`Example`](/api/example) objects. ~~Dict[str, Callable[[Language], Iterator[Example]]]~~ | + +### training {id="config-training",tag="section"} + +This section defines settings and controls for the training and evaluation +process that are used when you run [`spacy train`](/api/cli#train). + +| Name | Description | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~ | +| `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ | +| `before_to_disk` | Optional callback to modify `nlp` object right before it is saved to disk during and after training. Can be used to remove or reset config values or disable components. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | +| `before_update` 3.5 | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | +| `dev_corpus` | Dot notation of the config location defining the dev corpus. Defaults to `corpora.dev`. ~~str~~ | +| `dropout` | The dropout rate. Defaults to `0.1`. ~~float~~ | +| `eval_frequency` | How often to evaluate during training (steps). Defaults to `200`. ~~int~~ | +| `frozen_components` | Pipeline component names that are "frozen" and shouldn't be initialized or updated during training. See [here](/usage/training#config-components) for details. Defaults to `[]`. ~~List[str]~~ | +| `annotating_components` 3.1 | Pipeline component names that should set annotations on the predicted docs during training. See [here](/usage/training#annotating-components) for details. Defaults to `[]`. ~~List[str]~~ | +| `gpu_allocator` | Library for cupy to route GPU memory allocation to. Can be `"pytorch"` or `"tensorflow"`. Defaults to variable `${system.gpu_allocator}`. ~~str~~ | +| `logger` | Callable that takes the `nlp` and stdout and stderr `IO` objects, sets up the logger, and returns two new callables to log a training step and to finalize the logger. Defaults to [`ConsoleLogger`](/api/top-level#ConsoleLogger). ~~Callable[[Language, IO, IO], [Tuple[Callable[[Dict[str, Any]], None], Callable[[], None]]]]~~ | +| `max_epochs` | Maximum number of epochs to train for. `0` means an unlimited number of epochs. `-1` means that the train corpus should be streamed rather than loaded into memory with no shuffling within the training loop. Defaults to `0`. ~~int~~ | +| `max_steps` | Maximum number of update steps to train for. `0` means an unlimited number of steps. Defaults to `20000`. ~~int~~ | +| `optimizer` | The optimizer. The learning rate schedule and other settings can be configured as part of the optimizer. Defaults to [`Adam`](https://thinc.ai/docs/api-optimizers#adam). ~~Optimizer~~ | +| `patience` | How many steps to continue without improvement in evaluation score. `0` disables early stopping. Defaults to `1600`. ~~int~~ | +| `score_weights` | Score names shown in metrics mapped to their weight towards the final weighted score. See [here](/usage/training#metrics) for details. Defaults to `{}`. ~~Dict[str, float]~~ | +| `seed` | The random seed. Defaults to variable `${system.seed}`. ~~int~~ | +| `train_corpus` | Dot notation of the config location defining the train corpus. Defaults to `corpora.train`. ~~str~~ | + +### pretraining {id="config-pretraining",tag="section,optional"} + +This section is optional and defines settings and controls for +[language model pretraining](/usage/embeddings-transformers#pretraining). It's +used when you run [`spacy pretrain`](/api/cli#pretrain). + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `max_epochs` | Maximum number of epochs. Defaults to `1000`. ~~int~~ | +| `dropout` | The dropout rate. Defaults to `0.2`. ~~float~~ | +| `n_save_every` | Saving frequency. Defaults to `null`. ~~Optional[int]~~ | +| `objective` | The pretraining objective. Defaults to `{"type": "characters", "n_characters": 4}`. ~~Dict[str, Any]~~ | +| `optimizer` | The optimizer. The learning rate schedule and other settings can be configured as part of the optimizer. Defaults to [`Adam`](https://thinc.ai/docs/api-optimizers#adam). ~~Optimizer~~ | +| `corpus` | Dot notation of the config location defining the corpus with raw text. Defaults to `corpora.pretrain`. ~~str~~ | +| `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ | +| `component` | Component name to identify the layer with the model to pretrain. Defaults to `"tok2vec"`. ~~str~~ | +| `layer` | The specific layer of the model to pretrain. If empty, the whole model will be used. ~~str~~ | + +### initialize {id="config-initialize",tag="section"} + +This config block lets you define resources for **initializing the pipeline**. +It's used by [`Language.initialize`](/api/language#initialize) and typically +called right before training (but not at runtime). The section allows you to +specify local file paths or custom functions to load data resources from, +without requiring them at runtime when you load the trained pipeline back in. +Also see the usage guides on the +[config lifecycle](/usage/training#config-lifecycle) and +[custom initialization](/usage/training#initialization). + +> #### Example +> +> ```ini +> [initialize] +> vectors = "/path/to/vectors_nlp" +> init_tok2vec = "/path/to/pretrain.bin" +> +> [initialize_components] +> +> [initialize.components.my_component] +> data_path = "/path/to/component_data" +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `after_init` | Optional callback to modify the `nlp` object after initialization. ~~Optional[Callable[[Language], Language]]~~ | +| `before_init` | Optional callback to modify the `nlp` object before initialization. ~~Optional[Callable[[Language], Language]]~~ | +| `components` | Additional arguments passed to the `initialize` method of a pipeline component, keyed by component name. If type annotations are available on the method, the config will be validated against them. The `initialize` methods will always receive the `get_examples` callback and the current `nlp` object. ~~Dict[str, Dict[str, Any]]~~ | +| `init_tok2vec` | Optional path to pretrained tok2vec weights created with [`spacy pretrain`](/api/cli#pretrain). Defaults to variable `${paths.init_tok2vec}`. Ignored when actually running pretraining, as you're creating the file to be used later. ~~Optional[str]~~ | +| `lookups` | Additional lexeme and vocab data from [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). Defaults to `null`. ~~Optional[Lookups]~~ | +| `tokenizer` | Additional arguments passed to the `initialize` method of the specified tokenizer. Can be used for languages like Chinese that depend on dictionaries or trained models for tokenization. If type annotations are available on the method, the config will be validated against them. The `initialize` method will always receive the `get_examples` callback and the current `nlp` object. ~~Dict[str, Any]~~ | +| `vectors` | Name or path of pipeline containing pretrained word vectors to use, e.g. created with [`init vectors`](/api/cli#init-vectors). Defaults to `null`. ~~Optional[str]~~ | +| `vocab_data` | Path to JSONL-formatted [vocabulary file](/api/data-formats#vocab-jsonl) to initialize vocabulary. ~~Optional[str]~~ | + +## Training data {id="training"} + +### Binary training format {id="binary-training",version="3"} + +> #### Example +> +> ```python +> from spacy.tokens import DocBin +> from spacy.training import Corpus +> +> doc_bin = DocBin(docs=docs) +> doc_bin.to_disk("./data.spacy") +> reader = Corpus("./data.spacy") +> ``` + +The main data format used in spaCy v3.0 is a **binary format** created by +serializing a [`DocBin`](/api/docbin), which represents a collection of `Doc` +objects. This means that you can train spaCy pipelines using the same format it +outputs: annotated `Doc` objects. The binary format is extremely **efficient in +storage**, especially when packing multiple documents together. + +Typically, the extension for these binary files is `.spacy`, and they are used +as input format for specifying a [training corpus](/api/corpus) and for spaCy's +CLI [`train`](/api/cli#train) command. The built-in +[`convert`](/api/cli#convert) command helps you convert spaCy's previous +[JSON format](#json-input) to the new binary format. It also supports conversion +of the `.conllu` format used by the +[Universal Dependencies corpora](https://github.com/UniversalDependencies). + +Note that while this is the format used to save training data, you do not have +to understand the internal details to use it or create training data. See the +section on [preparing training data](/usage/training#training-data). + +### JSON training format {id="json-input",tag="deprecated"} + + + +As of v3.0, the JSON input format is deprecated and is replaced by the +[binary format](#binary-training). Instead of converting [`Doc`](/api/doc) +objects to JSON, you can now serialize them directly using the +[`DocBin`](/api/docbin) container and then use them as input data. + +[`spacy convert`](/api/cli) lets you convert your JSON data to the new `.spacy` +format: + +```bash +$ python -m spacy convert ./data.json . +``` + + + +> #### Annotating entities +> +> Named entities are provided in the +> [BILUO](/usage/linguistic-features#accessing-ner) notation. Tokens outside an +> entity are set to `"O"` and tokens that are part of an entity are set to the +> entity label, prefixed by the BILUO marker. For example `"B-ORG"` describes +> the first token of a multi-token `ORG` entity and `"U-PERSON"` a single token +> representing a `PERSON` entity. The +> [`offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags) function can +> help you convert entity offsets to the right format. + +```python {title="Example structure"} +[{ + "id": int, # ID of the document within the corpus + "paragraphs": [{ # list of paragraphs in the corpus + "raw": string, # raw text of the paragraph + "sentences": [{ # list of sentences in the paragraph + "tokens": [{ # list of tokens in the sentence + "id": int, # index of the token in the document + "dep": string, # dependency label + "head": int, # offset of token head relative to token index + "tag": string, # part-of-speech tag + "orth": string, # verbatim text of the token + "ner": string # BILUO label, e.g. "O" or "B-ORG" + }], + "brackets": [{ # phrase structure (NOT USED by current models) + "first": int, # index of first token + "last": int, # index of last token + "label": string # phrase label + }] + }], + "cats": [{ # new in v2.2: categories for text classifier + "label": string, # text category label + "value": float / bool # label applies (1.0/true) or not (0.0/false) + }] + }] +}] +``` + + + +Here's an example of dependencies, part-of-speech tags and named entities, taken +from the English Wall Street Journal portion of the Penn Treebank: + +```json +https://github.com/explosion/spaCy/blob/v2.3.x/examples/training/training-data.json +``` + + + +### Annotation format for creating training examples {id="dict-input"} + +An [`Example`](/api/example) object holds the information for one training +instance. It stores two [`Doc`](/api/doc) objects: one for holding the +gold-standard reference data, and one for holding the predictions of the +pipeline. Examples can be created using the +[`Example.from_dict`](/api/example#from_dict) method with a reference `Doc` and +a dictionary of gold-standard annotations. + +> #### Example +> +> ```python +> example = Example.from_dict(doc, gold_dict) +> ``` + + + +`Example` objects are used as part of the +[internal training API](/usage/training#api) and they're expected when you call +[`nlp.update`](/api/language#update). However, for most use cases, you +**shouldn't** have to write your own training scripts. It's recommended to train +your pipelines via the [`spacy train`](/api/cli#train) command with a config +file to keep track of your settings and hyperparameters and your own +[registered functions](/usage/training/#custom-code) to customize the setup. + + + +> #### Example +> +> ```python +> { +> "text": str, +> "words": List[str], +> "lemmas": List[str], +> "spaces": List[bool], +> "tags": List[str], +> "pos": List[str], +> "morphs": List[str], +> "sent_starts": List[Optional[bool]], +> "deps": List[str], +> "heads": List[int], +> "entities": List[str], +> "entities": List[(int, int, str)], +> "cats": Dict[str, float], +> "links": Dict[(int, int), dict], +> "spans": Dict[str, List[Tuple]], +> } +> ``` + +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `text` | Raw text. ~~str~~ | +| `words` | List of gold-standard tokens. ~~List[str]~~ | +| `lemmas` | List of lemmas. ~~List[str]~~ | +| `spaces` | List of boolean values indicating whether the corresponding tokens is followed by a space or not. ~~List[bool]~~ | +| `tags` | List of fine-grained [POS tags](/usage/linguistic-features#pos-tagging). ~~List[str]~~ | +| `pos` | List of coarse-grained [POS tags](/usage/linguistic-features#pos-tagging). ~~List[str]~~ | +| `morphs` | List of [morphological features](/usage/linguistic-features#rule-based-morphology). ~~List[str]~~ | +| `sent_starts` | List of boolean values indicating whether each token is the first of a sentence or not. ~~List[bool]~~ | +| `deps` | List of string values indicating the [dependency relation](/usage/linguistic-features#dependency-parse) of a token to its head. ~~List[str]~~ | +| `heads` | List of integer values indicating the dependency head of each token, referring to the absolute index of each token in the text. ~~List[int]~~ | +| `entities` | **Option 1:** List of [BILUO tags](/usage/linguistic-features#accessing-ner) per token of the format `"{action}-{label}"`, or `None` for unannotated tokens. ~~List[str]~~ | +| `entities` | **Option 2:** List of `(start_char, end_char, label)` tuples defining all entities in the text. ~~List[Tuple[int, int, str]]~~ | +| `cats` | Dictionary of `label`/`value` pairs indicating how relevant a certain [text category](/api/textcategorizer) is for the text. ~~Dict[str, float]~~ | +| `links` | Dictionary of `offset`/`dict` pairs defining [named entity links](/usage/linguistic-features#entity-linking). The character offsets are linked to a dictionary of relevant knowledge base IDs. ~~Dict[Tuple[int, int], Dict]~~ | +| `spans` | Dictionary of `spans_key`/`List[Tuple]` pairs defining the spans for each spans key as `(start_char, end_char, label, kb_id)` tuples. ~~Dict[str, List[Tuple[int, int, str, str]]~~ | + + + +- Multiple formats are possible for the "entities" entry, but you have to pick + one. +- Any values for sentence starts will be ignored if there are annotations for + dependency relations. +- If the dictionary contains values for `"text"` and `"words"`, but not + `"spaces"`, the latter are inferred automatically. If "words" is not provided + either, the values are inferred from the `Doc` argument. + + + +```python {title="Examples"} +# Training data for a part-of-speech tagger +doc = Doc(vocab, words=["I", "like", "stuff"]) +gold_dict = {"tags": ["NOUN", "VERB", "NOUN"]} +example = Example.from_dict(doc, gold_dict) + +# Training data for an entity recognizer (option 1) +doc = nlp("Laura flew to Silicon Valley.") +gold_dict = {"entities": ["U-PERS", "O", "O", "B-LOC", "L-LOC"]} +example = Example.from_dict(doc, gold_dict) + +# Training data for an entity recognizer (option 2) +doc = nlp("Laura flew to Silicon Valley.") +gold_dict = {"entities": [(0, 5, "PERSON"), (14, 28, "LOC")]} +example = Example.from_dict(doc, gold_dict) + +# Training data for text categorization +doc = nlp("I'm pretty happy about that!") +gold_dict = {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}} +example = Example.from_dict(doc, gold_dict) + +# Training data for an Entity Linking component (also requires entities & sentences) +doc = nlp("Russ Cochran his reprints include EC Comics.") +gold_dict = {"entities": [(0, 12, "PERSON")], + "links": {(0, 12): {"Q7381115": 1.0, "Q2146908": 0.0}}, + "sent_starts": [1, -1, -1, -1, -1, -1, -1, -1]} +example = Example.from_dict(doc, gold_dict) +``` + +## Lexical data for vocabulary {id="vocab-jsonl",version="2"} + +This data file can be provided via the `vocab_data` setting in the +`[initialize]` block of the training config to pre-define the lexical data to +initialize the `nlp` object's vocabulary with. The file should contain one +lexical entry per line. The first line defines the language and vocabulary +settings. All other lines are expected to be JSON objects describing an +individual lexeme. The lexical attributes will be then set as attributes on +spaCy's [`Lexeme`](/api/lexeme#attributes) object. + +> #### Example config +> +> ```ini +> [initialize] +> vocab_data = "/path/to/vocab-data.jsonl" +> ``` + +```python {title="First line"} +{"lang": "en", "settings": {"oov_prob": -20.502029418945312}} +``` + +```python {title="Entry structure"} +{ + "orth": string, # the word text + "id": int, # can correspond to row in vectors table + "lower": string, + "norm": string, + "shape": string + "prefix": string, + "suffix": string, + "length": int, + "cluster": string, + "prob": float, + "is_alpha": bool, + "is_ascii": bool, + "is_digit": bool, + "is_lower": bool, + "is_punct": bool, + "is_space": bool, + "is_title": bool, + "is_upper": bool, + "like_url": bool, + "like_num": bool, + "like_email": bool, + "is_stop": bool, + "is_oov": bool, + "is_quote": bool, + "is_left_punct": bool, + "is_right_punct": bool +} +``` + +Here's an example of the 20 most frequent lexemes in the English training data: + +```json +%%GITHUB_SPACY/extra/example_data/vocab-data.jsonl +``` + +## Pipeline meta {id="meta"} + +The pipeline meta is available as the file `meta.json` and exported +automatically when you save an `nlp` object to disk. Its contents are available +as [`nlp.meta`](/api/language#meta). + + + +As of spaCy v3.0, the `meta.json` **isn't** used to construct the language class +and pipeline anymore and only contains meta information for reference and for +creating a Python package with [`spacy package`](/api/cli#package). How to set +up the `nlp` object is now defined in the +[config file](/api/data-formats#config), which includes detailed information +about the pipeline components and their model architectures, and all other +settings and hyperparameters used to train the pipeline. It's the **single +source of truth** used for loading a pipeline. + + + +> #### Example +> +> ```json +> { +> "name": "example_pipeline", +> "lang": "en", +> "version": "1.0.0", +> "spacy_version": ">=3.0.0,<3.1.0", +> "parent_package": "spacy", +> "requirements": ["spacy-transformers>=1.0.0,<1.1.0"], +> "description": "Example pipeline for spaCy", +> "author": "You", +> "email": "you@example.com", +> "url": "https://example.com", +> "license": "CC BY-SA 3.0", +> "sources": [{ "name": "My Corpus", "license": "MIT" }], +> "vectors": { "width": 0, "vectors": 0, "keys": 0, "name": null }, +> "pipeline": ["tok2vec", "ner", "textcat"], +> "labels": { +> "ner": ["PERSON", "ORG", "PRODUCT"], +> "textcat": ["POSITIVE", "NEGATIVE"] +> }, +> "performance": { +> "ents_f": 82.7300930714, +> "ents_p": 82.135523614, +> "ents_r": 83.3333333333, +> "textcat_score": 88.364323811 +> }, +> "speed": { "cpu": 7667.8, "gpu": null, "nwords": 10329 }, +> "spacy_git_version": "61dfdd9fb" +> } +> ``` + +| Name | Description | +| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lang` | Pipeline language [ISO code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). Defaults to `"en"`. ~~str~~ | +| `name` | Pipeline name, e.g. `"core_web_sm"`. The final package name will be `{lang}_{name}`. Defaults to `"pipeline"`. ~~str~~ | +| `version` | Pipeline version. Will be used to version a Python package created with [`spacy package`](/api/cli#package). Defaults to `"0.0.0"`. ~~str~~ | +| `spacy_version` | spaCy version range the package is compatible with. Defaults to the spaCy version used to create the pipeline, up to next minor version, which is the default compatibility for the available [trained pipelines](/models). For instance, a pipeline trained with v3.0.0 will have the version range `">=3.0.0,<3.1.0"`. ~~str~~ | +| `parent_package` | Name of the spaCy package. Typically `"spacy"` or `"spacy_nightly"`. Defaults to `"spacy"`. ~~str~~ | +| `requirements` | Python package requirements that the pipeline depends on. Will be used for the Python package setup in [`spacy package`](/api/cli#package). Should be a list of package names with optional version specifiers, just like you'd define them in a `setup.cfg` or `requirements.txt`. Defaults to `[]`. ~~List[str]~~ | +| `description` | Pipeline description. Also used for Python package. Defaults to `""`. ~~str~~ | +| `author` | Pipeline author name. Also used for Python package. Defaults to `""`. ~~str~~ | +| `email` | Pipeline author email. Also used for Python package. Defaults to `""`. ~~str~~ | +| `url` | Pipeline author URL. Also used for Python package. Defaults to `""`. ~~str~~ | +| `license` | Pipeline license. Also used for Python package. Defaults to `""`. ~~str~~ | +| `sources` | Data sources used to train the pipeline. Typically a list of dicts with the keys `"name"`, `"url"`, `"author"` and `"license"`. [See here](https://github.com/explosion/spacy-models/tree/master/meta) for examples. Defaults to `None`. ~~Optional[List[Dict[str, str]]]~~ | +| `vectors` | Information about the word vectors included with the pipeline. Typically a dict with the keys `"width"`, `"vectors"` (number of vectors), `"keys"` and `"name"`. ~~Dict[str, Any]~~ | +| `pipeline` | Names of pipeline component names, in order. Corresponds to [`nlp.pipe_names`](/api/language#pipe_names). Only exists for reference and is not used to create the components. This information is defined in the [`config.cfg`](/api/data-formats#config). Defaults to `[]`. ~~List[str]~~ | +| `labels` | Label schemes of the trained pipeline components, keyed by component name. Corresponds to [`nlp.pipe_labels`](/api/language#pipe_labels). [See here](https://github.com/explosion/spacy-models/tree/master/meta) for examples. Defaults to `{}`. ~~Dict[str, Dict[str, List[str]]]~~ | +| `performance` | Training accuracy, added automatically by [`spacy train`](/api/cli#train). Dictionary of [score names](/usage/training#metrics) mapped to scores. Defaults to `{}`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | +| `speed` | Inference speed, added automatically by [`spacy train`](/api/cli#train). Typically a dictionary with the keys `"cpu"`, `"gpu"` and `"nwords"` (words per second). Defaults to `{}`. ~~Dict[str, Optional[Union[float, str]]]~~ | +| `spacy_git_version` 3 | Git commit of [`spacy`](https://github.com/explosion/spaCy) used to create pipeline. ~~str~~ | +| other | Any other custom meta information you want to add. The data is preserved in [`nlp.meta`](/api/language#meta). ~~Any~~ | diff --git a/website/docs/api/dependencymatcher.mdx b/website/docs/api/dependencymatcher.mdx new file mode 100644 index 0000000..d0971da --- /dev/null +++ b/website/docs/api/dependencymatcher.mdx @@ -0,0 +1,230 @@ +--- +title: DependencyMatcher +teaser: Match subtrees within a dependency parse +tag: class +version: 3 +source: spacy/matcher/dependencymatcher.pyx +--- + +The `DependencyMatcher` follows the same API as the [`Matcher`](/api/matcher) +and [`PhraseMatcher`](/api/phrasematcher) and lets you match on dependency trees +using +[Semgrex operators](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html). +It requires a pretrained [`DependencyParser`](/api/parser) or other component +that sets the `Token.dep` and `Token.head` attributes. See the +[usage guide](/usage/rule-based-matching#dependencymatcher) for examples. + +## Pattern format {id="patterns"} + +> ```python +> ### Example +> # pattern: "[subject] ... initially founded" +> [ +> # anchor token: founded +> { +> "RIGHT_ID": "founded", +> "RIGHT_ATTRS": {"ORTH": "founded"} +> }, +> # founded -> subject +> { +> "LEFT_ID": "founded", +> "REL_OP": ">", +> "RIGHT_ID": "subject", +> "RIGHT_ATTRS": {"DEP": "nsubj"} +> }, +> # "founded" follows "initially" +> { +> "LEFT_ID": "founded", +> "REL_OP": ";", +> "RIGHT_ID": "initially", +> "RIGHT_ATTRS": {"ORTH": "initially"} +> } +> ] +> ``` + +A pattern added to the `DependencyMatcher` consists of a list of dictionaries, +with each dictionary describing a token to match. Except for the first +dictionary, which defines an anchor token using only `RIGHT_ID` and +`RIGHT_ATTRS`, each pattern should have the following keys: + +| Name | Description | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LEFT_ID` | The name of the left-hand node in the relation, which has been defined in an earlier node. ~~str~~ | +| `REL_OP` | An operator that describes how the two nodes are related. ~~str~~ | +| `RIGHT_ID` | A unique name for the right-hand node in the relation. ~~str~~ | +| `RIGHT_ATTRS` | The token attributes to match for the right-hand node in the same format as patterns provided to the regular token-based [`Matcher`](/api/matcher). ~~Dict[str, Any]~~ | + + + +For examples of how to construct dependency matcher patterns for different types +of relations, see the usage guide on +[dependency matching](/usage/rule-based-matching#dependencymatcher). + + + +### Operators {id="operators"} + +The following operators are supported by the `DependencyMatcher`, most of which +come directly from +[Semgrex](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html): + +| Symbol | Description | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `A < B` | `A` is the immediate dependent of `B`. | +| `A > B` | `A` is the immediate head of `B`. | +| `A << B` | `A` is the dependent in a chain to `B` following dep → head paths. | +| `A >> B` | `A` is the head in a chain to `B` following head → dep paths. | +| `A . B` | `A` immediately precedes `B`, i.e. `A.i == B.i - 1`, and both are within the same dependency tree. | +| `A .* B` | `A` precedes `B`, i.e. `A.i < B.i`, and both are within the same dependency tree _(Semgrex counterpart: `..`)_. | +| `A ; B` | `A` immediately follows `B`, i.e. `A.i == B.i + 1`, and both are within the same dependency tree _(Semgrex counterpart: `-`)_. | +| `A ;* B` | `A` follows `B`, i.e. `A.i > B.i`, and both are within the same dependency tree _(Semgrex counterpart: `--`)_. | +| `A $+ B` | `B` is a right immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i - 1`. | +| `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. | +| `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. | +| `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. | +| `A >+ B` 3.5.1 | `B` is a right immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A >- B` 3.5.1 | `B` is a left immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i`. | +| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i`. | +| `A <+ B` 3.5.1 | `B` is a right immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A <- B` 3.5.1 | `B` is a left immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i`. | +| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i`. | + +## DependencyMatcher.\_\_init\_\_ {id="init",tag="method"} + +Create a `DependencyMatcher`. + +> #### Example +> +> ```python +> from spacy.matcher import DependencyMatcher +> matcher = DependencyMatcher(nlp.vocab) +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------- | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| _keyword-only_ | | +| `validate` | Validate all patterns added to this matcher. ~~bool~~ | + +## DependencyMatcher.\_\_call\_\_ {id="call",tag="method"} + +Find all tokens matching the supplied patterns on the `Doc` or `Span`. + +> #### Example +> +> ```python +> from spacy.matcher import DependencyMatcher +> +> matcher = DependencyMatcher(nlp.vocab) +> pattern = [{"RIGHT_ID": "founded_id", +> "RIGHT_ATTRS": {"ORTH": "founded"}}] +> matcher.add("FOUNDED", [pattern]) +> doc = nlp("Bill Gates founded Microsoft.") +> matches = matcher(doc) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ | +| **RETURNS** | A list of `(match_id, token_ids)` tuples, describing the matches. The `match_id` is the ID of the match pattern and `token_ids` is a list of token indices matched by the pattern, where the position of each token in the list corresponds to the position of the node specification in the pattern. ~~List[Tuple[int, List[int]]]~~ | + +## DependencyMatcher.\_\_len\_\_ {id="len",tag="method"} + +Get the number of rules added to the dependency matcher. Note that this only +returns the number of rules (identical with the number of IDs), not the number +of individual patterns. + +> #### Example +> +> ```python +> matcher = DependencyMatcher(nlp.vocab) +> assert len(matcher) == 0 +> pattern = [{"RIGHT_ID": "founded_id", +> "RIGHT_ATTRS": {"ORTH": "founded"}}] +> matcher.add("FOUNDED", [pattern]) +> assert len(matcher) == 1 +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The number of rules. ~~int~~ | + +## DependencyMatcher.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether the matcher contains rules for a match ID. + +> #### Example +> +> ```python +> matcher = DependencyMatcher(nlp.vocab) +> assert "FOUNDED" not in matcher +> matcher.add("FOUNDED", [pattern]) +> assert "FOUNDED" in matcher +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------- | +| `key` | The match ID. ~~str~~ | +| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | + +## DependencyMatcher.add {id="add",tag="method"} + +Add a rule to the matcher, consisting of an ID key, one or more patterns, and an +optional callback function to act on the matches. The callback function will +receive the arguments `matcher`, `doc`, `i` and `matches`. If a pattern already +exists for the given ID, the patterns will be extended. An `on_match` callback +will be overwritten. + +> #### Example +> +> ```python +> def on_match(matcher, doc, id, matches): +> print('Matched!', matches) +> +> matcher = DependencyMatcher(nlp.vocab) +> matcher.add("FOUNDED", patterns, on_match=on_match) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `match_id` | An ID for the patterns. ~~str~~ | +| `patterns` | A list of match patterns. A pattern consists of a list of dicts, where each dict describes a token in the tree. ~~List[List[Dict[str, Union[str, Dict]]]]~~ | +| _keyword-only_ | | +| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[DependencyMatcher, Doc, int, List[Tuple], Any]]~~ | + +## DependencyMatcher.get {id="get",tag="method"} + +Retrieve the pattern stored for a key. Returns the rule as an +`(on_match, patterns)` tuple containing the callback and available patterns. + +> #### Example +> +> ```python +> matcher.add("FOUNDED", patterns, on_match=on_match) +> on_match, patterns = matcher.get("FOUNDED") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------- | +| `key` | The ID of the match rule. ~~str~~ | +| **RETURNS** | The rule, as an `(on_match, patterns)` tuple. ~~Tuple[Optional[Callable], List[List[Union[Dict, Tuple]]]]~~ | + +## DependencyMatcher.remove {id="remove",tag="method"} + +Remove a rule from the dependency matcher. A `KeyError` is raised if the match +ID does not exist. + +> #### Example +> +> ```python +> matcher.add("FOUNDED", patterns) +> assert "FOUNDED" in matcher +> matcher.remove("FOUNDED") +> assert "FOUNDED" not in matcher +> ``` + +| Name | Description | +| ----- | --------------------------------- | +| `key` | The ID of the match rule. ~~str~~ | diff --git a/website/docs/api/dependencyparser.mdx b/website/docs/api/dependencyparser.mdx new file mode 100644 index 0000000..a6bc48c --- /dev/null +++ b/website/docs/api/dependencyparser.mdx @@ -0,0 +1,469 @@ +--- +title: DependencyParser +tag: class +source: spacy/pipeline/dep_parser.pyx +teaser: 'Pipeline component for syntactic dependency parsing' +api_base_class: /api/pipe +api_string_name: parser +api_trainable: true +--- + +A transition-based dependency parser component. The dependency parser jointly +learns sentence segmentation and labelled dependency parsing, and can optionally +learn to merge tokens that had been over-segmented by the tokenizer. The parser +uses a variant of the **non-monotonic arc-eager transition-system** described by +[Honnibal and Johnson (2014)](https://www.aclweb.org/anthology/D15-1162/), with +the addition of a "break" transition to perform the sentence segmentation. +[Nivre (2005)](https://www.aclweb.org/anthology/P05-1013/)'s **pseudo-projective +dependency transformation** is used to allow the parser to predict +non-projective parses. + +The parser is trained using an **imitation learning objective**. It follows the +actions predicted by the current weights, and at each state, determines which +actions are compatible with the optimal parse that could be reached from the +current state. The weights are updated such that the scores assigned to the set +of optimal actions is increased, while scores assigned to other actions are +decreased. Note that more than one action may be optimal for a given state. + +## Assigned Attributes {id="assigned-attributes"} + +Dependency predictions are assigned to the `Token.dep` and `Token.head` fields. +Beside the dependencies themselves, the parser decides sentence boundaries, +which are saved in `Token.is_sent_start` and accessible via `Doc.sents`. + +| Location | Value | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| `Token.dep` | The type of dependency relation (hash). ~~int~~ | +| `Token.dep_` | The type of dependency relation. ~~str~~ | +| `Token.head` | The syntactic parent, or "governor", of this token. ~~Token~~ | +| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. After the parser runs this will be `True` or `False` for all tokens. ~~bool~~ | +| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL +> config = { +> "moves": None, +> "update_with_oracle_cut_size": 100, +> "learn_tokens": False, +> "min_action_freq": 30, +> "model": DEFAULT_PARSER_MODEL, +> } +> nlp.add_pipe("parser", config=config) +> ``` + +| Setting | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `moves` | A list of transition names. Inferred from the data if not provided. Defaults to `None`. ~~Optional[TransitionSystem]~~ | +| `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | +| `learn_tokens` | Whether to learn to merge subtokens that are split relative to the gold standard. Experimental. Defaults to `False`. ~~bool~~ | +| `min_action_freq` | The minimum frequency of labelled actions to retain. Rarer labelled actions have their label backed-off to "dep". While this primarily affects the label accuracy, it can also affect the attachment structure, as the labels are used to represent the pseudo-projectivity transformation. Defaults to `30`. ~~int~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [TransitionBasedParser](/api/architectures#TransitionBasedParser). ~~Model[List[Doc], List[Floats2d]]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/dep_parser.pyx +``` + +## DependencyParser.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> parser = nlp.add_pipe("parser") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_parser"}} +> parser = nlp.add_pipe("parser", config=config) +> +> # Construction from class +> from spacy.pipeline import DependencyParser +> parser = DependencyParser(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| `moves` | A list of transition names. Inferred from the data if not provided. ~~Optional[TransitionSystem]~~ | +| _keyword-only_ | | +| `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | +| `learn_tokens` | Whether to learn to merge subtokens that are split relative to the gold standard. Experimental. Defaults to `False`. ~~bool~~ | +| `min_action_freq` | The minimum frequency of labelled actions to retain. Rarer labelled actions have their label backed-off to "dep". While this primarily affects the label accuracy, it can also affect the attachment structure, as the labels are used to represent the pseudo-projectivity transformation. ~~int~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_deps`](/api/scorer#score_deps) for the attribute `"dep"` ignoring the labels `p` and `punct` and [`Scorer.score_spans`](/api/scorer/#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | + +## DependencyParser.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/dependencyparser#call) and +[`pipe`](/api/dependencyparser#pipe) delegate to the +[`predict`](/api/dependencyparser#predict) and +[`set_annotations`](/api/dependencyparser#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> parser = nlp.add_pipe("parser") +> # This usually happens under the hood +> processed = parser(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## DependencyParser.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/dependencyparser#call) and +[`pipe`](/api/dependencyparser#pipe) delegate to the +[`predict`](/api/dependencyparser#predict) and +[`set_annotations`](/api/dependencyparser#set_annotations) methods. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> for doc in parser.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `docs` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## DependencyParser.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.parser] +> +> [initialize.components.parser.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/parser.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | + +## DependencyParser.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> scores = parser.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | + +## DependencyParser.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> scores = parser.predict([doc1, doc2]) +> parser.set_annotations([doc1, doc2], scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `DependencyParser.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | + +## DependencyParser.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects, updating the pipe's +model. Delegates to [`predict`](/api/dependencyparser#predict) and +[`get_loss`](/api/dependencyparser#get_loss). + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> optimizer = nlp.initialize() +> losses = parser.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## DependencyParser.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> scores = parser.predict([eg.predicted for eg in examples]) +> loss, d_loss = parser.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. ~~StateClass~~ | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## DependencyParser.create_optimizer {id="create_optimizer",tag="method"} + +Create an [`Optimizer`](https://thinc.ai/docs/api-optimizers) for the pipeline +component. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> optimizer = parser.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## DependencyParser.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> parser = DependencyParser(nlp.vocab) +> with parser.use_params(optimizer.averages): +> parser.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## DependencyParser.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. Note that you don't have to call this method if you +provide a **representative data sample** to the [`initialize`](#initialize) +method. In this case, all labels found in the sample will be automatically added +to the model, and the output dimension will be +[inferred](/usage/layers-architectures#thinc-shape-inference) automatically. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## DependencyParser.set_output {id="set_output",tag="method"} + +Change the output dimension of the component's model by calling the model's +attribute `resize_output`. This is a function that takes the original model and +the new output dimension `nO`, and changes the model in place. When resizing an +already trained model, care should be taken to avoid the "catastrophic +forgetting" problem. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser.set_output(512) +> ``` + +| Name | Description | +| ---- | --------------------------------- | +| `nO` | The new output dimension. ~~int~~ | + +## DependencyParser.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser.to_disk("/path/to/parser") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## DependencyParser.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser.from_disk("/path/to/parser") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `DependencyParser` object. ~~DependencyParser~~ | + +## DependencyParser.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> parser = nlp.add_pipe("parser") +> parser_bytes = parser.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `DependencyParser` object. ~~bytes~~ | + +## DependencyParser.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> parser_bytes = parser.to_bytes() +> parser = nlp.add_pipe("parser") +> parser.from_bytes(parser_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `DependencyParser` object. ~~DependencyParser~~ | + +## DependencyParser.labels {id="labels",tag="property"} + +The labels currently added to the component. + +> #### Example +> +> ```python +> parser.add_label("MY_LABEL") +> assert "MY_LABEL" in parser.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## DependencyParser.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`DependencyParser.initialize`](/api/dependencyparser#initialize) to initialize +the model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = parser.label_data +> parser.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = parser.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/doc.mdx b/website/docs/api/doc.mdx new file mode 100644 index 0000000..0a58265 --- /dev/null +++ b/website/docs/api/doc.mdx @@ -0,0 +1,792 @@ +--- +title: Doc +tag: class +teaser: A container for accessing linguistic annotations. +source: spacy/tokens/doc.pyx +--- + +A `Doc` is a sequence of [`Token`](/api/token) objects. Access sentences and +named entities, export annotations to numpy arrays, losslessly serialize to +compressed binary strings. The `Doc` object holds an array of +[`TokenC`](/api/cython-structs#tokenc) structs. The Python-level `Token` and +[`Span`](/api/span) objects are views of this array, i.e. they don't own the +data themselves. + +## Doc.\_\_init\_\_ {id="init",tag="method"} + +Construct a `Doc` object. The most common way to get a `Doc` object is via the +`nlp` object. + +> #### Example +> +> ```python +> # Construction 1 +> doc = nlp("Some text") +> +> # Construction 2 +> from spacy.tokens import Doc +> +> words = ["hello", "world", "!"] +> spaces = [True, False, False] +> doc = Doc(nlp.vocab, words=words, spaces=spaces) +> ``` + +| Name | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | A storage container for lexical types. ~~Vocab~~ | +| `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ | +| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ | +| _keyword-only_ | | +| `user_data` | Optional extra data to attach to the Doc. ~~Dict~~ | +| `tags` 3 | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `pos` 3 | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `morphs` 3 | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `lemmas` 3 | A list of strings, of the same length as `words`, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `heads` 3 | A list of values, of the same length as `words`, to assign as the head for each word. Head indices are the absolute position of the head in the `Doc`. Defaults to `None`. ~~Optional[List[int]]~~ | +| `deps` 3 | A list of strings, of the same length as `words`, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `sent_starts` 3 | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ | +| `ents` 3 | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ | + +## Doc.\_\_getitem\_\_ {id="getitem",tag="method"} + +Get a [`Token`](/api/token) object at position `i`, where `i` is an integer. +Negative indexing is supported, and follows the usual Python semantics, i.e. +`doc[-2]` is `doc[len(doc) - 2]`. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> assert doc[0].text == "Give" +> assert doc[-1].text == "." +> span = doc[1:3] +> assert span.text == "it back" +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `i` | The index of the token. ~~int~~ | +| **RETURNS** | The token at `doc[i]`. ~~Token~~ | + +Get a [`Span`](/api/span) object, starting at position `start` (token index) and +ending at position `end` (token index). For instance, `doc[2:5]` produces a span +consisting of tokens 2, 3 and 4. Stepped slices (e.g. `doc[start : end : step]`) +are not supported, as `Span` objects must be contiguous (cannot have gaps). You +can use negative indices and open-ended ranges, which have their normal Python +semantics. + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| `start_end` | The slice of the document to get. ~~Tuple[int, int]~~ | +| **RETURNS** | The span at `doc[start:end]`. ~~Span~~ | + +## Doc.\_\_iter\_\_ {id="iter",tag="method"} + +Iterate over `Token` objects, from which the annotations can be easily accessed. + +> #### Example +> +> ```python +> doc = nlp("Give it back") +> assert [t.text for t in doc] == ["Give", "it", "back"] +> ``` + +This is the main way of accessing [`Token`](/api/token) objects, which are the +main way annotations are accessed from Python. If faster-than-Python speeds are +required, you can instead access the annotations as a numpy array, or access the +underlying C data directly from Cython. + +| Name | Description | +| ---------- | --------------------------- | +| **YIELDS** | A `Token` object. ~~Token~~ | + +## Doc.\_\_len\_\_ {id="len",tag="method"} + +Get the number of tokens in the document. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> assert len(doc) == 7 +> ``` + +| Name | Description | +| ----------- | --------------------------------------------- | +| **RETURNS** | The number of tokens in the document. ~~int~~ | + +## Doc.set_extension {id="set_extension",tag="classmethod",version="2"} + +Define a custom attribute on the `Doc` which becomes available via `Doc._`. For +details, see the documentation on +[custom attributes](/usage/processing-pipelines#custom-components-attributes). + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> city_getter = lambda doc: any(city in doc.text for city in ("New York", "Paris", "Berlin")) +> Doc.set_extension("has_city", getter=city_getter) +> doc = nlp("I like New York") +> assert doc._.has_city +> ``` + +| Name | Description | +| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the attribute to set by the extension. For example, `"my_attr"` will be available as `doc._.my_attr`. ~~str~~ | +| `default` | Optional default value of the attribute if no getter or method is defined. ~~Optional[Any]~~ | +| `method` | Set a custom method on the object, for example `doc._.compare(other_doc)`. ~~Optional[Callable[[Doc, ...], Any]]~~ | +| `getter` | Getter function that takes the object and returns an attribute value. Is called when the user accesses the `._` attribute. ~~Optional[Callable[[Doc], Any]]~~ | +| `setter` | Setter function that takes the `Doc` and a value, and modifies the object. Is called when the user writes to the `Doc._` attribute. ~~Optional[Callable[[Doc, Any], None]]~~ | +| `force` | Force overwriting existing attribute. ~~bool~~ | + +## Doc.get_extension {id="get_extension",tag="classmethod",version="2"} + +Look up a previously registered extension by name. Returns a 4-tuple +`(default, method, getter, setter)` if the extension is registered. Raises a +`KeyError` otherwise. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> Doc.set_extension("has_city", default=False) +> extension = Doc.get_extension("has_city") +> assert extension == (False, None, None, None) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Doc.has_extension {id="has_extension",tag="classmethod",version="2"} + +Check whether an extension has been registered on the `Doc` class. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> Doc.set_extension("has_city", default=False) +> assert Doc.has_extension("has_city") +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `name` | Name of the extension to check. ~~str~~ | +| **RETURNS** | Whether the extension has been registered. ~~bool~~ | + +## Doc.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} + +Remove a previously registered extension. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> Doc.set_extension("has_city", default=False) +> removed = Doc.remove_extension("has_city") +> assert not Doc.has_extension("has_city") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Doc.char_span {id="char_span",tag="method",version="2"} + +Create a `Span` object from the slice `doc.text[start_idx:end_idx]`. Returns +`None` if the character indices don't map to a valid span using the default +alignment mode `"strict". + +> #### Example +> +> ```python +> doc = nlp("I like New York") +> span = doc.char_span(7, 15, label="GPE") +> assert span.text == "New York" +> ``` + +| Name | Description | +| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | +| `span_id` 3.3.1 | An identifier to associate with the span. ~~Union[int, str]~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | + +## Doc.set_ents {id="set_ents",tag="method",version="3"} + +Set the named entities in the document. + +> #### Example +> +> ```python +> from spacy.tokens import Span +> doc = nlp("Mr. Best flew to New York on Saturday morning.") +> doc.set_ents([Span(doc, 0, 2, "PERSON")]) +> ents = list(doc.ents) +> assert ents[0].label_ == "PERSON" +> assert ents[0].text == "Mr. Best" +> ``` + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `entities` | Spans with labels to set as entities. ~~List[Span]~~ | +| _keyword-only_ | | +| `blocked` | Spans to set as "blocked" (never an entity) for spacy's built-in NER component. Other components may ignore this setting. ~~Optional[List[Span]]~~ | +| `missing` | Spans with missing/unknown entity information. ~~Optional[List[Span]]~~ | +| `outside` | Spans outside of entities (O in IOB). ~~Optional[List[Span]]~~ | +| `default` | How to set entity annotation for tokens outside of any provided spans. Options: `"blocked"`, `"missing"`, `"outside"` and `"unmodified"` (preserve current state). Defaults to `"outside"`. ~~str~~ | + +## Doc.similarity {id="similarity",tag="method",model="vectors"} + +Make a semantic similarity estimate. The default estimate is cosine similarity +using an average of word vectors. + +> #### Example +> +> ```python +> apples = nlp("I like apples") +> oranges = nlp("I like oranges") +> apples_oranges = apples.similarity(oranges) +> oranges_apples = oranges.similarity(apples) +> assert apples_oranges == oranges_apples +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | +| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | + +## Doc.count_by {id="count_by",tag="method"} + +Count the frequencies of a given attribute. Produces a dict of +`{attr (int): count (ints)}` frequencies, keyed by the values of the given +attribute ID. + +> #### Example +> +> ```python +> from spacy.attrs import ORTH +> doc = nlp("apple apple orange banana") +> assert doc.count_by(ORTH) == {7024: 1, 119552: 1, 2087: 2} +> doc.to_array([ORTH]) +> # array([[11880], [11880], [7561], [12800]]) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------- | +| `attr_id` | The attribute ID. ~~int~~ | +| **RETURNS** | A dictionary mapping attributes to integer counts. ~~Dict[int, int]~~ | + +## Doc.get_lca_matrix {id="get_lca_matrix",tag="method"} + +Calculates the lowest common ancestor matrix for a given `Doc`. Returns LCA +matrix containing the integer index of the ancestor, or `-1` if no common +ancestor is found, e.g. if span excludes a necessary ancestor. + +> #### Example +> +> ```python +> doc = nlp("This is a test") +> matrix = doc.get_lca_matrix() +> # array([[0, 1, 1, 1], [1, 1, 1, 1], [1, 1, 2, 3], [1, 1, 3, 3]], dtype=int32) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------- | +| **RETURNS** | The lowest common ancestor matrix of the `Doc`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | + +## Doc.has_annotation {id="has_annotation",tag="method"} + +Check whether the doc contains annotation on a +[`Token` attribute](/api/token#attributes). + + + +This method replaces the previous boolean attributes like `Doc.is_tagged`, +`Doc.is_parsed` or `Doc.is_sentenced`. + +```diff +doc = nlp("This is a text") +- assert doc.is_parsed ++ assert doc.has_annotation("DEP") +``` + + + +| Name | Description | +| ------------------ | --------------------------------------------------------------------------------------------------- | +| `attr` | The attribute string name or int ID. ~~Union[int, str]~~ | +| _keyword-only_ | | +| `require_complete` | Whether to check that the attribute is set on every token in the doc. Defaults to `False`. ~~bool~~ | +| **RETURNS** | Whether specified annotation is present in the doc. ~~bool~~ | + +## Doc.to_array {id="to_array",tag="method"} + +Export given token attributes to a numpy `ndarray`. If `attr_ids` is a sequence +of `M` attributes, the output array will be of shape `(N, M)`, where `N` is the +length of the `Doc` (in tokens). If `attr_ids` is a single attribute, the output +shape will be `(N,)`. You can specify attributes by integer ID (e.g. +`spacy.attrs.LEMMA`) or string name (e.g. "LEMMA" or "lemma"). The values will +be 64-bit integers. + +Returns a 2D array with one row per token and one column per attribute (when +`attr_ids` is a list), or as a 1D numpy array, with one item per attribute (when +`attr_ids` is a single value). + +> #### Example +> +> ```python +> from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA +> doc = nlp(text) +> # All strings mapped to integers, for easy export to numpy +> np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA]) +> np_array = doc.to_array("POS") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | +| **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | + +## Doc.from_array {id="from_array",tag="method"} + +Load attributes from a numpy array. Write to a `Doc` object, from an `(M, N)` +array of attributes. + +> #### Example +> +> ```python +> from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA +> from spacy.tokens import Doc +> doc = nlp("Hello world!") +> np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA]) +> doc2 = Doc(doc.vocab, words=[t.text for t in doc]) +> doc2.from_array([LOWER, POS, ENT_TYPE, IS_ALPHA], np_array) +> assert doc[0].pos_ == doc2[0].pos_ +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------- | +| `attrs` | A list of attribute ID ints. ~~List[int]~~ | +| `array` | The attribute values to load. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Doc` itself. ~~Doc~~ | + +## Doc.from_docs {id="from_docs",tag="staticmethod",version="3"} + +Concatenate multiple `Doc` objects to form a new one. Raises an error if the +`Doc` objects do not all share the same `Vocab`. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> texts = ["London is the capital of the United Kingdom.", +> "The River Thames flows through London.", +> "The famous Tower Bridge crosses the River Thames."] +> docs = list(nlp.pipe(texts)) +> c_doc = Doc.from_docs(docs) +> assert str(c_doc) == " ".join(texts) +> assert len(list(c_doc.sents)) == len(docs) +> assert [str(ent) for ent in c_doc.ents] == \ +> [str(ent) for doc in docs for ent in doc.ents] +> ``` + +| Name | Description | +| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `docs` | A list of `Doc` objects. ~~List[Doc]~~ | +| `ensure_whitespace` | Insert a space between two adjacent docs whenever the first doc does not end in whitespace. ~~bool~~ | +| `attrs` | Optional list of attribute ID ints or attribute name strings. ~~Optional[List[Union[str, int]]]~~ | +| _keyword-only_ | | +| `exclude` 3.3 | String names of Doc attributes to exclude. Supported: `spans`, `tensor`, `user_data`. ~~Iterable[str]~~ | +| **RETURNS** | The new `Doc` object that is containing the other docs or `None`, if `docs` is empty or `None`. ~~Optional[Doc]~~ | + +## Doc.to_disk {id="to_disk",tag="method",version="2"} + +Save the current state to a directory. + +> #### Example +> +> ```python +> doc.to_disk("/path/to/doc") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Doc.from_disk {id="from_disk",tag="method",version="2"} + +Loads state from a directory. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> from spacy.vocab import Vocab +> doc = Doc(Vocab()).from_disk("/path/to/doc") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Doc` object. ~~Doc~~ | + +## Doc.to_bytes {id="to_bytes",tag="method"} + +Serialize, i.e. export the document contents to a binary string. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> doc_bytes = doc.to_bytes() +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | A losslessly serialized copy of the `Doc`, including all annotations. ~~bytes~~ | + +## Doc.from_bytes {id="from_bytes",tag="method"} + +Deserialize, i.e. import the document contents from a binary string. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> doc = nlp("Give it back! He pleaded.") +> doc_bytes = doc.to_bytes() +> doc2 = Doc(doc.vocab).from_bytes(doc_bytes) +> assert doc.text == doc2.text +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `data` | The string to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Doc` object. ~~Doc~~ | + +## Doc.to_json {id="to_json",tag="method"} + +Serializes a document to JSON. Note that this is format differs from the +deprecated [`JSON training format`](/api/data-formats#json-input). + +> #### Example +> +> ```python +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> assert doc.to_json()["text"] == doc.text +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ | +| **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ | + +## Doc.from_json {id="from_json",tag="method",version="3.3.1"} + +Deserializes a document from JSON, i.e. generates a document from the provided +JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> doc = nlp("All we have to decide is what to do with the time that is given us.") +> doc_json = doc.to_json() +> deserialized_doc = Doc(nlp.vocab).from_json(doc_json) +> assert deserialized_doc.text == doc.text == doc_json["text"] +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------- | +| `doc_json` | The Doc data in JSON format from [`Doc.to_json`](#to_json). ~~Dict[str, Any]~~ | +| _keyword-only_ | | +| `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ | + +## Doc.retokenize {id="retokenize",tag="contextmanager",version="2.1"} + +Context manager to handle retokenization of the `Doc`. Modifications to the +`Doc`'s tokenization are stored, and then made all at once when the context +manager exits. This is much more efficient, and less error-prone. All views of +the `Doc` (`Span` and `Token`) created before the retokenization are +invalidated, although they may accidentally continue to work. + +> #### Example +> +> ```python +> doc = nlp("Hello world!") +> with doc.retokenize() as retokenizer: +> retokenizer.merge(doc[0:2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| **RETURNS** | The retokenizer. ~~Retokenizer~~ | + +### Retokenizer.merge {id="retokenizer.merge",tag="method"} + +Mark a span for merging. The `attrs` will be applied to the resulting token (if +they're context-dependent token attributes like `LEMMA` or `DEP`) or to the +underlying lexeme (if they're context-independent lexical attributes like +`LOWER` or `IS_STOP`). Writable custom extension attributes can be provided +using the `"_"` key and specifying a dictionary that maps attribute names to +values. + +> #### Example +> +> ```python +> doc = nlp("I like David Bowie") +> with doc.retokenize() as retokenizer: +> attrs = {"LEMMA": "David Bowie"} +> retokenizer.merge(doc[2:4], attrs=attrs) +> ``` + +| Name | Description | +| ------- | --------------------------------------------------------------------- | +| `span` | The span to merge. ~~Span~~ | +| `attrs` | Attributes to set on the merged token. ~~Dict[Union[str, int], Any]~~ | + +### Retokenizer.split {id="retokenizer.split",tag="method"} + +Mark a token for splitting, into the specified `orths`. The `heads` are required +to specify how the new subtokens should be integrated into the dependency tree. +The list of per-token heads can either be a token in the original document, e.g. +`doc[2]`, or a tuple consisting of the token in the original document and its +subtoken index. For example, `(doc[3], 1)` will attach the subtoken to the +second subtoken of `doc[3]`. + +This mechanism allows attaching subtokens to other newly created subtokens, +without having to keep track of the changing token indices. If the specified +head token will be split within the retokenizer block and no subtoken index is +specified, it will default to `0`. Attributes to set on subtokens can be +provided as a list of values. They'll be applied to the resulting token (if +they're context-dependent token attributes like `LEMMA` or `DEP`) or to the +underlying lexeme (if they're context-independent lexical attributes like +`LOWER` or `IS_STOP`). + +> #### Example +> +> ```python +> doc = nlp("I live in NewYork") +> with doc.retokenize() as retokenizer: +> heads = [(doc[3], 1), doc[2]] +> attrs = {"POS": ["PROPN", "PROPN"], +> "DEP": ["pobj", "compound"]} +> retokenizer.split(doc[3], ["New", "York"], heads=heads, attrs=attrs) +> ``` + +| Name | Description | +| ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `token` | The token to split. ~~Token~~ | +| `orths` | The verbatim text of the split tokens. Needs to match the text of the original token. ~~List[str]~~ | +| `heads` | List of `token` or `(token, subtoken)` tuples specifying the tokens to attach the newly split subtokens to. ~~List[Union[Token, Tuple[Token, int]]]~~ | +| `attrs` | Attributes to set on all split tokens. Attribute names mapped to list of per-token attribute values. ~~Dict[Union[str, int], List[Any]]~~ | + +## Doc.ents {id="ents",tag="property",model="NER"} + +The named entities in the document. Returns a tuple of named entity `Span` +objects, if the entity recognizer has been applied. + +> #### Example +> +> ```python +> doc = nlp("Mr. Best flew to New York on Saturday morning.") +> ents = list(doc.ents) +> assert ents[0].label_ == "PERSON" +> assert ents[0].text == "Mr. Best" +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------- | +| **RETURNS** | Entities in the document, one `Span` per entity. ~~Tuple[Span]~~ | + +## Doc.spans {id="spans",tag="property"} + +A dictionary of named span groups, to store and access additional span +annotations. You can write to it by assigning a list of [`Span`](/api/span) +objects or a [`SpanGroup`](/api/spangroup) to a given key. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------ | +| **RETURNS** | The span groups assigned to the document. ~~Dict[str, SpanGroup]~~ | + +## Doc.cats {id="cats",tag="property",model="text classifier"} + +Maps a label to a score for categories applied to the document. Typically set by +the [`TextCategorizer`](/api/textcategorizer). + +> #### Example +> +> ```python +> doc = nlp("This is a text about football.") +> print(doc.cats) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The text categories mapped to scores. ~~Dict[str, float]~~ | + +## Doc.noun_chunks {id="noun_chunks",tag="property",model="parser"} + +Iterate over the base noun phrases in the document. Yields base noun-phrase +`Span` objects, if the document has been syntactically parsed. A base noun +phrase, or "NP chunk", is a noun phrase that does not permit other NPs to be +nested within it – so no NP-level coordination, no prepositional phrases, and no +relative clauses. + +To customize the noun chunk iterator in a loaded pipeline, modify +[`nlp.vocab.get_noun_chunks`](/api/vocab#attributes). If the `noun_chunk` +[syntax iterator](/usage/linguistic-features#language-data) has not been +implemented for the given language, a `NotImplementedError` is raised. + +> #### Example +> +> ```python +> doc = nlp("A phrase with another phrase occurs.") +> chunks = list(doc.noun_chunks) +> assert len(chunks) == 2 +> assert chunks[0].text == "A phrase" +> assert chunks[1].text == "another phrase" +> ``` + +| Name | Description | +| ---------- | ------------------------------------- | +| **YIELDS** | Noun chunks in the document. ~~Span~~ | + +## Doc.sents {id="sents",tag="property",model="sentences"} + +Iterate over the sentences in the document. Sentence spans have no label. + +This property is only available when +[sentence boundaries](/usage/linguistic-features#sbd) have been set on the +document by the `parser`, `senter`, `sentencizer` or some custom function. It +will raise an error otherwise. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence. Here's another...") +> sents = list(doc.sents) +> assert len(sents) == 2 +> assert [s.root.text for s in sents] == ["is", "'s"] +> ``` + +| Name | Description | +| ---------- | ----------------------------------- | +| **YIELDS** | Sentences in the document. ~~Span~~ | + +## Doc.has_vector {id="has_vector",tag="property",model="vectors"} + +A boolean value indicating whether a word vector is associated with the object. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> assert doc.has_vector +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| **RETURNS** | Whether the document has a vector data attached. ~~bool~~ | + +## Doc.vector {id="vector",tag="property",model="vectors"} + +A real-valued meaning representation. Defaults to an average of the token +vectors. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> assert doc.vector.dtype == "float32" +> assert doc.vector.shape == (300,) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------- | +| **RETURNS** | A 1-dimensional array representing the document's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Doc.vector_norm {id="vector_norm",tag="property",model="vectors"} + +The L2 norm of the document's vector representation. + +> #### Example +> +> ```python +> doc1 = nlp("I like apples") +> doc2 = nlp("I like oranges") +> doc1.vector_norm # 4.54232424414368 +> doc2.vector_norm # 3.304373298575751 +> assert doc1.vector_norm != doc2.vector_norm +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| **RETURNS** | The L2 norm of the vector representation. ~~float~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `text` | A string representation of the document text. ~~str~~ | +| `text_with_ws` | An alias of `Doc.text`, provided for duck-type compatibility with `Span` and `Token`. ~~str~~ | +| `mem` | The document's local memory heap, for all C data it owns. ~~cymem.Pool~~ | +| `vocab` | The store of lexical types. ~~Vocab~~ | +| `tensor` | Container for dense vector representations. ~~numpy.ndarray~~ | +| `user_data` | A generic storage area, for user custom data. ~~Dict[str, Any]~~ | +| `lang` | Language of the document's vocabulary. ~~int~~ | +| `lang_` | Language of the document's vocabulary. ~~str~~ | +| `sentiment` | The document's positivity/negativity score, if available. ~~float~~ | +| `user_hooks` | A dictionary that allows customization of the `Doc`'s properties. ~~Dict[str, Callable]~~ | +| `user_token_hooks` | A dictionary that allows customization of properties of `Token` children. ~~Dict[str, Callable]~~ | +| `user_span_hooks` | A dictionary that allows customization of properties of `Span` children. ~~Dict[str, Callable]~~ | +| `has_unknown_spaces` | Whether the document was constructed without known spacing between tokens (typically when created from gold tokenization). ~~bool~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = doc.to_bytes(exclude=["text", "tensor"]) +> doc.from_disk("./doc.bin", exclude=["user_data"]) +> ``` + +| Name | Description | +| ------------------ | --------------------------------------------- | +| `text` | The value of the `Doc.text` attribute. | +| `sentiment` | The value of the `Doc.sentiment` attribute. | +| `tensor` | The value of the `Doc.tensor` attribute. | +| `user_data` | The value of the `Doc.user_data` dictionary. | +| `user_data_keys` | The keys of the `Doc.user_data` dictionary. | +| `user_data_values` | The values of the `Doc.user_data` dictionary. | diff --git a/website/docs/api/docbin.mdx b/website/docs/api/docbin.mdx new file mode 100644 index 0000000..b5cf29d --- /dev/null +++ b/website/docs/api/docbin.mdx @@ -0,0 +1,183 @@ +--- +title: DocBin +tag: class +version: 2.2 +teaser: Pack Doc objects for binary serialization +source: spacy/tokens/_serialize.py +--- + +The `DocBin` class lets you efficiently serialize the information from a +collection of `Doc` objects. You can control which information is serialized by +passing a list of attribute IDs, and optionally also specify whether the user +data is serialized. The `DocBin` is faster and produces smaller data sizes than +pickle, and allows you to deserialize without executing arbitrary Python code. A +notable downside to this format is that you can't easily extract just one +document from the `DocBin`. The serialization format is gzipped msgpack, where +the msgpack object has the following structure: + +```python {title="msgpack object structure"} +{ + "version": str, # DocBin version number + "attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE] + "tokens": bytes, # Serialized numpy uint64 array with the token data + "spaces": bytes, # Serialized numpy boolean array with spaces data + "lengths": bytes, # Serialized numpy int32 array with the doc lengths + "strings": List[str] # List of unique strings in the token data +} +``` + +Strings for the words, tags, labels etc are represented by 64-bit hashes in the +token data, and every string that occurs at least once is passed via the strings +object. This means the storage is more efficient if you pack more documents +together, because you have less duplication in the strings. For usage examples, +see the docs on [serializing `Doc` objects](/usage/saving-loading#docs). + +## DocBin.\_\_init\_\_ {id="init",tag="method"} + +Create a `DocBin` object to hold serialized annotations. + +> #### Example +> +> ```python +> from spacy.tokens import DocBin +> doc_bin = DocBin(attrs=["ENT_IOB", "ENT_TYPE"]) +> ``` + +| Argument | Description | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `attrs` | List of attributes to serialize. `ORTH` (hash of token text) and `SPACY` (whether the token is followed by whitespace) are always serialized, so they're not required. Defaults to `("ORTH", "TAG", "HEAD", "DEP", "ENT_IOB", "ENT_TYPE", "ENT_KB_ID", "LEMMA", "MORPH", "POS")`. ~~Iterable[str]~~ | +| `store_user_data` | Whether to write the `Doc.user_data` and the values of custom extension attributes to file/bytes. Defaults to `False`. ~~bool~~ | +| `docs` | `Doc` objects to add on initialization. ~~Iterable[Doc]~~ | + +## DocBin.\_\_len\_\_ {id="len",tag="method"} + +Get the number of `Doc` objects that were added to the `DocBin`. + +> #### Example +> +> ```python +> doc_bin = DocBin(attrs=["LEMMA"]) +> doc = nlp("This is a document to serialize.") +> doc_bin.add(doc) +> assert len(doc_bin) == 1 +> ``` + +| Argument | Description | +| ----------- | --------------------------------------------------- | +| **RETURNS** | The number of `Doc`s added to the `DocBin`. ~~int~~ | + +## DocBin.add {id="add",tag="method"} + +Add a `Doc`'s annotations to the `DocBin` for serialization. + +> #### Example +> +> ```python +> doc_bin = DocBin(attrs=["LEMMA"]) +> doc = nlp("This is a document to serialize.") +> doc_bin.add(doc) +> ``` + +| Argument | Description | +| -------- | -------------------------------- | +| `doc` | The `Doc` object to add. ~~Doc~~ | + +## DocBin.get_docs {id="get_docs",tag="method"} + +Recover `Doc` objects from the annotations, using the given vocab. + +> #### Example +> +> ```python +> docs = list(doc_bin.get_docs(nlp.vocab)) +> ``` + +| Argument | Description | +| ---------- | --------------------------- | +| `vocab` | The shared vocab. ~~Vocab~~ | +| **YIELDS** | The `Doc` objects. ~~Doc~~ | + +## DocBin.merge {id="merge",tag="method"} + +Extend the annotations of this `DocBin` with the annotations from another. Will +raise an error if the pre-defined `attrs` of the two `DocBin`s don't match. + +> #### Example +> +> ```python +> doc_bin1 = DocBin(attrs=["LEMMA", "POS"]) +> doc_bin1.add(nlp("Hello world")) +> doc_bin2 = DocBin(attrs=["LEMMA", "POS"]) +> doc_bin2.add(nlp("This is a sentence")) +> doc_bin1.merge(doc_bin2) +> assert len(doc_bin1) == 2 +> ``` + +| Argument | Description | +| -------- | ------------------------------------------------------ | +| `other` | The `DocBin` to merge into the current bin. ~~DocBin~~ | + +## DocBin.to_bytes {id="to_bytes",tag="method"} + +Serialize the `DocBin`'s annotations to a bytestring. + +> #### Example +> +> ```python +> docs = [nlp("Hello world!")] +> doc_bin = DocBin(docs=docs) +> doc_bin_bytes = doc_bin.to_bytes() +> ``` + +| Argument | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The serialized `DocBin`. ~~bytes~~ | + +## DocBin.from_bytes {id="from_bytes",tag="method"} + +Deserialize the `DocBin`'s annotations from a bytestring. + +> #### Example +> +> ```python +> doc_bin_bytes = doc_bin.to_bytes() +> new_doc_bin = DocBin().from_bytes(doc_bin_bytes) +> ``` + +| Argument | Description | +| ------------ | -------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The loaded `DocBin`. ~~DocBin~~ | + +## DocBin.to_disk {id="to_disk",tag="method",version="3"} + +Save the serialized `DocBin` to a file. Typically uses the `.spacy` extension +and the result can be used as the input data for +[`spacy train`](/api/cli#train). + +> #### Example +> +> ```python +> docs = [nlp("Hello world!")] +> doc_bin = DocBin(docs=docs) +> doc_bin.to_disk("./data.spacy") +> ``` + +| Argument | Description | +| -------- | -------------------------------------------------------------------------- | +| `path` | The file path, typically with the `.spacy` extension. ~~Union[str, Path]~~ | + +## DocBin.from_disk {id="from_disk",tag="method",version="3"} + +Load a serialized `DocBin` from a file. Typically uses the `.spacy` extension. + +> #### Example +> +> ```python +> doc_bin = DocBin().from_disk("./data.spacy") +> ``` + +| Argument | Description | +| ----------- | -------------------------------------------------------------------------- | +| `path` | The file path, typically with the `.spacy` extension. ~~Union[str, Path]~~ | +| **RETURNS** | The loaded `DocBin`. ~~DocBin~~ | diff --git a/website/docs/api/edittreelemmatizer.mdx b/website/docs/api/edittreelemmatizer.mdx new file mode 100644 index 0000000..8296748 --- /dev/null +++ b/website/docs/api/edittreelemmatizer.mdx @@ -0,0 +1,409 @@ +--- +title: EditTreeLemmatizer +tag: class +source: spacy/pipeline/edit_tree_lemmatizer.py +version: 3.3 +teaser: 'Pipeline component for lemmatization' +api_base_class: /api/pipe +api_string_name: trainable_lemmatizer +api_trainable: true +--- + +A trainable component for assigning base forms to tokens. This lemmatizer uses +**edit trees** to transform tokens into base forms. The lemmatization model +predicts which edit tree is applicable to a token. The edit tree data structure +and construction method used by this lemmatizer were proposed in +[Joint Lemmatization and Morphological Tagging with Lemming](https://aclanthology.org/D15-1272.pdf) +(Thomas Müller et al., 2015). + +For a lookup and rule-based lemmatizer, see [`Lemmatizer`](/api/lemmatizer). + +## Assigned Attributes {id="assigned-attributes"} + +Predictions are assigned to `Token.lemma`. + +| Location | Value | +| -------------- | ------------------------- | +| `Token.lemma` | The lemma (hash). ~~int~~ | +| `Token.lemma_` | The lemma. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.edit_tree_lemmatizer import DEFAULT_EDIT_TREE_LEMMATIZER_MODEL +> config = {"model": DEFAULT_EDIT_TREE_LEMMATIZER_MODEL} +> nlp.add_pipe("trainable_lemmatizer", config=config, name="lemmatizer") +> ``` + +| Setting | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | A model instance that predicts the edit tree probabilities. The output vectors should match the number of edit trees in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `backoff` | ~~Token~~ attribute to use when no applicable edit tree is found. Defaults to `orth`. ~~str~~ | +| `min_tree_freq` | Minimum frequency of an edit tree in the training set to be used. Defaults to `3`. ~~int~~ | +| `overwrite` | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `top_k` | The number of most probable edit trees to try before resorting to `backoff`. Defaults to `1`. ~~int~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/edit_tree_lemmatizer.py +``` + +## EditTreeLemmatizer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> +> # Construction via create_pipe with custom model +> config = {"model": {"@architectures": "my_tagger"}} +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", config=config, name="lemmatizer") +> +> # Construction from class +> from spacy.pipeline import EditTreeLemmatizer +> lemmatizer = EditTreeLemmatizer(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | A model instance that predicts the edit tree probabilities. The output vectors should match the number of edit trees in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `backoff` | ~~Token~~ attribute to use when no applicable edit tree is found. Defaults to `orth`. ~~str~~ | +| `min_tree_freq` | Minimum frequency of an edit tree in the training set to be used. Defaults to `3`. ~~int~~ | +| `overwrite` | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `top_k` | The number of most probable edit trees to try before resorting to `backoff`. Defaults to `1`. ~~int~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ | + +## EditTreeLemmatizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/edittreelemmatizer#call) and +[`pipe`](/api/edittreelemmatizer#pipe) delegate to the +[`predict`](/api/edittreelemmatizer#predict) and +[`set_annotations`](/api/edittreelemmatizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> # This usually happens under the hood +> processed = lemmatizer(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## EditTreeLemmatizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/edittreelemmatizer#call) +and [`pipe`](/api/edittreelemmatizer#pipe) delegate to the +[`predict`](/api/edittreelemmatizer#predict) and +[`set_annotations`](/api/edittreelemmatizer#set_annotations) methods. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> for doc in lemmatizer.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## EditTreeLemmatizer.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> lemmatizer.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.lemmatizer] +> +> [initialize.components.lemmatizer.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/lemmatizer.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | + +## EditTreeLemmatizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> tree_ids = lemmatizer.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## EditTreeLemmatizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed tree +identifiers. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> tree_ids = lemmatizer.predict([doc1, doc2]) +> lemmatizer.set_annotations([doc1, doc2], tree_ids) +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `tree_ids` | The identifiers of the edit trees to apply, produced by `EditTreeLemmatizer.predict`. | + +## EditTreeLemmatizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/edittreelemmatizer#predict) and +[`get_loss`](/api/edittreelemmatizer#get_loss). + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> optimizer = nlp.initialize() +> losses = lemmatizer.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## EditTreeLemmatizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> scores = lemmatizer.model.begin_update([eg.predicted for eg in examples]) +> loss, d_loss = lemmatizer.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## EditTreeLemmatizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> optimizer = lemmatizer.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## EditTreeLemmatizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> with lemmatizer.use_params(optimizer.averages): +> lemmatizer.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## EditTreeLemmatizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> lemmatizer.to_disk("/path/to/lemmatizer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## EditTreeLemmatizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> lemmatizer.from_disk("/path/to/lemmatizer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | + +## EditTreeLemmatizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> lemmatizer_bytes = lemmatizer.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `EditTreeLemmatizer` object. ~~bytes~~ | + +## EditTreeLemmatizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> lemmatizer_bytes = lemmatizer.to_bytes() +> lemmatizer = nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +> lemmatizer.from_bytes(lemmatizer_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | + +## EditTreeLemmatizer.labels {id="labels",tag="property"} + +The labels currently added to the component. + + + +The `EditTreeLemmatizer` labels are not useful by themselves, since they are +identifiers of edit trees. + + + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## EditTreeLemmatizer.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`EditTreeLemmatizer.initialize`](/api/edittreelemmatizer#initialize) to +initialize the model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = lemmatizer.label_data +> lemmatizer.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = lemmatizer.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | +| `trees` | The edit trees. You usually don't want to exclude this. | diff --git a/website/docs/api/entitylinker.mdx b/website/docs/api/entitylinker.mdx new file mode 100644 index 0000000..f4b83d8 --- /dev/null +++ b/website/docs/api/entitylinker.mdx @@ -0,0 +1,403 @@ +--- +title: EntityLinker +tag: class +source: spacy/pipeline/entity_linker.py +version: 2.2 +teaser: 'Pipeline component for named entity linking and disambiguation' +api_base_class: /api/pipe +api_string_name: entity_linker +api_trainable: true +--- + +An `EntityLinker` component disambiguates textual mentions (tagged as named +entities) to unique identifiers, grounding the named entities into the "real +world". It requires a `KnowledgeBase`, as well as a function to generate +plausible candidates from that `KnowledgeBase` given a certain textual mention, +and a machine learning model to pick the right candidate, given the local +context of the mention. `EntityLinker` defaults to using the +[`InMemoryLookupKB`](/api/inmemorylookupkb) implementation. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions, in the form of knowledge base IDs, will be assigned to +`Token.ent_kb_id_`. + +| Location | Value | +| ------------------ | --------------------------------- | +| `Token.ent_kb_id` | Knowledge base ID (hash). ~~int~~ | +| `Token.ent_kb_id_` | Knowledge base ID. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.entity_linker import DEFAULT_NEL_MODEL +> config = { +> "labels_discard": [], +> "n_sents": 0, +> "incl_prior": True, +> "incl_context": True, +> "model": DEFAULT_NEL_MODEL, +> "entity_vector_length": 64, +> "get_candidates": {'@misc': 'spacy.CandidateGenerator.v1'}, +> "threshold": None, +> } +> nlp.add_pipe("entity_linker", config=config) +> ``` + +| Setting | Description | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `labels_discard` | NER labels that will automatically get a "NIL" prediction. Defaults to `[]`. ~~Iterable[str]~~ | +| `n_sents` | The number of neighbouring sentences to take into account. Defaults to 0. ~~int~~ | +| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. Defaults to `True`. ~~bool~~ | +| `incl_context` | Whether or not to include the local context in the model. Defaults to `True`. ~~bool~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [EntityLinker](/api/architectures#EntityLinker). ~~Model~~ | +| `entity_vector_length` | Size of encoding vectors in the KB. Defaults to `64`. ~~int~~ | +| `use_gold_ents` | Whether to copy entities from the gold docs or not. Defaults to `True`. If `False`, entities must be set in the training data or by an annotating component in the pipeline. ~~bool~~ | +| `get_candidates` | Function that generates plausible candidates for a given `Span` object. Defaults to [CandidateGenerator](/api/architectures#CandidateGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | +| `get_candidates_batch` 3.5 | Function that generates plausible candidates for a given batch of `Span` objects. Defaults to [CandidateBatchGenerator](/api/architectures#CandidateBatchGenerator), a function looking up exact, case-dependent aliases in the KB. ~~Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]]~~ | +| `generate_empty_kb` 3.5.1 | Function that generates an empty `KnowledgeBase` object. Defaults to [`spacy.EmptyKB.v2`](/api/architectures#EmptyKB), which generates an empty [`InMemoryLookupKB`](/api/inmemorylookupkb). ~~Callable[[Vocab, int], KnowledgeBase]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the threshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/entity_linker.py +``` + +## EntityLinker.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> entity_linker = nlp.add_pipe("entity_linker") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_el.v1"}} +> entity_linker = nlp.add_pipe("entity_linker", config=config) +> +> # Construction from class +> from spacy.pipeline import EntityLinker +> entity_linker = EntityLinker(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +Upon construction of the entity linker component, an empty knowledge base is +constructed with the provided `entity_vector_length`. If you want to use a +custom knowledge base, you should either call +[`set_kb`](/api/entitylinker#set_kb) or provide a `kb_loader` in the +[`initialize`](/api/entitylinker#initialize) call. + +| Name | Description | +| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `entity_vector_length` | Size of encoding vectors in the KB. ~~int~~ | +| `get_candidates` | Function that generates plausible candidates for a given `Span` object. ~~Callable[[KnowledgeBase, Span], Iterable[Candidate]]~~ | +| `labels_discard` | NER labels that will automatically get a `"NIL"` prediction. ~~Iterable[str]~~ | +| `n_sents` | The number of neighbouring sentences to take into account. ~~int~~ | +| `incl_prior` | Whether or not to include prior probabilities from the KB in the model. ~~bool~~ | +| `incl_context` | Whether or not to include the local context in the model. ~~bool~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `True`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | +| `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the threshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | + +## EntityLinker.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/entitylinker#call) and [`pipe`](/api/entitylinker#pipe) +delegate to the [`predict`](/api/entitylinker#predict) and +[`set_annotations`](/api/entitylinker#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> entity_linker = nlp.add_pipe("entity_linker") +> # This usually happens under the hood +> processed = entity_linker(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## EntityLinker.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/entitylinker#call) and +[`pipe`](/api/entitylinker#pipe) delegate to the +[`predict`](/api/entitylinker#predict) and +[`set_annotations`](/api/entitylinker#set_annotations) methods. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> for doc in entity_linker.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## EntityLinker.set_kb {id="set_kb",tag="method",version="3"} + +The `kb_loader` should be a function that takes a `Vocab` instance and creates +the `KnowledgeBase`, ensuring that the strings of the knowledge base are synced +with the current vocab. + +> #### Example +> +> ```python +> def create_kb(vocab): +> kb = InMemoryLookupKB(vocab, entity_vector_length=128) +> kb.add_entity(...) +> kb.add_alias(...) +> return kb +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker.set_kb(create_kb) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------- | +| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | + +## EntityLinker.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +Optionally, a `kb_loader` argument may be specified to change the internal +knowledge base. This argument should be a function that takes a `Vocab` instance +and creates the `KnowledgeBase`, ensuring that the strings of the knowledge base +are synced with the current vocab. + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker.initialize(lambda: examples, nlp=nlp, kb_loader=my_kb) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | + +## EntityLinker.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. Returns the KB IDs for each entity in each doc, including `NIL` +if there is no prediction. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> kb_ids = entity_linker.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The predicted KB identifiers for the entities in the `docs`. ~~List[str]~~ | + +## EntityLinker.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of documents, using pre-computed entity IDs for a list of named +entities. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> kb_ids = entity_linker.predict([doc1, doc2]) +> entity_linker.set_annotations([doc1, doc2], kb_ids) +> ``` + +| Name | Description | +| -------- | --------------------------------------------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `kb_ids` | The knowledge base identifiers for the entities in the docs, predicted by `EntityLinker.predict`. ~~List[str]~~ | + +## EntityLinker.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects, updating both the +pipe's entity linking model and context encoder. Delegates to +[`predict`](/api/entitylinker#predict). + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> optimizer = nlp.initialize() +> losses = entity_linker.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## EntityLinker.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> optimizer = entity_linker.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## EntityLinker.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> with entity_linker.use_params(optimizer.averages): +> entity_linker.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## EntityLinker.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker.to_disk("/path/to/entity_linker") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## EntityLinker.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker.from_disk("/path/to/entity_linker") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `EntityLinker` object. ~~EntityLinker~~ | + +## EntityLinker.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker_bytes = entity_linker.to_bytes() +> ``` + +Serialize the pipe to a bytestring, including the `KnowledgeBase`. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `EntityLinker` object. ~~bytes~~ | + +## EntityLinker.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> entity_linker_bytes = entity_linker.to_bytes() +> entity_linker = nlp.add_pipe("entity_linker") +> entity_linker.from_bytes(entity_linker_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `EntityLinker` object. ~~EntityLinker~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = entity_linker.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | +| `kb` | The knowledge base. You usually don't want to exclude this. | diff --git a/website/docs/api/entityrecognizer.mdx b/website/docs/api/entityrecognizer.mdx new file mode 100644 index 0000000..c80406a --- /dev/null +++ b/website/docs/api/entityrecognizer.mdx @@ -0,0 +1,464 @@ +--- +title: EntityRecognizer +tag: class +source: spacy/pipeline/ner.pyx +teaser: 'Pipeline component for named entity recognition' +api_base_class: /api/pipe +api_string_name: ner +api_trainable: true +--- + +A transition-based named entity recognition component. The entity recognizer +identifies **non-overlapping labelled spans** of tokens. The transition-based +algorithm used encodes certain assumptions that are effective for "traditional" +named entity recognition tasks, but may not be a good fit for every span +identification problem. Specifically, the loss function optimizes for **whole +entity accuracy**, so if your inter-annotator agreement on boundary tokens is +low, the component will likely perform poorly on your problem. The +transition-based algorithm also assumes that the most decisive information about +your entities will be close to their initial tokens. If your entities are long +and characterized by tokens in their middle, the component will likely not be a +good fit for your task. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `Doc.ents` as a tuple. Each label will also be +reflected to each underlying token, where it is saved in the `Token.ent_type` +and `Token.ent_iob` fields. Note that by definition each token can only have one +label. + +When setting `Doc.ents` to create training data, all the spans must be valid and +non-overlapping, or an error will be thrown. + +| Location | Value | +| ----------------- | ----------------------------------------------------------------- | +| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ | +| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ | +| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ | +| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | +| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.ner import DEFAULT_NER_MODEL +> config = { +> "moves": None, +> "update_with_oracle_cut_size": 100, +> "model": DEFAULT_NER_MODEL, +> "incorrect_spans_key": "incorrect_spans", +> } +> nlp.add_pipe("ner", config=config) +> ``` + +| Setting | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `moves` | A list of transition names. Inferred from the data if not provided. Defaults to `None`. ~~Optional[TransitionSystem]~~ | +| `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [TransitionBasedParser](/api/architectures#TransitionBasedParser). ~~Model[List[Doc], List[Floats2d]]~~ | +| `incorrect_spans_key` | This key refers to a `SpanGroup` in `doc.spans` that specifies incorrect spans. The NER will learn not to predict (exactly) those spans. Defaults to `None`. ~~Optional[str]~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/ner.pyx +``` + +## EntityRecognizer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> ner = nlp.add_pipe("ner") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_ner"}} +> parser = nlp.add_pipe("ner", config=config) +> +> # Construction from class +> from spacy.pipeline import EntityRecognizer +> ner = EntityRecognizer(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| `moves` | A list of transition names. Inferred from the data if set to `None`, which is the default. ~~Optional[TransitionSystem]~~ | +| _keyword-only_ | | +| `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | +| `incorrect_spans_key` | Identifies spans that are known to be incorrect entity annotations. The incorrect entity annotations can be stored in the span group in [`Doc.spans`](/api/doc#spans), under this key. Defaults to `None`. ~~Optional[str]~~ | + +## EntityRecognizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/entityrecognizer#call) and +[`pipe`](/api/entityrecognizer#pipe) delegate to the +[`predict`](/api/entityrecognizer#predict) and +[`set_annotations`](/api/entityrecognizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> ner = nlp.add_pipe("ner") +> # This usually happens under the hood +> processed = ner(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## EntityRecognizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/entityrecognizer#call) and +[`pipe`](/api/entityrecognizer#pipe) delegate to the +[`predict`](/api/entityrecognizer#predict) and +[`set_annotations`](/api/entityrecognizer#set_annotations) methods. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> for doc in ner.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `docs` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## EntityRecognizer.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.ner] +> +> [initialize.components.ner.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/ner.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | + +## EntityRecognizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> scores = ner.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | + +## EntityRecognizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> scores = ner.predict([doc1, doc2]) +> ner.set_annotations([doc1, doc2], scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `EntityRecognizer.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | + +## EntityRecognizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects, updating the pipe's +model. Delegates to [`predict`](/api/entityrecognizer#predict) and +[`get_loss`](/api/entityrecognizer#get_loss). + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> optimizer = nlp.initialize() +> losses = ner.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## EntityRecognizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> scores = ner.predict([eg.predicted for eg in examples]) +> loss, d_loss = ner.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. ~~StateClass~~ | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## EntityRecognizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> optimizer = ner.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## EntityRecognizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> ner = EntityRecognizer(nlp.vocab) +> with ner.use_params(optimizer.averages): +> ner.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## EntityRecognizer.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. Note that you don't have to call this method if you +provide a **representative data sample** to the [`initialize`](#initialize) +method. In this case, all labels found in the sample will be automatically added +to the model, and the output dimension will be +[inferred](/usage/layers-architectures#thinc-shape-inference) automatically. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## EntityRecognizer.set_output {id="set_output",tag="method"} + +Change the output dimension of the component's model by calling the model's +attribute `resize_output`. This is a function that takes the original model and +the new output dimension `nO`, and changes the model in place. When resizing an +already trained model, care should be taken to avoid the "catastrophic +forgetting" problem. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner.set_output(512) +> ``` + +| Name | Description | +| ---- | --------------------------------- | +| `nO` | The new output dimension. ~~int~~ | + +## EntityRecognizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner.to_disk("/path/to/ner") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## EntityRecognizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner.from_disk("/path/to/ner") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `EntityRecognizer` object. ~~EntityRecognizer~~ | + +## EntityRecognizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> ner_bytes = ner.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `EntityRecognizer` object. ~~bytes~~ | + +## EntityRecognizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ner_bytes = ner.to_bytes() +> ner = nlp.add_pipe("ner") +> ner.from_bytes(ner_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `EntityRecognizer` object. ~~EntityRecognizer~~ | + +## EntityRecognizer.labels {id="labels",tag="property"} + +The labels currently added to the component. + +> #### Example +> +> ```python +> ner.add_label("MY_LABEL") +> assert "MY_LABEL" in ner.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## EntityRecognizer.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`EntityRecognizer.initialize`](/api/entityrecognizer#initialize) to initialize +the model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = ner.label_data +> ner.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = ner.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/entityruler.mdx b/website/docs/api/entityruler.mdx new file mode 100644 index 0000000..335e876 --- /dev/null +++ b/website/docs/api/entityruler.mdx @@ -0,0 +1,336 @@ +--- +title: EntityRuler +tag: class +source: spacy/pipeline/entityruler.py +version: 2.1 +teaser: 'Pipeline component for rule-based named entity recognition' +api_string_name: entity_ruler +api_trainable: false +--- + +The entity ruler lets you add spans to the [`Doc.ents`](/api/doc#ents) using +token-based rules or exact phrase matches. It can be combined with the +statistical [`EntityRecognizer`](/api/entityrecognizer) to boost accuracy, or +used on its own to implement a purely rule-based entity recognition system. For +usage examples, see the docs on +[rule-based entity recognition](/usage/rule-based-matching#entityruler). + +## Assigned Attributes {id="assigned-attributes"} + +This component assigns predictions basically the same way as the +[`EntityRecognizer`](/api/entityrecognizer). + +Predictions can be accessed under `Doc.ents` as a tuple. Each label will also be +reflected in each underlying token, where it is saved in the `Token.ent_type` +and `Token.ent_iob` fields. Note that by definition each token can only have one +label. + +When setting `Doc.ents` to create training data, all the spans must be valid and +non-overlapping, or an error will be thrown. + +| Location | Value | +| ----------------- | ----------------------------------------------------------------- | +| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ | +| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ | +| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ | +| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | +| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). + +> #### Example +> +> ```python +> config = { +> "phrase_matcher_attr": None, +> "validate": True, +> "overwrite_ents": False, +> "ent_id_sep": "||", +> } +> nlp.add_pipe("entity_ruler", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/entityruler.py +``` + +## EntityRuler.\_\_init\_\_ {id="init",tag="method"} + +Initialize the entity ruler. If patterns are supplied here, they need to be a +list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either +be a token pattern (list) or a phrase pattern (string). For example: +`{"label": "ORG", "pattern": "Apple"}`. + +> #### Example +> +> ```python +> # Construction via add_pipe +> ruler = nlp.add_pipe("entity_ruler") +> +> # Construction from class +> from spacy.pipeline import EntityRuler +> ruler = EntityRuler(nlp, overwrite_ents=True) +> ``` + +| Name | Description | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` 3 | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | + +## EntityRuler.initialize {id="initialize",tag="method",version="3"} + +Initialize the component with data and used before training to load in rules +from a [pattern file](/usage/rule-based-matching/#entityruler-files). This +method is typically called by [`Language.initialize`](/api/language#initialize) +and lets you customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> entity_ruler = nlp.add_pipe("entity_ruler") +> entity_ruler.initialize(lambda: [], nlp=nlp, patterns=patterns) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.entity_ruler] +> +> [initialize.components.entity_ruler.patterns] +> @readers = "srsly.read_jsonl.v1" +> path = "corpus/entity_ruler_patterns.jsonl +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Not used by the `EntityRuler`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | + +## EntityRuler.\_\_len\_\_ {id="len",tag="method"} + +The number of all patterns added to the entity ruler. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> assert len(ruler) == 0 +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert len(ruler) == 1 +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The number of patterns. ~~int~~ | + +## EntityRuler.\_\_contains\_\_ {id="contains",tag="method"} + +Whether a label is present in the patterns. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert "ORG" in ruler +> assert not "PERSON" in ruler +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| `label` | The label to check. ~~str~~ | +| **RETURNS** | Whether the entity ruler contains the label. ~~bool~~ | + +## EntityRuler.\_\_call\_\_ {id="call",tag="method"} + +Find matches in the `Doc` and add them to the `doc.ents`. Typically, this +happens automatically after the component has been added to the pipeline using +[`nlp.add_pipe`](/api/language#add_pipe). If the entity ruler was initialized +with `overwrite_ents=True`, existing entities will be replaced if they overlap +with the matches. When matches overlap in a Doc, the entity ruler prioritizes +longer patterns over shorter, and if equal the match occurring first in the Doc +is chosen. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> +> doc = nlp("A text about Apple.") +> ents = [(ent.text, ent.label_) for ent in doc.ents] +> assert ents == [("Apple", "ORG")] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with added entities, if available. ~~Doc~~ | + +## EntityRuler.add_patterns {id="add_patterns",tag="method"} + +Add patterns to the entity ruler. A pattern can either be a token pattern (list +of dicts) or a phrase pattern (string). For more details, see the usage guide on +[rule-based matching](/usage/rule-based-matching). + +> #### Example +> +> ```python +> patterns = [ +> {"label": "ORG", "pattern": "Apple"}, +> {"label": "GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]} +> ] +> ruler = nlp.add_pipe("entity_ruler") +> ruler.add_patterns(patterns) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------- | +| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | + +## EntityRuler.remove {id="remove",tag="method",version="3.2.1"} + +Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if +the ID does not exist. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("entity_ruler") +> ruler.add_patterns(patterns) +> ruler.remove("apple") +> ``` + +| Name | Description | +| ---- | ----------------------------------- | +| `id` | The ID of the pattern rule. ~~str~~ | + +## EntityRuler.to_disk {id="to_disk",tag="method"} + +Save the entity ruler patterns to a directory. The patterns will be saved as +newline-delimited JSON (JSONL). If a file with the suffix `.jsonl` is provided, +only the patterns are saved as JSONL. If a directory name is provided, a +`patterns.jsonl` and `cfg` file with the component configuration is exported. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> ruler.to_disk("/path/to/patterns.jsonl") # saves patterns only +> ruler.to_disk("/path/to/entity_ruler") # saves patterns and config +> ``` + +| Name | Description | +| ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | A path to a JSONL file or directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## EntityRuler.from_disk {id="from_disk",tag="method"} + +Load the entity ruler from a path. Expects either a file containing +newline-delimited JSON (JSONL) with one entry per line, or a directory +containing a `patterns.jsonl` file and a `cfg` file with the component +configuration. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> ruler.from_disk("/path/to/patterns.jsonl") # loads patterns only +> ruler.from_disk("/path/to/entity_ruler") # loads patterns and config +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------- | +| `path` | A path to a JSONL file or directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | + +## EntityRuler.to_bytes {id="to_bytes",tag="method"} + +Serialize the entity ruler patterns to a bytestring. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("entity_ruler") +> ruler_bytes = ruler.to_bytes() +> ``` + +| Name | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The serialized patterns. ~~bytes~~ | + +## EntityRuler.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler_bytes = ruler.to_bytes() +> ruler = nlp.add_pipe("entity_ruler") +> ruler.from_bytes(ruler_bytes) +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------- | +| `bytes_data` | The bytestring to load. ~~bytes~~ | +| **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | + +## EntityRuler.labels {id="labels",tag="property"} + +All labels present in the match patterns. + +| Name | Description | +| ----------- | -------------------------------------- | +| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | + +## EntityRuler.ent_ids {id="ent_ids",tag="property",version="2.2.2"} + +All entity IDs present in the `id` properties of the match patterns. + +| Name | Description | +| ----------- | ----------------------------------- | +| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | + +## EntityRuler.patterns {id="patterns",tag="property"} + +Get all patterns that were added to the entity ruler. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------- | +| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ----------------- | --------------------------------------------------------------------------------------------------------------------- | +| `matcher` | The underlying matcher used to process token patterns. ~~Matcher~~ | +| `phrase_matcher` | The underlying phrase matcher used to process phrase patterns. ~~PhraseMatcher~~ | +| `token_patterns` | The token patterns present in the entity ruler, keyed by label. ~~Dict[str, List[Dict[str, Union[str, List[dict]]]]~~ | +| `phrase_patterns` | The phrase patterns present in the entity ruler, keyed by label. ~~Dict[str, List[Doc]]~~ | diff --git a/website/docs/api/example.mdx b/website/docs/api/example.mdx new file mode 100644 index 0000000..a29d5a7 --- /dev/null +++ b/website/docs/api/example.mdx @@ -0,0 +1,330 @@ +--- +title: Example +teaser: A training instance +tag: class +source: spacy/training/example.pyx +version: 3.0 +--- + +An `Example` holds the information for one training instance. It stores two +`Doc` objects: one for holding the gold-standard reference data, and one for +holding the predictions of the pipeline. An +[`Alignment`](/api/example#alignment-object) object stores the alignment between +these two documents, as they can differ in tokenization. + +## Example.\_\_init\_\_ {id="init",tag="method"} + +Construct an `Example` object from the `predicted` document and the `reference` +document. If `alignment` is `None`, it will be initialized from the words in +both documents. + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> from spacy.training import Example +> pred_words = ["Apply", "some", "sunscreen"] +> pred_spaces = [True, True, False] +> gold_words = ["Apply", "some", "sun", "screen"] +> gold_spaces = [True, True, False, False] +> gold_tags = ["VERB", "DET", "NOUN", "NOUN"] +> predicted = Doc(nlp.vocab, words=pred_words, spaces=pred_spaces) +> reference = Doc(nlp.vocab, words=gold_words, spaces=gold_spaces, tags=gold_tags) +> example = Example(predicted, reference) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `predicted` | The document containing (partial) predictions. Cannot be `None`. ~~Doc~~ | +| `reference` | The document containing gold-standard annotations. Cannot be `None`. ~~Doc~~ | +| _keyword-only_ | | +| `alignment` | An object holding the alignment between the tokens of the `predicted` and `reference` documents. ~~Optional[Alignment]~~ | + +## Example.from_dict {id="from_dict",tag="classmethod"} + +Construct an `Example` object from the `predicted` document and the reference +annotations provided as a dictionary. For more details on the required format, +see the [training format documentation](/api/data-formats#dict-input). + +> #### Example +> +> ```python +> from spacy.tokens import Doc +> from spacy.training import Example +> +> predicted = Doc(vocab, words=["Apply", "some", "sunscreen"]) +> token_ref = ["Apply", "some", "sun", "screen"] +> tags_ref = ["VERB", "DET", "NOUN", "NOUN"] +> example = Example.from_dict(predicted, {"words": token_ref, "tags": tags_ref}) +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------- | +| `predicted` | The document containing (partial) predictions. Cannot be `None`. ~~Doc~~ | +| `example_dict` | The gold-standard annotations as a dictionary. Cannot be `None`. ~~Dict[str, Any]~~ | +| **RETURNS** | The newly constructed object. ~~Example~~ | + +## Example.text {id="text",tag="property"} + +The text of the `predicted` document in this `Example`. + +> #### Example +> +> ```python +> raw_text = example.text +> ``` + +| Name | Description | +| ----------- | --------------------------------------------- | +| **RETURNS** | The text of the `predicted` document. ~~str~~ | + +## Example.predicted {id="predicted",tag="property"} + +The `Doc` holding the predictions. Occasionally also referred to as `example.x`. + +> #### Example +> +> ```python +> docs = [eg.predicted for eg in examples] +> predictions, _ = model.begin_update(docs) +> set_annotations(docs, predictions) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The document containing (partial) predictions. ~~Doc~~ | + +## Example.reference {id="reference",tag="property"} + +The `Doc` holding the gold-standard annotations. Occasionally also referred to +as `example.y`. + +> #### Example +> +> ```python +> for i, eg in enumerate(examples): +> for j, label in enumerate(all_labels): +> gold_labels[i][j] = eg.reference.cats.get(label, 0.0) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The document containing gold-standard annotations. ~~Doc~~ | + +## Example.alignment {id="alignment",tag="property"} + +The [`Alignment`](/api/example#alignment-object) object mapping the tokens of +the `predicted` document to those of the `reference` document. + +> #### Example +> +> ```python +> tokens_x = ["Apply", "some", "sunscreen"] +> x = Doc(vocab, words=tokens_x) +> tokens_y = ["Apply", "some", "sun", "screen"] +> example = Example.from_dict(x, {"words": tokens_y}) +> alignment = example.alignment +> assert list(alignment.y2x.data) == [[0], [1], [2], [2]] +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------- | +| **RETURNS** | The document containing gold-standard annotations. ~~Alignment~~ | + +## Example.get_aligned {id="get_aligned",tag="method"} + +Get the aligned view of a certain token attribute, denoted by its int ID or +string name. + +> #### Example +> +> ```python +> predicted = Doc(vocab, words=["Apply", "some", "sunscreen"]) +> token_ref = ["Apply", "some", "sun", "screen"] +> tags_ref = ["VERB", "DET", "NOUN", "NOUN"] +> example = Example.from_dict(predicted, {"words": token_ref, "tags": tags_ref}) +> assert example.get_aligned("TAG", as_string=True) == ["VERB", "DET", "NOUN"] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------- | +| `field` | Attribute ID or string name. ~~Union[int, str]~~ | +| `as_string` | Whether or not to return the list of values as strings. Defaults to `False`. ~~bool~~ | +| **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | + +## Example.get_aligned_parse {id="get_aligned_parse",tag="method"} + +Get the aligned view of the dependency parse. If `projectivize` is set to +`True`, non-projective dependency trees are made projective through the +Pseudo-Projective Dependency Parsing algorithm by Nivre and Nilsson (2005). + +> #### Example +> +> ```python +> doc = nlp("He pretty quickly walks away") +> example = Example.from_dict(doc, {"heads": [3, 2, 3, 0, 2]}) +> proj_heads, proj_labels = example.get_aligned_parse(projectivize=True) +> assert proj_heads == [3, 2, 3, 0, 3] +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------- | +| `projectivize` | Whether or not to projectivize the dependency trees. Defaults to `True`. ~~bool~~ | +| **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | + +## Example.get_aligned_ner {id="get_aligned_ner",tag="method"} + +Get the aligned view of the NER +[BILUO](/usage/linguistic-features#accessing-ner) tags. + +> #### Example +> +> ```python +> words = ["Mrs", "Smith", "flew", "to", "New York"] +> doc = Doc(en_vocab, words=words) +> entities = [(0, 9, "PERSON"), (18, 26, "LOC")] +> gold_words = ["Mrs Smith", "flew", "to", "New", "York"] +> example = Example.from_dict(doc, {"words": gold_words, "entities": entities}) +> ner_tags = example.get_aligned_ner() +> assert ner_tags == ["B-PERSON", "L-PERSON", "O", "O", "U-LOC"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------- | +| **RETURNS** | List of BILUO values, denoting whether tokens are part of an NER annotation or not. ~~List[str]~~ | + +## Example.get_aligned_spans_y2x {id="get_aligned_spans_y2x",tag="method"} + +Get the aligned view of any set of [`Span`](/api/span) objects defined over +[`Example.reference`](/api/example#reference). The resulting span indices will +align to the tokenization in [`Example.predicted`](/api/example#predicted). + +> #### Example +> +> ```python +> words = ["Mr and Mrs Smith", "flew", "to", "New York"] +> doc = Doc(en_vocab, words=words) +> entities = [(0, 16, "PERSON")] +> tokens_ref = ["Mr", "and", "Mrs", "Smith", "flew", "to", "New", "York"] +> example = Example.from_dict(doc, {"words": tokens_ref, "entities": entities}) +> ents_ref = example.reference.ents +> assert [(ent.start, ent.end) for ent in ents_ref] == [(0, 4)] +> ents_y2x = example.get_aligned_spans_y2x(ents_ref) +> assert [(ent.start, ent.end) for ent in ents_y2x] == [(0, 1)] +> ``` + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------- | +| `y_spans` | `Span` objects aligned to the tokenization of `reference`. ~~Iterable[Span]~~ | +| `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | +| **RETURNS** | `Span` objects aligned to the tokenization of `predicted`. ~~List[Span]~~ | + +## Example.get_aligned_spans_x2y {id="get_aligned_spans_x2y",tag="method"} + +Get the aligned view of any set of [`Span`](/api/span) objects defined over +[`Example.predicted`](/api/example#predicted). The resulting span indices will +align to the tokenization in [`Example.reference`](/api/example#reference). This +method is particularly useful to assess the accuracy of predicted entities +against the original gold-standard annotation. + +> #### Example +> +> ```python +> nlp.add_pipe("my_ner") +> doc = nlp("Mr and Mrs Smith flew to New York") +> tokens_ref = ["Mr and Mrs", "Smith", "flew", "to", "New York"] +> example = Example.from_dict(doc, {"words": tokens_ref}) +> ents_pred = example.predicted.ents +> # Assume the NER model has found "Mr and Mrs Smith" as a named entity +> assert [(ent.start, ent.end) for ent in ents_pred] == [(0, 4)] +> ents_x2y = example.get_aligned_spans_x2y(ents_pred) +> assert [(ent.start, ent.end) for ent in ents_x2y] == [(0, 2)] +> ``` + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------- | +| `x_spans` | `Span` objects aligned to the tokenization of `predicted`. ~~Iterable[Span]~~ | +| `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | +| **RETURNS** | `Span` objects aligned to the tokenization of `reference`. ~~List[Span]~~ | + +## Example.to_dict {id="to_dict",tag="method"} + +Return a [dictionary representation](/api/data-formats#dict-input) of the +reference annotation contained in this `Example`. + +> #### Example +> +> ```python +> eg_dict = example.to_dict() +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------- | +| **RETURNS** | Dictionary representation of the reference annotation. ~~Dict[str, Any]~~ | + +## Example.split_sents {id="split_sents",tag="method"} + +Split one `Example` into multiple `Example` objects, one for each sentence. + +> #### Example +> +> ```python +> doc = nlp("I went yesterday had lots of fun") +> tokens_ref = ["I", "went", "yesterday", "had", "lots", "of", "fun"] +> sents_ref = [True, False, False, True, False, False, False] +> example = Example.from_dict(doc, {"words": tokens_ref, "sent_starts": sents_ref}) +> split_examples = example.split_sents() +> assert split_examples[0].text == "I went yesterday " +> assert split_examples[1].text == "had lots of fun" +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| **RETURNS** | List of `Example` objects, one for each original sentence. ~~List[Example]~~ | + +## Alignment {id="alignment-object",version="3"} + +Calculate alignment tables between two tokenizations. + +### Alignment attributes {id="alignment-attributes"} + +Alignment attributes are managed using `AlignmentArray`, which is a simplified +version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) type that +only supports the `data` and `length` attributes. + +| Name | Description | +| ----- | ------------------------------------------------------------------------------------- | +| `x2y` | The `AlignmentArray` object holding the alignment from `x` to `y`. ~~AlignmentArray~~ | +| `y2x` | The `AlignmentArray` object holding the alignment from `y` to `x`. ~~AlignmentArray~~ | + + + +The current implementation of the alignment algorithm assumes that both +tokenizations add up to the same string. For example, you'll be able to align +`["I", "'", "m"]` and `["I", "'m"]`, which both add up to `"I'm"`, but not +`["I", "'m"]` and `["I", "am"]`. + + + +> #### Example +> +> ```python +> from spacy.training import Alignment +> +> bert_tokens = ["obama", "'", "s", "podcast"] +> spacy_tokens = ["obama", "'s", "podcast"] +> alignment = Alignment.from_strings(bert_tokens, spacy_tokens) +> a2b = alignment.x2y +> assert list(a2b.data) == [0, 1, 1, 2] +> ``` +> +> If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and +> `A[2]` (`"s"`) both align to `B[1]` (`"'s"`). + +### Alignment.from_strings {id="classmethod",tag="function"} + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `A` | String values of candidate tokens to align. ~~List[str]~~ | +| `B` | String values of reference tokens to align. ~~List[str]~~ | +| **RETURNS** | An `Alignment` object describing the alignment. ~~Alignment~~ | diff --git a/website/docs/api/index.mdx b/website/docs/api/index.mdx new file mode 100644 index 0000000..6c6e1ff --- /dev/null +++ b/website/docs/api/index.mdx @@ -0,0 +1,6 @@ +--- +title: Library Architecture +next: /api/architectures +--- + + diff --git a/website/docs/api/inmemorylookupkb.mdx b/website/docs/api/inmemorylookupkb.mdx new file mode 100644 index 0000000..15b1d3b --- /dev/null +++ b/website/docs/api/inmemorylookupkb.mdx @@ -0,0 +1,303 @@ +--- +title: InMemoryLookupKB +teaser: + The default implementation of the KnowledgeBase interface. Stores all + information in-memory. +tag: class +source: spacy/kb/kb_in_memory.pyx +version: 3.5 +--- + +The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and +implements all of its methods. It stores all KB data in-memory and generates +[`Candidate`](/api/kb#candidate) objects by exactly matching mentions with +entity names. It's highly optimized for both a low memory footprint and speed of +retrieval. + +## InMemoryLookupKB.\_\_init\_\_ {id="init",tag="method"} + +Create the knowledge base. + +> #### Example +> +> ```python +> from spacy.kb import InMemoryLookupKB +> vocab = nlp.vocab +> kb = InMemoryLookupKB(vocab=vocab, entity_vector_length=64) +> ``` + +| Name | Description | +| ---------------------- | ------------------------------------------------ | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.entity_vector_length {id="entity_vector_length",tag="property"} + +The length of the fixed-size entity vectors in the knowledge base. + +| Name | Description | +| ----------- | ------------------------------------------------ | +| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.add_entity {id="add_entity",tag="method"} + +Add an entity to the knowledge base, specifying its corpus frequency and entity +vector, which should be of length +[`entity_vector_length`](/api/inmemorylookupkb#entity_vector_length). + +> #### Example +> +> ```python +> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) +> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> ``` + +| Name | Description | +| --------------- | ---------------------------------------------------------- | +| `entity` | The unique entity identifier. ~~str~~ | +| `freq` | The frequency of the entity in a typical corpus. ~~float~~ | +| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.set_entities {id="set_entities",tag="method"} + +Define the full list of entities in the knowledge base, specifying the corpus +frequency and entity vector for each entity. + +> #### Example +> +> ```python +> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> ``` + +| Name | Description | +| ------------- | ---------------------------------------------------------------- | +| `entity_list` | List of unique entity identifiers. ~~Iterable[Union[str, int]]~~ | +| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | +| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | + +## InMemoryLookupKB.add_alias {id="add_alias",tag="method"} + +Add an alias or mention to the knowledge base, specifying its potential KB +identifiers and their prior probabilities. The entity identifiers should refer +to entities previously added with +[`add_entity`](/api/inmemorylookupkb#add_entity) or +[`set_entities`](/api/inmemorylookupkb#set_entities). The sum of the prior +probabilities should not exceed 1. Note that an empty string can not be used as +alias. + +> #### Example +> +> ```python +> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) +> ``` + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------- | +| `alias` | The textual mention or alias. Can not be the empty string. ~~str~~ | +| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | +| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | + +## InMemoryLookupKB.\_\_len\_\_ {id="len",tag="method"} + +Get the total number of entities in the knowledge base. + +> #### Example +> +> ```python +> total_entities = len(kb) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| **RETURNS** | The number of entities in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_entity_strings {id="get_entity_strings",tag="method"} + +Get a list of all entity IDs in the knowledge base. + +> #### Example +> +> ```python +> all_entities = kb.get_entity_strings() +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_size_aliases {id="get_size_aliases",tag="method"} + +Get the total number of aliases in the knowledge base. + +> #### Example +> +> ```python +> total_aliases = kb.get_size_aliases() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------- | +| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_alias_strings {id="get_alias_strings",tag="method"} + +Get a list of all aliases in the knowledge base. + +> #### Example +> +> ```python +> all_aliases = kb.get_alias_strings() +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_candidates {id="get_candidates",tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). Wraps +[`get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | + +## InMemoryLookupKB.get_candidates_batch {id="get_candidates_batch",tag="method"} + +Same as [`get_candidates()`](/api/inmemorylookupkb#get_candidates), but for an +arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component +will call `get_candidates_batch()` instead of `get_candidates()`, if the config +parameter `candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | + +## InMemoryLookupKB.get_alias_candidates {id="get_alias_candidates",tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). + +> #### Example +> +> ```python +> candidates = kb.get_alias_candidates("Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | + +## InMemoryLookupKB.get_vector {id="get_vector",tag="method"} + +Given a certain entity ID, retrieve its pretrained entity vector. + +> #### Example +> +> ```python +> vector = kb.get_vector("Q42") +> ``` + +| Name | Description | +| ----------- | ------------------------------------ | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.get_vectors {id="get_vectors",tag="method"} + +Same as [`get_vector()`](/api/inmemorylookupkb#get_vector), but for an arbitrary +number of entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. + +> #### Example +> +> ```python +> vectors = kb.get_vectors(("Q42", "Q3107329")) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | + +## InMemoryLookupKB.get_prior_prob {id="get_prior_prob",tag="method"} + +Given a certain entity ID and a certain textual mention, retrieve the prior +probability of the fact that the mention links to the entity ID. + +> #### Example +> +> ```python +> probability = kb.get_prior_prob("Q42", "Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | + +## InMemoryLookupKB.to_disk {id="to_disk",tag="method"} + +Save the current state of the knowledge base to a directory. + +> #### Example +> +> ```python +> kb.to_disk(path) +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | + +## InMemoryLookupKB.from_disk {id="from_disk",tag="method"} + +Restore the state of the knowledge base from a given directory. Note that the +[`Vocab`](/api/vocab) should also be the same as the one used to create the KB. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> kb = InMemoryLookupKB(vocab=vocab, entity_vector_length=64) +> kb.from_disk("/path/to/kb") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | diff --git a/website/docs/api/kb.mdx b/website/docs/api/kb.mdx new file mode 100644 index 0000000..2b0d4d9 --- /dev/null +++ b/website/docs/api/kb.mdx @@ -0,0 +1,232 @@ +--- +title: KnowledgeBase +teaser: + A storage class for entities and aliases of a specific knowledge base + (ontology) +tag: class +source: spacy/kb/kb.pyx +version: 2.2 +--- + +The `KnowledgeBase` object is an abstract class providing a method to generate +[`Candidate`](/api/kb#candidate) objects, which are plausible external +identifiers given a certain textual mention. Each such `Candidate` holds +information from the relevant KB entities, such as its frequency in text and +possible aliases. Each entity in the knowledge base also has a pretrained entity +vector of a fixed size. + +Beyond that, `KnowledgeBase` classes have to implement a number of utility +functions called by the [`EntityLinker`](/api/entitylinker) component. + + + +This class was not abstract up to spaCy version 3.5. The `KnowledgeBase` +implementation up to that point is available as +[`InMemoryLookupKB`](/api/inmemorylookupkb) from 3.5 onwards. + + + +## KnowledgeBase.\_\_init\_\_ {id="init",tag="method"} + +`KnowledgeBase` is an abstract class and cannot be instantiated. Its child +classes should call `__init__()` to set up some necessary attributes. + +> #### Example +> +> ```python +> from spacy.kb import KnowledgeBase +> from spacy.vocab import Vocab +> +> class FullyImplementedKB(KnowledgeBase): +> def __init__(self, vocab: Vocab, entity_vector_length: int): +> super().__init__(vocab, entity_vector_length) +> ... +> vocab = nlp.vocab +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) +> ``` + +| Name | Description | +| ---------------------- | ------------------------------------------------ | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | + +## KnowledgeBase.entity_vector_length {id="entity_vector_length",tag="property"} + +The length of the fixed-size entity vectors in the knowledge base. + +| Name | Description | +| ----------- | ------------------------------------------------ | +| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | + +## KnowledgeBase.get_candidates {id="get_candidates",tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | + +## KnowledgeBase.get_candidates_batch {id="get_candidates_batch",tag="method"} + +Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary +number of mentions. The [`EntityLinker`](/api/entitylinker) component will call +`get_candidates_batch()` instead of `get_candidates()`, if the config parameter +`candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | + +## KnowledgeBase.get_alias_candidates {id="get_alias_candidates",tag="method"} + + + This method is _not_ available from spaCy 3.5 onwards. + + +From spaCy 3.5 on `KnowledgeBase` is an abstract class (with +[`InMemoryLookupKB`](/api/inmemorylookupkb) being a drop-in replacement) to +allow more flexibility in customizing knowledge bases. Some of its methods were +moved to [`InMemoryLookupKB`](/api/inmemorylookupkb) during this refactoring, +one of those being `get_alias_candidates()`. This method is now available as +[`InMemoryLookupKB.get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). +Note: +[`InMemoryLookupKB.get_candidates()`](/api/inmemorylookupkb#get_candidates) +defaults to +[`InMemoryLookupKB.get_alias_candidates()`](/api/inmemorylookupkb#get_alias_candidates). + +## KnowledgeBase.get_vector {id="get_vector",tag="method"} + +Given a certain entity ID, retrieve its pretrained entity vector. + +> #### Example +> +> ```python +> vector = kb.get_vector("Q42") +> ``` + +| Name | Description | +| ----------- | -------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~Iterable[float]~~ | + +## KnowledgeBase.get_vectors {id="get_vectors",tag="method"} + +Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of +entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. + +> #### Example +> +> ```python +> vectors = kb.get_vectors(("Q42", "Q3107329")) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | + +## KnowledgeBase.to_disk {id="to_disk",tag="method"} + +Save the current state of the knowledge base to a directory. + +> #### Example +> +> ```python +> kb.to_disk(path) +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | + +## KnowledgeBase.from_disk {id="from_disk",tag="method"} + +Restore the state of the knowledge base from a given directory. Note that the +[`Vocab`](/api/vocab) should also be the same as the one used to create the KB. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) +> kb.from_disk("/path/to/kb") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | + +## Candidate {id="candidate",tag="class"} + +A `Candidate` object refers to a textual mention (alias) that may or may not be +resolved to a specific entity from a `KnowledgeBase`. This will be used as input +for the entity linking algorithm which will disambiguate the various candidates +to the correct one. Each candidate `(alias, entity)` pair is assigned to a +certain prior probability. + +### Candidate.\_\_init\_\_ {id="candidate-init",tag="method"} + +Construct a `Candidate` object. Usually this constructor is not called directly, +but instead these objects are returned by the `get_candidates` method of the +[`entity_linker`](/api/entitylinker) pipe. + +> #### Example +> +> ```python +> from spacy.kb import Candidate +> candidate = Candidate(kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob) +> ``` + +| Name | Description | +| ------------- | ------------------------------------------------------------------------- | +| `kb` | The knowledge base that defined this candidate. ~~KnowledgeBase~~ | +| `entity_hash` | The hash of the entity's KB ID. ~~int~~ | +| `entity_freq` | The entity frequency as recorded in the KB. ~~float~~ | +| `alias_hash` | The hash of the textual mention or alias. ~~int~~ | +| `prior_prob` | The prior probability of the `alias` referring to the `entity`. ~~float~~ | + +## Candidate attributes {id="candidate-attributes"} + +| Name | Description | +| --------------- | ------------------------------------------------------------------------ | +| `entity` | The entity's unique KB identifier. ~~int~~ | +| `entity_` | The entity's unique KB identifier. ~~str~~ | +| `alias` | The alias or textual mention. ~~int~~ | +| `alias_` | The alias or textual mention. ~~str~~ | +| `prior_prob` | The prior probability of the `alias` referring to the `entity`. ~~long~~ | +| `entity_freq` | The frequency of the entity in a typical corpus. ~~long~~ | +| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | diff --git a/website/docs/api/language.mdx b/website/docs/api/language.mdx new file mode 100644 index 0000000..a1c6601 --- /dev/null +++ b/website/docs/api/language.mdx @@ -0,0 +1,1163 @@ +--- +title: Language +teaser: A text-processing pipeline +tag: class +source: spacy/language.py +--- + +Usually you'll load this once per process as `nlp` and pass the instance around +your application. The `Language` class is created when you call +[`spacy.load`](/api/top-level#spacy.load) and contains the shared vocabulary and +[language data](/usage/linguistic-features#language-data), optional binary +weights, e.g. provided by a [trained pipeline](/models), and the +[processing pipeline](/usage/processing-pipelines) containing components like +the tagger or parser that are called on a document in order. You can also add +your own processing pipeline components that take a `Doc` object, modify it and +return it. + +## Language.\_\_init\_\_ {id="init",tag="method"} + +Initialize a `Language` object. Note that the `meta` is only used for meta +information in [`Language.meta`](/api/language#meta) and not to configure the +`nlp` object or to override the config. To initialize from a config, use +[`Language.from_config`](/api/language#from_config) instead. + +> #### Example +> +> ```python +> # Construction from subclass +> from spacy.lang.en import English +> nlp = English() +> +> # Construction from scratch +> from spacy.vocab import Vocab +> from spacy.language import Language +> nlp = Language(Vocab()) +> ``` + +| Name | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------ | +| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | +| _keyword-only_ | | +| `max_length` | Maximum number of characters allowed in a single text. Defaults to `10 ** 6`. ~~int~~ | +| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | +| `create_tokenizer` | Optional function that receives the `nlp` object and returns a tokenizer. ~~Callable[[Language], Callable[[str], Doc]]~~ | +| `batch_size` | Default batch size for [`pipe`](#pipe) and [`evaluate`](#evaluate). Defaults to `1000`. ~~int~~ | + +## Language.from_config {id="from_config",tag="classmethod",version="3"} + +Create a `Language` object from a loaded config. Will set up the tokenizer and +language data, add pipeline components based on the pipeline and add pipeline +components based on the definitions specified in the config. If no config is +provided, the default config of the given language is used. This is also how +spaCy loads a model under the hood based on its +[`config.cfg`](/api/data-formats#config). + +> #### Example +> +> ```python +> from thinc.api import Config +> from spacy.language import Language +> +> config = Config().from_disk("./config.cfg") +> nlp = Language.from_config(config) +> ``` + +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | +| _keyword-only_ | | +| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | +| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | +| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The initialized object. ~~Language~~ | + +## Language.component {id="component",tag="classmethod",version="3"} + +Register a custom pipeline component under a given name. This allows +initializing the component by name using +[`Language.add_pipe`](/api/language#add_pipe) and referring to it in +[config files](/usage/training#config). This classmethod and decorator is +intended for **simple stateless functions** that take a `Doc` and return it. For +more complex stateful components that allow settings and need access to the +shared `nlp` object, use the [`Language.factory`](/api/language#factory) +decorator. For more details and examples, see the +[usage documentation](/usage/processing-pipelines#custom-components). + +> #### Example +> +> ```python +> from spacy.language import Language +> +> # Usage as a decorator +> @Language.component("my_component") +> def my_component(doc): +> # Do something to the doc +> return doc +> +> # Usage as a function +> Language.component("my_component2", func=my_component) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | The name of the component factory. ~~str~~ | +| _keyword-only_ | | +| `assigns` | `Doc` or `Token` attributes assigned by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | +| `func` | Optional function if not used as a decorator. ~~Optional[Callable[[Doc], Doc]]~~ | + +## Language.factory {id="factory",tag="classmethod"} + +Register a custom pipeline component factory under a given name. This allows +initializing the component by name using +[`Language.add_pipe`](/api/language#add_pipe) and referring to it in +[config files](/usage/training#config). The registered factory function needs to +take at least two **named arguments** which spaCy fills in automatically: `nlp` +for the current `nlp` object and `name` for the component instance name. This +can be useful to distinguish multiple instances of the same component and allows +trainable components to add custom losses using the component instance name. The +`default_config` defines the default values of the remaining factory arguments. +It's merged into the [`nlp.config`](/api/language#config). For more details and +examples, see the +[usage documentation](/usage/processing-pipelines#custom-components). + +> #### Example +> +> ```python +> from spacy.language import Language +> +> # Usage as a decorator +> @Language.factory( +> "my_component", +> default_config={"some_setting": True}, +> ) +> def create_my_component(nlp, name, some_setting): +> return MyComponent(some_setting) +> +> # Usage as function +> Language.factory( +> "my_component", +> default_config={"some_setting": True}, +> func=create_my_component +> ) +> ``` + +| Name | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | The name of the component factory. ~~str~~ | +| _keyword-only_ | | +| `default_config` | The default config, describing the default values of the factory arguments. ~~Dict[str, Any]~~ | +| `assigns` | `Doc` or `Token` attributes assigned by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | +| `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ | +| `func` | Optional function if not used as a decorator. ~~Optional[Callable[[...], Callable[[Doc], Doc]]]~~ | + +## Language.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipeline to some text. The text can span multiple sentences, and can +contain arbitrary whitespace. Alignment into the original string is preserved. + +Instead of text, a `Doc` can be passed as input, in which case tokenization is +skipped, but the rest of the pipeline is run. + +> #### Example +> +> ```python +> doc = nlp("An example sentence. Another sentence.") +> assert (doc[0].text, doc[0].head.tag_) == ("An", "NN") +> ``` + +| Name | Description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `text` | The text to be processed, or a Doc. ~~Union[str, Doc]~~ | +| _keyword-only_ | | +| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). ~~List[str]~~ | +| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | +| **RETURNS** | A container for accessing the annotations. ~~Doc~~ | + +## Language.pipe {id="pipe",tag="method"} + +Process texts as a stream, and yield `Doc` objects in order. This is usually +more efficient than processing texts one-by-one. + +Instead of text, a `Doc` object can be passed as input. In this case +tokenization is skipped but the rest of the pipeline is run. + +> #### Example +> +> ```python +> texts = ["One document.", "...", "Lots of documents"] +> for doc in nlp.pipe(texts, batch_size=50): +> assert doc.has_annotation("DEP") +> ``` + +| Name | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `texts` | A sequence of strings (or `Doc` objects). ~~Iterable[Union[str, Doc]]~~ | +| _keyword-only_ | | +| `as_tuples` | If set to `True`, inputs should be a sequence of `(text, context)` tuples. Output will then be a sequence of `(doc, context)` tuples. Defaults to `False`. ~~bool~~ | +| `batch_size` | The number of texts to buffer. ~~Optional[int]~~ | +| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). ~~List[str]~~ | +| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | +| `n_process` | Number of processors to use. Defaults to `1`. ~~int~~ | +| **YIELDS** | Documents in the order of the original text. ~~Doc~~ | + +## Language.set_error_handler {id="set_error_handler",tag="method",version="3"} + +Define a callback that will be invoked when an error is thrown during processing +of one or more documents. Specifically, this function will call +[`set_error_handler`](/api/pipe#set_error_handler) on all the pipeline +components that define that function. The error handler will be invoked with the +original component's name, the component itself, the list of documents that was +being processed, and the original error. + +> #### Example +> +> ```python +> def warn_error(proc_name, proc, docs, e): +> print(f"An error occurred when applying component {proc_name}.") +> +> nlp.set_error_handler(warn_error) +> ``` + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------------------------- | +| `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | + +## Language.initialize {id="initialize",tag="method",version="3"} + +Initialize the pipeline for training and return an +[`Optimizer`](https://thinc.ai/docs/api-optimizers). Under the hood, it uses the +settings defined in the [`[initialize]`](/api/data-formats#config-initialize) +config block to set up the vocabulary, load in vectors and tok2vec weights and +pass optional arguments to the `initialize` methods implemented by pipeline +components or the tokenizer. This method is typically called automatically when +you run [`spacy train`](/api/cli#train). See the usage guide on the +[config lifecycle](/usage/training#config-lifecycle) and +[initialization](/usage/training#initialization) for details. + +`get_examples` should be a function that returns an iterable of +[`Example`](/api/example) objects. The data examples can either be the full +training data or a representative sample. They are used to **initialize the +models** of trainable pipeline components and are passed each component's +[`initialize`](/api/pipe#initialize) method, if available. Initialization +includes validating the network, +[inferring missing shapes](/usage/layers-architectures#thinc-shape-inference) +and setting up the label scheme based on the data. + +If no `get_examples` function is provided when calling `nlp.initialize`, the +pipeline components will be initialized with generic data. In this case, it is +crucial that the output dimension of each component has already been defined +either in the [config](/usage/training#config), or by calling +[`pipe.add_label`](/api/pipe#add_label) for each possible output label (e.g. for +the tagger or textcat). + + + +This method was previously called `begin_training`. It now also takes a +**function** that is called with no arguments and returns a sequence of +[`Example`](/api/example) objects instead of tuples of `Doc` and `GoldParse` +objects. + + + +> #### Example +> +> ```python +> get_examples = lambda: examples +> optimizer = nlp.initialize(get_examples) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Optional function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Optional[Callable[[], Iterable[Example]]]~~ | +| _keyword-only_ | | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Language.resume_training {id="resume_training",tag="method,experimental",version="3"} + +Continue training a trained pipeline. Create and return an optimizer, and +initialize "rehearsal" for any pipeline component that has a `rehearse` method. +Rehearsal is used to prevent models from "forgetting" their initialized +"knowledge". To perform rehearsal, collect samples of text you want the models +to retain performance on, and call [`nlp.rehearse`](/api/language#rehearse) with +a batch of [Example](/api/example) objects. + +> #### Example +> +> ```python +> optimizer = nlp.resume_training() +> nlp.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Language.update {id="update",tag="method"} + +Update the models in the pipeline. + + + +The `Language.update` method now takes a batch of [`Example`](/api/example) +objects instead of the raw texts and annotations or `Doc` and `GoldParse` +objects. An [`Example`](/api/example) streamlines how data is passed around. It +stores two `Doc` objects: one for holding the gold-standard reference data, and +one for holding the predictions of the pipeline. + +For most use cases, you shouldn't have to write your own training scripts +anymore. Instead, you can use [`spacy train`](/api/cli#train) with a config file +and custom registered functions if needed. See the +[training documentation](/usage/training) for details. + + + +> #### Example +> +> ```python +> for raw_text, entity_offsets in train_data: +> doc = nlp.make_doc(raw_text) +> example = Example.from_dict(doc, {"entities": entity_offsets}) +> nlp.update([example], sgd=optimizer) +> ``` + +| Name | Description | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Dictionary to update with the loss, keyed by pipeline component. ~~Optional[Dict[str, float]]~~ | +| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Language.rehearse {id="rehearse",tag="method,experimental",version="3"} + +Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the +current model to make predictions similar to an initial model, to try to address +the "catastrophic forgetting" problem. This feature is experimental. + +> #### Example +> +> ```python +> optimizer = nlp.resume_training() +> losses = nlp.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------- | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Dictionary to update with the loss, keyed by pipeline component. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Language.evaluate {id="evaluate",tag="method"} + +Evaluate a pipeline's components. + + + +The `Language.evaluate` method now takes a batch of [`Example`](/api/example) +objects instead of tuples of `Doc` and `GoldParse` objects. + + + +> #### Example +> +> ```python +> scores = nlp.evaluate(examples) +> print(scores) +> ``` + +| Name | Description | +| -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `batch_size` | The batch size to use. ~~Optional[int]~~ | +| `scorer` | Optional [`Scorer`](/api/scorer) to use. If not passed in, a new one will be created. ~~Optional[Scorer]~~ | +| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | +| `scorer_cfg` | Optional dictionary of keyword arguments for the `Scorer`. Defaults to `None`. ~~Optional[Dict[str, Any]]~~ | +| `per_component` 3.6 | Whether to return the scores keyed by component name. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A dictionary of evaluation scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## Language.use_params {id="use_params",tag="contextmanager, method"} + +Replace weights of models in the pipeline with those provided in the params +dictionary. Can be used as a context manager, in which case, models go back to +their original weights after the block. + +> #### Example +> +> ```python +> with nlp.use_params(optimizer.averages): +> nlp.to_disk("/tmp/checkpoint") +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------ | +| `params` | A dictionary of parameters keyed by model ID. ~~dict~~ | + +## Language.add_pipe {id="add_pipe",tag="method",version="2"} + +Add a component to the processing pipeline. Expects a name that maps to a +component factory registered using +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory). Components should be callables +that take a `Doc` object, modify it and return it. Only one of `before`, +`after`, `first` or `last` can be set. Default behavior is `last=True`. + + + +As of v3.0, the [`Language.add_pipe`](/api/language#add_pipe) method doesn't +take callables anymore and instead expects the **name of a component factory** +registered using [`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory). It now takes care of creating the +component, adds it to the pipeline and returns it. + + + +> #### Example +> +> ```python +> @Language.component("component") +> def component_func(doc): +> # modify Doc and return it +> return doc +> +> nlp.add_pipe("component", before="ner") +> component = nlp.add_pipe("component", name="custom_name", last=True) +> +> # Add component from source pipeline +> source_nlp = spacy.load("en_core_web_sm") +> nlp.add_pipe("ner", source=source_nlp) +> ``` + +| Name | Description | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `factory_name` | Name of the registered component factory. ~~str~~ | +| `name` | Optional unique name of pipeline component instance. If not set, the factory name is used. An error is raised if the name already exists in the pipeline. ~~Optional[str]~~ | +| _keyword-only_ | | +| `before` | Component name or index to insert component directly before. ~~Optional[Union[str, int]]~~ | +| `after` | Component name or index to insert component directly after. ~~Optional[Union[str, int]]~~ | +| `first` | Insert component first / not first in the pipeline. ~~Optional[bool]~~ | +| `last` | Insert component last / not last in the pipeline. ~~Optional[bool]~~ | +| `config` 3 | Optional config parameters to use for this component. Will be merged with the `default_config` specified by the component factory. ~~Dict[str, Any]~~ | +| `source` 3 | Optional source pipeline to copy component from. If a source is provided, the `factory_name` is interpreted as the name of the component in the source pipeline. Make sure that the vocab, vectors and settings of the source pipeline match the target pipeline. ~~Optional[Language]~~ | +| `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | + +## Language.create_pipe {id="create_pipe",tag="method",version="2"} + +Create a pipeline component from a factory. + + + +As of v3.0, the [`Language.add_pipe`](/api/language#add_pipe) method also takes +the string name of the factory, creates the component, adds it to the pipeline +and returns it. The `Language.create_pipe` method is now mostly used internally. +To create a component and add it to the pipeline, you should always use +`Language.add_pipe`. + + + +> #### Example +> +> ```python +> parser = nlp.create_pipe("parser") +> ``` + +| Name | Description | +| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `factory_name` | Name of the registered component factory. ~~str~~ | +| `name` | Optional unique name of pipeline component instance. If not set, the factory name is used. An error is raised if the name already exists in the pipeline. ~~Optional[str]~~ | +| _keyword-only_ | | +| `config` 3 | Optional config parameters to use for this component. Will be merged with the `default_config` specified by the component factory. ~~Dict[str, Any]~~ | +| `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | + +## Language.has_factory {id="has_factory",tag="classmethod",version="3"} + +Check whether a factory name is registered on the `Language` class or subclass. +Will check for +[language-specific factories](/usage/processing-pipelines#factories-language) +registered on the subclass, as well as general-purpose factories registered on +the `Language` base class, available to all subclasses. + +> #### Example +> +> ```python +> from spacy.language import Language +> from spacy.lang.en import English +> +> @English.component("component") +> def component(doc): +> return doc +> +> assert English.has_factory("component") +> assert not Language.has_factory("component") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------- | +| `name` | Name of the pipeline factory to check. ~~str~~ | +| **RETURNS** | Whether a factory of that name is registered on the class. ~~bool~~ | + +## Language.has_pipe {id="has_pipe",tag="method",version="2"} + +Check whether a component is present in the pipeline. Equivalent to +`name in nlp.pipe_names`. + +> #### Example +> +> ```python +> @Language.component("component") +> def component(doc): +> return doc +> +> nlp.add_pipe("component", name="my_component") +> assert "my_component" in nlp.pipe_names +> assert nlp.has_pipe("my_component") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------- | +| `name` | Name of the pipeline component to check. ~~str~~ | +| **RETURNS** | Whether a component of that name exists in the pipeline. ~~bool~~ | + +## Language.get_pipe {id="get_pipe",tag="method",version="2"} + +Get a pipeline component for a given component name. + +> #### Example +> +> ```python +> parser = nlp.get_pipe("parser") +> custom_component = nlp.get_pipe("custom_component") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------ | +| `name` | Name of the pipeline component to get. ~~str~~ | +| **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | + +## Language.replace_pipe {id="replace_pipe",tag="method",version="2"} + +Replace a component in the pipeline and return the new component. + + + +As of v3.0, the `Language.replace_pipe` method doesn't take callables anymore +and instead expects the **name of a component factory** registered using +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory). + + + +> #### Example +> +> ```python +> new_parser = nlp.replace_pipe("parser", "my_custom_parser") +> ``` + +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Name of the component to replace. ~~str~~ | +| `component` | The factory name of the component to insert. ~~str~~ | +| _keyword-only_ | | +| `config` 3 | Optional config parameters to use for the new component. Will be merged with the `default_config` specified by the component factory. ~~Optional[Dict[str, Any]]~~ | +| `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The new pipeline component. ~~Callable[[Doc], Doc]~~ | + +## Language.rename_pipe {id="rename_pipe",tag="method",version="2"} + +Rename a component in the pipeline. Useful to create custom names for +pre-defined and pre-loaded components. To change the default name of a component +added to the pipeline, you can also use the `name` argument on +[`add_pipe`](/api/language#add_pipe). + +> #### Example +> +> ```python +> nlp.rename_pipe("parser", "spacy_parser") +> ``` + +| Name | Description | +| ---------- | ---------------------------------------- | +| `old_name` | Name of the component to rename. ~~str~~ | +| `new_name` | New name of the component. ~~str~~ | + +## Language.remove_pipe {id="remove_pipe",tag="method",version="2"} + +Remove a component from the pipeline. Returns the removed component name and +component function. + +> #### Example +> +> ```python +> name, component = nlp.remove_pipe("parser") +> assert name == "parser" +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------ | +| `name` | Name of the component to remove. ~~str~~ | +| **RETURNS** | A `(name, component)` tuple of the removed component. ~~Tuple[str, Callable[[Doc], Doc]]~~ | + +## Language.disable_pipe {id="disable_pipe",tag="method",version="3"} + +Temporarily disable a pipeline component so it's not run as part of the +pipeline. Disabled components are listed in +[`nlp.disabled`](/api/language#attributes) and included in +[`nlp.components`](/api/language#attributes), but not in +[`nlp.pipeline`](/api/language#pipeline), so they're not run when you process a +`Doc` with the `nlp` object. If the component is already disabled, this method +does nothing. + +> #### Example +> +> ```python +> nlp.add_pipe("ner") +> nlp.add_pipe("textcat") +> assert nlp.pipe_names == ["ner", "textcat"] +> nlp.disable_pipe("ner") +> assert nlp.pipe_names == ["textcat"] +> assert nlp.component_names == ["ner", "textcat"] +> assert nlp.disabled == ["ner"] +> ``` + +| Name | Description | +| ------ | ----------------------------------------- | +| `name` | Name of the component to disable. ~~str~~ | + +## Language.enable_pipe {id="enable_pipe",tag="method",version="3"} + +Enable a previously disabled component (e.g. via +[`Language.disable_pipes`](/api/language#disable_pipes)) so it's run as part of +the pipeline, [`nlp.pipeline`](/api/language#pipeline). If the component is +already enabled, this method does nothing. + +> #### Example +> +> ```python +> nlp.disable_pipe("ner") +> assert "ner" in nlp.disabled +> assert not "ner" in nlp.pipe_names +> nlp.enable_pipe("ner") +> assert not "ner" in nlp.disabled +> assert "ner" in nlp.pipe_names +> ``` + +| Name | Description | +| ------ | ---------------------------------------- | +| `name` | Name of the component to enable. ~~str~~ | + +## Language.select_pipes {id="select_pipes",tag="contextmanager, method",version="3"} + +Disable one or more pipeline components. If used as a context manager, the +pipeline will be restored to the initial state at the end of the block. +Otherwise, a `DisabledPipes` object is returned, that has a `.restore()` method +you can use to undo your changes. You can specify either `disable` (as a list or +string), or `enable`. In the latter case, all components not in the `enable` +list will be disabled. Under the hood, this method calls into +[`disable_pipe`](/api/language#disable_pipe) and +[`enable_pipe`](/api/language#enable_pipe). + +> #### Example +> +> ```python +> with nlp.select_pipes(disable=["tagger", "parser"]): +> nlp.initialize() +> +> with nlp.select_pipes(enable="ner"): +> nlp.initialize() +> +> disabled = nlp.select_pipes(disable=["tagger", "parser"]) +> nlp.initialize() +> disabled.restore() +> ``` + + + +As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`: + +```diff +- nlp.disable_pipes(["tagger", "parser"]) ++ nlp.select_pipes(disable=["tagger", "parser"]) +``` + + + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------ | +| _keyword-only_ | | +| `disable` | Name(s) of pipeline component(s) to disable. ~~Optional[Union[str, Iterable[str]]]~~ | +| `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ | +| **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ | + +## Language.get_factory_meta {id="get_factory_meta",tag="classmethod",version="3"} + +Get the factory meta information for a given pipeline component name. Expects +the name of the component **factory**. The factory meta is an instance of the +[`FactoryMeta`](/api/language#factorymeta) dataclass and contains the +information about the component and its default provided by the +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorator. + +> #### Example +> +> ```python +> factory_meta = Language.get_factory_meta("ner") +> assert factory_meta.factory == "ner" +> print(factory_meta.default_config) +> ``` + +| Name | Description | +| ----------- | --------------------------------- | +| `name` | The factory name. ~~str~~ | +| **RETURNS** | The factory meta. ~~FactoryMeta~~ | + +## Language.get_pipe_meta {id="get_pipe_meta",tag="method",version="3"} + +Get the factory meta information for a given pipeline component name. Expects +the name of the component **instance** in the pipeline. The factory meta is an +instance of the [`FactoryMeta`](/api/language#factorymeta) dataclass and +contains the information about the component and its default provided by the +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorator. + +> #### Example +> +> ```python +> nlp.add_pipe("ner", name="entity_recognizer") +> factory_meta = nlp.get_pipe_meta("entity_recognizer") +> assert factory_meta.factory == "ner" +> print(factory_meta.default_config) +> ``` + +| Name | Description | +| ----------- | ------------------------------------ | +| `name` | The pipeline component name. ~~str~~ | +| **RETURNS** | The factory meta. ~~FactoryMeta~~ | + +## Language.analyze_pipes {id="analyze_pipes",tag="method",version="3"} + +Analyze the current pipeline components and show a summary of the attributes +they assign and require, and the scores they set. The data is based on the +information provided in the [`@Language.component`](/api/language#component) and +[`@Language.factory`](/api/language#factory) decorator. If requirements aren't +met, e.g. if a component specifies a required property that is not set by a +previous component, a warning is shown. + + + +The pipeline analysis is static and does **not actually run the components**. +This means that it relies on the information provided by the components +themselves. If a custom component declares that it assigns an attribute but it +doesn't, the pipeline analysis won't catch that. + + + +> #### Example +> +> ```python +> nlp = spacy.blank("en") +> nlp.add_pipe("tagger") +> nlp.add_pipe("entity_linker") +> analysis = nlp.analyze_pipes() +> ``` + + + +```json {title="Structured"} +{ + "summary": { + "tagger": { + "assigns": ["token.tag"], + "requires": [], + "scores": ["tag_acc", "pos_acc", "lemma_acc"], + "retokenizes": false + }, + "entity_linker": { + "assigns": ["token.ent_kb_id"], + "requires": ["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"], + "scores": [], + "retokenizes": false + } + }, + "problems": { + "tagger": [], + "entity_linker": [ + "doc.ents", + "doc.sents", + "token.ent_iob", + "token.ent_type" + ] + }, + "attrs": { + "token.ent_iob": { "assigns": [], "requires": ["entity_linker"] }, + "doc.ents": { "assigns": [], "requires": ["entity_linker"] }, + "token.ent_kb_id": { "assigns": ["entity_linker"], "requires": [] }, + "doc.sents": { "assigns": [], "requires": ["entity_linker"] }, + "token.tag": { "assigns": ["tagger"], "requires": [] }, + "token.ent_type": { "assigns": [], "requires": ["entity_linker"] } + } +} +``` + +``` +### Pretty +============================= Pipeline Overview ============================= + +# Component Assigns Requires Scores Retokenizes +- ------------- --------------- -------------- ----------- ----------- +0 tagger token.tag tag_acc False + +1 entity_linker token.ent_kb_id doc.ents nel_micro_f False + doc.sents nel_micro_r + token.ent_iob nel_micro_p + token.ent_type + + +================================ Problems (4) ================================ +⚠ 'entity_linker' requirements not met: doc.ents, doc.sents, +token.ent_iob, token.ent_type +``` + + + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `keys` | The values to display in the table. Corresponds to attributes of the [`FactoryMeta`](/api/language#factorymeta). Defaults to `["assigns", "requires", "scores", "retokenizes"]`. ~~List[str]~~ | +| `pretty` | Pretty-print the results as a table. Defaults to `False`. ~~bool~~ | +| **RETURNS** | Dictionary containing the pipe analysis, keyed by `"summary"` (component meta by pipe), `"problems"` (attribute names by pipe) and `"attrs"` (pipes that assign and require an attribute, keyed by attribute). ~~Optional[Dict[str, Any]]~~ | + +## Language.replace_listeners {id="replace_listeners",tag="method",version="3"} + +Find [listener layers](/usage/embeddings-transformers#embedding-layers) +(connecting to a shared token-to-vector embedding component) of a given pipeline +component model and replace them with a standalone copy of the token-to-vector +layer. The listener layer allows other components to connect to a shared +token-to-vector embedding component like [`Tok2Vec`](/api/tok2vec) or +[`Transformer`](/api/transformer). Replacing listeners can be useful when +training a pipeline with components sourced from an existing pipeline: if +multiple components (e.g. tagger, parser, NER) listen to the same +token-to-vector component, but some of them are frozen and not updated, their +performance may degrade significantly as the token-to-vector component is updated +with new data. To prevent this, listeners can be replaced with a standalone +token-to-vector layer that is owned by the component and doesn't change if the +component isn't updated. + +This method is typically not called directly and only executed under the hood +when loading a config with +[sourced components](/usage/training#config-components) that define +`replace_listeners`. + +> ```python +> ### Example +> nlp = spacy.load("en_core_web_sm") +> nlp.replace_listeners("tok2vec", "tagger", ["model.tok2vec"]) +> ``` +> +> ```ini +> ### config.cfg (excerpt) +> [training] +> frozen_components = ["tagger"] +> +> [components] +> +> [components.tagger] +> source = "en_core_web_sm" +> replace_listeners = ["model.tok2vec"] +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec_name` | Name of the token-to-vector component, typically `"tok2vec"` or `"transformer"`.~~str~~ | +| `pipe_name` | Name of pipeline component to replace listeners for. ~~str~~ | +| `listeners` | The paths to the listeners, relative to the component config, e.g. `["model.tok2vec"]`. Typically, implementations will only connect to one tok2vec component, `model.tok2vec`, but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a _complete_ list of the paths to all listener layers used by the model that should be replaced.~~Iterable[str]~~ | + +## Language.memory_zone {id="memory_zone",tag="contextmanager",version="3.8"} + +Begin a block where all resources allocated during the block will be freed at +the end of it. If a resources was created within the memory zone block, +accessing it outside the block is invalid. Behavior of this invalid access is +undefined. Memory zones should not be nested. The memory zone is helpful for +services that need to process large volumes of text with a defined memory budget. + +> ```python +> ### Example +> counts = Counter() +> with nlp.memory_zone(): +> for doc in nlp.pipe(texts): +> for token in doc: +> counts[token.text] += 1 +> ``` + +| Name | Description | +| --- | --- | +| `mem` | Optional `cymem.Pool` object to own allocations (created if not provided). This argument is not required for ordinary usage. Defaults to `None`. ~~Optional[cymem.Pool]~~ | +| **RETURNS** | The memory pool that owns the allocations. This object is not required for ordinary usage. ~~Iterator[cymem.Pool]~~ | + +## Language.meta {id="meta",tag="property"} + +Meta data for the `Language` class, including name, version, data sources, +license, author information and more. If a trained pipeline is loaded, this +contains meta data of the pipeline. The `Language.meta` is also what's +serialized as the `meta.json` when you save an `nlp` object to disk. See the +[meta data format](/api/data-formats#meta) for more details. + + + +As of v3.0, the meta only contains **meta information** about the pipeline and +isn't used to construct the language class and pipeline components. This +information is expressed in the [`config.cfg`](/api/data-formats#config). + + + +> #### Example +> +> ```python +> print(nlp.meta) +> ``` + +| Name | Description | +| ----------- | --------------------------------- | +| **RETURNS** | The meta data. ~~Dict[str, Any]~~ | + +## Language.config {id="config",tag="property",version="3"} + +Export a trainable [`config.cfg`](/api/data-formats#config) for the current +`nlp` object. Includes the current pipeline, all configs used to create the +currently active pipeline components, as well as the default training config +that can be used with [`spacy train`](/api/cli#train). `Language.config` returns +a [Thinc `Config` object](https://thinc.ai/docs/api-config#config), which is a +subclass of the built-in `dict`. It supports the additional methods `to_disk` +(serialize the config to a file) and `to_str` (output the config as a string). + +> #### Example +> +> ```python +> nlp.config.to_disk("./config.cfg") +> print(nlp.config.to_str()) +> ``` + +| Name | Description | +| ----------- | ---------------------- | +| **RETURNS** | The config. ~~Config~~ | + +## Language.to_disk {id="to_disk",tag="method",version="2"} + +Save the current state to a directory. Under the hood, this method delegates to +the `to_disk` methods of the individual pipeline components, if available. This +means that if a trained pipeline is loaded, all components and their weights +will be saved to disk. + +> #### Example +> +> ```python +> nlp.to_disk("/path/to/pipeline") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Language.from_disk {id="from_disk",tag="method",version="2"} + +Loads state from a directory, including all data that was saved with the +`Language` object. Modifies the object in place and returns it. + + + +Keep in mind that this method **only loads the serialized state** and doesn't +set up the `nlp` object. This means that it requires the correct language class +to be initialized and all pipeline components to be added to the pipeline. If +you want to load a serialized pipeline from a directory, you should use +[`spacy.load`](/api/top-level#spacy.load), which will set everything up for you. + + + +> #### Example +> +> ```python +> from spacy.language import Language +> nlp = Language().from_disk("/path/to/pipeline") +> +> # Using language-specific subclass +> from spacy.lang.en import English +> nlp = English().from_disk("/path/to/pipeline") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Language` object. ~~Language~~ | + +## Language.to_bytes {id="to_bytes",tag="method"} + +Serialize the current state to a binary string. + +> #### Example +> +> ```python +> nlp_bytes = nlp.to_bytes() +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------ | +| _keyword-only_ | | +| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~iterable~~ | +| **RETURNS** | The serialized form of the `Language` object. ~~bytes~~ | + +## Language.from_bytes {id="from_bytes",tag="method"} + +Load state from a binary string. Note that this method is commonly used via the +subclasses like `English` or `German` to make language-specific functionality +like the [lexical attribute getters](/usage/linguistic-features#language-data) +available to the loaded object. + +Note that if you want to serialize and reload a whole pipeline, using this alone +won't work, you also need to handle the config. See +["Serializing the pipeline"](https://spacy.io/usage/saving-loading#pipeline) for +details. + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp_bytes = nlp.to_bytes() +> nlp2 = English() +> nlp2.from_bytes(nlp_bytes) +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Language` object. ~~Language~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | A container for the lexical types. ~~Vocab~~ | +| `tokenizer` | The tokenizer. ~~Tokenizer~~ | +| `make_doc` | Callable that takes a string and returns a `Doc`. ~~Callable[[str], Doc]~~ | +| `pipeline` | List of `(name, component)` tuples describing the current processing pipeline, in order. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | +| `pipe_names` | List of pipeline component names, in order. ~~List[str]~~ | +| `pipe_labels` | List of labels set by the pipeline components, if available, keyed by component name. ~~Dict[str, List[str]]~~ | +| `pipe_factories` | Dictionary of pipeline component names, mapped to their factory names. ~~Dict[str, str]~~ | +| `factories` | All available factory functions, keyed by name. ~~Dict[str, Callable[[...], Callable[[Doc], Doc]]]~~ | +| `factory_names` 3 | List of all available factory names. ~~List[str]~~ | +| `components` 3 | List of all available `(name, component)` tuples, including components that are currently disabled. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | +| `component_names` 3 | List of all available component names, including components that are currently disabled. ~~List[str]~~ | +| `disabled` 3 | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ | +| `path` | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ | + +## Class attributes {id="class-attributes"} + +| Name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Defaults` | Settings, data and factory methods for creating the `nlp` object and processing pipeline. ~~Defaults~~ | +| `lang` | Two-letter [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) or three-letter [ISO 639-3](https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes) language codes, such as 'en' and 'eng' for English. ~~str~~ | +| `default_config` | Base [config](/usage/training#config) to use for [Language.config](/api/language#config). Defaults to [`default_config.cfg`](%%GITHUB_SPACY/spacy/default_config.cfg). ~~Config~~ | + +## Defaults {id="defaults"} + +The following attributes can be set on the `Language.Defaults` class to +customize the default language data: + +> #### Example +> +> ```python +> from spacy.language import language +> from spacy.lang.tokenizer_exceptions import URL_MATCH +> from thinc.api import Config +> +> DEFAULT_CONFIFG = """ +> [nlp.tokenizer] +> @tokenizers = "MyCustomTokenizer.v1" +> """ +> +> class Defaults(Language.Defaults): +> stop_words = set() +> tokenizer_exceptions = {} +> prefixes = tuple() +> suffixes = tuple() +> infixes = tuple() +> token_match = None +> url_match = URL_MATCH +> lex_attr_getters = {} +> syntax_iterators = {} +> writing_system = {"direction": "ltr", "has_case": True, "has_letters": True} +> config = Config().from_str(DEFAULT_CONFIG) +> ``` + +| Name | Description | +| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `stop_words` | List of stop words, used for `Token.is_stop`.
**Example:** [`stop_words.py`](%%GITHUB_SPACY/spacy/lang/en/stop_words.py) ~~Set[str]~~ | +| `tokenizer_exceptions` | Tokenizer exception rules, string mapped to list of token attributes.
**Example:** [`de/tokenizer_exceptions.py`](%%GITHUB_SPACY/spacy/lang/de/tokenizer_exceptions.py) ~~Dict[str, List[dict]]~~ | +| `prefixes`, `suffixes`, `infixes` | Prefix, suffix and infix rules for the default tokenizer.
**Example:** [`puncutation.py`](%%GITHUB_SPACY/spacy/lang/punctuation.py) ~~Optional[Sequence[Union[str, Pattern]]]~~ | +| `token_match` | Optional regex for matching strings that should never be split, overriding the infix rules.
**Example:** [`fr/tokenizer_exceptions.py`](%%GITHUB_SPACY/spacy/lang/fr/tokenizer_exceptions.py) ~~Optional[Callable]~~ | +| `url_match` | Regular expression for matching URLs. Prefixes and suffixes are removed before applying the match.
**Example:** [`tokenizer_exceptions.py`](%%GITHUB_SPACY/spacy/lang/tokenizer_exceptions.py) ~~Optional[Callable]~~ | +| `lex_attr_getters` | Custom functions for setting lexical attributes on tokens, e.g. `like_num`.
**Example:** [`lex_attrs.py`](%%GITHUB_SPACY/spacy/lang/en/lex_attrs.py) ~~Dict[int, Callable[[str], Any]]~~ | +| `syntax_iterators` | Functions that compute views of a `Doc` object based on its syntax. At the moment, only used for [noun chunks](/usage/linguistic-features#noun-chunks).
**Example:** [`syntax_iterators.py`](%%GITHUB_SPACY/spacy/lang/en/syntax_iterators.py). ~~Dict[str, Callable[[Union[Doc, Span]], Iterator[Span]]]~~ | +| `writing_system` | Information about the language's writing system, available via `Vocab.writing_system`. Defaults to: `{"direction": "ltr", "has_case": True, "has_letters": True}.`.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Dict[str, Any]~~ | +| `config` | Default [config](/usage/training#config) added to `nlp.config`. This can include references to custom tokenizers or lemmatizers.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Config~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = nlp.to_bytes(exclude=["tokenizer", "vocab"]) +> nlp.from_disk("/pipeline", exclude=["ner"]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------ | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `tokenizer` | Tokenization rules and exceptions. | +| `meta` | The meta data, available as [`Language.meta`](/api/language#meta). | +| ... | String names of pipeline components, e.g. `"ner"`. | + +## FactoryMeta {id="factorymeta",version="3",tag="dataclass"} + +The `FactoryMeta` contains the information about the component and its default +provided by the [`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorator. It's created whenever a +component is defined and stored on the `Language` class for each component +instance and factory instance. + +| Name | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `factory` | The name of the registered component factory. ~~str~~ | +| `default_config` | The default config, describing the default values of the factory arguments. ~~Dict[str, Any]~~ | +| `assigns` | `Doc` or `Token` attributes assigned by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `requires` | `Doc` or `Token` attributes required by this component, e.g. `["token.ent_id"]`. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | +| `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | +| `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ | +| `scores` | All scores set by the components if it's trainable, e.g. `["ents_f", "ents_r", "ents_p"]`. Based on the `default_score_weights` and used for [pipe analysis](/usage/processing-pipelines#analysis). ~~Iterable[str]~~ | diff --git a/website/docs/api/large-language-models.mdx b/website/docs/api/large-language-models.mdx new file mode 100644 index 0000000..6e2436c --- /dev/null +++ b/website/docs/api/large-language-models.mdx @@ -0,0 +1,1689 @@ +--- +title: Large Language Models +teaser: Integrating LLMs into structured NLP pipelines +menu: + - ['Config and implementation', 'config'] + - ['Tasks', 'tasks'] + - ['Models', 'models'] + - ['Cache', 'cache'] + - ['Various Functions', 'various-functions'] +--- + +[The `spacy-llm` package](https://github.com/explosion/spacy-llm) integrates +Large Language Models (LLMs) into spaCy, featuring a modular system for **fast +prototyping** and **prompting**, and turning unstructured responses into +**robust outputs** for various NLP tasks, **no training data** required. + +## Config and implementation {id="config"} + +An LLM component is implemented through the `LLMWrapper` class. It is accessible +through a generic `llm` +[component factory](https://spacy.io/usage/processing-pipelines#custom-components-factories) +as well as through task-specific component factories: `llm_ner`, `llm_spancat`, +`llm_rel`, `llm_textcat`, `llm_sentiment`, `llm_summarization`, +`llm_entity_linker`, `llm_raw` and `llm_translation`. For these factories, the +GPT-3-5 model from OpenAI is used by default, but this can be customized. + +> #### Example +> +> ```python +> # Construction via add_pipe with the default GPT 3.5 model and an explicitly defined task +> config = {"task": {"@llm_tasks": "spacy.NER.v3", "labels": ["PERSON", "ORGANISATION", "LOCATION"]}} +> llm = nlp.add_pipe("llm", config=config) +> +> # Construction via add_pipe with a task-specific factory and default GPT3.5 model +> llm = nlp.add_pipe("llm_ner") +> +> # Construction via add_pipe with a task-specific factory and custom model +> llm = nlp.add_pipe("llm_ner", config={"model": {"@llm_models": "spacy.Dolly.v1", "name": "dolly-v2-12b"}}) +> +> # Construction from class +> from spacy_llm.pipeline import LLMWrapper +> llm = LLMWrapper(vocab=nlp.vocab, task=task, model=model, cache=cache, save_io=True) +> ``` + +### LLMWrapper.\_\_init\_\_ {id="init",tag="method"} + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------- | +| `name` | String name of the component instance. `llm` by default. ~~str~~ | +| _keyword-only_ | | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `task` | An [LLM Task](#tasks) can generate prompts and parse LLM responses. ~~LLMTask~~ | +| `model` | The [LLM Model](#models) queries a specific LLM API.. ~~Callable[[Iterable[Any]], Iterable[Any]]~~ | +| `cache` | [Cache](#cache) to use for caching prompts and responses per doc. ~~Cache~~ | +| `save_io` | Whether to save LLM I/O (prompts and responses) in the `Doc._.llm_io` custom attribute. ~~bool~~ | + +### LLMWrapper.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. + +> #### Example +> +> ```python +> doc = nlp("Ingrid visited Paris.") +> llm_ner = nlp.add_pipe("llm_ner") +> # This usually happens under the hood +> processed = llm_ner(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +### LLMWrapper.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. + +> #### Example +> +> ```python +> llm_ner = nlp.add_pipe("llm_ner") +> for doc in llm_ner.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `docs` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +### LLMWrapper.add_label {id="add_label",tag="method"} + +Add a new label to the pipe's task. Alternatively, provide the labels upon the +[task](#task) definition, or through the `[initialize]` block of the +[config](#config). + +> #### Example +> +> ```python +> llm_ner = nlp.add_pipe("llm_ner") +> llm_ner.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +### LLMWrapper.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> llm_ner = nlp.add_pipe("llm_ner") +> llm_ner.to_disk("/path/to/llm_ner") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +### LLMWrapper.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> llm_ner = nlp.add_pipe("llm_ner") +> llm_ner.from_disk("/path/to/llm_ner") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `LLMWrapper` object. ~~LLMWrapper~~ | + +### LLMWrapper.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> llm_ner = nlp.add_pipe("llm_ner") +> ner_bytes = llm_ner.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `LLMWrapper` object. ~~bytes~~ | + +### LLMWrapper.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ner_bytes = llm_ner.to_bytes() +> llm_ner = nlp.add_pipe("llm_ner") +> llm_ner.from_bytes(ner_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `LLMWrapper` object. ~~LLMWrapper~~ | + +### LLMWrapper.labels {id="labels",tag="property"} + +The labels currently added to the component. Empty tuple if the LLM's task does +not require labels. + +> #### Example +> +> ```python +> llm_ner.add_label("MY_LABEL") +> assert "MY_LABEL" in llm_ner.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## Tasks {id="tasks"} + +In `spacy-llm`, a _task_ defines an NLP problem or question and its solution +using an LLM. It does so by implementing the following responsibilities: + +1. Loading a prompt template and injecting documents' data into the prompt. + Optionally, include fewshot examples in the prompt. +2. Splitting the prompt into several pieces following a map-reduce paradigm, + _if_ the prompt is too long to fit into the model's context and the task + supports sharding prompts. +3. Parsing the LLM's responses back into structured information and validating + the parsed output. + +Two different task interfaces are supported: `ShardingLLMTask` and +`NonShardingLLMTask`. Only the former supports the sharding of documents, i. e. +splitting up prompts if they are too long. + +All tasks are registered in the `llm_tasks` registry. + +### On Sharding {id="task-sharding"} + +"Sharding" describes, generally speaking, the process of distributing parts of a +dataset across multiple storage units for easier processing and lookups. In +`spacy-llm` we use this term (synonymously: "mapping") to describe the splitting +up of prompts if they are too long for a model to handle, and "fusing" +(synonymously: "reducing") to describe how the model responses for several +shards are merged back together into a single document. + +Prompts are broken up in a manner that _always_ keeps the prompt in the template +intact, meaning that the instructions to the LLM will always stay complete. The +document content however will be split, if the length of the fully rendered +prompt exceeds a model context length. + +A toy example: let's assume a model has a context window of 25 tokens and the +prompt template for our fictional, sharding-supporting task looks like this: + +``` +Estimate the sentiment of this text: +"{text}" +Estimated sentiment: +``` + +Depending on how tokens are counted exactly (this is a config setting), we might +come up with `n = 12` tokens for the number of tokens in the prompt +instructions. Furthermore let's assume that our `text` is "This has been +amazing - I can't remember the last time I left the cinema so impressed." - +which has roughly 19 tokens. + +Considering we only have 13 tokens to add to our prompt before we hit the +context limit, we'll have to split our prompt into two parts. Thus `spacy-llm`, +assuming the task used supports sharding, will split the prompt into two (the +default splitting strategy splits by tokens, but alternative splitting +strategies splitting e. g. by sentences can be configured): + +_(Prompt 1/2)_ + +``` +Estimate the sentiment of this text: +"This has been amazing - I can't remember " +Estimated sentiment: +``` + +_(Prompt 2/2)_ + +``` +Estimate the sentiment of this text: +"the last time I left the cinema so impressed." +Estimated sentiment: +``` + +The reduction step is task-specific - a sentiment estimation task might e. g. do +a weighted average of the sentiment scores. Note that prompt sharding introduces +potential inaccuracies, as the LLM won't have access to the entire document at +once. Depending on your use case this might or might not be problematic. + +### `NonShardingLLMTask` {id="task-nonsharding"} + +#### task.generate_prompts {id="task-nonsharding-generate-prompts"} + +Takes a collection of documents, and returns a collection of "prompts", which +can be of type `Any`. Often, prompts are of type `str` - but this is not +enforced to allow for maximum flexibility in the framework. + +| Argument | Description | +| ----------- | ---------------------------------------- | +| `docs` | The input documents. ~~Iterable[Doc]~~ | +| **RETURNS** | The generated prompts. ~~Iterable[Any]~~ | + +#### task.parse_responses {id="task-non-sharding-parse-responses"} + +Takes a collection of LLM responses and the original documents, parses the +responses into structured information, and sets the annotations on the +documents. The `parse_responses` function is free to set the annotations in any +way, including `Doc` fields like `ents`, `spans` or `cats`, or using custom +defined fields. + +The `responses` are of type `Iterable[Any]`, though they will often be `str` +objects. This depends on the return type of the [model](#models). + +| Argument | Description | +| ----------- | ------------------------------------------------------ | +| `docs` | The input documents. ~~Iterable[Doc]~~ | +| `responses` | The responses received from the LLM. ~~Iterable[Any]~~ | +| **RETURNS** | The annotated documents. ~~Iterable[Doc]~~ | + +### `ShardingLLMTask` {id="task-sharding"} + +#### task.generate_prompts {id="task-sharding-generate-prompts"} + +Takes a collection of documents, breaks them up into shards if necessary to fit +all content into the model's context, and returns a collection of collections of +"prompts" (i. e. each doc can have multiple shards, each of which have exactly +one prompt), which can be of type `Any`. Often, prompts are of type `str` - but +this is not enforced to allow for maximum flexibility in the framework. + +| Argument | Description | +| ----------- | -------------------------------------------------- | +| `docs` | The input documents. ~~Iterable[Doc]~~ | +| **RETURNS** | The generated prompts. ~~Iterable[Iterable[Any]]~~ | + +#### task.parse_responses {id="task-sharding-parse-responses"} + +Receives a collection of collections of LLM responses (i. e. each doc can have +multiple shards, each of which have exactly one prompt / prompt response) and +the original shards, parses the responses into structured information, sets the +annotations on the shards, and merges back doc shards into single docs. The +`parse_responses` function is free to set the annotations in any way, including +`Doc` fields like `ents`, `spans` or `cats`, or using custom defined fields. + +The `responses` are of type `Iterable[Iterable[Any]]`, though they will often be +`str` objects. This depends on the return type of the [model](#models). + +| Argument | Description | +| ----------- | ---------------------------------------------------------------- | +| `shards` | The input document shards. ~~Iterable[Iterable[Doc]]~~ | +| `responses` | The responses received from the LLM. ~~Iterable[Iterable[Any]]~~ | +| **RETURNS** | The annotated documents. ~~Iterable[Doc]~~ | + +### Translation {id="translation"} + +The translation task translates texts from a defined or inferred source to a +defined target language. + +#### spacy.Translation.v1 {id="translation-v1"} + +`spacy.Translation.v1` supports both zero-shot and few-shot prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.Translation.v1" +> examples = null +> target_lang = "Spanish" +> ``` + +| Argument | Description | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [translation.v1.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/translation.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[TranslationTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `TranslationExample`. ~~Optional[Type[FewshotExample]]~~ | +| `source_lang` | Language to translate from. Doesn't have to be set. ~~Optional[str]~~ | +| `target_lang` | Language to translate to. No default value, has to be set. ~~str~~ | +| `field` | Name of extension attribute to store translation in (i. e. the translation will be available in `doc._.{field}`). Defaults to `translation`. ~~str~~ | + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +- text: 'Top of the morning to you!' + translation: '¡Muy buenos días!' +- text: 'The weather is great today.' + translation: 'El clima está fantástico hoy.' +- text: 'Do you know what will happen tomorrow?' + translation: '¿Sabes qué pasará mañana?' +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.Translation.v1" +target_lang = "Spanish" +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "translation_examples.yml" +``` + +### Raw prompting {id="raw"} + +Different to all other tasks `spacy.Raw.vX` doesn't provide a specific prompt, +wrapping doc data, to the model. Instead it instructs the model to reply to the +doc content. This is handy for use cases like question answering (where each doc +contains one question) or if you want to include customized prompts for each +doc. + +#### spacy.Raw.v1 {id="raw-v1"} + +Note that since this task may request arbitrary information, it doesn't do any +parsing per se - the model response is stored in a custom `Doc` attribute (i. e. +can be accessed via `doc._.{field}`). + +It supports both zero-shot and few-shot prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.Raw.v1" +> examples = null +> ``` + +| Argument | Description | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [raw.v1.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/raw.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[RawTask]]~~ | +| `prompt_example_type` | Type to use for fewshot examples. Defaults to `RawExample`. ~~Optional[Type[FewshotExample]]~~ | +| `field` | Name of extension attribute to store model reply in (i. e. the reply will be available in `doc._.{field}`). Defaults to `reply`. ~~str~~ | + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +# Each example can follow an arbitrary pattern. It might help the prompt performance though if the examples resemble +# the actual docs' content. +- text: "3 + 5 = x. What's x?" + reply: '8' + +- text: 'Write me a limerick.' + reply: + "There was an Old Man with a beard, Who said, 'It is just as I feared! Two + Owls and a Hen, Four Larks and a Wren, Have all built their nests in my + beard!" + +- text: "Analyse the sentiment of the text 'This is great'." + reply: "'This is great' expresses a very positive sentiment." +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.Raw.v1" +field = "llm_reply" +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "raw_examples.yml" +``` + +### Summarization {id="summarization"} + +A summarization task takes a document as input and generates a summary that is +stored in an extension attribute. + +#### spacy.Summarization.v1 {id="summarization-v1"} + +The `spacy.Summarization.v1` task supports both zero-shot and few-shot +prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.Summarization.v1" +> examples = null +> max_n_words = null +> ``` + +| Argument | Description | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [summarization.v1.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/summarization.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SummarizationTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `SummarizationExample`. ~~Optional[Type[FewshotExample]]~~ | +| `max_n_words` | Maximum number of words to be used in summary. Note that this should not expected to work exactly. Defaults to `None`. ~~Optional[int]~~ | +| `field` | Name of extension attribute to store summary in (i. e. the summary will be available in `doc._.{field}`). Defaults to `summary`. ~~str~~ | + +The summarization task prompts the model for a concise summary of the provided +text. It optionally allows to limit the response to a certain number of tokens - +note that this requirement will be included in the prompt, but the task doesn't +perform a hard cut-off. It's hence possible that your summary exceeds +`max_n_words`. + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +- text: > + The United Nations, referred to informally as the UN, is an + intergovernmental organization whose stated purposes are to maintain + international peace and security, develop friendly relations among nations, + achieve international cooperation, and serve as a centre for harmonizing the + actions of nations. It is the world's largest international organization. + The UN is headquartered on international territory in New York City, and the + organization has other offices in Geneva, Nairobi, Vienna, and The Hague, + where the International Court of Justice is headquartered.\n\n The UN was + established after World War II with the aim of preventing future world wars, + and succeeded the League of Nations, which was characterized as + ineffective. + summary: + 'The UN is an international organization that promotes global peace, + cooperation, and harmony. Established after WWII, its purpose is to prevent + future world wars.' +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.Summarization.v1" +max_n_words = 20 +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "summarization_examples.yml" +``` + +### EL (Entity Linking) {id="nel"} + +The EL links recognized entities (see [NER](#ner)) to those in a knowledge base +(KB). The EL task prompts the LLM to select the most likely candidate from the +KB, whose structure can be arbitrary. + +Note that the documents processed by the entity linking task are expected to +have recognized entities in their `.ents` attribute. This can be achieved by +either running the [NER task](#ner), using a trained spaCy NER model or setting +the entities manually prior to running the EL task. + +In order to be able to pull data from the KB, an object implementing the +`CandidateSelector` protocol has to be provided. This requires two functions: +(1) `__call__()` to fetch candidate entities for entity mentions in the text +(assumed to be available in `Doc.ents`) and (2) `get_entity_description()` to +fetch descriptions for any given entity ID. Descriptions can be empty, but +ideally provide more context for entities stored in the KB. + +`spacy-llm` provides a `CandidateSelector` implementation +(`spacy.CandidateSelector.v1`) that leverages a spaCy knowledge base - as used +in an `entity_linking` component - to select candidates. This knowledge base can +be loaded from an existing spaCy pipeline (note that the pipeline's EL component +doesn't have to be trained) or from a separate .yaml file. + +#### spacy.EntityLinker.v1 {id="el-v1"} + +Supports zero- and few-shot prompting. Relies on a configurable component +suggesting viable entities before letting the LLM pick the most likely +candidate. + +> #### Example config for spacy.EntityLinker.v1 +> +> ```ini +> [paths] +> el_nlp = null +> +> ... +> +> [components.llm.task] +> @llm_tasks = "spacy.EntityLinker.v1" +> +> [initialize] +> [initialize.components] +> [initialize.components.llm] +> [initialize.components.llm.candidate_selector] +> @llm_misc = "spacy.CandidateSelector.v1" +> +> # Load a KB from a KB file. For loading KBs from spaCy pipelines see spacy.KBObjectLoader.v1. +> [initialize.components.llm.candidate_selector.kb_loader] +> @llm_misc = "spacy.KBFileLoader.v1" +> # Path to knowledge base .yaml file. +> path = ${paths.el_kb} +> ``` + +| Argument | Description | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [entity_linker.v1.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/entity_linker.v1.jinja). ~~str~~ | +| `parse_responses` | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[EntityLinkerTask]]~~ | +| `prompt_example_type` | Type to use for fewshot examples. Defaults to `ELExample`. ~~Optional[Type[FewshotExample]]~~ | +| `examples` | Optional callable that reads a file containing task examples for few-shot learning. If `None` is passed, zero-shot learning will be used. Defaults to `None`. ~~ExamplesConfigType~~ | +| `scorer` | Scorer function. Defaults to the metric used by spaCy to evaluate entity linking performance. ~~Optional[Scorer]~~ | + +##### spacy.CandidateSelector.v1 {id="candidate-selector-v1"} + +`spacy.CandidateSelector.v1` is an implementation of the `CandidateSelector` +protocol required by [`spacy.EntityLinker.v1`](#el-v1). The built-in candidate +selector method allows loading existing knowledge bases in several ways, e. g. +loading from a spaCy pipeline with a (not necessarily trained) entity linking +component, and loading from a file describing the knowlege base as a .yaml file. +Either way the loaded data will be converted to a spaCy `InMemoryLookupKB` +instance. The KB's selection capabilities are used to select the most likely +entity candidates for the specified mentions. + +> #### Example config for spacy.CandidateSelector.v1 +> +> ```ini +> [initialize] +> [initialize.components] +> [initialize.components.llm] +> [initialize.components.llm.candidate_selector] +> @llm_misc = "spacy.CandidateSelector.v1" +> +> # Load a KB from a KB file. For loading KBs from spaCy pipelines see spacy.KBObjectLoader.v1. +> [initialize.components.llm.candidate_selector.kb_loader] +> @llm_misc = "spacy.KBFileLoader.v1" +> # Path to knowledge base .yaml file. +> path = ${paths.el_kb} +> ``` + +| Argument | Description | +| ----------- | ----------------------------------------------------------------- | +| `kb_loader` | KB loader object. ~~InMemoryLookupKBLoader~~ | +| `top_n` | Top-n candidates to include in the prompt. Defaults to 5. ~~int~~ | + +##### spacy.KBObjectLoader.v1 {id="kb-object-loader-v1"} + +Adheres to the `InMemoryLookupKBLoader` interface required by +[`spacy.CandidateSelector.v1`](#candidate-selector-v1). Loads a knowledge base +from an existing spaCy pipeline. + +> #### Example config for spacy.KBObjectLoader.v1 +> +> ```ini +> [initialize.components.llm.candidate_selector.kb_loader] +> @llm_misc = "spacy.KBObjectLoader.v1" +> # Path to knowledge base directory in serialized spaCy pipeline. +> path = ${paths.el_kb} +> # Path to spaCy pipeline. If this is not specified, spacy-llm tries to determine this automatically (but may fail). +> nlp_path = ${paths.el_nlp} +> # Path to file with descriptions for entity. +> desc_path = ${paths.el_desc} +> ``` + +| Argument | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | Path to KB file. ~~Union[str, Path]~~ | +| `nlp_path` | Path to serialized NLP pipeline. If None, path will be guessed. ~~Optional[Union[Path, str]]~~ | +| `desc_path` | Path to file with descriptions for entities. ~~int~~ | +| `ent_desc_reader` | Entity description reader. Defaults to an internal method expecting a CSV file without header row, with ";" as delimiters, and with two columns - one for the entitys' IDs, one for their descriptions. ~~Optional[EntDescReader]~~ | + +##### spacy.KBFileLoader.v1 {id="kb-file-loader-v1"} + +Adheres to the `InMemoryLookupKBLoader` interface required by +[`spacy.CandidateSelector.v1`](#candidate-selector-v1). Loads a knowledge base +from a knowledge base file. The KB .yaml file has to stick to the following +format: + +```yaml +entities: + # The key should be whatever ID identifies this entity uniquely in your knowledge base. + ID1: + name: "..." + desc: "..." + ID2: + ... +# Data on aliases in your knowledge base - e. g. "Apple" for the entity "Apple Inc.". +aliases: + - alias: "..." + # List of all entities that this alias refers to. + entities: ["ID1", "ID2", ...] + # Optional: prior probabilities that this alias refers to the n-th entity in the "entities" attribute. + probabilities: [0.5, 0.2, ...] + - alias: "..." + entities: [...] + probabilities: [...] + ... +``` + +See +[here](https://github.com/explosion/spacy-llm/blob/main/usage_examples/el_openai/el_kb_data.yml) +for a toy example of how such a KB file might look like. + +> #### Example config for spacy.KBFileLoader.v1 +> +> ```ini +> [initialize.components.llm.candidate_selector.kb_loader] +> @llm_misc = "spacy.KBFileLoader.v1" +> # Path to knowledge base file. +> path = ${paths.el_kb} +> ``` + +| Argument | Description | +| -------- | ------------------------------------- | +| `path` | Path to KB file. ~~Union[str, Path]~~ | + +### NER {id="ner"} + +The NER task identifies non-overlapping entities in text. + +#### spacy.NER.v3 {id="ner-v3"} + +Version 3 is fundamentally different to v1 and v2, as it implements +Chain-of-Thought prompting, based on the +[PromptNER paper](https://arxiv.org/pdf/2305.15444.pdf) by Ashok and Lipton +(2023). On an internal use-case, we have found this implementation to obtain +significant better accuracy - with an increase of F-score of up to 15 percentage +points. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.NER.v3" +> labels = ["PERSON", "ORGANISATION", "LOCATION"] +> ``` + +When no examples are [specified](/usage/large-language-models#few-shot-prompts), +the v3 implementation will use a dummy example in the prompt. Technically this +means that the task will always perform few-shot prompting under the hood. + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `template` | Custom prompt template to send to LLM model. Defaults to [ner.v3.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/ner.v3.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[NERTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `NERExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` | Optional dict mapping a label to a description of that label. These descriptions are added to the prompt to help instruct the LLM on what to extract. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `description` (NEW) | A description of what to recognize or not recognize as entities. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. Defaults to `None`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | + +Note that the `single_match` parameter, used in v1 and v2, is not supported +anymore, as the CoT parsing algorithm takes care of this automatically. + +New to v3 is the fact that you can provide an explicit description of what +entities should look like. You can use this feature in addition to +`label_definitions`. + +```ini +[components.llm.task] +@llm_tasks = "spacy.NER.v3" +labels = ["DISH", "INGREDIENT", "EQUIPMENT"] +description = Entities are the names food dishes, + ingredients, and any kind of cooking equipment. + Adjectives, verbs, adverbs are not entities. + Pronouns are not entities. + +[components.llm.task.label_definitions] +DISH = "Known food dishes, e.g. Lobster Ravioli, garlic bread" +INGREDIENT = "Individual parts of a food dish, including herbs and spices." +EQUIPMENT = "Any kind of cooking equipment. e.g. oven, cooking pot, grill" +``` + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +While not required, this task works best when both positive and negative +examples are provided. The format is different than the files required for v1 +and v2, as additional fields such as `is_entity` and `reason` should now be +provided. + +```json +[ + { + "text": "You can't get a great chocolate flavor with carob.", + "spans": [ + { + "text": "chocolate", + "is_entity": false, + "label": "==NONE==", + "reason": "is a flavor in this context, not an ingredient" + }, + { + "text": "carob", + "is_entity": true, + "label": "INGREDIENT", + "reason": "is an ingredient to add chocolate flavor" + } + ] + }, + ... +] +``` + +```ini +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "${paths.examples}" +``` + +For a fully working example, see this +[usage example](https://github.com/explosion/spacy-llm/tree/main/usage_examples/ner_v3_openai). + +#### spacy.NER.v2 {id="ner-v2"} + +This version supports explicitly defining the provided labels with custom +descriptions, and further supports zero-shot and few-shot prompting just like +v1. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.NER.v2" +> labels = ["PERSON", "ORGANISATION", "LOCATION"] +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `template` (NEW) | Custom prompt template to send to LLM model. Defaults to [ner.v2.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/ner.v2.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[NERTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `NERExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` (NEW) | Optional dict mapping a label to a description of that label. These descriptions are added to the prompt to help instruct the LLM on what to extract. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. Defaults to `None`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | +| `single_match` | Whether to match an entity in the LLM's response only once (the first hit) or multiple times. Defaults to `False`. ~~bool~~ | + +The parameters `alignment_mode`, `case_sensitive_matching` and `single_match` +are identical to the [v1](#ner-v1) implementation. The format of few-shot +examples are also the same. + +> Label descriptions can also be used with explicit examples to give as much +> info to the LLM model as possible. + +New to v2 is the fact that you can write definitions for each label and provide +them via the `label_definitions` argument. This lets you tell the LLM exactly +what you're looking for rather than relying on the LLM to interpret its task +given just the label name. Label descriptions are freeform so you can write +whatever you want here, but a brief description along with some examples and +counter examples seems to work quite well. + +```ini +[components.llm.task] +@llm_tasks = "spacy.NER.v2" +labels = PERSON,SPORTS_TEAM + +[components.llm.task.label_definitions] +PERSON = "Extract any named individual in the text." +SPORTS_TEAM = "Extract the names of any professional sports team. e.g. Golden State Warriors, LA Lakers, Man City, Real Madrid" +``` + +For a fully working example, see this +[usage example](https://github.com/explosion/spacy-llm/tree/main/usage_examples/ner_dolly). + +#### spacy.NER.v1 {id="ner-v1"} + +The original version of the built-in NER task supports both zero-shot and +few-shot prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.NER.v1" +> labels = PERSON,ORGANISATION,LOCATION +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[NERTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `NERExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | Comma-separated list of labels. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | +| `single_match` | Whether to match an entity in the LLM's response only once (the first hit) or multiple times. Defaults to `False`. ~~bool~~ | + +The NER task implementation doesn't currently ask the LLM for specific offsets, +but simply expects a list of strings that represent the enties in the document. +This means that a form of string matching is required. This can be configured by +the following parameters: + +- The `single_match` parameter is typically set to `False` to allow for multiple + matches. For instance, the response from the LLM might only mention the entity + "Paris" once, but you'd still want to mark it every time it occurs in the + document. +- The case-sensitive matching is typically set to `False` to be robust against + case variances in the LLM's output. +- The `alignment_mode` argument is used to match entities as returned by the LLM + to the tokens from the original `Doc` - specifically it's used as argument in + the call to [`doc.char_span()`](/api/doc#char_span). The `"strict"` mode will + only keep spans that strictly adhere to the given token boundaries. + `"contract"` will only keep those tokens that are fully within the given + range, e.g. reducing `"New Y"` to `"New"`. Finally, `"expand"` will expand the + span to the next token boundaries, e.g. expanding `"New Y"` out to + `"New York"`. + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +- text: Jack and Jill went up the hill. + entities: + PERSON: + - Jack + - Jill + LOCATION: + - hill +- text: Jack fell down and broke his crown. + entities: + PERSON: + - Jack +``` + +```ini +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "ner_examples.yml" +``` + +### SpanCat {id="spancat"} + +The SpanCat task identifies potentially overlapping entities in text. + +#### spacy.SpanCat.v3 {id="spancat-v3"} + +The built-in SpanCat v3 task is a simple adaptation of the NER v3 task to +support overlapping entities and store its annotations in `doc.spans`. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.SpanCat.v3" +> labels = ["PERSON", "ORGANISATION", "LOCATION"] +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `template` | Custom prompt template to send to LLM model. Defaults to [`spancat.v3.jinja`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/spancat.v3.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `SpanCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` | Optional dict mapping a label to a description of that label. These descriptions are added to the prompt to help instruct the LLM on what to extract. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `description` (NEW) | A description of what to recognize or not recognize as entities. ~~str~~ | +| `spans_key` | Key of the `Doc.spans` dict to save the spans under. Defaults to `"sc"`. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | + +Note that the `single_match` parameter, used in v1 and v2, is not supported +anymore, as the CoT parsing algorithm takes care of this automatically. + +#### spacy.SpanCat.v2 {id="spancat-v2"} + +The built-in SpanCat v2 task is a simple adaptation of the NER v2 task to +support overlapping entities and store its annotations in `doc.spans`. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.SpanCat.v2" +> labels = ["PERSON", "ORGANISATION", "LOCATION"] +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `template` (NEW) | Custom prompt template to send to LLM model. Defaults to [`spancat.v2.jinja`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/spancat.v2.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `SpanCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` (NEW) | Optional dict mapping a label to a description of that label. These descriptions are added to the prompt to help instruct the LLM on what to extract. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `spans_key` | Key of the `Doc.spans` dict to save the spans under. Defaults to `"sc"`. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | +| `single_match` | Whether to match an entity in the LLM's response only once (the first hit) or multiple times. Defaults to `False`. ~~bool~~ | + +Except for the `spans_key` parameter, the SpanCat v2 task reuses the +configuration from the NER v2 task. Refer to [its documentation](#ner-v2) for +more insight. + +#### spacy.SpanCat.v1 {id="spancat-v1"} + +The original version of the built-in SpanCat task is a simple adaptation of the +v1 NER task to support overlapping entities and store its annotations in +`doc.spans`. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.SpanCat.v1" +> labels = PERSON,ORGANISATION,LOCATION +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `SpanCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | Comma-separated list of labels. ~~str~~ | +| `spans_key` | Key of the `Doc.spans` dict to save the spans under. Defaults to `"sc"`. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, defaults to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `alignment_mode` | Alignment mode in case the LLM returns entities that do not align with token boundaries. Options are `"strict"`, `"contract"` or `"expand"`. Defaults to `"contract"`. ~~str~~ | +| `case_sensitive_matching` | Whether to search without case sensitivity. Defaults to `False`. ~~bool~~ | +| `single_match` | Whether to match an entity in the LLM's response only once (the first hit) or multiple times. Defaults to `False`. ~~bool~~ | + +Except for the `spans_key` parameter, the SpanCat v1 task reuses the +configuration from the NER v1 task. Refer to [its documentation](#ner-v1) for +more insight. + +### TextCat {id="textcat"} + +The TextCat task labels documents with relevant categories. + +#### spacy.TextCat.v3 {id="textcat-v3"} + +On top of the functionality from v2, version 3 of the built-in TextCat tasks +allows setting definitions of labels. Those definitions are included in the +prompt. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.TextCat.v3" +> labels = ["COMPLIMENT", "INSULT"] +> +> [components.llm.task.label_definitions] +> "COMPLIMENT" = "a polite expression of praise or admiration.", +> "INSULT" = "a disrespectful or scornfully abusive remark or act." +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [`textcat.v3.jinja`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/textcat.v3.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `TextCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` (NEW) | Dictionary of label definitions. Included in the prompt, if set. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, falls back to `spacy.LowercaseNormalizer.v1`. Defaults to `None`. ~~Optional[Callable[[str], str]]~~ | +| `exclusive_classes` | If set to `True`, only one label per document should be valid. If set to `False`, one document can have multiple labels. Defaults to `False`. ~~bool~~ | +| `allow_none` | When set to `True`, allows the LLM to not return any of the given label. The resulting dict in `doc.cats` will have `0.0` scores for all labels. Defaults to `True`. ~~bool~~ | +| `verbose` | If set to `True`, warnings will be generated when the LLM returns invalid responses. Defaults to `False`. ~~bool~~ | + +The formatting of few-shot examples is the same as those for the +[v1](#textcat-v1) implementation. + +#### spacy.TextCat.v2 {id="textcat-v2"} + +V2 includes all v1 functionality, with an improved prompt template. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.TextCat.v2" +> labels = ["COMPLIMENT", "INSULT"] +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` (NEW) | Custom prompt template to send to LLM model. Defaults to [`textcat.v2.jinja`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/textcat.v2.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `TextCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, falls back to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `exclusive_classes` | If set to `True`, only one label per document should be valid. If set to `False`, one document can have multiple labels. Defaults to `False`. ~~bool~~ | +| `allow_none` | When set to `True`, allows the LLM to not return any of the given label. The resulting dict in `doc.cats` will have `0.0` scores for all labels. Defaults to `True`. ~~bool~~ | +| `verbose` | If set to `True`, warnings will be generated when the LLM returns invalid responses. Defaults to `False`. ~~bool~~ | + +The formatting of few-shot examples is the same as those for the +[v1](#textcat-v1) implementation. + +#### spacy.TextCat.v1 {id="textcat-v1"} + +Version 1 of the built-in TextCat task supports both zero-shot and few-shot +prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.TextCat.v1" +> labels = COMPLIMENT,INSULT +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | Optional function that generates examples for few-shot learning. Deafults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SpanCatTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `TextCatExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | Comma-separated list of labels. ~~str~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, falls back to `spacy.LowercaseNormalizer.v1`. ~~Optional[Callable[[str], str]]~~ | +| `exclusive_classes` | If set to `True`, only one label per document should be valid. If set to `False`, one document can have multiple labels. Defaults to `False`. ~~bool~~ | +| `allow_none` | When set to `True`, allows the LLM to not return any of the given label. The resulting dict in `doc.cats` will have `0.0` scores for all labels. Defaults to `True`. ~~bool~~ | +| `verbose` | If set to `True`, warnings will be generated when the LLM returns invalid responses. Defaults to `False`. ~~bool~~ | + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```json +[ + { + "text": "You look great!", + "answer": "Compliment" + }, + { + "text": "You are not very clever at all.", + "answer": "Insult" + } +] +``` + +```ini +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "textcat_examples.json" +``` + +If you want to perform few-shot learning with a binary classifier (i. e. a text +either should or should not be assigned to a given class), you can provide +positive and negative examples with answers of "POS" or "NEG". "POS" means that +this example should be assigned the class label defined in the configuration, +"NEG" means it shouldn't. E. g. for spam classification: + +```json +[ + { + "text": "You won the lottery! Wire a fee of 200$ to be able to withdraw your winnings.", + "answer": "POS" + }, + { + "text": "Your order #123456789 has arrived", + "answer": "NEG" + } +] +``` + +### REL {id="rel"} + +The REL task extracts relations between named entities. + +#### spacy.REL.v1 {id="rel-v1"} + +The built-in REL task supports both zero-shot and few-shot prompting. It relies +on an upstream NER component for entities extraction. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.REL.v1" +> labels = ["LivesIn", "Visits"] +> ``` + +| Argument | Description | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [`rel.v3.jinja`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/rel.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[RELTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `RELExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `labels` | List of labels or str of comma-separated list of labels. ~~Union[List[str], str]~~ | +| `label_definitions` | Dictionary providing a description for each relation label. Defaults to `None`. ~~Optional[Dict[str, str]]~~ | +| `normalizer` | Function that normalizes the labels as returned by the LLM. If `None`, falls back to `spacy.LowercaseNormalizer.v1`. Defaults to `None`. ~~Optional[Callable[[str], str]]~~ | +| `verbose` | If set to `True`, warnings will be generated when the LLM returns invalid responses. Defaults to `False`. ~~bool~~ | + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```json +{"text": "Laura bought a house in Boston with her husband Mark.", "ents": [{"start_char": 0, "end_char": 5, "label": "PERSON"}, {"start_char": 24, "end_char": 30, "label": "GPE"}, {"start_char": 48, "end_char": 52, "label": "PERSON"}], "relations": [{"dep": 0, "dest": 1, "relation": "LivesIn"}, {"dep": 2, "dest": 1, "relation": "LivesIn"}]} +{"text": "Michael travelled through South America by bike.", "ents": [{"start_char": 0, "end_char": 7, "label": "PERSON"}, {"start_char": 26, "end_char": 39, "label": "LOC"}], "relations": [{"dep": 0, "dest": 1, "relation": "Visits"}]} +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.REL.v1" +labels = ["LivesIn", "Visits"] + +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "rel_examples.jsonl" +``` + +Note: the REL task relies on pre-extracted entities to make its prediction. +Hence, you'll need to add a component that populates `doc.ents` with recognized +spans to your spaCy pipeline and put it _before_ the REL component. + +For a fully working example, see this +[usage example](https://github.com/explosion/spacy-llm/tree/main/usage_examples/rel_openai). + +### Lemma {id="lemma"} + +The Lemma task lemmatizes the provided text and updates the `lemma_` attribute +in the doc's tokens accordingly. + +#### spacy.Lemma.v1 {id="lemma-v1"} + +This task supports both zero-shot and few-shot prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.Lemma.v1" +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [lemma.v1.jinja](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/tasks/templates/lemma.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[LemmaTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `LemmaExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | + +The task prompts the LLM to lemmatize the passed text and return the lemmatized +version as a list of tokens and their corresponding lemma. E. g. the text +`I'm buying ice cream for my friends` should invoke the response + +``` +I: I +'m: be +buying: buy +ice: ice +cream: cream +for: for +my: my +friends: friend +.: . +``` + +If for any given text/doc instance the number of lemmas returned by the LLM +doesn't match the number of tokens from the pipeline's tokenizer, no lemmas are +stored in the corresponding doc's tokens. Otherwise the tokens `.lemma_` +property is updated with the lemma suggested by the LLM. + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +- text: I'm buying ice cream. + lemmas: + - 'I': 'I' + - "'m": 'be' + - 'buying': 'buy' + - 'ice': 'ice' + - 'cream': 'cream' + - '.': '.' + +- text: I've watered the plants. + lemmas: + - 'I': 'I' + - "'ve": 'have' + - 'watered': 'water' + - 'the': 'the' + - 'plants': 'plant' + - '.': '.' +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.Lemma.v1" +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "lemma_examples.yml" +``` + +### Sentiment {id="sentiment"} + +Performs sentiment analysis on provided texts. Scores between 0 and 1 are stored +in `Doc._.sentiment` - the higher, the more positive. Note in cases of parsing +issues (e. g. in case of unexpected LLM responses) the value might be `None`. + +#### spacy.Sentiment.v1 {id="sentiment-v1"} + +This task supports both zero-shot and few-shot prompting. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.Sentiment.v1" +> examples = null +> ``` + +| Argument | Description | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `template` | Custom prompt template to send to LLM model. Defaults to [sentiment.v1.jinja](./spacy_llm/tasks/templates/sentiment.v1.jinja). ~~str~~ | +| `examples` | Optional function that generates examples for few-shot learning. Defaults to `None`. ~~Optional[Callable[[], Iterable[Any]]]~~ | +| `parse_responses` (NEW) | Callable for parsing LLM responses for this task. Defaults to the internal parsing method for this task. ~~Optional[TaskResponseParser[SentimentTask]]~~ | +| `prompt_example_type` (NEW) | Type to use for fewshot examples. Defaults to `SentimentExample`. ~~Optional[Type[FewshotExample]]~~ | +| `scorer` (NEW) | Scorer function that evaluates the task performance on provided examples. Defaults to the metric used by spaCy. ~~Optional[Scorer]~~ | +| `field` | Name of extension attribute to store summary in (i. e. the summary will be available in `doc._.{field}`). Defaults to `sentiment`. ~~str~~ | + +To perform [few-shot learning](/usage/large-language-models#few-shot-prompts), +you can write down a few examples in a separate file, and provide these to be +injected into the prompt to the LLM. The default reader `spacy.FewShotReader.v1` +supports `.yml`, `.yaml`, `.json` and `.jsonl`. + +```yaml +- text: 'This is horrifying.' + score: 0 +- text: 'This is underwhelming.' + score: 0.25 +- text: 'This is ok.' + score: 0.5 +- text: "I'm looking forward to this!" + score: 1.0 +``` + +```ini +[components.llm.task] +@llm_tasks = "spacy.Sentiment.v1" +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "sentiment_examples.yml" +``` + +### NoOp {id="noop"} + +This task is only useful for testing - it tells the LLM to do nothing, and does +not set any fields on the `docs`. + +> #### Example config +> +> ```ini +> [components.llm.task] +> @llm_tasks = "spacy.NoOp.v1" +> ``` + +#### spacy.NoOp.v1 {id="noop-v1"} + +This task needs no further configuration. + +## Models {id="models"} + +A _model_ defines which LLM model to query, and how to query it. It can be a +simple function taking a collection of prompts (consistent with the output type +of `task.generate_prompts()`) and returning a collection of responses +(consistent with the expected input of `parse_responses`). Generally speaking, +it's a function of type +`Callable[[Iterable[Iterable[Any]]], Iterable[Iterable[Any]]]`, but specific +implementations can have other signatures, like +`Callable[[Iterable[Iterable[str]]], Iterable[Iterable[str]]]`. + +Note: the model signature expects a nested iterable so it's able to deal with +sharded docs. Unsharded docs (i. e. those produced by (nonsharding +tasks)[/api/large-language-models#task-nonsharding]) are reshaped to fit the +expected data structure. + +### Models via REST API {id="models-rest"} + +These models all take the same parameters, but note that the `config` should +contain provider-specific keys and values, as it will be passed onwards to the +provider's API. + +| Argument | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Model name, i. e. any supported variant for this particular model. Default depends on the specific model (cf. below) ~~str~~ | +| `config` | Further configuration passed on to the model. Default depends on the specific model (cf. below). ~~Dict[Any, Any]~~ | +| `strict` | If `True`, raises an error if the LLM API returns a malformed response. Otherwise, return the error responses as is. Defaults to `True`. ~~bool~~ | +| `max_tries` | Max. number of tries for API request. Defaults to `5`. ~~int~~ | +| `max_request_time` | Max. time (in seconds) to wait for request to terminate before raising an exception. Defaults to `30.0`. ~~float~~ | +| `interval` | Time interval (in seconds) for API retries in seconds. Defaults to `1.0`. ~~float~~ | +| `endpoint` | Endpoint URL. Defaults to the provider's standard URL, if available (which is not the case for providers with exclusively custom deployments, such as Azure) ~~Optional[str]~~ | + +> #### Example config: +> +> ```ini +> [components.llm.model] +> @llm_models = "spacy.GPT-4.v1" +> name = "gpt-4" +> config = {"temperature": 0.0} +> ``` + +Currently, these models are provided as part of the core library: + +| Model | Provider | Supported names | Default name | Default config | +| ----------------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------- | ------------------------------------ | +| `spacy.GPT-4.v1` | OpenAI | `["gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314"]` | `"gpt-4"` | `{}` | +| `spacy.GPT-4.v2` | OpenAI | `["gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314"]` | `"gpt-4"` | `{temperature=0.0}` | +| `spacy.GPT-4.v3` | OpenAI | All names of [GPT-4 models](https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo) offered by OpenAI | `"gpt-4"` | `{temperature=0.0}` | +| `spacy.GPT-3-5.v1` | OpenAI | `["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-0613-16k", "gpt-3.5-turbo-instruct"]` | `"gpt-3.5-turbo"` | `{}` | +| `spacy.GPT-3-5.v2` | OpenAI | `["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-0613-16k", "gpt-3.5-turbo-instruct"]` | `"gpt-3.5-turbo"` | `{temperature=0.0}` | +| `spacy.GPT-3-5.v3` | OpenAI | All names of [GPT-3.5 models](https://platform.openai.com/docs/models/gpt-3-5) offered by OpenAI | `"gpt-3.5-turbo"` | `{temperature=0.0}` | +| `spacy.Davinci.v1` | OpenAI | `["davinci"]` | `"davinci"` | `{}` | +| `spacy.Davinci.v2` | OpenAI | `["davinci"]` | `"davinci"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Text-Davinci.v1` | OpenAI | `["text-davinci-003", "text-davinci-002"]` | `"text-davinci-003"` | `{}` | +| `spacy.Text-Davinci.v2` | OpenAI | `["text-davinci-003", "text-davinci-002"]` | `"text-davinci-003"` | `{temperature=0.0, max_tokens=1000}` | +| `spacy.Code-Davinci.v1` | OpenAI | `["code-davinci-002"]` | `"code-davinci-002"` | `{}` | +| `spacy.Code-Davinci.v2` | OpenAI | `["code-davinci-002"]` | `"code-davinci-002"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Curie.v1` | OpenAI | `["curie"]` | `"curie"` | `{}` | +| `spacy.Curie.v2` | OpenAI | `["curie"]` | `"curie"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Text-Curie.v1` | OpenAI | `["text-curie-001"]` | `"text-curie-001"` | `{}` | +| `spacy.Text-Curie.v2` | OpenAI | `["text-curie-001"]` | `"text-curie-001"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Babbage.v1` | OpenAI | `["babbage"]` | `"babbage"` | `{}` | +| `spacy.Babbage.v2` | OpenAI | `["babbage"]` | `"babbage"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Text-Babbage.v1` | OpenAI | `["text-babbage-001"]` | `"text-babbage-001"` | `{}` | +| `spacy.Text-Babbage.v2` | OpenAI | `["text-babbage-001"]` | `"text-babbage-001"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Ada.v1` | OpenAI | `["ada"]` | `"ada"` | `{}` | +| `spacy.Ada.v2` | OpenAI | `["ada"]` | `"ada"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Text-Ada.v1` | OpenAI | `["text-ada-001"]` | `"text-ada-001"` | `{}` | +| `spacy.Text-Ada.v2` | OpenAI | `["text-ada-001"]` | `"text-ada-001"` | `{temperature=0.0, max_tokens=500}` | +| `spacy.Azure.v1` | Microsoft, OpenAI | Arbitrary values | No default | `{temperature=0.0}` | +| `spacy.Command.v1` | Cohere | `["command", "command-light", "command-light-nightly", "command-nightly"]` | `"command"` | `{}` | +| `spacy.Claude-2-1.v1` | Anthropic | `["claude-2-1"]` | `"claude-2-1"` | `{}` | +| `spacy.Claude-2.v1` | Anthropic | `["claude-2", "claude-2-100k"]` | `"claude-2"` | `{}` | +| `spacy.Claude-1.v1` | Anthropic | `["claude-1", "claude-1-100k"]` | `"claude-1"` | `{}` | +| `spacy.Claude-1-0.v1` | Anthropic | `["claude-1.0"]` | `"claude-1.0"` | `{}` | +| `spacy.Claude-1-2.v1` | Anthropic | `["claude-1.2"]` | `"claude-1.2"` | `{}` | +| `spacy.Claude-1-3.v1` | Anthropic | `["claude-1.3", "claude-1.3-100k"]` | `"claude-1.3"` | `{}` | +| `spacy.Claude-instant-1.v1` | Anthropic | `["claude-instant-1", "claude-instant-1-100k"]` | `"claude-instant-1"` | `{}` | +| `spacy.Claude-instant-1-1.v1` | Anthropic | `["claude-instant-1.1", "claude-instant-1.1-100k"]` | `"claude-instant-1.1"` | `{}` | +| `spacy.PaLM.v1` | Google | `["chat-bison-001", "text-bison-001"]` | `"text-bison-001"` | `{temperature=0.0}` | + +To use these models, make sure that you've [set the relevant API](#api-keys) +keys as environment variables. + +**⚠️ A note on `spacy.Azure.v1`.** Working with Azure OpenAI is slightly +different than working with models from other providers: + +- In Azure LLMs have to be made available by creating a _deployment_ of a given + model (e. g. GPT-3.5). This deployment can have an arbitrary name. The `name` + argument, which everywhere else denotes the model name (e. g. `claude-1.0`, + `gpt-3.5`), here refers to the _deployment name_. +- Deployed Azure OpenAI models are reachable via a resource-specific base URL, + usually of the form `https://{resource}.openai.azure.com`. Hence the URL has + to be specified via the `base_url` argument. +- Azure further expects the _API version_ to be specified. The default value for + this, via the `api_version` argument, is currently `2023-05-15` but may be + updated in the future. +- Finally, since we can't infer information about the model from the deployment + name, `spacy-llm` requires the `model_type` to be set to either + `"completions"` or `"chat"`, depending on whether the deployed model is a + completion or chat model. + +#### API Keys {id="api-keys"} + +Note that when using hosted services, you have to ensure that the proper API +keys are set as environment variables as described by the corresponding +provider's documentation. + +E. g. when using OpenAI, you have to get an API key from openai.com, and ensure +that the keys are set as environmental variables: + +```shell +export OPENAI_API_KEY="sk-..." +export OPENAI_API_ORG="org-..." +``` + +For Cohere: + +```shell +export CO_API_KEY="..." +``` + +For Anthropic: + +```shell +export ANTHROPIC_API_KEY="..." +``` + +For PaLM: + +```shell +export PALM_API_KEY="..." +``` + +### Models via HuggingFace {id="models-hf"} + +These models all take the same parameters: + +| Argument | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Model name, i. e. any supported variant for this particular model. ~~str~~ | +| `config_init` | Further configuration passed on to the construction of the model with `transformers.pipeline()`. Defaults to `{}`. ~~Dict[str, Any]~~ | +| `config_run` | Further configuration used during model inference. Defaults to `{}`. ~~Dict[str, Any]~~ | + +> #### Example config +> +> ```ini +> [components.llm.model] +> @llm_models = "spacy.Llama2.v1" +> name = "Llama-2-7b-hf" +> ``` + +Currently, these models are provided as part of the core library: + +| Model | Provider | Supported names | HF directory | +| -------------------- | --------------- | ------------------------------------------------------------------------------------------------------------ | -------------------------------------- | +| `spacy.Dolly.v1` | Databricks | `["dolly-v2-3b", "dolly-v2-7b", "dolly-v2-12b"]` | https://huggingface.co/databricks | +| `spacy.Falcon.v1` | TII | `["falcon-rw-1b", "falcon-7b", "falcon-7b-instruct", "falcon-40b-instruct"]` | https://huggingface.co/tiiuae | +| `spacy.Llama2.v1` | Meta AI | `["Llama-2-7b-hf", "Llama-2-13b-hf", "Llama-2-70b-hf"]` | https://huggingface.co/meta-llama | +| `spacy.Mistral.v1` | Mistral AI | `["Mistral-7B-v0.1", "Mistral-7B-Instruct-v0.1"]` | https://huggingface.co/mistralai | +| `spacy.StableLM.v1` | Stability AI | `["stablelm-base-alpha-3b", "stablelm-base-alpha-7b", "stablelm-tuned-alpha-3b", "stablelm-tuned-alpha-7b"]` | https://huggingface.co/stabilityai | +| `spacy.OpenLLaMA.v1` | OpenLM Research | `["open_llama_3b", "open_llama_7b", "open_llama_7b_v2", "open_llama_13b"]` | https://huggingface.co/openlm-research | + + + +Some models available on Hugging Face (HF), such as Llama 2, are _gated models_. +That means that users have to fulfill certain requirements to be allowed access +to these models. In the case of Llama 2 you'll need to request agree to Meta's +Terms of Service while logged in with your HF account. After Meta grants you +permission to use Llama 2, you'll be able to download and use the model. + +This requires that you are logged in with your HF account on your local +machine - check out the HF quick start documentation. In a nutshell, you'll need +to create an access token on HF and log in to HF using your access token, e. g. +with `huggingface-cli login`. + + + +Note that Hugging Face will download the model the first time you use it - you +can +[define the cached directory](https://huggingface.co/docs/huggingface_hub/main/en/guides/manage-cache) +by setting the environmental variable `HF_HOME`. + +#### Installation with HuggingFace {id="install-hf"} + +To use models from HuggingFace, ideally you have a GPU enabled and have +installed `transformers`, `torch` and CUDA in your virtual environment. This +allows you to have the setting `device=cuda:0` in your config, which ensures +that the model is loaded entirely on the GPU (and fails otherwise). + +You can do so with + +```shell +python -m pip install "spacy-llm[transformers]" "transformers[sentencepiece]" +``` + +If you don't have access to a GPU, you can install `accelerate` and +set`device_map=auto` instead, but be aware that this may result in some layers +getting distributed to the CPU or even the hard drive, which may ultimately +result in extremely slow queries. + +```shell +python -m pip install "accelerate>=0.16.0,<1.0" +``` + +### LangChain models {id="langchain-models"} + +To use [LangChain](https://github.com/hwchase17/langchain) for the API retrieval +part, make sure you have installed it first: + +```shell +python -m pip install "langchain==0.0.191" +# Or install with spacy-llm directly +python -m pip install "spacy-llm[extras]" +``` + +Note that LangChain currently only supports Python 3.9 and beyond. + +LangChain models in `spacy-llm` work slightly differently. `langchain`'s models +are parsed automatically, each LLM class in `langchain` has one entry in +`spacy-llm`'s registry. As `langchain`'s design has one class per API and not +per model, this results in registry entries like `langchain.OpenAI.v1` - i. e. +there is one registry entry per API and not per model (family), as for the REST- +and HuggingFace-based entries. + +The name of the model to be used has to be passed in via the `name` attribute. + +> #### Example config +> +> ```ini +> [components.llm.model] +> @llm_models = "langchain.OpenAI.v1" +> name = "gpt-3.5-turbo" +> query = {"@llm_queries": "spacy.CallLangChain.v1"} +> config = {"temperature": 0.0} +> ``` + +| Argument | Description | +| -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | The name of a model supported by LangChain for this API. ~~str~~ | +| `config` | Configuration passed on to the LangChain model. Defaults to `{}`. ~~Dict[Any, Any]~~ | +| `query` | Function that executes the prompts. If `None`, defaults to `spacy.CallLangChain.v1`. ~~Optional[Callable[["langchain.llms.BaseLLM", Iterable[Any]], Iterable[Any]]]~~ | + +The default `query` (`spacy.CallLangChain.v1`) executes the prompts by running +`model(text)` for each given textual prompt. + +## Cache {id="cache"} + +Interacting with LLMs, either through an external API or a local instance, is +costly. Since developing an NLP pipeline generally means a lot of exploration +and prototyping, `spacy-llm` implements a built-in cache to avoid reprocessing +the same documents at each run that keeps batches of documents stored on disk. + +> #### Example config +> +> ```ini +> [components.llm.cache] +> @llm_misc = "spacy.BatchCache.v1" +> path = "path/to/cache" +> batch_size = 64 +> max_batches_in_mem = 4 +> ``` + +| Argument | Description | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | Cache directory. If `None`, no caching is performed, and this component will act as a NoOp. Defaults to `None`. ~~Optional[Union[str, Path]]~~ | +| `batch_size` | Number of docs in one batch (file). Once a batch is full, it will be peristed to disk. Defaults to 64. ~~int~~ | +| `max_batches_in_mem` | Max. number of batches to hold in memory. Allows you to limit the effect on your memory if you're handling a lot of docs. Defaults to 4. ~~int~~ | + +When retrieving a document, the `BatchCache` will first figure out what batch +the document belongs to. If the batch isn't in memory it will try to load the +batch from disk and then move it into memory. + +Note that since the cache is generated by a registered function, you can also +provide your own registered function returning your own cache implementation. If +you wish to do so, ensure that your cache object adheres to the `Protocol` +defined in `spacy_llm.ty.Cache`. + +## Various functions {id="various-functions"} + +### spacy.FewShotReader.v1 {id="fewshotreader-v1"} + +This function is registered in spaCy's `misc` registry, and reads in examples +from a `.yml`, `.yaml`, `.json` or `.jsonl` file. It uses +[`srsly`](https://github.com/explosion/srsly) to read in these files and parses +them depending on the file extension. + +> #### Example config +> +> ```ini +> [components.llm.task.examples] +> @misc = "spacy.FewShotReader.v1" +> path = "ner_examples.yml" +> ``` + +| Argument | Description | +| -------- | ----------------------------------------------------------------------------------------------- | +| `path` | Path to an examples file with suffix `.yml`, `.yaml`, `.json` or `.jsonl`. ~~Union[str, Path]~~ | + +### spacy.FileReader.v1 {id="filereader-v1"} + +This function is registered in spaCy's `misc` registry, and reads a file +provided to the `path` to return a `str` representation of its contents. This +function is typically used to read +[Jinja](https://jinja.palletsprojects.com/en/3.1.x/) files containing the prompt +template. + +> #### Example config +> +> ```ini +> [components.llm.task.template] +> @misc = "spacy.FileReader.v1" +> path = "ner_template.jinja2" +> ``` + +| Argument | Description | +| -------- | ------------------------------------------------- | +| `path` | Path to the file to be read. ~~Union[str, Path]~~ | + +### Normalizer functions {id="normalizer-functions"} + +These functions provide simple normalizations for string comparisons, e.g. +between a list of specified labels and a label given in the raw text of the LLM +response. They are registered in spaCy's `misc` registry and have the signature +`Callable[[str], str]`. + +- `spacy.StripNormalizer.v1`: only apply `text.strip()` +- `spacy.LowercaseNormalizer.v1`: applies `text.strip().lower()` to compare + strings in a case-insensitive way. diff --git a/website/docs/api/legacy.mdx b/website/docs/api/legacy.mdx new file mode 100644 index 0000000..b44df53 --- /dev/null +++ b/website/docs/api/legacy.mdx @@ -0,0 +1,386 @@ +--- +title: Legacy functions and architectures +teaser: Archived implementations available through spacy-legacy +source: spacy/legacy +--- + +The [`spacy-legacy`](https://github.com/explosion/spacy-legacy) package includes +outdated registered functions and architectures. It is installed automatically +as a dependency of spaCy, and provides backwards compatibility for archived +functions that may still be used in projects. + +You can find the detailed documentation of each such legacy function on this +page. + +## Architectures {id="architectures"} + +These functions are available from `@spacy.registry.architectures`. + +### spacy.Tok2Vec.v1 {id="Tok2Vec_v1"} + +The `spacy.Tok2Vec.v1` architecture was expecting an `encode` model of type +`Model[Floats2D, Floats2D]` such as `spacy.MaxoutWindowEncoder.v1` or +`spacy.MishWindowEncoder.v1`. + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.Tok2Vec.v1" +> +> [model.embed] +> @architectures = "spacy.CharacterEmbed.v1" +> # ... +> +> [model.encode] +> @architectures = "spacy.MaxoutWindowEncoder.v1" +> # ... +> ``` + +Construct a tok2vec model out of two subnetworks: one for embedding and one for +encoding. See the +["Embed, Encode, Attend, Predict"](https://explosion.ai/blog/deep-learning-formula-nlp) +blog post for background. + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `embed` | Embed tokens into context-independent word vector representations. For example, [CharacterEmbed](/api/architectures#CharacterEmbed) or [MultiHashEmbed](/api/architectures#MultiHashEmbed). ~~Model[List[Doc], List[Floats2d]]~~ | +| `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder.v1](/api/legacy#MaxoutWindowEncoder_v1). ~~Model[Floats2d, Floats2d]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | + +### spacy.MaxoutWindowEncoder.v1 {id="MaxoutWindowEncoder_v1"} + +The `spacy.MaxoutWindowEncoder.v1` architecture was producing a model of type +`Model[Floats2D, Floats2D]`. Since `spacy.MaxoutWindowEncoder.v2`, this has been +changed to output type `Model[List[Floats2d], List[Floats2d]]`. + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.MaxoutWindowEncoder.v1" +> width = 128 +> window_size = 1 +> maxout_pieces = 3 +> depth = 4 +> ``` + +Encode context using convolutions with maxout activation, layer normalization +and residual connections. + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between `64` and `300`. ~~int~~ | +| `window_size` | The number of words to concatenate around each token to construct the convolution. Recommended value is `1`. ~~int~~ | +| `maxout_pieces` | The number of maxout pieces to use. Recommended values are `2` or `3`. ~~int~~ | +| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | + +### spacy.MishWindowEncoder.v1 {id="MishWindowEncoder_v1"} + +The `spacy.MishWindowEncoder.v1` architecture was producing a model of type +`Model[Floats2D, Floats2D]`. Since `spacy.MishWindowEncoder.v2`, this has been +changed to output type `Model[List[Floats2d], List[Floats2d]]`. + +> #### Example config +> +> ```ini +> [model] +> @architectures = "spacy.MishWindowEncoder.v1" +> width = 64 +> window_size = 1 +> depth = 4 +> ``` + +Encode context using convolutions with +[`Mish`](https://thinc.ai/docs/api-layers#mish) activation, layer normalization +and residual connections. + +| Name | Description | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `width` | The input and output width. These are required to be the same, to allow residual connections. This value will be determined by the width of the inputs. Recommended values are between `64` and `300`. ~~int~~ | +| `window_size` | The number of words to concatenate around each token to construct the convolution. Recommended value is `1`. ~~int~~ | +| `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | + +### spacy.HashEmbedCNN.v1 {id="HashEmbedCNN_v1"} + +Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except +using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. + +### spacy.MultiHashEmbed.v1 {id="MultiHashEmbed_v1"} + +Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed) +except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are +included. + +### spacy.CharacterEmbed.v1 {id="CharacterEmbed_v1"} + +Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed) +except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are +included. + +### spacy.TextCatEnsemble.v1 {id="TextCatEnsemble_v1"} + +The `spacy.TextCatEnsemble.v1` architecture built an internal `tok2vec` and +`linear_model`. Since `spacy.TextCatEnsemble.v2`, this has been refactored so +that the `TextCatEnsemble` takes these two sublayers as input. + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatEnsemble.v1" +> exclusive_classes = false +> pretrained_vectors = null +> width = 64 +> embed_size = 2000 +> conv_depth = 2 +> window_size = 1 +> ngram_size = 1 +> dropout = null +> nO = null +> ``` + +Stacked ensemble of a bag-of-words model and a neural network model. The neural +network has an internal CNN Tok2Vec layer and uses attention. + +| Name | Description | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `pretrained_vectors` | Whether or not pretrained vectors will be used in addition to the feature vectors. ~~bool~~ | +| `width` | Output dimension of the feature encoding step. ~~int~~ | +| `embed_size` | Input dimension of the feature encoding step. ~~int~~ | +| `conv_depth` | Depth of the tok2vec layer. ~~int~~ | +| `window_size` | The number of contextual vectors to [concatenate](https://thinc.ai/docs/api-layers#expand_window) from the left and from the right. ~~int~~ | +| `ngram_size` | Determines the maximum length of the n-grams in the BOW model. For instance, `ngram_size=3`would give unigram, trigram and bigram features. ~~int~~ | +| `dropout` | The dropout rate. ~~float~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.TextCatCNN.v1 {id="TextCatCNN_v1"} + +Since `spacy.TextCatCNN.v2`, this architecture has become resizable, which means +that you can add labels to a previously trained textcat. `TextCatCNN` v1 did not +yet support that. `TextCatCNN` has been replaced by the more general +[`TextCatReduce`](/api/architectures#TextCatReduce) layer. `TextCatCNN` is +identical to `TextCatReduce` with `use_reduce_mean=true`, +`use_reduce_first=false`, `reduce_last=false` and `use_reduce_max=false`. + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatCNN.v1" +> exclusive_classes = false +> nO = null +> +> [model.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v1" +> pretrained_vectors = null +> width = 96 +> depth = 4 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +A neural network model where token vectors are calculated using a CNN. The +vectors are mean pooled and used as features in a feed-forward network. This +architecture is usually less accurate than the ensemble, but runs faster. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.TextCatCNN.v2 {id="TextCatCNN_v2"} + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatCNN.v2" +> exclusive_classes = false +> nO = null +> +> [model.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v2" +> pretrained_vectors = null +> width = 96 +> depth = 4 +> embed_size = 2000 +> window_size = 1 +> maxout_pieces = 3 +> subword_features = true +> ``` + +A neural network model where token vectors are calculated using a CNN. The +vectors are mean pooled and used as features in a feed-forward network. This +architecture is usually less accurate than the ensemble, but runs faster. + +`TextCatCNN` has been replaced by the more general +[`TextCatReduce`](/api/architectures#TextCatReduce) layer. `TextCatCNN` is +identical to `TextCatReduce` with `use_reduce_mean=true`, +`use_reduce_first=false`, `reduce_last=false` and `use_reduce_max=false`. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + + + +[TextCatCNN.v1](/api/legacy#TextCatCNN_v1) had the exact same signature, but was +not yet resizable. Since v2, new labels can be added to this component, even +after training. + + + +### spacy.TextCatBOW.v1 {id="TextCatBOW_v1"} + +Since `spacy.TextCatBOW.v2`, this architecture has become resizable, which means +that you can add labels to a previously trained textcat. `TextCatBOW` v1 did not +yet support that. Versions of this model before `spacy.TextCatBOW.v3` used an +erroneous sparse linear layer that only used a small number of the allocated +parameters. + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatBOW.v1" +> exclusive_classes = false +> ngram_size = 1 +> no_output_layer = false +> nO = null +> ``` + +An n-gram "bag-of-words" model. This architecture should run much faster than +the others, but may not be as accurate, especially if texts are short. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `ngram_size` | Determines the maximum length of the n-grams in the BOW model. For instance, `ngram_size=3` would give unigram, trigram and bigram features. ~~int~~ | +| `no_output_layer` | Whether or not to add an output layer to the model (`Softmax` activation if `exclusive_classes` is `True`, else `Logistic`). ~~bool~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.TextCatBOW.v2 {id="TextCatBOW"} + +Versions of this model before `spacy.TextCatBOW.v3` used an erroneous sparse +linear layer that only used a small number of the allocated parameters. + +> #### Example Config +> +> ```ini +> [model] +> @architectures = "spacy.TextCatBOW.v2" +> exclusive_classes = false +> ngram_size = 1 +> no_output_layer = false +> nO = null +> ``` + +An n-gram "bag-of-words" model. This architecture should run much faster than +the others, but may not be as accurate, especially if texts are short. + +| Name | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `exclusive_classes` | Whether or not categories are mutually exclusive. ~~bool~~ | +| `ngram_size` | Determines the maximum length of the n-grams in the BOW model. For instance, `ngram_size=3` would give unigram, trigram and bigram features. ~~int~~ | +| `no_output_layer` | Whether or not to add an output layer to the model (`Softmax` activation if `exclusive_classes` is `True`, else `Logistic`). ~~bool~~ | +| `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy.TransitionBasedParser.v1 {id="TransitionBasedParser_v1"} + +Identical to +[`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser) +except the `use_upper` was set to `True` by default. + +## Layers {id="layers"} + +These functions are available from `@spacy.registry.layers`. + +### spacy.StaticVectors.v1 {id="StaticVectors_v1"} + +Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except +for the handling of tokens without vectors. + + + +`spacy.StaticVectors.v1` maps tokens without vectors to the final row in the +vectors table, which causes the model predictions to change if new vectors are +added to an existing vectors table. See more details in +[issue #7662](https://github.com/explosion/spaCy/issues/7662#issuecomment-813925655). + + + +## Loggers {id="loggers"} + +These functions are available from `@spacy.registry.loggers`. + +### spacy.ConsoleLogger.v1 {id="ConsoleLogger_v1"} + +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.ConsoleLogger.v1" +> progress_bar = true +> ``` + +Writes the results of a training step to the console in a tabular format. + + + +```bash +$ python -m spacy train config.cfg +``` + +``` +ℹ Using CPU +ℹ Loading config and nlp from: config.cfg +ℹ Pipeline: ['tok2vec', 'tagger'] +ℹ Start training +ℹ Training. Initial learn rate: 0.0 + +E # LOSS TOK2VEC LOSS TAGGER TAG_ACC SCORE +--- ------ ------------ ----------- ------- ------ + 0 0 0.00 86.20 0.22 0.00 + 0 200 3.08 18968.78 34.00 0.34 + 0 400 31.81 22539.06 33.64 0.34 + 0 600 92.13 22794.91 43.80 0.44 + 0 800 183.62 21541.39 56.05 0.56 + 0 1000 352.49 25461.82 65.15 0.65 + 0 1200 422.87 23708.82 71.84 0.72 + 0 1400 601.92 24994.79 76.57 0.77 + 0 1600 662.57 22268.02 80.20 0.80 + 0 1800 1101.50 28413.77 82.56 0.83 + 0 2000 1253.43 28736.36 85.00 0.85 + 0 2200 1411.02 28237.53 87.42 0.87 + 0 2400 1605.35 28439.95 88.70 0.89 +``` + +Note that the cumulative loss keeps increasing within one epoch, but should +start decreasing across epochs. + + + +| Name | Description | +| -------------- | --------------------------------------------------------- | +| `progress_bar` | Whether the logger should print the progress bar ~~bool~~ | + +Logging utilities for spaCy are implemented in the +[`spacy-loggers`](https://github.com/explosion/spacy-loggers) repo, and the +functions are typically available from `@spacy.registry.loggers`. + +More documentation can be found in that repo's +[readme](https://github.com/explosion/spacy-loggers/blob/main/README.md) file. diff --git a/website/docs/api/lemmatizer.mdx b/website/docs/api/lemmatizer.mdx new file mode 100644 index 0000000..f6657db --- /dev/null +++ b/website/docs/api/lemmatizer.mdx @@ -0,0 +1,328 @@ +--- +title: Lemmatizer +tag: class +source: spacy/pipeline/lemmatizer.py +version: 3 +teaser: 'Pipeline component for lemmatization' +api_string_name: lemmatizer +api_trainable: false +--- + +Component for assigning base forms to tokens using rules based on part-of-speech +tags, or lookup tables. Different [`Language`](/api/language) subclasses can +implement their own lemmatizer components via +[language-specific factories](/usage/processing-pipelines#factories-language). +The default data used is provided by the +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) +extension package. + +For a trainable lemmatizer, see [`EditTreeLemmatizer`](/api/edittreelemmatizer). + + + +As of v3.0, the `Lemmatizer` is a **standalone pipeline component** that can be +added to your pipeline, and not a hidden part of the vocab that runs behind the +scenes. This makes it easier to customize how lemmas should be assigned in your +pipeline. + +If the lemmatization mode is set to `"rule"`, which requires coarse-grained POS +(`Token.pos`) to be assigned, make sure a [`Tagger`](/api/tagger), +[`Morphologizer`](/api/morphologizer) or another component assigning POS is +available in the pipeline and runs _before_ the lemmatizer. + + + +## Assigned Attributes {id="assigned-attributes"} + +Lemmas generated by rules or predicted will be saved to `Token.lemma`. + +| Location | Value | +| -------------- | ------------------------- | +| `Token.lemma` | The lemma (hash). ~~int~~ | +| `Token.lemma_` | The lemma. ~~str~~ | + +## Config and implementation + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). For examples of the lookups +data format used by the lookup and rule-based lemmatizers, see +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). + +> #### Example +> +> ```python +> config = {"mode": "rule"} +> nlp.add_pipe("lemmatizer", config=config) +> ``` + +| Setting | Description | +| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mode` | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `lookup` if no language-specific lemmatizer is available (see the following table). ~~str~~ | +| `overwrite` | Whether to overwrite existing lemmas. Defaults to `False`. ~~bool~~ | +| `model` | **Not yet implemented:** the model to use. ~~Model~~ | +| _keyword-only_ | | +| `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ | + +Many languages specify a default lemmatizer mode other than `lookup` if a better +lemmatizer is available. The lemmatizer modes `rule` and `pos_lookup` require +[`token.pos`](/api/token) from a previous pipeline component (see example +pipeline configurations in the +[pretrained pipeline design details](/models#design-cnn)) or rely on third-party +libraries (`pymorphy3`). + +| Language | Default Mode | +| -------- | ------------ | +| `bn` | `rule` | +| `ca` | `pos_lookup` | +| `el` | `rule` | +| `en` | `rule` | +| `es` | `rule` | +| `fa` | `rule` | +| `fr` | `rule` | +| `it` | `pos_lookup` | +| `mk` | `rule` | +| `nb` | `rule` | +| `nl` | `rule` | +| `pl` | `pos_lookup` | +| `ru` | `pymorphy3` | +| `sv` | `rule` | +| `uk` | `pymorphy3` | + +```python +%%GITHUB_SPACY/spacy/pipeline/lemmatizer.py +``` + +## Lemmatizer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> lemmatizer = nlp.add_pipe("lemmatizer") +> +> # Construction via add_pipe with custom settings +> config = {"mode": "rule", "overwrite": True} +> lemmatizer = nlp.add_pipe("lemmatizer", config=config) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| -------------- | --------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | **Not yet implemented:** The model to use. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| mode | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `"lookup"`. ~~str~~ | +| overwrite | Whether to overwrite existing lemmas. ~~bool~~ | + +## Lemmatizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> lemmatizer = nlp.add_pipe("lemmatizer") +> # This usually happens under the hood +> processed = lemmatizer(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## Lemmatizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("lemmatizer") +> for doc in lemmatizer.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Lemmatizer.initialize {id="initialize",tag="method"} + +Initialize the lemmatizer and load any data resources. This method is typically +called by [`Language.initialize`](/api/language#initialize) and lets you +customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. The loading only happens during initialization, typically before +training. At runtime, all data is loaded from disk. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("lemmatizer") +> lemmatizer.initialize(lookups=lookups) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.lemmatizer] +> +> [initialize.components.lemmatizer.lookups] +> @misc = "load_my_lookups.v1" +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Defaults to `None`. ~~Optional[Callable[[], Iterable[Example]]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `lookups` | The lookups object containing the tables such as `"lemma_rules"`, `"lemma_index"`, `"lemma_exc"` and `"lemma_lookup"`. If `None`, default tables are loaded from [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). Defaults to `None`. ~~Optional[Lookups]~~ | + +## Lemmatizer.lookup_lemmatize {id="lookup_lemmatize",tag="method"} + +Lemmatize a token using a lookup-based approach. If no lemma is found, the +original string is returned. + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `token` | The token to lemmatize. ~~Token~~ | +| **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | + +## Lemmatizer.rule_lemmatize {id="rule_lemmatize",tag="method"} + +Lemmatize a token using a rule-based approach. Typically relies on POS tags. + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `token` | The token to lemmatize. ~~Token~~ | +| **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | + +## Lemmatizer.is_base_form {id="is_base_form",tag="method"} + +Check whether we're dealing with an uninflected paradigm, so we can avoid +lemmatization entirely. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------- | +| `token` | The token to analyze. ~~Token~~ | +| **RETURNS** | Whether the token's attributes (e.g., part-of-speech tag, morphological features) describe a base form. ~~bool~~ | + +## Lemmatizer.get_lookups_config {id="get_lookups_config",tag="classmethod"} + +Returns the lookups configuration settings for a given mode for use in +[`Lemmatizer.load_lookups`](/api/lemmatizer#load_lookups). + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------- | +| `mode` | The lemmatizer mode. ~~str~~ | +| **RETURNS** | The required table names and the optional table names. ~~Tuple[List[str], List[str]]~~ | + +## Lemmatizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("lemmatizer") +> lemmatizer.to_disk("/path/to/lemmatizer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Lemmatizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("lemmatizer") +> lemmatizer.from_disk("/path/to/lemmatizer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Lemmatizer` object. ~~Lemmatizer~~ | + +## Lemmatizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> lemmatizer = nlp.add_pipe("lemmatizer") +> lemmatizer_bytes = lemmatizer.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Lemmatizer` object. ~~bytes~~ | + +## Lemmatizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> lemmatizer_bytes = lemmatizer.to_bytes() +> lemmatizer = nlp.add_pipe("lemmatizer") +> lemmatizer.from_bytes(lemmatizer_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Lemmatizer` object. ~~Lemmatizer~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| --------- | ------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). ~~Vocab~~ | +| `lookups` | The lookups object. ~~Lookups~~ | +| `mode` | The lemmatizer mode. ~~str~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = lemmatizer.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| --------- | ---------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `lookups` | The lookups. You usually don't want to exclude this. | diff --git a/website/docs/api/lexeme.mdx b/website/docs/api/lexeme.mdx new file mode 100644 index 0000000..539f502 --- /dev/null +++ b/website/docs/api/lexeme.mdx @@ -0,0 +1,164 @@ +--- +title: Lexeme +teaser: An entry in the vocabulary +tag: class +source: spacy/lexeme.pyx +--- + +A `Lexeme` has no string context – it's a word type, as opposed to a word token. +It therefore has no part-of-speech tag, dependency parse, or lemma (if +lemmatization depends on the part-of-speech tag). + +## Lexeme.\_\_init\_\_ {id="init",tag="method"} + +Create a `Lexeme` object. + +| Name | Description | +| ------- | ---------------------------------- | +| `vocab` | The parent vocabulary. ~~Vocab~~ | +| `orth` | The orth id of the lexeme. ~~int~~ | + +## Lexeme.set_flag {id="set_flag",tag="method"} + +Change the value of a boolean flag. + +> #### Example +> +> ```python +> COOL_FLAG = nlp.vocab.add_flag(lambda text: False) +> nlp.vocab["spaCy"].set_flag(COOL_FLAG, True) +> ``` + +| Name | Description | +| --------- | -------------------------------------------- | +| `flag_id` | The attribute ID of the flag to set. ~~int~~ | +| `value` | The new value of the flag. ~~bool~~ | + +## Lexeme.check_flag {id="check_flag",tag="method"} + +Check the value of a boolean flag. + +> #### Example +> +> ```python +> is_my_library = lambda text: text in ["spaCy", "Thinc"] +> MY_LIBRARY = nlp.vocab.add_flag(is_my_library) +> assert nlp.vocab["spaCy"].check_flag(MY_LIBRARY) == True +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------- | +| `flag_id` | The attribute ID of the flag to query. ~~int~~ | +| **RETURNS** | The value of the flag. ~~bool~~ | + +## Lexeme.similarity {id="similarity",tag="method",model="vectors"} + +Compute a semantic similarity estimate. Defaults to cosine over vectors. + +> #### Example +> +> ```python +> apple = nlp.vocab["apple"] +> orange = nlp.vocab["orange"] +> apple_orange = apple.similarity(orange) +> orange_apple = orange.similarity(apple) +> assert apple_orange == orange_apple +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | +| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | + +## Lexeme.has_vector {id="has_vector",tag="property",model="vectors"} + +A boolean value indicating whether a word vector is associated with the lexeme. + +> #### Example +> +> ```python +> apple = nlp.vocab["apple"] +> assert apple.has_vector +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------- | +| **RETURNS** | Whether the lexeme has a vector data attached. ~~bool~~ | + +## Lexeme.vector {id="vector",tag="property",model="vectors"} + +A real-valued meaning representation. + +> #### Example +> +> ```python +> apple = nlp.vocab["apple"] +> assert apple.vector.dtype == "float32" +> assert apple.vector.shape == (300,) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------ | +| **RETURNS** | A 1-dimensional array representing the lexeme's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Lexeme.vector_norm {id="vector_norm",tag="property",model="vectors"} + +The L2 norm of the lexeme's vector representation. + +> #### Example +> +> ```python +> apple = nlp.vocab["apple"] +> pasta = nlp.vocab["pasta"] +> apple.vector_norm # 7.1346845626831055 +> pasta.vector_norm # 7.759851932525635 +> assert apple.vector_norm != pasta.vector_norm +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| **RETURNS** | The L2 norm of the vector representation. ~~float~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The lexeme's vocabulary. ~~Vocab~~ | +| `text` | Verbatim text content. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Lexeme.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `rank` | Sequential ID of the lexeme's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `flags` | Container of the lexeme's binary flags. ~~int~~ | +| `norm` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~int~~ | +| `norm_` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~str~~ | +| `lower` | Lowercase form of the word. ~~int~~ | +| `lower_` | Lowercase form of the word. ~~str~~ | +| `shape` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | +| `shape_` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | +| `prefix` | Length-N substring from the start of the word. Defaults to `N=1`. ~~int~~ | +| `prefix_` | Length-N substring from the start of the word. Defaults to `N=1`. ~~str~~ | +| `suffix` | Length-N substring from the end of the word. Defaults to `N=3`. ~~int~~ | +| `suffix_` | Length-N substring from the end of the word. Defaults to `N=3`. ~~str~~ | +| `is_alpha` | Does the lexeme consist of alphabetic characters? Equivalent to `lexeme.text.isalpha()`. ~~bool~~ | +| `is_ascii` | Does the lexeme consist of ASCII characters? Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. ~~bool~~ | +| `is_digit` | Does the lexeme consist of digits? Equivalent to `lexeme.text.isdigit()`. ~~bool~~ | +| `is_lower` | Is the lexeme in lowercase? Equivalent to `lexeme.text.islower()`. ~~bool~~ | +| `is_upper` | Is the lexeme in uppercase? Equivalent to `lexeme.text.isupper()`. ~~bool~~ | +| `is_title` | Is the lexeme in titlecase? Equivalent to `lexeme.text.istitle()`. ~~bool~~ | +| `is_punct` | Is the lexeme punctuation? ~~bool~~ | +| `is_left_punct` | Is the lexeme a left punctuation mark, e.g. `(`? ~~bool~~ | +| `is_right_punct` | Is the lexeme a right punctuation mark, e.g. `)`? ~~bool~~ | +| `is_space` | Does the lexeme consist of whitespace characters? Equivalent to `lexeme.text.isspace()`. ~~bool~~ | +| `is_bracket` | Is the lexeme a bracket? ~~bool~~ | +| `is_quote` | Is the lexeme a quotation mark? ~~bool~~ | +| `is_currency` | Is the lexeme a currency symbol? ~~bool~~ | +| `like_url` | Does the lexeme resemble a URL? ~~bool~~ | +| `like_num` | Does the lexeme represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | +| `like_email` | Does the lexeme resemble an email address? ~~bool~~ | +| `is_oov` | Is the lexeme out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | +| `is_stop` | Is the lexeme part of a "stop list"? ~~bool~~ | +| `lang` | Language of the parent vocabulary. ~~int~~ | +| `lang_` | Language of the parent vocabulary. ~~str~~ | +| `prob` | Smoothed log probability estimate of the lexeme's word type (context-independent entry in the vocabulary). ~~float~~ | +| `cluster` | Brown cluster ID. ~~int~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the lexeme. ~~float~~ | diff --git a/website/docs/api/lookups.mdx b/website/docs/api/lookups.mdx new file mode 100644 index 0000000..71a857c --- /dev/null +++ b/website/docs/api/lookups.mdx @@ -0,0 +1,313 @@ +--- +title: Lookups +teaser: A container for large lookup tables and dictionaries +tag: class +source: spacy/lookups.py +version: 2.2 +--- + +This class allows convenient access to large lookup tables and dictionaries, +e.g. lemmatization data or tokenizer exception lists using Bloom filters. +Lookups are available via the [`Vocab`](/api/vocab) as `vocab.lookups`, so they +can be accessed before the pipeline components are applied (e.g. in the +tokenizer and lemmatizer), as well as within the pipeline components via +`doc.vocab.lookups`. + +## Lookups.\_\_init\_\_ {id="init",tag="method"} + +Create a `Lookups` object. + +> #### Example +> +> ```python +> from spacy.lookups import Lookups +> lookups = Lookups() +> ``` + +## Lookups.\_\_len\_\_ {id="len",tag="method"} + +Get the current number of tables in the lookups. + +> #### Example +> +> ```python +> lookups = Lookups() +> assert len(lookups) == 0 +> ``` + +| Name | Description | +| ----------- | -------------------------------------------- | +| **RETURNS** | The number of tables in the lookups. ~~int~~ | + +## Lookups.\_\_contains\_\_ {id="contains",tag="method"} + +Check if the lookups contain a table of a given name. Delegates to +[`Lookups.has_table`](/api/lookups#has_table). + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table") +> assert "some_table" in lookups +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| `name` | Name of the table. ~~str~~ | +| **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | + +## Lookups.tables {id="tables",tag="property"} + +Get the names of all tables in the lookups. + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table") +> assert lookups.tables == ["some_table"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------- | +| **RETURNS** | Names of the tables in the lookups. ~~List[str]~~ | + +## Lookups.add_table {id="add_table",tag="method"} + +Add a new table with optional data to the lookups. Raises an error if the table +exists. + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table", {"foo": "bar"}) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `name` | Unique name of the table. ~~str~~ | +| `data` | Optional data to add to the table. ~~dict~~ | +| **RETURNS** | The newly added table. ~~Table~~ | + +## Lookups.get_table {id="get_table",tag="method"} + +Get a table from the lookups. Raises an error if the table doesn't exist. + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table", {"foo": "bar"}) +> table = lookups.get_table("some_table") +> assert table["foo"] == "bar" +> ``` + +| Name | Description | +| ----------- | -------------------------- | +| `name` | Name of the table. ~~str~~ | +| **RETURNS** | The table. ~~Table~~ | + +## Lookups.remove_table {id="remove_table",tag="method"} + +Remove a table from the lookups. Raises an error if the table doesn't exist. + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table") +> removed_table = lookups.remove_table("some_table") +> assert "some_table" not in lookups +> ``` + +| Name | Description | +| ----------- | ------------------------------------ | +| `name` | Name of the table to remove. ~~str~~ | +| **RETURNS** | The removed table. ~~Table~~ | + +## Lookups.has_table {id="has_table",tag="method"} + +Check if the lookups contain a table of a given name. Equivalent to +[`Lookups.__contains__`](/api/lookups#contains). + +> #### Example +> +> ```python +> lookups = Lookups() +> lookups.add_table("some_table") +> assert lookups.has_table("some_table") +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| `name` | Name of the table. ~~str~~ | +| **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | + +## Lookups.to_bytes {id="to_bytes",tag="method"} + +Serialize the lookups to a bytestring. + +> #### Example +> +> ```python +> lookup_bytes = lookups.to_bytes() +> ``` + +| Name | Description | +| ----------- | --------------------------------- | +| **RETURNS** | The serialized lookups. ~~bytes~~ | + +## Lookups.from_bytes {id="from_bytes",tag="method"} + +Load the lookups from a bytestring. + +> #### Example +> +> ```python +> lookup_bytes = lookups.to_bytes() +> lookups = Lookups() +> lookups.from_bytes(lookup_bytes) +> ``` + +| Name | Description | +| ------------ | -------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The loaded lookups. ~~Lookups~~ | + +## Lookups.to_disk {id="to_disk",tag="method"} + +Save the lookups to a directory as `lookups.bin`. Expects a path to a directory, +which will be created if it doesn't exist. + +> #### Example +> +> ```python +> lookups.to_disk("/path/to/lookups") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## Lookups.from_disk {id="from_disk",tag="method"} + +Load lookups from a directory containing a `lookups.bin`. Will skip loading if +the file doesn't exist. + +> #### Example +> +> ```python +> from spacy.lookups import Lookups +> lookups = Lookups() +> lookups.from_disk("/path/to/lookups") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The loaded lookups. ~~Lookups~~ | + +## Table {id="table",tag="class, ordererddict"} + +A table in the lookups. Subclass of `OrderedDict` that implements a slightly +more consistent and unified API and includes a Bloom filter to speed up missed +lookups. Supports **all other methods and attributes** of `OrderedDict` / +`dict`, and the customized methods listed here. Methods that get or set keys +accept both integers and strings (which will be hashed before being added to the +table). + +### Table.\_\_init\_\_ {id="table.init",tag="method"} + +Initialize a new table. + +> #### Example +> +> ```python +> from spacy.lookups import Table +> data = {"foo": "bar", "baz": 100} +> table = Table(name="some_table", data=data) +> assert "foo" in table +> assert table["foo"] == "bar" +> ``` + +| Name | Description | +| ------ | ------------------------------------------ | +| `name` | Optional table name for reference. ~~str~~ | + +### Table.from_dict {id="table.from_dict",tag="classmethod"} + +Initialize a new table from a dict. + +> #### Example +> +> ```python +> from spacy.lookups import Table +> data = {"foo": "bar", "baz": 100} +> table = Table.from_dict(data, name="some_table") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------ | +| `data` | The dictionary. ~~dict~~ | +| `name` | Optional table name for reference. ~~str~~ | +| **RETURNS** | The newly constructed object. ~~Table~~ | + +### Table.set {id="table.set",tag="method"} + +Set a new key / value pair. String keys will be hashed. Same as +`table[key] = value`. + +> #### Example +> +> ```python +> from spacy.lookups import Table +> table = Table() +> table.set("foo", "bar") +> assert table["foo"] == "bar" +> ``` + +| Name | Description | +| ------- | ---------------------------- | +| `key` | The key. ~~Union[str, int]~~ | +| `value` | The value. | + +### Table.to_bytes {id="table.to_bytes",tag="method"} + +Serialize the table to a bytestring. + +> #### Example +> +> ```python +> table_bytes = table.to_bytes() +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The serialized table. ~~bytes~~ | + +### Table.from_bytes {id="table.from_bytes",tag="method"} + +Load a table from a bytestring. + +> #### Example +> +> ```python +> table_bytes = table.to_bytes() +> table = Table() +> table.from_bytes(table_bytes) +> ``` + +| Name | Description | +| ------------ | --------------------------- | +| `bytes_data` | The data to load. ~~bytes~~ | +| **RETURNS** | The loaded table. ~~Table~~ | + +### Attributes {id="table-attributes"} + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `name` | Table name. ~~str~~ | +| `default_size` | Default size of bloom filters if no data is provided. ~~int~~ | +| `bloom` | The bloom filters. ~~preshed.BloomFilter~~ | diff --git a/website/docs/api/matcher.mdx b/website/docs/api/matcher.mdx new file mode 100644 index 0000000..c66579d --- /dev/null +++ b/website/docs/api/matcher.mdx @@ -0,0 +1,269 @@ +--- +title: Matcher +teaser: Match sequences of tokens, based on pattern rules +tag: class +source: spacy/matcher/matcher.pyx +--- + +The `Matcher` lets you find words and phrases using rules describing their token +attributes. Rules can refer to token annotations (like the text or +part-of-speech tags), as well as lexical attributes like `Token.is_punct`. +Applying the matcher to a [`Doc`](/api/doc) gives you access to the matched +tokens in context. For in-depth examples and workflows for combining rules and +statistical models, see the [usage guide](/usage/rule-based-matching) on +rule-based matching. + +## Pattern format {id="patterns"} + +> ```json +> ### Example +> [ +> {"LOWER": "i"}, +> {"LEMMA": {"IN": ["like", "love"]}}, +> {"POS": "NOUN", "OP": "+"} +> ] +> ``` + +A pattern added to the `Matcher` consists of a list of dictionaries. Each +dictionary describes **one token** and its attributes. The available token +pattern keys correspond to a number of +[`Token` attributes](/api/token#attributes). The supported attributes for +rule-based matching are: + +| Attribute | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `TEXT` | The exact verbatim text of a token. ~~str~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | +| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | +| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | +| `IS_SENT_START` | Token is start of sentence. ~~bool~~ | +| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ | +| `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | +| `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ | +| `_` | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ | + +Operators and quantifiers define **how often** a token pattern should be +matched: + +> ```json +> ### Example +> [ +> {"POS": "ADJ", "OP": "*"}, +> {"POS": "NOUN", "OP": "+"} +> {"POS": "PROPN", "OP": "{2}"} +> ] +> ``` + +| OP | Description | +| ------- | ---------------------------------------------------------------------- | +| `!` | Negate the pattern, by requiring it to match exactly 0 times. | +| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | +| `+` | Require the pattern to match 1 or more times. | +| `*` | Allow the pattern to match 0 or more times. | +| `{n}` | Require the pattern to match exactly _n_ times. | +| `{n,m}` | Require the pattern to match at least _n_ but not more than _m_ times. | +| `{n,}` | Require the pattern to match at least _n_ times. | +| `{,m}` | Require the pattern to match at most _m_ times. | + +Token patterns can also map to a **dictionary of properties** instead of a +single value to indicate whether the expected value is a member of a list or how +it compares to another value. + +> ```json +> ### Example +> [ +> {"LEMMA": {"IN": ["like", "love", "enjoy"]}}, +> {"POS": "PROPN", "LENGTH": {">=": 10}}, +> ] +> ``` + +| Attribute | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REGEX` | Attribute value matches the regular expression at any position in the string. ~~Any~~ | +| `FUZZY` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, -1)`. The default method allows a Levenshtein edit distance of at least 2 and up to 30% of the pattern string length. ~~Any~~ | +| `FUZZY1`, `FUZZY2`, ... `FUZZY9` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, N)`. The default method allows a Levenshtein edit distance of at most N (1-9). ~~Any~~ | +| `IN` | Attribute value is member of a list. ~~Any~~ | +| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | +| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | +| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | +| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ | +| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | + +As of spaCy v3.5, `REGEX` and `FUZZY` can be used in combination with `IN` and +`NOT_IN`. + +## Matcher.\_\_init\_\_ {id="init",tag="method"} + +Create the rule-based `Matcher`. If `validate=True` is set, all patterns added +to the matcher will be validated against a JSON schema and a `MatchPatternError` +is raised if problems are found. Those can include incorrect types (e.g. a +string where an integer is expected) or unexpected property names. + +> #### Example +> +> ```python +> from spacy.matcher import Matcher +> matcher = Matcher(nlp.vocab) +> ``` + +| Name | Description | +| --------------- | ----------------------------------------------------------------------------------------------------- | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `validate` | Validate all patterns added to this matcher. ~~bool~~ | +| `fuzzy_compare` | The comparison method used for the `FUZZY` operators. ~~Callable[[str, str, int], bool]~~ | + +## Matcher.\_\_call\_\_ {id="call",tag="method"} + +Find all token sequences matching the supplied patterns on the `Doc` or `Span`. + +Note that if a single label has multiple patterns associated with it, the +returned matches don't provide a way to tell which pattern was responsible for +the match. + +> #### Example +> +> ```python +> from spacy.matcher import Matcher +> +> matcher = Matcher(nlp.vocab) +> pattern = [{"LOWER": "hello"}, {"LOWER": "world"}] +> matcher.add("HelloWorld", [pattern]) +> doc = nlp("hello world!") +> matches = matcher(doc) +> ``` + +| Name | Description | +| ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ | +| _keyword-only_ | | +| `as_spans` 3 | Instead of tuples, return a list of [`Span`](/api/span) objects of the matches, with the `match_id` assigned as the span label. Defaults to `False`. ~~bool~~ | +| `allow_missing` 3 | Whether to skip checks for missing annotation for attributes included in patterns. Defaults to `False`. ~~bool~~ | +| `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | + +## Matcher.\_\_len\_\_ {id="len",tag="method",version="2"} + +Get the number of rules added to the matcher. Note that this only returns the +number of rules (identical with the number of IDs), not the number of individual +patterns. + +> #### Example +> +> ```python +> matcher = Matcher(nlp.vocab) +> assert len(matcher) == 0 +> matcher.add("Rule", [[{"ORTH": "test"}]]) +> assert len(matcher) == 1 +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The number of rules. ~~int~~ | + +## Matcher.\_\_contains\_\_ {id="contains",tag="method",version="2"} + +Check whether the matcher contains rules for a match ID. + +> #### Example +> +> ```python +> matcher = Matcher(nlp.vocab) +> assert "Rule" not in matcher +> matcher.add("Rule", [[{'ORTH': 'test'}]]) +> assert "Rule" in matcher +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------- | +| `key` | The match ID. ~~str~~ | +| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | + +## Matcher.add {id="add",tag="method",version="2"} + +Add a rule to the matcher, consisting of an ID key, one or more patterns, and an +optional callback function to act on the matches. The callback function will +receive the arguments `matcher`, `doc`, `i` and `matches`. If a pattern already +exists for the given ID, the patterns will be extended. An `on_match` callback +will be overwritten. + +> #### Example +> +> ```python +> def on_match(matcher, doc, id, matches): +> print('Matched!', matches) +> +> matcher = Matcher(nlp.vocab) +> patterns = [ +> [{"LOWER": "hello"}, {"LOWER": "world"}], +> [{"ORTH": "Google"}, {"ORTH": "Maps"}] +> ] +> matcher.add("TEST_PATTERNS", patterns, on_match=on_match) +> doc = nlp("HELLO WORLD on Google Maps.") +> matches = matcher(doc) +> ``` + + + +As of spaCy v3.0, `Matcher.add` takes a list of patterns as the second argument +(instead of a variable number of arguments). The `on_match` callback becomes an +optional keyword argument. + +```diff +patterns = [[{"TEXT": "Google"}, {"TEXT": "Now"}], [{"TEXT": "GoogleNow"}]] +- matcher.add("GoogleNow", on_match, *patterns) ++ matcher.add("GoogleNow", patterns, on_match=on_match) +``` + + + +| Name | Description | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `match_id` | An ID for the thing you're matching. ~~str~~ | +| `patterns` | Match pattern. A pattern consists of a list of dicts, where each dict describes a token. ~~List[List[Dict[str, Any]]]~~ | +| _keyword-only_ | | +| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | +| `greedy` 3 | Optional filter for greedy matches. Can either be `"FIRST"` or `"LONGEST"`. ~~Optional[str]~~ | + +## Matcher.remove {id="remove",tag="method",version="2"} + +Remove a rule from the matcher. A `KeyError` is raised if the match ID does not +exist. + +> #### Example +> +> ```python +> matcher.add("Rule", [[{"ORTH": "test"}]]) +> assert "Rule" in matcher +> matcher.remove("Rule") +> assert "Rule" not in matcher +> ``` + +| Name | Description | +| ----- | --------------------------------- | +| `key` | The ID of the match rule. ~~str~~ | + +## Matcher.get {id="get",tag="method",version="2"} + +Retrieve the pattern stored for a key. Returns the rule as an +`(on_match, patterns)` tuple containing the callback and available patterns. + +> #### Example +> +> ```python +> matcher.add("Rule", [[{"ORTH": "test"}]]) +> on_match, patterns = matcher.get("Rule") +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------------- | +| `key` | The ID of the match rule. ~~str~~ | +| **RETURNS** | The rule, as an `(on_match, patterns)` tuple. ~~Tuple[Optional[Callable], List[List[dict]]]~~ | diff --git a/website/docs/api/morphologizer.mdx b/website/docs/api/morphologizer.mdx new file mode 100644 index 0000000..8f189d1 --- /dev/null +++ b/website/docs/api/morphologizer.mdx @@ -0,0 +1,441 @@ +--- +title: Morphologizer +tag: class +source: spacy/pipeline/morphologizer.pyx +version: 3 +teaser: 'Pipeline component for predicting morphological features' +api_base_class: /api/tagger +api_string_name: morphologizer +api_trainable: true +--- + +A trainable pipeline component to predict morphological features and +coarse-grained POS tags following the Universal Dependencies +[UPOS](https://universaldependencies.org/u/pos/index.html) and +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +annotation guidelines. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions are saved to `Token.morph` and `Token.pos`. + +| Location | Value | +| ------------- | ----------------------------------------- | +| `Token.pos` | The UPOS part of speech (hash). ~~int~~ | +| `Token.pos_` | The UPOS part of speech. ~~str~~ | +| `Token.morph` | Morphological features. ~~MorphAnalysis~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.morphologizer import DEFAULT_MORPH_MODEL +> config = {"model": DEFAULT_MORPH_MODEL} +> nlp.add_pipe("morphologizer", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The model to use. Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `overwrite` 3.2 | Whether the values of existing features are overwritten. Defaults to `True`. ~~bool~~ | +| `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | +| `label_smoothing` 3.6 | [Label smoothing](https://arxiv.org/abs/1906.02629) factor. Defaults to `0.0`. ~~float~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/morphologizer.pyx +``` + +## Morphologizer.\_\_init\_\_ {id="init",tag="method"} + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +The `overwrite` and `extend` settings determine how existing annotation is +handled (with the example for existing annotation `A=B|C=D` + predicted +annotation `C=E|X=Y`): + +- `overwrite=True, extend=True`: overwrite values of existing features, add any + new features (`A=B|C=D` + `C=E|X=Y` → `A=B|C=E|X=Y`) +- `overwrite=True, extend=False`: overwrite completely, removing any existing + features (`A=B|C=D` + `C=E|X=Y` → `C=E|X=Y`) +- `overwrite=False, extend=True`: keep values of existing features, add any new + features (`A=B|C=D` + `C=E|X=Y` → `A=B|C=D|X=Y`) +- `overwrite=False, extend=False`: do not modify the existing annotation if set + (`A=B|C=D` + `C=E|X=Y` → `A=B|C=D`) + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> morphologizer = nlp.add_pipe("morphologizer") +> +> # Construction via create_pipe with custom model +> config = {"model": {"@architectures": "my_morphologizer"}} +> morphologizer = nlp.add_pipe("morphologizer", config=config) +> +> # Construction from class +> from spacy.pipeline import Morphologizer +> morphologizer = Morphologizer(nlp.vocab, model) +> ``` + +| Name | Description | +| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `overwrite` 3.2 | Whether the values of existing features are overwritten. Defaults to `True`. ~~bool~~ | +| `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | + +## Morphologizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/morphologizer#call) and [`pipe`](/api/morphologizer#pipe) +delegate to the [`predict`](/api/morphologizer#predict) and +[`set_annotations`](/api/morphologizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> morphologizer = nlp.add_pipe("morphologizer") +> # This usually happens under the hood +> processed = morphologizer(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## Morphologizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/morphologizer#call) and +[`pipe`](/api/morphologizer#pipe) delegate to the +[`predict`](/api/morphologizer#predict) and +[`set_annotations`](/api/morphologizer#set_annotations) methods. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> for doc in morphologizer.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Morphologizer.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.morphologizer] +> +> [initialize.components.morphologizer.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/morphologizer.json +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[dict]~~ | + +## Morphologizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> scores = morphologizer.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## Morphologizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> scores = morphologizer.predict([doc1, doc2]) +> morphologizer.set_annotations([doc1, doc2], scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `Morphologizer.predict`. | + +## Morphologizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/morphologizer#predict) and +[`get_loss`](/api/morphologizer#get_loss). + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> optimizer = nlp.initialize() +> losses = morphologizer.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Morphologizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> scores = morphologizer.predict([eg.predicted for eg in examples]) +> loss, d_loss = morphologizer.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## Morphologizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> optimizer = morphologizer.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Morphologizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> with morphologizer.use_params(optimizer.averages): +> morphologizer.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## Morphologizer.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. If the `Morphologizer` should set annotations for +both `pos` and `morph`, the label should include the UPOS as the feature `POS`. +Raises an error if the output dimension is already set, or if the model has +already been fully [initialized](#initialize). Note that you don't have to call +this method if you provide a **representative data sample** to the +[`initialize`](#initialize) method. In this case, all labels found in the sample +will be automatically added to the model, and the output dimension will be +[inferred](/usage/layers-architectures#thinc-shape-inference) automatically. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer.add_label("Mood=Ind|POS=VERB|Tense=Past|VerbForm=Fin") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## Morphologizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer.to_disk("/path/to/morphologizer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Morphologizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer.from_disk("/path/to/morphologizer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Morphologizer` object. ~~Morphologizer~~ | + +## Morphologizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer_bytes = morphologizer.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Morphologizer` object. ~~bytes~~ | + +## Morphologizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> morphologizer_bytes = morphologizer.to_bytes() +> morphologizer = nlp.add_pipe("morphologizer") +> morphologizer.from_bytes(morphologizer_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Morphologizer` object. ~~Morphologizer~~ | + +## Morphologizer.labels {id="labels",tag="property"} + +The labels currently added to the component in the Universal Dependencies +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +format. Note that even for a blank component, this will always include the +internal empty label `_`. If POS features are used, the labels will include the +coarse-grained POS as the feature `POS`. + +> #### Example +> +> ```python +> morphologizer.add_label("Mood=Ind|POS=VERB|Tense=Past|VerbForm=Fin") +> assert "Mood=Ind|POS=VERB|Tense=Past|VerbForm=Fin" in morphologizer.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## Morphologizer.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`Morphologizer.initialize`](/api/morphologizer#initialize) to initialize the +model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = morphologizer.label_data +> morphologizer.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~dict~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = morphologizer.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/morphology.mdx b/website/docs/api/morphology.mdx new file mode 100644 index 0000000..7f68020 --- /dev/null +++ b/website/docs/api/morphology.mdx @@ -0,0 +1,256 @@ +--- +title: Morphology +tag: class +source: spacy/morphology.pyx +--- + +Store the possible morphological analyses for a language, and index them by +hash. To save space on each token, tokens only know the hash of their +morphological analysis, so queries of morphological attributes are delegated to +this class. See [`MorphAnalysis`](/api/morphology#morphanalysis) for the +container storing a single morphological analysis. + +## Morphology.\_\_init\_\_ {id="init",tag="method"} + +Create a `Morphology` object. + +> #### Example +> +> ```python +> from spacy.morphology import Morphology +> +> morphology = Morphology(strings) +> ``` + +| Name | Description | +| --------- | --------------------------------- | +| `strings` | The string store. ~~StringStore~~ | + +## Morphology.add {id="add",tag="method"} + +Insert a morphological analysis in the morphology table, if not already present. +The morphological analysis may be provided in the Universal Dependencies +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +format as a string or in the tag map dictionary format. Returns the hash of the +new analysis. + +> #### Example +> +> ```python +> feats = "Feat1=Val1|Feat2=Val2" +> hash = nlp.vocab.morphology.add(feats) +> assert hash == nlp.vocab.strings[feats] +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------ | +| `features` | The morphological features. ~~Union[Dict, str]~~ | + +## Morphology.get {id="get",tag="method"} + +> #### Example +> +> ```python +> feats = "Feat1=Val1|Feat2=Val2" +> hash = nlp.vocab.morphology.add(feats) +> assert nlp.vocab.morphology.get(hash) == feats +> ``` + +Get the +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +string for the hash of the morphological analysis. + +| Name | Description | +| ------- | ----------------------------------------------- | +| `morph` | The hash of the morphological analysis. ~~int~~ | + +## Morphology.feats_to_dict {id="feats_to_dict",tag="staticmethod"} + +Convert a string +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +representation to a dictionary of features and values in the same format as the +tag map. + +> #### Example +> +> ```python +> from spacy.morphology import Morphology +> d = Morphology.feats_to_dict("Feat1=Val1|Feat2=Val2") +> assert d == {"Feat1": "Val1", "Feat2": "Val2"} +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `feats` | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | +| **RETURNS** | The morphological features as a dictionary. ~~Dict[str, str]~~ | + +## Morphology.dict_to_feats {id="dict_to_feats",tag="staticmethod"} + +Convert a dictionary of features and values to a string +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +representation. + +> #### Example +> +> ```python +> from spacy.morphology import Morphology +> f = Morphology.dict_to_feats({"Feat1": "Val1", "Feat2": "Val2"}) +> assert f == "Feat1=Val1|Feat2=Val2" +> ``` + +| Name | Description | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `feats_dict` | The morphological features as a dictionary. ~~Dict[str, str]~~ | +| **RETURNS** | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `\|`. ~~str~~ | +| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ | +| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ | + +## MorphAnalysis {id="morphanalysis",tag="class",source="spacy/tokens/morphanalysis.pyx"} + +Stores a single morphological analysis. + +### MorphAnalysis.\_\_init\_\_ {id="morphanalysis-init",tag="method"} + +Initialize a MorphAnalysis object from a Universal Dependencies +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +string or a dictionary of morphological features. + +> #### Example +> +> ```python +> from spacy.tokens import MorphAnalysis +> +> feats = "Feat1=Val1|Feat2=Val2" +> m = MorphAnalysis(nlp.vocab, feats) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------- | +| `vocab` | The vocab. ~~Vocab~~ | +| `features` | The morphological features. ~~Union[Dict[str, str], str]~~ | + +### MorphAnalysis.\_\_contains\_\_ {id="morphanalysis-contains",tag="method"} + +Whether a feature/value pair is in the analysis. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val2|Feat2=Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert "Feat1=Val1" in morph +> ``` + +| Name | Description | +| ------------ | --------------------------------------------------------------------- | +| `feature` | A feature/value pair. ~~str~~ | +| **RETURNS** | Whether the feature/value pair is contained in the analysis. ~~bool~~ | + +### MorphAnalysis.\_\_iter\_\_ {id="morphanalysis-iter",tag="method"} + +Iterate over the feature/value pairs in the analysis. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val3|Feat2=Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert list(morph) == ["Feat1=Va1", "Feat1=Val3", "Feat2=Val2"] +> ``` + +| Name | Description | +| ---------- | --------------------------------------------- | +| **YIELDS** | A feature/value pair in the analysis. ~~str~~ | + +### MorphAnalysis.\_\_len\_\_ {id="morphanalysis-len",tag="method"} + +Returns the number of features in the analysis. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val2|Feat2=Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert len(morph) == 3 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------- | +| **RETURNS** | The number of features in the analysis. ~~int~~ | + +### MorphAnalysis.\_\_str\_\_ {id="morphanalysis-str",tag="method"} + +Returns the morphological analysis in the Universal Dependencies +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +string format. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val2|Feat2=Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert str(morph) == feats +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| **RETURNS** | The analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | + +### MorphAnalysis.get {id="morphanalysis-get",tag="method"} + +Retrieve values for a feature by field. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert morph.get("Feat1") == ["Val1", "Val2"] +> ``` + +| Name | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `field` | The field to retrieve. ~~str~~ | +| `default` 3.5.3 | The value to return if the field is not present. If unset or `None`, the default return value is `[]`. ~~Optional[List[str]]~~ | +| **RETURNS** | A list of the individual features. ~~List[str]~~ | + +### MorphAnalysis.to_dict {id="morphanalysis-to_dict",tag="method"} + +Produce a dict representation of the analysis, in the same format as the tag +map. + +> #### Example +> +> ```python +> feats = "Feat1=Val1,Val2|Feat2=Val2" +> morph = MorphAnalysis(nlp.vocab, feats) +> assert morph.to_dict() == {"Feat1": "Val1,Val2", "Feat2": "Val2"} +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| **RETURNS** | The dict representation of the analysis. ~~Dict[str, str]~~ | + +### MorphAnalysis.from_id {id="morphanalysis-from_id",tag="classmethod"} + +Create a morphological analysis from a given hash ID. + +> #### Example +> +> ```python +> feats = "Feat1=Val1|Feat2=Val2" +> hash = nlp.vocab.strings[feats] +> morph = MorphAnalysis.from_id(nlp.vocab, hash) +> assert str(morph) == feats +> ``` + +| Name | Description | +| ------- | ---------------------------------------- | +| `vocab` | The vocab. ~~Vocab~~ | +| `key` | The hash of the features string. ~~int~~ | diff --git a/website/docs/api/phrasematcher.mdx b/website/docs/api/phrasematcher.mdx new file mode 100644 index 0000000..14ccefb --- /dev/null +++ b/website/docs/api/phrasematcher.mdx @@ -0,0 +1,175 @@ +--- +title: PhraseMatcher +teaser: Match sequences of tokens, based on documents +tag: class +source: spacy/matcher/phrasematcher.pyx +version: 2 +--- + +The `PhraseMatcher` lets you efficiently match large terminology lists. While +the [`Matcher`](/api/matcher) lets you match sequences based on lists of token +descriptions, the `PhraseMatcher` accepts match patterns in the form of `Doc` +objects. See the [usage guide](/usage/rule-based-matching#phrasematcher) for +examples. + +## PhraseMatcher.\_\_init\_\_ {id="init",tag="method"} + +Create the rule-based `PhraseMatcher`. Setting a different `attr` to match on +will change the token attributes that will be compared to determine a match. By +default, the incoming `Doc` is checked for sequences of tokens with the same +`ORTH` value, i.e. the verbatim token text. Matching on the attribute `LOWER` +will result in case-insensitive matching, since only the lowercase token texts +are compared. In theory, it's also possible to match on sequences of the same +part-of-speech tags or dependency labels. + +If `validate=True` is set, additional validation is performed when pattern are +added. At the moment, it will check whether a `Doc` has attributes assigned that +aren't necessary to produce the matches (for example, part-of-speech tags if the +`PhraseMatcher` matches on the token text). Since this can often lead to +significantly worse performance when creating the pattern, a `UserWarning` will +be shown. + +> #### Example +> +> ```python +> from spacy.matcher import PhraseMatcher +> matcher = PhraseMatcher(nlp.vocab) +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------------------------ | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `attr` | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ | +| `validate` | Validate patterns added to the matcher. ~~bool~~ | + +## PhraseMatcher.\_\_call\_\_ {id="call",tag="method"} + +Find all token sequences matching the supplied patterns on the `Doc` or `Span`. + +> #### Example +> +> ```python +> from spacy.matcher import PhraseMatcher +> +> matcher = PhraseMatcher(nlp.vocab) +> matcher.add("OBAMA", [nlp("Barack Obama")]) +> doc = nlp("Barack Obama lifts America one last time in emotional farewell") +> matches = matcher(doc) +> ``` + +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ | +| _keyword-only_ | | +| `as_spans` 3 | Instead of tuples, return a list of [`Span`](/api/span) objects of the matches, with the `match_id` assigned as the span label. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | + + + +Because spaCy stores all strings as integers, the `match_id` you get back will +be an integer, too – but you can always get the string representation by looking +it up in the vocabulary's `StringStore`, i.e. `nlp.vocab.strings`: + +```python +match_id_string = nlp.vocab.strings[match_id] +``` + + + +## PhraseMatcher.\_\_len\_\_ {id="len",tag="method"} + +Get the number of rules added to the matcher. Note that this only returns the +number of rules (identical with the number of IDs), not the number of individual +patterns. + +> #### Example +> +> ```python +> matcher = PhraseMatcher(nlp.vocab) +> assert len(matcher) == 0 +> matcher.add("OBAMA", [nlp("Barack Obama")]) +> assert len(matcher) == 1 +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The number of rules. ~~int~~ | + +## PhraseMatcher.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether the matcher contains rules for a match ID. + +> #### Example +> +> ```python +> matcher = PhraseMatcher(nlp.vocab) +> assert "OBAMA" not in matcher +> matcher.add("OBAMA", [nlp("Barack Obama")]) +> assert "OBAMA" in matcher +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------- | +| `key` | The match ID. ~~str~~ | +| **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | + +## PhraseMatcher.add {id="add",tag="method"} + +Add a rule to the matcher, consisting of an ID key, one or more patterns, and a +callback function to act on the matches. The callback function will receive the +arguments `matcher`, `doc`, `i` and `matches`. If a pattern already exists for +the given ID, the patterns will be extended. An `on_match` callback will be +overwritten. + +> #### Example +> +> ```python +> def on_match(matcher, doc, id, matches): +> print('Matched!', matches) +> +> matcher = PhraseMatcher(nlp.vocab) +> matcher.add("OBAMA", [nlp("Barack Obama")], on_match=on_match) +> matcher.add("HEALTH", [nlp("health care reform"), nlp("healthcare reform")], on_match=on_match) +> doc = nlp("Barack Obama urges Congress to find courage to defend his healthcare reforms") +> matches = matcher(doc) +> ``` + + + +As of spaCy v3.0, `PhraseMatcher.add` takes a list of patterns as the second +argument (instead of a variable number of arguments). The `on_match` callback +becomes an optional keyword argument. + +```diff +patterns = [nlp("health care reform"), nlp("healthcare reform")] +- matcher.add("HEALTH", on_match, *patterns) ++ matcher.add("HEALTH", patterns, on_match=on_match) +``` + + + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `key` | An ID for the thing you're matching. ~~str~~ | +| `docs` | `Doc` objects of the phrases to match. ~~List[Doc]~~ | +| _keyword-only_ | | +| `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | + +## PhraseMatcher.remove {id="remove",tag="method",version="2.2"} + +Remove a rule from the matcher by match ID. A `KeyError` is raised if the key +does not exist. + +> #### Example +> +> ```python +> matcher = PhraseMatcher(nlp.vocab) +> matcher.add("OBAMA", [nlp("Barack Obama")]) +> assert "OBAMA" in matcher +> matcher.remove("OBAMA") +> assert "OBAMA" not in matcher +> ``` + +| Name | Description | +| ----- | --------------------------------- | +| `key` | The ID of the match rule. ~~str~~ | diff --git a/website/docs/api/pipe.mdx b/website/docs/api/pipe.mdx new file mode 100644 index 0000000..c2777ed --- /dev/null +++ b/website/docs/api/pipe.mdx @@ -0,0 +1,539 @@ +--- +title: TrainablePipe +tag: class +teaser: Base class for trainable pipeline components +--- + +This class is a base class and **not instantiated directly**. Trainable pipeline +components like the [`EntityRecognizer`](/api/entityrecognizer) or +[`TextCategorizer`](/api/textcategorizer) inherit from it and it defines the +interface that components should follow to function as trainable components in a +spaCy pipeline. See the docs on +[writing trainable components](/usage/processing-pipelines#trainable-components) +for how to use the `TrainablePipe` base class to implement custom components. + +{/* TODO: Pipe vs TrainablePipe, check methods below (all renamed to TrainablePipe for now) */} + +> #### Why is it implemented in Cython? +> +> The `TrainablePipe` class is implemented in a `.pyx` module, the extension +> used by [Cython](/api/cython). This is needed so that **other** Cython +> classes, like the [`EntityRecognizer`](/api/entityrecognizer) can inherit from +> it. But it doesn't mean you have to implement trainable components in Cython – +> pure Python components like the [`TextCategorizer`](/api/textcategorizer) can +> also inherit from `TrainablePipe`. + +```python +%%GITHUB_SPACY/spacy/pipeline/trainable_pipe.pyx +``` + +## TrainablePipe.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> from spacy.pipeline import TrainablePipe +> from spacy.language import Language +> +> class CustomPipe(TrainablePipe): +> ... +> +> @Language.factory("your_custom_pipe", default_config={"model": MODEL}) +> def make_custom_pipe(nlp, name, model): +> return CustomPipe(nlp.vocab, model, name) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| ------- | -------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], Any]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| `**cfg` | Additional config parameters and settings. Will be available as the dictionary `cfg` and is serialized with the component. | + +## TrainablePipe.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/pipe#call) and [`pipe`](/api/pipe#pipe) delegate to the +[`predict`](/api/pipe#predict) and +[`set_annotations`](/api/pipe#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> pipe = nlp.add_pipe("your_custom_pipe") +> # This usually happens under the hood +> processed = pipe(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## TrainablePipe.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/pipe#call) and +[`pipe`](/api/pipe#pipe) delegate to the [`predict`](/api/pipe#predict) and +[`set_annotations`](/api/pipe#set_annotations) methods. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> for doc in pipe.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## TrainablePipe.set_error_handler {id="set_error_handler",tag="method",version="3"} + +Define a callback that will be invoked when an error is thrown during processing +of one or more documents with either [`__call__`](/api/pipe#call) or +[`pipe`](/api/pipe#pipe). The error handler will be invoked with the original +component's name, the component itself, the list of documents that was being +processed, and the original error. + +> #### Example +> +> ```python +> def warn_error(proc_name, proc, docs, e): +> print(f"An error occurred when applying component {proc_name}.") +> +> pipe = nlp.add_pipe("ner") +> pipe.set_error_handler(warn_error) +> ``` + +| Name | Description | +| --------------- | -------------------------------------------------------------------------------------------------------------- | +| `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | + +## TrainablePipe.get_error_handler {id="get_error_handler",tag="method",version="3"} + +Retrieve the callback that performs error handling for this component's +[`__call__`](/api/pipe#call) and [`pipe`](/api/pipe#pipe) methods. If no custom +function was previously defined with +[`set_error_handler`](/api/pipe#set_error_handler), a default function is +returned that simply reraises the exception. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("ner") +> error_handler = pipe.get_error_handler() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------- | +| **RETURNS** | The function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | + +## TrainablePipe.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. The data examples are +used to **initialize the model** of the component and can either be the full +training data or a representative sample. Initialization includes validating the +network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe.initialize(lambda: [], pipeline=nlp.pipeline) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## TrainablePipe.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + + + +This method needs to be overwritten with your own custom `predict` method. + + + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> scores = pipe.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## TrainablePipe.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + + + +This method needs to be overwritten with your own custom `set_annotations` +method. + + + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> scores = pipe.predict(docs) +> pipe.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------ | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `Tagger.predict`. | + +## TrainablePipe.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> optimizer = nlp.initialize() +> losses = pipe.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## TrainablePipe.rehearse {id="rehearse",tag="method,experimental",version="3"} + +Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the +current model to make predictions similar to an initial model, to try to address +the "catastrophic forgetting" problem. This feature is experimental. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> optimizer = nlp.resume_training() +> losses = pipe.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## TrainablePipe.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + + + +This method needs to be overwritten with your own custom `get_loss` method. + + + +> #### Example +> +> ```python +> ner = nlp.add_pipe("ner") +> scores = ner.predict([eg.predicted for eg in examples]) +> loss, d_loss = ner.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## TrainablePipe.score {id="score",tag="method",version="3"} + +Score a batch of examples. + +> #### Example +> +> ```python +> scores = pipe.score(examples) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------- | +| `examples` | The examples to score. ~~Iterable[Example]~~ | +| _keyword-only_ | +| `\*\*kwargs` | Any additional settings to pass on to the scorer. ~~Any~~ | +| **RETURNS** | The scores, e.g. produced by the [`Scorer`](/api/scorer). ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## TrainablePipe.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. Defaults to +[`Adam`](https://thinc.ai/docs/api-optimizers#adam) with default settings. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> optimizer = pipe.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## TrainablePipe.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> with pipe.use_params(optimizer.averages): +> pipe.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## TrainablePipe.finish_update {id="finish_update",tag="method"} + +Update parameters using the current parameter gradients. Defaults to calling +[`self.model.finish_update`](https://thinc.ai/docs/api-model#finish_update). + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> optimizer = nlp.initialize() +> losses = pipe.update(examples, sgd=None) +> pipe.finish_update(sgd) +> ``` + +| Name | Description | +| ----- | ------------------------------------- | +| `sgd` | An optimizer. ~~Optional[Optimizer]~~ | + +## TrainablePipe.add_label {id="add_label",tag="method"} + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe.add_label("MY_LABEL") +> ``` + +Add a new label to the pipe, to be predicted by the model. The actual +implementation depends on the specific component, but in general `add_label` +shouldn't be called if the output dimension is already set, or if the model has +already been fully [initialized](#initialize). If these conditions are violated, +the function will raise an Error. The exception to this rule is when the +component is [resizable](#is_resizable), in which case +[`set_output`](#set_output) should be called to ensure that the model is +properly resized. + + + +This method needs to be overwritten with your own custom `add_label` method. + + + +| Name | Description | +| ----------- | ------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | 0 if the label is already present, otherwise 1. ~~int~~ | + +Note that in general, you don't have to call `pipe.add_label` if you provide a +representative data sample to the [`initialize`](#initialize) method. In this +case, all labels found in the sample will be automatically added to the model, +and the output dimension will be +[inferred](/usage/layers-architectures#thinc-shape-inference) automatically. + +## TrainablePipe.is_resizable {id="is_resizable",tag="property"} + +> #### Example +> +> ```python +> can_resize = pipe.is_resizable +> ``` +> +> With custom resizing implemented by a component: +> +> ```python +> def custom_resize(model, new_nO): +> # adjust model +> return model +> +> custom_model.attrs["resize_output"] = custom_resize +> ``` + +Check whether or not the output dimension of the component's model can be +resized. If this method returns `True`, [`set_output`](#set_output) can be +called to change the model's output dimension. + +For built-in components that are not resizable, you have to create and train a +new model from scratch with the appropriate architecture and output dimension. +For custom components, you can implement a `resize_output` function and add it +as an attribute to the component's model. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------- | +| **RETURNS** | Whether or not the output dimension of the model can be changed after initialization. ~~bool~~ | + +## TrainablePipe.set_output {id="set_output",tag="method"} + +Change the output dimension of the component's model. If the component is not +[resizable](#is_resizable), this method will raise a `NotImplementedError`. If a +component is resizable, the model's attribute `resize_output` will be called. +This is a function that takes the original model and the new output dimension +`nO`, and changes the model in place. When resizing an already trained model, +care should be taken to avoid the "catastrophic forgetting" problem. + +> #### Example +> +> ```python +> if pipe.is_resizable: +> pipe.set_output(512) +> ``` + +| Name | Description | +| ---- | --------------------------------- | +| `nO` | The new output dimension. ~~int~~ | + +## TrainablePipe.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe.to_disk("/path/to/pipe") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## TrainablePipe.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe.from_disk("/path/to/pipe") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified pipe. ~~TrainablePipe~~ | + +## TrainablePipe.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe_bytes = pipe.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the pipe. ~~bytes~~ | + +## TrainablePipe.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> pipe_bytes = pipe.to_bytes() +> pipe = nlp.add_pipe("your_custom_pipe") +> pipe.from_bytes(pipe_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The pipe. ~~TrainablePipe~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary that's passed in on initialization. ~~Vocab~~ | +| `model` | The model powering the component. ~~Model[List[Doc], Any]~~ | +| `name` | The name of the component instance in the pipeline. Can be used in the losses. ~~str~~ | +| `cfg` | Keyword arguments passed to [`TrainablePipe.__init__`](/api/pipe#init). Will be serialized with the component. ~~Dict[str, Any]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = pipe.to_disk("/path") +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/pipeline-functions.mdx b/website/docs/api/pipeline-functions.mdx new file mode 100644 index 0000000..545ace2 --- /dev/null +++ b/website/docs/api/pipeline-functions.mdx @@ -0,0 +1,188 @@ +--- +title: Pipeline Functions +teaser: Other built-in pipeline components and helpers +source: spacy/pipeline/functions.py +menu: + - ['merge_noun_chunks', 'merge_noun_chunks'] + - ['merge_entities', 'merge_entities'] + - ['merge_subtokens', 'merge_subtokens'] + - ['token_splitter', 'token_splitter'] + - ['doc_cleaner', 'doc_cleaner'] +--- + +## merge_noun_chunks {id="merge_noun_chunks",tag="function"} + +Merge noun chunks into a single token. Also available via the string name +`"merge_noun_chunks"`. + +> #### Example +> +> ```python +> texts = [t.text for t in nlp("I have a blue car")] +> assert texts == ["I", "have", "a", "blue", "car"] +> +> nlp.add_pipe("merge_noun_chunks") +> texts = [t.text for t in nlp("I have a blue car")] +> assert texts == ["I", "have", "a blue car"] +> ``` + + + +Since noun chunks require part-of-speech tags and the dependency parse, make +sure to add this component _after_ the `"tagger"` and `"parser"` components. By +default, `nlp.add_pipe` will add components to the end of the pipeline and after +all other components. + + + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with merged noun chunks. ~~Doc~~ | + +## merge_entities {id="merge_entities",tag="function"} + +Merge named entities into a single token. Also available via the string name +`"merge_entities"`. + +> #### Example +> +> ```python +> texts = [t.text for t in nlp("I like David Bowie")] +> assert texts == ["I", "like", "David", "Bowie"] +> +> nlp.add_pipe("merge_entities") +> +> texts = [t.text for t in nlp("I like David Bowie")] +> assert texts == ["I", "like", "David Bowie"] +> ``` + + + +Since named entities are set by the entity recognizer, make sure to add this +component _after_ the `"ner"` component. By default, `nlp.add_pipe` will add +components to the end of the pipeline and after all other components. + + + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with merged entities. ~~Doc~~ | + +## merge_subtokens {id="merge_subtokens",tag="function",version="2.1"} + +Merge subtokens into a single token. Also available via the string name +`"merge_subtokens"`. As of v2.1, the parser is able to predict "subtokens" that +should be merged into one single token later on. This is especially relevant for +languages like Chinese, Japanese or Korean, where a "word" isn't defined as a +whitespace-delimited sequence of characters. Under the hood, this component uses +the [`Matcher`](/api/matcher) to find sequences of tokens with the dependency +label `"subtok"` and then merges them into a single token. + +> #### Example +> +> Note that this example assumes a custom Chinese model that oversegments and +> was trained to predict subtokens. +> +> ```python +> doc = nlp("拜托") +> print([(token.text, token.dep_) for token in doc]) +> # [('拜', 'subtok'), ('托', 'subtok')] +> +> nlp.add_pipe("merge_subtokens") +> doc = nlp("拜托") +> print([token.text for token in doc]) +> # ['拜托'] +> ``` + + + +Since subtokens are set by the parser, make sure to add this component _after_ +the `"parser"` component. By default, `nlp.add_pipe` will add components to the +end of the pipeline and after all other components. + + + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| `label` | The subtoken dependency label. Defaults to `"subtok"`. ~~str~~ | +| **RETURNS** | The modified `Doc` with merged subtokens. ~~Doc~~ | + +## token_splitter {id="token_splitter",tag="function",version="3.0"} + +Split tokens longer than a minimum length into shorter tokens. Intended for use +with transformer pipelines where long spaCy tokens lead to input text that +exceed the transformer model max length. + +> #### Example +> +> ```python +> config = {"min_length": 20, "split_length": 5} +> nlp.add_pipe("token_splitter", config=config, first=True) +> doc = nlp("aaaaabbbbbcccccdddddee") +> print([token.text for token in doc]) +> # ['aaaaa', 'bbbbb', 'ccccc', 'ddddd', 'ee'] +> ``` + +| Setting | Description | +| -------------- | --------------------------------------------------------------------- | +| `min_length` | The minimum length for a token to be split. Defaults to `25`. ~~int~~ | +| `split_length` | The length of the split tokens. Defaults to `5`. ~~int~~ | +| **RETURNS** | The modified `Doc` with the split tokens. ~~Doc~~ | + +## doc_cleaner {id="doc_cleaner",tag="function",version="3.2.1"} + +Clean up `Doc` attributes. Intended for use at the end of pipelines with +`tok2vec` or `transformer` pipeline components that store tensors and other +values that can require a lot of memory and frequently aren't needed after the +whole pipeline has run. + +> #### Example +> +> ```python +> config = {"attrs": {"tensor": None}} +> nlp.add_pipe("doc_cleaner", config=config) +> doc = nlp("text") +> assert doc.tensor is None +> ``` + +| Setting | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `attrs` | A dict of the `Doc` attributes and the values to set them to. Defaults to `{"tensor": None, "_.trf_data": None}` to clean up after `tok2vec` and `transformer` components. ~~dict~~ | +| `silent` | If `False`, show warnings if attributes aren't found or can't be set. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The modified `Doc` with the modified attributes. ~~Doc~~ | + +## span_cleaner {id="span_cleaner",tag="function,experimental"} + +Remove `SpanGroup`s from `doc.spans` based on a key prefix. This is used to +clean up after the [`CoreferenceResolver`](/api/coref) when it's paired with a +[`SpanResolver`](/api/span-resolver). + + + +This pipeline function is not yet integrated into spaCy core, and is available +via the extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "span_cleaner"` in your +[training config](/usage/training#config) or `nlp.add_pipe("span_cleaner")` will +work out-of-the-box. + + + +> #### Example +> +> ```python +> config = {"prefix": "coref_head_clusters"} +> nlp.add_pipe("span_cleaner", config=config) +> doc = nlp("text") +> assert "coref_head_clusters_1" not in doc.spans +> ``` + +| Setting | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------- | +| `prefix` | A prefix to check `SpanGroup` keys for. Any matching groups will be removed. Defaults to `"coref_head_clusters"`. ~~str~~ | +| **RETURNS** | The modified `Doc` with any matching spans removed. ~~Doc~~ | diff --git a/website/docs/api/scorer.mdx b/website/docs/api/scorer.mdx new file mode 100644 index 0000000..9bdd0a8 --- /dev/null +++ b/website/docs/api/scorer.mdx @@ -0,0 +1,334 @@ +--- +title: Scorer +teaser: Compute evaluation scores +tag: class +source: spacy/scorer.py +--- + +The `Scorer` computes evaluation scores. It's typically created by +[`Language.evaluate`](/api/language#evaluate). In addition, the `Scorer` +provides a number of evaluation methods for evaluating [`Token`](/api/token) and +[`Doc`](/api/doc) attributes. + +## Scorer.\_\_init\_\_ {id="init",tag="method"} + +Create a new `Scorer`. + +> #### Example +> +> ```python +> from spacy.scorer import Scorer +> +> # Default scoring pipeline +> scorer = Scorer() +> +> # Provided scoring pipeline +> nlp = spacy.load("en_core_web_sm") +> scorer = Scorer(nlp) +> ``` + +| Name | Description | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The pipeline to use for scoring, where each pipeline component may provide a scoring method. If none is provided, then a default pipeline is constructed using the `default_lang` and `default_pipeline` settings. ~~Optional[Language]~~ | +| `default_lang` | The language to use for a default pipeline if `nlp` is not provided. Defaults to `xx`. ~~str~~ | +| `default_pipeline` | The pipeline components to use for a default pipeline if `nlp` is not provided. Defaults to `("senter", "tagger", "morphologizer", "parser", "ner", "textcat")`. ~~Iterable[string]~~ | +| _keyword-only_ | | +| `**kwargs` | Any additional settings to pass on to the individual scoring methods. ~~Any~~ | + +## Scorer.score {id="score",tag="method"} + +Calculate the scores for a list of [`Example`](/api/example) objects using the +scoring methods provided by the components in the pipeline. + +The returned `Dict` contains the scores provided by the individual pipeline +components. For the scoring methods provided by the `Scorer` and used by the +core pipeline components, the individual score names start with the `Token` or +`Doc` attribute being scored: + +- `token_acc`, `token_p`, `token_r`, `token_f` +- `sents_p`, `sents_r`, `sents_f` +- `tag_acc` +- `pos_acc` +- `morph_acc`, `morph_micro_p`, `morph_micro_r`, `morph_micro_f`, + `morph_per_feat` +- `lemma_acc` +- `dep_uas`, `dep_las`, `dep_las_per_type` +- `ents_p`, `ents_r` `ents_f`, `ents_per_type` +- `spans_sc_p`, `spans_sc_r`, `spans_sc_f` +- `cats_score` (depends on config, description provided in `cats_score_desc`), + `cats_micro_p`, `cats_micro_r`, `cats_micro_f`, `cats_macro_p`, + `cats_macro_r`, `cats_macro_f`, `cats_macro_auc`, `cats_f_per_type`, + `cats_auc_per_type` + +> #### Example +> +> ```python +> scorer = Scorer() +> scores = scorer.score(examples) +> ``` + +| Name | Description | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `per_component` 3.6 | Whether to return the scores keyed by component name. Defaults to `False`. ~~bool~~ | +| **RETURNS** | A dictionary of scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## Scorer.score_tokenization {id="score_tokenization",tag="staticmethod",version="3"} + +Scores the tokenization: + +- `token_acc`: number of correct tokens / number of predicted tokens +- `token_p`, `token_r`, `token_f`: precision, recall and F-score for token + character spans + +Docs with `has_unknown_spaces` are skipped during scoring. + +> #### Example +> +> ```python +> scores = Scorer.score_tokenization(examples) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| **RETURNS** | `Dict` | A dictionary containing the scores `token_acc`, `token_p`, `token_r`, `token_f`. ~~Dict[str, float]]~~ | + +## Scorer.score_token_attr {id="score_token_attr",tag="staticmethod",version="3"} + +Scores a single token attribute. Tokens with missing values in the reference doc +are skipped during scoring. + +> #### Example +> +> ```python +> scores = Scorer.score_token_attr(examples, "pos") +> print(scores["pos_acc"]) +> ``` + +| Name | Description | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(token, attr)` should return the value of the attribute for an individual `Token`. ~~Callable[[Token, str], Any]~~ | +| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | +| **RETURNS** | A dictionary containing the score `{attr}_acc`. ~~Dict[str, float]~~ | + +## Scorer.score_token_attr_per_feat {id="score_token_attr_per_feat",tag="staticmethod",version="3"} + +Scores a single token attribute per feature for a token attribute in the +Universal Dependencies +[FEATS](https://universaldependencies.org/format.html#morphological-annotation) +format. Tokens with missing values in the reference doc are skipped during +scoring. + +> #### Example +> +> ```python +> scores = Scorer.score_token_attr_per_feat(examples, "morph") +> print(scores["morph_per_feat"]) +> ``` + +| Name | Description | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(token, attr)` should return the value of the attribute for an individual `Token`. ~~Callable[[Token, str], Any]~~ | +| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | +| **RETURNS** | A dictionary containing the micro PRF scores under the key `{attr}_micro_p/r/f` and the per-feature PRF scores under `{attr}_per_feat`. ~~Dict[str, Dict[str, float]]~~ | + +## Scorer.score_spans {id="score_spans",tag="staticmethod",version="3"} + +Returns PRF scores for labeled or unlabeled spans. + +> #### Example +> +> ```python +> scores = Scorer.score_spans(examples, "ents") +> print(scores["ents_f"]) +> ``` + +| Name | Description | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(doc, attr)` should return the `Span` objects for an individual `Doc`. ~~Callable[[Doc, str], Iterable[Span]]~~ | +| `has_annotation` | Defaults to `None`. If provided, `has_annotation(doc)` should return whether a `Doc` has annotation for this `attr`. Docs without annotation are skipped for scoring purposes. ~~str~~ | +| `labeled` | Defaults to `True`. If set to `False`, two spans will be considered equal if their start and end match, irrespective of their label. ~~bool~~ | +| `allow_overlap` | Defaults to `False`. Whether or not to allow overlapping spans. If set to `False`, the alignment will automatically resolve conflicts. ~~bool~~ | +| **RETURNS** | A dictionary containing the PRF scores under the keys `{attr}_p`, `{attr}_r`, `{attr}_f` and the per-type PRF scores under `{attr}_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## Scorer.score_deps {id="score_deps",tag="staticmethod",version="3"} + +Calculate the UAS, LAS, and LAS per type scores for dependency parses. Tokens +with missing values for the `attr` (typically `dep`) are skipped during scoring. + +> #### Example +> +> ```python +> def dep_getter(token, attr): +> dep = getattr(token, attr) +> dep = token.vocab.strings.as_string(dep).lower() +> return dep +> +> scores = Scorer.score_deps( +> examples, +> "dep", +> getter=dep_getter, +> ignore_labels=("p", "punct") +> ) +> print(scores["dep_uas"], scores["dep_las"]) +> ``` + +| Name | Description | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(token, attr)` should return the value of the attribute for an individual `Token`. ~~Callable[[Token, str], Any]~~ | +| `head_attr` | The attribute containing the head token. ~~str~~ | +| `head_getter` | Defaults to `getattr`. If provided, `head_getter(token, attr)` should return the head for an individual `Token`. ~~Callable[[Doc, str], Token]~~ | +| `ignore_labels` | Labels to ignore while scoring (e.g. `"punct"`). ~~Iterable[str]~~ | +| `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | +| **RETURNS** | A dictionary containing the scores: `{attr}_uas`, `{attr}_las`, and `{attr}_las_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## Scorer.score_cats {id="score_cats",tag="staticmethod",version="3"} + +Calculate PRF and ROC AUC scores for a doc-level attribute that is a dict +containing scores for each label like `Doc.cats`. The returned dictionary +contains the following scores: + +- `{attr}_micro_p`, `{attr}_micro_r` and `{attr}_micro_f`: each instance across + each label is weighted equally +- `{attr}_macro_p`, `{attr}_macro_r` and `{attr}_macro_f`: the average values + across evaluations per label +- `{attr}_f_per_type` and `{attr}_auc_per_type`: each contains a dictionary of + scores, keyed by label +- A final `{attr}_score` and corresponding `{attr}_score_desc` (text + description) + +The reported `{attr}_score` depends on the classification properties: + +- **binary exclusive with positive label:** `{attr}_score` is set to the F-score + of the positive label +- **3+ exclusive classes**, macro-averaged F-score: + `{attr}_score = {attr}_macro_f` +- **multilabel**, macro-averaged AUC: `{attr}_score = {attr}_macro_auc` + +> #### Example +> +> ```python +> labels = ["LABEL_A", "LABEL_B", "LABEL_C"] +> scores = Scorer.score_cats( +> examples, +> "cats", +> labels=labels +> ) +> print(scores["cats_macro_auc"]) +> ``` + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(doc, attr)` should return the cats for an individual `Doc`. ~~Callable[[Doc, str], Dict[str, float]]~~ | +| labels | The set of possible labels. Defaults to `[]`. ~~Iterable[str]~~ | +| `multi_label` | Whether the attribute allows multiple labels. Defaults to `True`. When set to `False` (exclusive labels), missing gold labels are interpreted as `0.0` and the threshold is set to `0.0`. ~~bool~~ | +| `positive_label` | The positive label for a binary task with exclusive classes. Defaults to `None`. ~~Optional[str]~~ | +| `threshold` | Cutoff to consider a prediction "positive". Defaults to `0.5` for multi-label, and `0.0` (i.e. whatever's highest scoring) otherwise. ~~float~~ | +| **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ | + +## Scorer.score_links {id="score_links",tag="staticmethod",version="3"} + +Returns PRF for predicted links on the entity level. To disentangle the +performance of the NEL from the NER, this method only evaluates NEL links for +entities that overlap between the gold reference and the predictions. + +> #### Example +> +> ```python +> scores = Scorer.score_links( +> examples, +> negative_labels=["NIL", ""] +> ) +> print(scores["nel_micro_f"]) +> ``` + +| Name | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `negative_labels` | The string values that refer to no annotation (e.g. "NIL"). ~~Iterable[str]~~ | +| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | + +## get_ner_prf {id="get_ner_prf",version="3"} + +Compute micro-PRF and per-entity PRF scores. + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | + +## score_coref_clusters {id="score_coref_clusters",tag="experimental"} + +Returns LEA ([Moosavi and Strube, 2016](https://aclanthology.org/P16-1060/)) PRF +scores for coreference clusters. + + + +Note this scoring function is not yet included in spaCy core - for details, see +the [CoreferenceResolver](/api/coref) docs. + + + +> #### Example +> +> ```python +> scores = score_coref_clusters( +> examples, +> span_cluster_prefix="coref_clusters", +> ) +> print(scores["coref_f"]) +> ``` + +| Name | Description | +| --------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `span_cluster_prefix` | The prefix used for spans representing coreference clusters. ~~str~~ | +| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | + +## score_span_predictions {id="score_span_predictions",tag="experimental"} + +Return accuracy for reconstructions of spans from single tokens. Only exactly +correct predictions are counted as correct, there is no partial credit for near +answers. Used by the [SpanResolver](/api/span-resolver). + + + +Note this scoring function is not yet included in spaCy core - for details, see +the [SpanResolver](/api/span-resolver) docs. + + + +> #### Example +> +> ```python +> scores = score_span_predictions( +> examples, +> output_prefix="coref_clusters", +> ) +> print(scores["span_coref_clusters_accuracy"]) +> ``` + +| Name | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `output_prefix` | The prefix used for spans representing the final predicted spans. ~~str~~ | +| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | diff --git a/website/docs/api/sentencerecognizer.mdx b/website/docs/api/sentencerecognizer.mdx new file mode 100644 index 0000000..5435399 --- /dev/null +++ b/website/docs/api/sentencerecognizer.mdx @@ -0,0 +1,375 @@ +--- +title: SentenceRecognizer +tag: class +source: spacy/pipeline/senter.pyx +version: 3 +teaser: 'Pipeline component for sentence segmentation' +api_base_class: /api/tagger +api_string_name: senter +api_trainable: true +--- + +A trainable pipeline component for sentence segmentation. For a simpler, +rule-based strategy, see the [`Sentencizer`](/api/sentencizer). + +## Assigned Attributes {id="assigned-attributes"} + +Predicted values will be assigned to `Token.is_sent_start`. The resulting +sentences can be accessed using `Doc.sents`. + +| Location | Value | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | +| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.senter import DEFAULT_SENTER_MODEL +> config = {"model": DEFAULT_SENTER_MODEL,} +> nlp.add_pipe("senter", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/senter.pyx +``` + +## SentenceRecognizer.\_\_init\_\_ {id="init",tag="method"} + +Initialize the sentence recognizer. + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> senter = nlp.add_pipe("senter") +> +> # Construction via create_pipe with custom model +> config = {"model": {"@architectures": "my_senter"}} +> senter = nlp.add_pipe("senter", config=config) +> +> # Construction from class +> from spacy.pipeline import SentenceRecognizer +> senter = SentenceRecognizer(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | + +## SentenceRecognizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/sentencerecognizer#call) and +[`pipe`](/api/sentencerecognizer#pipe) delegate to the +[`predict`](/api/sentencerecognizer#predict) and +[`set_annotations`](/api/sentencerecognizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> senter = nlp.add_pipe("senter") +> # This usually happens under the hood +> processed = senter(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## SentenceRecognizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/sentencerecognizer#call) +and [`pipe`](/api/sentencerecognizer#pipe) delegate to the +[`predict`](/api/sentencerecognizer#predict) and +[`set_annotations`](/api/sentencerecognizer#set_annotations) methods. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> for doc in senter.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## SentenceRecognizer.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> senter.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## SentenceRecognizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> scores = senter.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## SentenceRecognizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> scores = senter.predict([doc1, doc2]) +> senter.set_annotations([doc1, doc2], scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------------------ | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `SentenceRecognizer.predict`. | + +## SentenceRecognizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/sentencerecognizer#predict) and +[`get_loss`](/api/sentencerecognizer#get_loss). + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> optimizer = nlp.initialize() +> losses = senter.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SentenceRecognizer.rehearse {id="rehearse",tag="method,experimental",version="3"} + +Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the +current model to make predictions similar to an initial model to try to address +the "catastrophic forgetting" problem. This feature is experimental. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> optimizer = nlp.resume_training() +> losses = senter.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SentenceRecognizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> scores = senter.predict([eg.predicted for eg in examples]) +> loss, d_loss = senter.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## SentenceRecognizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> optimizer = senter.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## SentenceRecognizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> with senter.use_params(optimizer.averages): +> senter.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## SentenceRecognizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> senter.to_disk("/path/to/senter") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## SentenceRecognizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> senter.from_disk("/path/to/senter") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `SentenceRecognizer` object. ~~SentenceRecognizer~~ | + +## SentenceRecognizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> senter = nlp.add_pipe("senter") +> senter_bytes = senter.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `SentenceRecognizer` object. ~~bytes~~ | + +## SentenceRecognizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> senter_bytes = senter.to_bytes() +> senter = nlp.add_pipe("senter") +> senter.from_bytes(senter_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `SentenceRecognizer` object. ~~SentenceRecognizer~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = senter.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/sentencizer.mdx b/website/docs/api/sentencizer.mdx new file mode 100644 index 0000000..9fb5ea7 --- /dev/null +++ b/website/docs/api/sentencizer.mdx @@ -0,0 +1,195 @@ +--- +title: Sentencizer +tag: class +source: spacy/pipeline/sentencizer.pyx +teaser: 'Pipeline component for rule-based sentence boundary detection' +api_string_name: sentencizer +api_trainable: false +--- + +A simple pipeline component to allow custom sentence boundary detection logic +that doesn't require the dependency parse. By default, sentence segmentation is +performed by the [`DependencyParser`](/api/dependencyparser), so the +`Sentencizer` lets you implement a simpler, rule-based strategy that doesn't +require a statistical model to be loaded. + +## Assigned Attributes {id="assigned-attributes"} + +Calculated values will be assigned to `Token.is_sent_start`. The resulting +sentences can be accessed using `Doc.sents`. + +| Location | Value | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | +| `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). + +> #### Example +> +> ```python +> config = {"punct_chars": None} +> nlp.add_pipe("sentencizer", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/sentencizer.pyx +``` + +## Sentencizer.\_\_init\_\_ {id="init",tag="method"} + +Initialize the sentencizer. + +> #### Example +> +> ```python +> # Construction via add_pipe +> sentencizer = nlp.add_pipe("sentencizer") +> +> # Construction from class +> from spacy.pipeline import Sentencizer +> sentencizer = Sentencizer() +> ``` + +| Name | Description | +| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults. ~~Optional[List[str]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | + +```python {title="punct_chars defaults"} +['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', '।', '॥', '၊', '။', '።', + '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', '᥅', '᪨', '᪩', '᪪', '᪫', + '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', '‼', '‽', '⁇', '⁈', '⁉', + '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶', '꡷', '꣎', '꣏', '꤯', '꧈', + '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒', '﹖', '﹗', '！', '．', '？', + '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀', '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', + '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼', '𑊩', '𑑋', '𑑌', '𑗂', + '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐', '𑗑', '𑗒', '𑗓', + '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂', '𑩃', '𑪛', + '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', '｡', '。'] +``` + +## Sentencizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the sentencizer on a `Doc`. Typically, this happens automatically after +the component has been added to the pipeline using +[`nlp.add_pipe`](/api/language#add_pipe). + +> #### Example +> +> ```python +> from spacy.lang.en import English +> +> nlp = English() +> nlp.add_pipe("sentencizer") +> doc = nlp("This is a sentence. This is another sentence.") +> assert len(list(doc.sents)) == 2 +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with added sentence boundaries. ~~Doc~~ | + +## Sentencizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. + +> #### Example +> +> ```python +> sentencizer = nlp.add_pipe("sentencizer") +> for doc in sentencizer.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Sentencizer.to_disk {id="to_disk",tag="method"} + +Save the sentencizer settings (punctuation characters) to a directory. Will +create a file `sentencizer.json`. This also happens automatically when you save +an `nlp` object with a sentencizer added to its pipeline. + +> #### Example +> +> ```python +> config = {"punct_chars": [".", "?", "!", "。"]} +> sentencizer = nlp.add_pipe("sentencizer", config=config) +> sentencizer.to_disk("/path/to/sentencizer.json") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a JSON file, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## Sentencizer.from_disk {id="from_disk",tag="method"} + +Load the sentencizer settings from a file. Expects a JSON file. This also +happens automatically when you load an `nlp` object or model with a sentencizer +added to its pipeline. + +> #### Example +> +> ```python +> sentencizer = nlp.add_pipe("sentencizer") +> sentencizer.from_disk("/path/to/sentencizer.json") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a JSON file. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `Sentencizer` object. ~~Sentencizer~~ | + +## Sentencizer.to_bytes {id="to_bytes",tag="method"} + +Serialize the sentencizer settings to a bytestring. + +> #### Example +> +> ```python +> config = {"punct_chars": [".", "?", "!", "。"]} +> sentencizer = nlp.add_pipe("sentencizer", config=config) +> sentencizer_bytes = sentencizer.to_bytes() +> ``` + +| Name | Description | +| ----------- | ------------------------------ | +| **RETURNS** | The serialized data. ~~bytes~~ | + +## Sentencizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> sentencizer_bytes = sentencizer.to_bytes() +> sentencizer = nlp.add_pipe("sentencizer") +> sentencizer.from_bytes(sentencizer_bytes) +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------- | +| `bytes_data` | The bytestring to load. ~~bytes~~ | +| **RETURNS** | The modified `Sentencizer` object. ~~Sentencizer~~ | diff --git a/website/docs/api/span-resolver.mdx b/website/docs/api/span-resolver.mdx new file mode 100644 index 0000000..f061d8d --- /dev/null +++ b/website/docs/api/span-resolver.mdx @@ -0,0 +1,356 @@ +--- +title: SpanResolver +tag: class,experimental +source: spacy-experimental/coref/span_resolver_component.py +teaser: 'Pipeline component for resolving tokens into spans' +api_base_class: /api/pipe +api_string_name: span_resolver +api_trainable: true +--- + +> #### Installation +> +> ```bash +> $ pip install -U spacy-experimental +> ``` + + + +This component not yet integrated into spaCy core, and is available via the +extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "experimental_span_resolver"` in your +[training config](/usage/training#config) or +`nlp.add_pipe("experimental_span_resolver")` will work out-of-the-box. + + + +A `SpanResolver` component takes in tokens (represented as `Span` objects of +length 1) and resolves them into `Span` objects of arbitrary length. The initial +use case is as a post-processing step on word-level +[coreference resolution](/api/coref). The input and output keys used to store +`Span` objects are configurable. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `Doc.spans` as [`SpanGroup`s](/api/spangroup). + +Input token spans will be read in using an input prefix, by default +`"coref_head_clusters"`, and output spans will be saved using an output prefix +(default `"coref_clusters"`) plus a serial number starting from one. The +prefixes are configurable. + +| Location | Value | +| ------------------------------------------------- | ------------------------------------------------------------------------- | +| `Doc.spans[output_prefix + "_" + cluster_number]` | One group of predicted spans. Cluster number starts from 1. ~~SpanGroup~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#coref-architectures) documentation for +details on the architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy_experimental.coref.span_resolver_component import DEFAULT_SPAN_RESOLVER_MODEL +> from spacy_experimental.coref.coref_util import DEFAULT_CLUSTER_PREFIX, DEFAULT_CLUSTER_HEAD_PREFIX +> config={ +> "model": DEFAULT_SPAN_RESOLVER_MODEL, +> "input_prefix": DEFAULT_CLUSTER_HEAD_PREFIX, +> "output_prefix": DEFAULT_CLUSTER_PREFIX, +> }, +> nlp.add_pipe("experimental_span_resolver", config=config) +> ``` + +| Setting | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [SpanResolver](/api/architectures#SpanResolver). ~~Model~~ | +| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | +| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | + +## SpanResolver.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_span_resolver.v1"}} +> span_resolver = nlp.add_pipe("experimental_span_resolver", config=config) +> +> # Construction from class +> from spacy_experimental.coref.span_resolver_component import SpanResolver +> span_resolver = SpanResolver(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | +| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | + +## SpanResolver.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](#call) and [`pipe`](#pipe) delegate to the [`predict`](#predict) +and [`set_annotations`](#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> # This usually happens under the hood +> processed = span_resolver(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## SpanResolver.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/span-resolver#call) and +[`pipe`](/api/span-resolver#pipe) delegate to the +[`predict`](/api/span-resolver#predict) and +[`set_annotations`](/api/span-resolver#set_annotations) methods. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> for doc in span_resolver.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## SpanResolver.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## SpanResolver.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. Predictions are returned as a list of `MentionClusters`, one for +each input `Doc`. A `MentionClusters` instance is just a list of lists of pairs +of `int`s, where each item corresponds to an input `SpanGroup`, and the `int`s +correspond to token indices. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> spans = span_resolver.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The predicted spans for the `Doc`s. ~~List[MentionClusters]~~ | + +## SpanResolver.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of documents, saving predictions using the output prefix in +`Doc.spans`. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> spans = span_resolver.predict([doc1, doc2]) +> span_resolver.set_annotations([doc1, doc2], spans) +> ``` + +| Name | Description | +| ------- | ------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `spans` | The predicted spans for the `docs`. ~~List[MentionClusters]~~ | + +## SpanResolver.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects. Delegates to +[`predict`](/api/span-resolver#predict). + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> optimizer = nlp.initialize() +> losses = span_resolver.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SpanResolver.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> optimizer = span_resolver.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## SpanResolver.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> with span_resolver.use_params(optimizer.averages): +> span_resolver.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## SpanResolver.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.to_disk("/path/to/span_resolver") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## SpanResolver.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.from_disk("/path/to/span_resolver") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `SpanResolver` object. ~~SpanResolver~~ | + +## SpanResolver.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver_bytes = span_resolver.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `SpanResolver` object. ~~bytes~~ | + +## SpanResolver.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_resolver_bytes = span_resolver.to_bytes() +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.from_bytes(span_resolver_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `SpanResolver` object. ~~SpanResolver~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = span_resolver.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/span.mdx b/website/docs/api/span.mdx new file mode 100644 index 0000000..d440153 --- /dev/null +++ b/website/docs/api/span.mdx @@ -0,0 +1,572 @@ +--- +title: Span +tag: class +source: spacy/tokens/span.pyx +--- + +A slice from a [`Doc`](/api/doc) object. + +## Span.\_\_init\_\_ {id="init",tag="method"} + +Create a `Span` object from the slice `doc[start : end]`. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:4] +> assert [t.text for t in span] == ["it", "back", "!"] +> ``` + +| Name | Description | +| ------------- | --------------------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `start` | The index of the first token of the span. ~~int~~ | +| `end` | The index of the first token after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `vector_norm` | The L2 norm of the document's vector representation. ~~float~~ | +| `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | +| `span_id` | An ID to associate with the span. ~~Union[str, int]~~ | + +## Span.\_\_getitem\_\_ {id="getitem",tag="method"} + +Get a `Token` object. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:4] +> assert span[1].text == "back" +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------- | +| `i` | The index of the token within the span. ~~int~~ | +| **RETURNS** | The token at `span[i]`. ~~Token~~ | + +Get a `Span` object. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:4] +> assert span[1:3].text == "back!" +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------- | +| `start_end` | The slice of the span to get. ~~Tuple[int, int]~~ | +| **RETURNS** | The span at `span[start : end]`. ~~Span~~ | + +## Span.\_\_iter\_\_ {id="iter",tag="method"} + +Iterate over `Token` objects. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:4] +> assert [t.text for t in span] == ["it", "back", "!"] +> ``` + +| Name | Description | +| ---------- | --------------------------- | +| **YIELDS** | A `Token` object. ~~Token~~ | + +## Span.\_\_len\_\_ {id="len",tag="method"} + +Get the number of tokens in the span. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:4] +> assert len(span) == 3 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------- | +| **RETURNS** | The number of tokens in the span. ~~int~~ | + +## Span.set_extension {id="set_extension",tag="classmethod",version="2"} + +Define a custom attribute on the `Span` which becomes available via `Span._`. +For details, see the documentation on +[custom attributes](/usage/processing-pipelines#custom-components-attributes). + +> #### Example +> +> ```python +> from spacy.tokens import Span +> city_getter = lambda span: any(city in span.text for city in ("New York", "Paris", "Berlin")) +> Span.set_extension("has_city", getter=city_getter) +> doc = nlp("I like New York in Autumn") +> assert doc[1:4]._.has_city +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the attribute to set by the extension. For example, `"my_attr"` will be available as `span._.my_attr`. ~~str~~ | +| `default` | Optional default value of the attribute if no getter or method is defined. ~~Optional[Any]~~ | +| `method` | Set a custom method on the object, for example `span._.compare(other_span)`. ~~Optional[Callable[[Span, ...], Any]]~~ | +| `getter` | Getter function that takes the object and returns an attribute value. Is called when the user accesses the `._` attribute. ~~Optional[Callable[[Span], Any]]~~ | +| `setter` | Setter function that takes the `Span` and a value, and modifies the object. Is called when the user writes to the `Span._` attribute. ~~Optional[Callable[[Span, Any], None]]~~ | +| `force` | Force overwriting existing attribute. ~~bool~~ | + +## Span.get_extension {id="get_extension",tag="classmethod",version="2"} + +Look up a previously registered extension by name. Returns a 4-tuple +`(default, method, getter, setter)` if the extension is registered. Raises a +`KeyError` otherwise. + +> #### Example +> +> ```python +> from spacy.tokens import Span +> Span.set_extension("is_city", default=False) +> extension = Span.get_extension("is_city") +> assert extension == (False, None, None, None) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Span.has_extension {id="has_extension",tag="classmethod",version="2"} + +Check whether an extension has been registered on the `Span` class. + +> #### Example +> +> ```python +> from spacy.tokens import Span +> Span.set_extension("is_city", default=False) +> assert Span.has_extension("is_city") +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `name` | Name of the extension to check. ~~str~~ | +| **RETURNS** | Whether the extension has been registered. ~~bool~~ | + +## Span.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} + +Remove a previously registered extension. + +> #### Example +> +> ```python +> from spacy.tokens import Span +> Span.set_extension("is_city", default=False) +> removed = Span.remove_extension("is_city") +> assert not Span.has_extension("is_city") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Span.char_span {id="char_span",tag="method",version="2.2.4"} + +Create a `Span` object from the slice `span.text[start:end]`. Returns `None` if +the character indices don't map to a valid span. + +> #### Example +> +> ```python +> doc = nlp("I like New York") +> span = doc[1:4].char_span(5, 13, label="GPE") +> assert span.text == "New York" +> ``` + +| Name | Description | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `id` | Unused. ~~Union[int, str]~~ | +| `alignment_mode` 3.5.1 | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | +| `span_id` 3.5.1 | An identifier to associate with the span. ~~Union[int, str]~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | + +## Span.similarity {id="similarity",tag="method",model="vectors"} + +Make a semantic similarity estimate. The default estimate is cosine similarity +using an average of word vectors. + +> #### Example +> +> ```python +> doc = nlp("green apples and red oranges") +> green_apples = doc[:2] +> red_oranges = doc[3:] +> apples_oranges = green_apples.similarity(red_oranges) +> oranges_apples = red_oranges.similarity(green_apples) +> assert apples_oranges == oranges_apples +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | +| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | + +## Span.get_lca_matrix {id="get_lca_matrix",tag="method"} + +Calculates the lowest common ancestor matrix for a given `Span`. Returns LCA +matrix containing the integer index of the ancestor, or `-1` if no common +ancestor is found, e.g. if span excludes a necessary ancestor. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn") +> span = doc[1:4] +> matrix = span.get_lca_matrix() +> # array([[0, 0, 0], [0, 1, 2], [0, 2, 2]], dtype=int32) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| **RETURNS** | The lowest common ancestor matrix of the `Span`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | + +## Span.to_array {id="to_array",tag="method",version="2"} + +Given a list of `M` attribute IDs, export the tokens to a numpy `ndarray` of +shape `(N, M)`, where `N` is the length of the document. The values will be +32-bit integers. + +> #### Example +> +> ```python +> from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA +> doc = nlp("I like New York in Autumn.") +> span = doc[2:3] +> # All strings mapped to integers, for easy export to numpy +> np_array = span.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA]) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | +| **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | + +## Span.ents {id="ents",tag="property",version="2.0.13",model="ner"} + +The named entities that fall completely within the span. Returns a tuple of +`Span` objects. + +> #### Example +> +> ```python +> doc = nlp("Mr. Best flew to New York on Saturday morning.") +> span = doc[0:6] +> ents = list(span.ents) +> assert ents[0].label == 346 +> assert ents[0].label_ == "PERSON" +> assert ents[0].text == "Mr. Best" +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------- | +| **RETURNS** | Entities in the span, one `Span` per entity. ~~Tuple[Span, ...]~~ | + +## Span.noun_chunks {id="noun_chunks",tag="property",model="parser"} + +Iterate over the base noun phrases in the span. Yields base noun-phrase `Span` +objects, if the document has been syntactically parsed. A base noun phrase, or +"NP chunk", is a noun phrase that does not permit other NPs to be nested within +it – so no NP-level coordination, no prepositional phrases, and no relative +clauses. + +If the `noun_chunk` [syntax iterator](/usage/linguistic-features#language-data) +has not been implemented for the given language, a `NotImplementedError` is +raised. + +> #### Example +> +> ```python +> doc = nlp("A phrase with another phrase occurs.") +> span = doc[3:5] +> chunks = list(span.noun_chunks) +> assert len(chunks) == 1 +> assert chunks[0].text == "another phrase" +> ``` + +| Name | Description | +| ---------- | --------------------------------- | +| **YIELDS** | Noun chunks in the span. ~~Span~~ | + +## Span.as_doc {id="as_doc",tag="method"} + +Create a new `Doc` object corresponding to the `Span`, with a copy of the data. + +When calling this on many spans from the same doc, passing in a precomputed +array representation of the doc using the `array_head` and `array` args can save +time. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> span = doc[2:4] +> doc2 = span.as_doc() +> assert doc2.text == "New York" +> ``` + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------- | +| `copy_user_data` | Whether or not to copy the original doc's user data. ~~bool~~ | +| `array_head` | Precomputed array attributes (headers) of the original doc, as generated by `Doc._get_array_attrs()`. ~~Tuple~~ | +| `array` | Precomputed array version of the original doc as generated by [`Doc.to_array`](/api/doc#to_array). ~~numpy.ndarray~~ | +| **RETURNS** | A `Doc` object of the `Span`'s content. ~~Doc~~ | + +## Span.root {id="root",tag="property",model="parser"} + +The token with the shortest path to the root of the sentence (or the root +itself). If multiple tokens are equally high in the tree, the first token is +taken. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> i, like, new, york, in_, autumn, dot = range(len(doc)) +> assert doc[new].head.text == "York" +> assert doc[york].head.text == "like" +> new_york = doc[new:york+1] +> assert new_york.root.text == "York" +> ``` + +| Name | Description | +| ----------- | ------------------------- | +| **RETURNS** | The root token. ~~Token~~ | + +## Span.conjuncts {id="conjuncts",tag="property",model="parser"} + +A tuple of tokens coordinated to `span.root`. + +> #### Example +> +> ```python +> doc = nlp("I like apples and oranges") +> apples_conjuncts = doc[2:3].conjuncts +> assert [t.text for t in apples_conjuncts] == ["oranges"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------- | +| **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | + +## Span.lefts {id="lefts",tag="property",model="parser"} + +Tokens that are to the left of the span, whose heads are within the span. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> lefts = [t.text for t in doc[3:7].lefts] +> assert lefts == ["New"] +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------- | +| **YIELDS** | A left-child of a token of the span. ~~Token~~ | + +## Span.rights {id="rights",tag="property",model="parser"} + +Tokens that are to the right of the span, whose heads are within the span. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> rights = [t.text for t in doc[2:4].rights] +> assert rights == ["in"] +> ``` + +| Name | Description | +| ---------- | ----------------------------------------------- | +| **YIELDS** | A right-child of a token of the span. ~~Token~~ | + +## Span.n_lefts {id="n_lefts",tag="property",model="parser"} + +The number of tokens that are to the left of the span, whose heads are within +the span. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> assert doc[3:7].n_lefts == 1 +> ``` + +| Name | Description | +| ----------- | ---------------------------------------- | +| **RETURNS** | The number of left-child tokens. ~~int~~ | + +## Span.n_rights {id="n_rights",tag="property",model="parser"} + +The number of tokens that are to the right of the span, whose heads are within +the span. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> assert doc[2:4].n_rights == 1 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------- | +| **RETURNS** | The number of right-child tokens. ~~int~~ | + +## Span.subtree {id="subtree",tag="property",model="parser"} + +Tokens within the span and tokens which descend from them. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> subtree = [t.text for t in doc[:3].subtree] +> assert subtree == ["Give", "it", "back", "!"] +> ``` + +| Name | Description | +| ---------- | ----------------------------------------------------------- | +| **YIELDS** | A token within the span, or a descendant from it. ~~Token~~ | + +## Span.has_vector {id="has_vector",tag="property",model="vectors"} + +A boolean value indicating whether a word vector is associated with the object. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> assert doc[1:].has_vector +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| **RETURNS** | Whether the span has a vector data attached. ~~bool~~ | + +## Span.vector {id="vector",tag="property",model="vectors"} + +A real-valued meaning representation. Defaults to an average of the token +vectors. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> assert doc[1:].vector.dtype == "float32" +> assert doc[1:].vector.shape == (300,) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| **RETURNS** | A 1-dimensional array representing the span's vector. ~~`numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Span.vector_norm {id="vector_norm",tag="property",model="vectors"} + +The L2 norm of the span's vector representation. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> doc[1:].vector_norm # 4.800883928527915 +> doc[2:].vector_norm # 6.895897646384268 +> assert doc[1:].vector_norm != doc[2:].vector_norm +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| **RETURNS** | The L2 norm of the vector representation. ~~float~~ | + +## Span.sent {id="sent",tag="property",model="sentences"} + +The sentence span that this span is a part of. This property is only available +when [sentence boundaries](/usage/linguistic-features#sbd) have been set on the +document by the `parser`, `senter`, `sentencizer` or some custom function. It +will raise an error otherwise. + +If the span happens to cross sentence boundaries, only the first sentence will +be returned. If it is required that the sentence always includes the full span, +the result can be adjusted as such: + +```python +sent = span.sent +sent = doc[sent.start : max(sent.end, span.end)] +``` + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[1:3] +> assert span.sent.text == "Give it back!" +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------- | +| **RETURNS** | The sentence span that this span is a part of. ~~Span~~ | + +## Span.sents {id="sents",tag="property",model="sentences",version="3.2.1"} + +Returns a generator over the sentences the span belongs to. This property is +only available when [sentence boundaries](/usage/linguistic-features#sbd) have +been set on the document by the `parser`, `senter`, `sentencizer` or some custom +function. It will raise an error otherwise. + +If the span happens to cross sentence boundaries, all sentences the span +overlaps with will be returned. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> span = doc[2:4] +> assert len(span.sents) == 2 +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------- | +| **RETURNS** | A generator yielding sentences this `Span` is a part of ~~Iterable[Span]~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `tensor` | The span's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | +| `start` | The token offset for the start of the span. ~~int~~ | +| `end` | The token offset for the end of the span. ~~int~~ | +| `start_char` | The character offset for the start of the span. ~~int~~ | +| `end_char` | The character offset for the end of the span. ~~int~~ | +| `text` | A string representation of the span text. ~~str~~ | +| `text_with_ws` | The text content of the span with a trailing whitespace character if the last token has one. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Span.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `label` | The hash value of the span's label. ~~int~~ | +| `label_` | The span's label. ~~str~~ | +| `lemma_` | The span's lemma. Equivalent to `"".join(token.lemma_ + token.whitespace_ for token in span).strip()`. ~~str~~ | +| `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ | +| `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ | +| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ | +| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ | +| `id` | The hash value of the span's ID. ~~int~~ | +| `id_` | The span's ID. ~~str~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/spancategorizer.mdx b/website/docs/api/spancategorizer.mdx new file mode 100644 index 0000000..98a1948 --- /dev/null +++ b/website/docs/api/spancategorizer.mdx @@ -0,0 +1,559 @@ +--- +title: SpanCategorizer +tag: class,experimental +source: spacy/pipeline/spancat.py +version: 3.1 +teaser: 'Pipeline component for labeling potentially overlapping spans of text' +api_base_class: /api/pipe +api_string_name: spancat +api_trainable: true +--- + +A span categorizer consists of two parts: a [suggester function](#suggesters) +that proposes candidate spans, which may or may not overlap, and a labeler model +that predicts zero or more labels for each candidate. + +This component comes in two forms: `spancat` and `spancat_singlelabel` (added in +spaCy v3.5.1). When you need to perform multi-label classification on your +spans, use `spancat`. The `spancat` component uses a `Logistic` layer where the +output class probabilities are independent for each class. However, if you need +to predict at most one true class for a span, then use `spancat_singlelabel`. It +uses a `Softmax` layer and treats the task as a multi-class problem. + +Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the doc +under `doc.spans[spans_key]`, where `spans_key` is a component config setting. +Individual span scores are stored in `doc.spans[spans_key].attrs["scores"]`. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `Doc.spans[spans_key]` as a +[`SpanGroup`](/api/spangroup). The scores for the spans in the `SpanGroup` will +be saved in `SpanGroup.attrs["scores"]`. + +`spans_key` defaults to `"sc"`, but can be passed as a parameter. The `spancat` +component will overwrite any existing spans under the spans key +`doc.spans[spans_key]`. + +| Location | Value | +| -------------------------------------- | -------------------------------------------------------- | +| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | +| `Doc.spans[spans_key].attrs["scores"]` | The score for each span in the `SpanGroup`. ~~Floats1d~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example (spancat) +> +> ```python +> from spacy.pipeline.spancat import DEFAULT_SPANCAT_MODEL +> config = { +> "threshold": 0.5, +> "spans_key": "labeled_spans", +> "max_positive": None, +> "model": DEFAULT_SPANCAT_MODEL, +> "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, +> } +> nlp.add_pipe("spancat", config=config) +> ``` + +> #### Example (spancat_singlelabel) +> +> ```python +> from spacy.pipeline.spancat import DEFAULT_SPANCAT_SINGLELABEL_MODEL +> config = { +> "spans_key": "labeled_spans", +> "model": DEFAULT_SPANCAT_SINGLELABEL_MODEL, +> "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]}, +> # Additional spancat_singlelabel parameters +> "negative_weight": 0.8, +> "allow_overlap": True, +> } +> nlp.add_pipe("spancat_singlelabel", config=config) +> ``` + +| Setting | Description | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. Defaults to [`ngram_suggester`](#ngram_suggester). ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | +| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. Defaults to [SpanCategorizer](/api/architectures#SpanCategorizer). ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Meant to be used in combination with the multi-class `spancat` component with a `Logistic` scoring layer. Defaults to `0.5`. ~~float~~ | +| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. Meant to be used together with the `spancat` component and defaults to 0 with `spancat_singlelabel`. ~~Optional[int]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| `add_negative_label` 3.5.1 | Whether to learn to predict a special negative label for each unannotated `Span` . This should be `True` when using a `Softmax` classifier layer and so its `True` by default for `spancat_singlelabel`. Spans with negative labels and their scores are not stored as annotations. ~~bool~~ | +| `negative_weight` 3.5.1 | Multiplier for the loss terms. It can be used to downweight the negative samples if there are too many. It is only used when `add_negative_label` is `True`. Defaults to `1.0`. ~~float~~ | +| `allow_overlap` 3.5.1 | If `True`, the data is assumed to contain overlapping spans. It is only available when `max_positive` is exactly 1. Defaults to `True`. ~~bool~~ | + + + +If you set a non-default value for `spans_key`, you'll have to update +`[training.score_weights]` as well so that weights are computed properly. E. g. +for `spans_key == "myspankey"`, include this in your config: + +```ini +[training.score_weights] +spans_myspankey_f = 1.0 +spans_myspankey_p = 0.0 +spans_myspankey_r = 0.0 +``` + + + +```python +%%GITHUB_SPACY/spacy/pipeline/spancat.py +``` + +## SpanCategorizer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> # Replace 'spancat' with 'spancat_singlelabel' for exclusive classes +> spancat = nlp.add_pipe("spancat") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_spancat"}} +> spancat = nlp.add_pipe("spancat", config=config) +> +> # Construction from class +> from spacy.pipeline import SpanCategorizer +> spancat = SpanCategorizer(nlp.vocab, model, suggester) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | A model instance that is given a a list of documents and `(start, end)` indices representing candidate span offsets. The model predicts a probability for each category for each span. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | +| `suggester` | A function that [suggests spans](#suggesters). Spans are returned as a ragged array with two integer columns, for the start and end positions. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#sans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | +| `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | +| `allow_overlap` 3.5.1 | If `True`, the data is assumed to contain overlapping spans. It is only available when `max_positive` is exactly 1. Defaults to `True`. ~~bool~~ | +| `add_negative_label` 3.5.1 | Whether to learn to predict a special negative label for each unannotated `Span`. This should be `True` when using a `Softmax` classifier layer and so its `True` by default for `spancat_singlelabel` . Spans with negative labels and their scores are not stored as annotations. ~~bool~~ | +| `negative_weight` 3.5.1 | Multiplier for the loss terms. It can be used to downweight the negative samples if there are too many . It is only used when `add_negative_label` is `True`. Defaults to `1.0`. ~~float~~ | + +## SpanCategorizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/spancategorizer#call) and [`pipe`](/api/spancategorizer#pipe) +delegate to the [`predict`](/api/spancategorizer#predict) and +[`set_annotations`](/api/spancategorizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> spancat = nlp.add_pipe("spancat") +> # This usually happens under the hood +> processed = spancat(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## SpanCategorizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/spancategorizer#call) and +[`pipe`](/api/spancategorizer#pipe) delegate to the +[`predict`](/api/spancategorizer#predict) and +[`set_annotations`](/api/spancategorizer#set_annotations) methods. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> for doc in spancat.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## SpanCategorizer.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.spancat] +> +> [initialize.components.spancat.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/spancat.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | + +## SpanCategorizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> scores = spancat.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## SpanCategorizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> scores = spancat.predict(docs) +> spancat.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | --------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `SpanCategorizer.predict`. | + +## SpanCategorizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/spancategorizer#predict) and +[`get_loss`](/api/spancategorizer#get_loss). + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> optimizer = nlp.initialize() +> losses = spancat.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SpanCategorizer.set_candidates {id="set_candidates",tag="method", version="3.3"} + +Use the suggester to add a list of [`Span`](/api/span) candidates to a list of +[`Doc`](/api/doc) objects. This method is intended to be used for debugging +purposes. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat.set_candidates(docs, "candidates") +> ``` + +| Name | Description | +| ---------------- | -------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `candidates_key` | Key of the Doc.spans dict to save the candidate spans under. ~~str~~ | + +## SpanCategorizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> scores = spancat.predict([eg.predicted for eg in examples]) +> loss, d_loss = spancat.get_loss(examples, scores) +> ``` + +| Name | Description | +| -------------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `spans_scores` | Scores representing the model's predictions. ~~Tuple[Ragged, Floats2d]~~ | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## SpanCategorizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> optimizer = spancat.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## SpanCategorizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> with spancat.use_params(optimizer.averages): +> spancat.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## SpanCategorizer.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. Raises an error if the output dimension is already +set, or if the model has already been fully [initialized](#initialize). Note +that you don't have to call this method if you provide a **representative data +sample** to the [`initialize`](#initialize) method. In this case, all labels +found in the sample will be automatically added to the model, and the output +dimension will be [inferred](/usage/layers-architectures#thinc-shape-inference) +automatically. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## SpanCategorizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat.to_disk("/path/to/spancat") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## SpanCategorizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat.from_disk("/path/to/spancat") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `SpanCategorizer` object. ~~SpanCategorizer~~ | + +## SpanCategorizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> spancat = nlp.add_pipe("spancat") +> spancat_bytes = spancat.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `SpanCategorizer` object. ~~bytes~~ | + +## SpanCategorizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> spancat_bytes = spancat.to_bytes() +> spancat = nlp.add_pipe("spancat") +> spancat.from_bytes(spancat_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `SpanCategorizer` object. ~~SpanCategorizer~~ | + +## SpanCategorizer.labels {id="labels",tag="property"} + +The labels currently added to the component. + +> #### Example +> +> ```python +> spancat.add_label("MY_LABEL") +> assert "MY_LABEL" in spancat.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## SpanCategorizer.label_data {id="label_data",tag="property"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`SpanCategorizer.initialize`](/api/spancategorizer#initialize) to initialize +the model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = spancat.label_data +> spancat.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = spancat.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | + +## Suggesters {id="suggesters",tag="registered functions",source="spacy/pipeline/spancat.py"} + +### spacy.ngram_suggester.v1 {id="ngram_suggester"} + +> #### Example Config +> +> ```ini +> [components.spancat.suggester] +> @misc = "spacy.ngram_suggester.v1" +> sizes = [1, 2, 3] +> ``` + +Suggest all spans of the given lengths. Spans are returned as a ragged array of +integers. The array has two columns, indicating the start and end position. + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------- | +| `sizes` | The phrase lengths to suggest. For example, `[1, 2]` will suggest phrases consisting of 1 or 2 tokens. ~~List[int]~~ | +| **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | + +### spacy.ngram_range_suggester.v1 {id="ngram_range_suggester"} + +> #### Example Config +> +> ```ini +> [components.spancat.suggester] +> @misc = "spacy.ngram_range_suggester.v1" +> min_size = 2 +> max_size = 4 +> ``` + +Suggest all spans of at least length `min_size` and at most length `max_size` +(both inclusive). Spans are returned as a ragged array of integers. The array +has two columns, indicating the start and end position. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| `min_size` | The minimal phrase lengths to suggest (inclusive). ~~[int]~~ | +| `max_size` | The maximal phrase lengths to suggest (inclusive). ~~[int]~~ | +| **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | + +### spacy.preset_spans_suggester.v1 {id="preset_spans_suggester"} + +> #### Example Config +> +> ```ini +> [components.spancat.suggester] +> @misc = "spacy.preset_spans_suggester.v1" +> spans_key = "my_spans" +> ``` + +Suggest all spans that are already stored in doc.spans[spans_key]. This is +useful when an upstream component is used to set the spans on the Doc such as a +[`SpanRuler`](/api/spanruler) or [`SpanFinder`](/api/spanfinder). + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------- | +| `spans_key` | Key of [`Doc.spans`](/api/doc/#spans) that provides spans to suggest. ~~str~~ | +| **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | diff --git a/website/docs/api/spanfinder.mdx b/website/docs/api/spanfinder.mdx new file mode 100644 index 0000000..ef4a6ba --- /dev/null +++ b/website/docs/api/spanfinder.mdx @@ -0,0 +1,372 @@ +--- +title: SpanFinder +tag: class,experimental +source: spacy/pipeline/span_finder.py +version: 3.6 +teaser: + 'Pipeline component for identifying potentially overlapping spans of text' +api_base_class: /api/pipe +api_string_name: span_finder +api_trainable: true +--- + +The span finder identifies potentially overlapping, unlabeled spans. It +identifies tokens that start or end spans and annotates unlabeled spans between +starts and ends, with optional filters for min and max span length. It is +intended for use in combination with a component like +[`SpanCategorizer`](/api/spancategorizer) that may further filter or label the +spans. Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the +doc under `doc.spans[spans_key]`, where `spans_key` is a component config +setting. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `Doc.spans[spans_key]` as a +[`SpanGroup`](/api/spangroup). + +`spans_key` defaults to `"sc"`, but can be passed as a parameter. The +`span_finder` component will overwrite any existing spans under the spans key +`doc.spans[spans_key]`. + +| Location | Value | +| ---------------------- | ---------------------------------- | +| `Doc.spans[spans_key]` | The unlabeled spans. ~~SpanGroup~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.span_finder import DEFAULT_SPAN_FINDER_MODEL +> config = { +> "threshold": 0.5, +> "spans_key": "my_spans", +> "max_length": None, +> "min_length": None, +> "model": DEFAULT_SPAN_FINDER_MODEL, +> } +> nlp.add_pipe("span_finder", config=config) +> ``` + +| Setting | Description | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | A model instance that is given a list of documents and predicts a probability for each token. ~~Model[List[Doc], Floats2d]~~ | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Defaults to `0.5`. ~~float~~ | +| `max_length` | Maximum length of the produced spans, defaults to `25`. ~~Optional[int]~~ | +| `min_length` | Minimum length of the produced spans, defaults to `None` meaning shortest span length is 1. ~~Optional[int]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/span_finder.py +``` + +## SpanFinder.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> span_finder = nlp.add_pipe("span_finder") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_span_finder"}} +> span_finder = nlp.add_pipe("span_finder", config=config) +> +> # Construction from class +> from spacy.pipeline import SpanFinder +> span_finder = SpanFinder(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | A model instance that is given a list of documents and predicts a probability for each token. ~~Model[List[Doc], Floats2d]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | Key of the [`Doc.spans`](/api/doc#spans) dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. Defaults to `"sc"`. ~~str~~ | +| `threshold` | Minimum probability to consider a prediction positive. Defaults to `0.5`. ~~float~~ | +| `max_length` | Maximum length of the produced spans, defaults to `None` meaning unlimited length. ~~Optional[int]~~ | +| `min_length` | Minimum length of the produced spans, defaults to `None` meaning shortest span length is 1. ~~Optional[int]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +## SpanFinder.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/spanfinder#call) and [`pipe`](/api/spanfinder#pipe) delegate +to the [`predict`](/api/spanfinder#predict) and +[`set_annotations`](/api/spanfinder#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> span_finder = nlp.add_pipe("span_finder") +> # This usually happens under the hood +> processed = span_finder(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## SpanFinder.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/spanfinder#call) and +[`pipe`](/api/spanfinder#pipe) delegate to the +[`predict`](/api/spanfinder#predict) and +[`set_annotations`](/api/spanfinder#set_annotations) methods. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> for doc in span_finder.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## SpanFinder.initialize {id="initialize",tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network and +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) This +method is typically called by [`Language.initialize`](/api/language#initialize) +and lets you customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> span_finder.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## SpanFinder.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> scores = span_finder.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## SpanFinder.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> scores = span_finder.predict(docs) +> span_finder.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | ---------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `SpanFinder.predict`. | + +## SpanFinder.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/spanfinder#predict) and +[`get_loss`](/api/spanfinder#get_loss). + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> optimizer = nlp.initialize() +> losses = span_finder.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SpanFinder.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> scores = span_finder.predict([eg.predicted for eg in examples]) +> loss, d_loss = span_finder.get_loss(examples, scores) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------ | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `spans_scores` | Scores representing the model's predictions. ~~Tuple[Ragged, Floats2d]~~ | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, Floats2d]~~ | + +## SpanFinder.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> optimizer = span_finder.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## SpanFinder.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> with span_finder.use_params(optimizer.averages): +> span_finder.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## SpanFinder.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> span_finder.to_disk("/path/to/span_finder") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## SpanFinder.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> span_finder.from_disk("/path/to/span_finder") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `SpanFinder` object. ~~SpanFinder~~ | + +## SpanFinder.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> span_finder = nlp.add_pipe("span_finder") +> span_finder_bytes = span_finder.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `SpanFinder` object. ~~bytes~~ | + +## SpanFinder.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_finder_bytes = span_finder.to_bytes() +> span_finder = nlp.add_pipe("span_finder") +> span_finder.from_bytes(span_finder_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `SpanFinder` object. ~~SpanFinder~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = span_finder.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/spangroup.mdx b/website/docs/api/spangroup.mdx new file mode 100644 index 0000000..cd0accb --- /dev/null +++ b/website/docs/api/spangroup.mdx @@ -0,0 +1,317 @@ +--- +title: SpanGroup +tag: class +source: spacy/tokens/span_group.pyx +version: 3 +--- + +A group of arbitrary, potentially overlapping [`Span`](/api/span) objects that +all belong to the same [`Doc`](/api/doc) object. The group can be named, and you +can attach additional attributes to it. Span groups are generally accessed via +the [`Doc.spans`](/api/doc#spans) attribute, which will convert lists of spans +into a `SpanGroup` object for you automatically on assignment. `SpanGroup` +objects behave similar to `list`s, so you can append `Span` objects to them or +access a member at a given index. + +## SpanGroup.\_\_init\_\_ {id="init",tag="method"} + +Create a `SpanGroup`. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> spans = [doc[0:1], doc[1:3]] +> +> # Construction 1 +> from spacy.tokens import SpanGroup +> +> group = SpanGroup(doc, name="errors", spans=spans, attrs={"annotator": "matt"}) +> doc.spans["errors"] = group +> +> # Construction 2 +> doc.spans["errors"] = spans +> assert isinstance(doc.spans["errors"], SpanGroup) +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doc` | The document the span group belongs to. ~~Doc~~ | +| _keyword-only_ | | +| `name` | The name of the span group. If the span group is created automatically on assignment to `doc.spans`, the key name is used. Defaults to `""`. ~~str~~ | +| `attrs` | Optional JSON-serializable attributes to attach to the span group. ~~Dict[str, Any]~~ | +| `spans` | The spans to add to the span group. ~~Iterable[Span]~~ | + +## SpanGroup.doc {id="doc",tag="property"} + +The [`Doc`](/api/doc) object the span group is referring to. + + + +When a `Doc` object is garbage collected, any related `SpanGroup` object won't +be functional anymore, as these objects use a `weakref` to refer to the +document. An error will be raised as the internal `doc` object will be `None`. +To avoid this, make sure that the original `Doc` objects are still available in +the scope of your function. + + + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> assert doc.spans["errors"].doc == doc +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The reference document. ~~Doc~~ | + +## SpanGroup.has_overlap {id="has_overlap",tag="property"} + +Check whether the span group contains overlapping spans. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> assert not doc.spans["errors"].has_overlap +> doc.spans["errors"].append(doc[2:4]) +> assert doc.spans["errors"].has_overlap +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------- | +| **RETURNS** | Whether the span group contains overlaps. ~~bool~~ | + +## SpanGroup.\_\_len\_\_ {id="len",tag="method"} + +Get the number of spans in the group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> assert len(doc.spans["errors"]) == 2 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------- | +| **RETURNS** | The number of spans in the group. ~~int~~ | + +## SpanGroup.\_\_getitem\_\_ {id="getitem",tag="method"} + +Get a span from the group. Note that a copy of the span is returned, so if any +changes are made to this span, they are not reflected in the corresponding +member of the span group. The item or group will need to be reassigned for +changes to be reflected in the span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> span = doc.spans["errors"][1] +> assert span.text == "goi ng" +> span.label_ = 'LABEL' +> assert doc.spans["errors"][1].label_ != 'LABEL' # The span within the group was not updated +> ``` + +| Name | Description | +| ----------- | ------------------------------------- | +| `i` | The item index. ~~int~~ | +| **RETURNS** | The span at the given index. ~~Span~~ | + +## SpanGroup.\_\_setitem\_\_ {id="setitem",tag="method", version="3.3"} + +Set a span in the span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> span = doc[0:2] +> doc.spans["errors"][0] = span +> assert doc.spans["errors"][0].text == "Their goi" +> ``` + +| Name | Description | +| ------ | ----------------------- | +| `i` | The item index. ~~int~~ | +| `span` | The new value. ~~Span~~ | + +## SpanGroup.\_\_delitem\_\_ {id="delitem",tag="method", version="3.3"} + +Delete a span from the span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> del doc.spans[0] +> assert len(doc.spans["errors"]) == 1 +> ``` + +| Name | Description | +| ---- | ----------------------- | +| `i` | The item index. ~~int~~ | + +## SpanGroup.\_\_add\_\_ {id="add",tag="method", version="3.3"} + +Concatenate the current span group with another span group and return the result +in a new span group. Any `attrs` from the first span group will have precedence +over `attrs` in the second. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> doc.spans["other"] = [doc[0:2], doc[2:4]] +> span_group = doc.spans["errors"] + doc.spans["other"] +> assert len(span_group) == 4 +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| `other` | The span group or spans to concatenate. ~~Union[SpanGroup, Iterable[Span]]~~ | +| **RETURNS** | The new span group. ~~SpanGroup~~ | + +## SpanGroup.\_\_iadd\_\_ {id="iadd",tag="method", version="3.3"} + +Append an iterable of spans or the content of a span group to the current span +group. Any `attrs` in the other span group will be added for keys that are not +already present in the current span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> doc.spans["errors"] += [doc[3:4], doc[2:3]] +> assert len(doc.spans["errors"]) == 4 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------- | +| `other` | The span group or spans to append. ~~Union[SpanGroup, Iterable[Span]]~~ | +| **RETURNS** | The span group. ~~SpanGroup~~ | + +## SpanGroup.\_\_iter\_\_ {id="iter",tag="method",version="3.5"} + +Iterate over the spans in this span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> for error_span in doc.spans["errors"]: +> print(error_span) +> ``` + +| Name | Description | +| ---------- | ----------------------------------- | +| **YIELDS** | A span in this span group. ~~Span~~ | + + +## SpanGroup.append {id="append",tag="method"} + +Add a [`Span`](/api/span) object to the group. The span must refer to the same +[`Doc`](/api/doc) object as the span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1]] +> doc.spans["errors"].append(doc[1:3]) +> assert len(doc.spans["errors"]) == 2 +> ``` + +| Name | Description | +| ------ | ---------------------------- | +| `span` | The span to append. ~~Span~~ | + +## SpanGroup.extend {id="extend",tag="method"} + +Add multiple [`Span`](/api/span) objects or contents of another `SpanGroup` to +the group. All spans must refer to the same [`Doc`](/api/doc) object as the span +group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [] +> doc.spans["errors"].extend([doc[1:3], doc[0:1]]) +> assert len(doc.spans["errors"]) == 2 +> span_group = SpanGroup(doc, spans=[doc[1:4], doc[0:3]]) +> doc.spans["errors"].extend(span_group) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------- | +| `spans` | The spans to add. ~~Union[SpanGroup, Iterable["Span"]]~~ | + +## SpanGroup.copy {id="copy",tag="method", version="3.3"} + +Return a copy of the span group. + +> #### Example +> +> ```python +> from spacy.tokens import SpanGroup +> +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[1:3], doc[0:3]] +> new_group = doc.spans["errors"].copy() +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------- | +| `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ | +| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | + +## SpanGroup.to_bytes {id="to_bytes",tag="method"} + +Serialize the span group to a bytestring. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> group_bytes = doc.spans["errors"].to_bytes() +> ``` + +| Name | Description | +| ----------- | ------------------------------------- | +| **RETURNS** | The serialized `SpanGroup`. ~~bytes~~ | + +## SpanGroup.from_bytes {id="from_bytes",tag="method"} + +Load the span group from a bytestring. Modifies the object in place and returns +it. + +> #### Example +> +> ```python +> from spacy.tokens import SpanGroup +> +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> group_bytes = doc.spans["errors"].to_bytes() +> new_group = SpanGroup() +> new_group.from_bytes(group_bytes) +> ``` + +| Name | Description | +| ------------ | ------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The `SpanGroup` object. ~~SpanGroup~~ | diff --git a/website/docs/api/spanruler.mdx b/website/docs/api/spanruler.mdx new file mode 100644 index 0000000..5889b19 --- /dev/null +++ b/website/docs/api/spanruler.mdx @@ -0,0 +1,353 @@ +--- +title: SpanRuler +tag: class +source: spacy/pipeline/span_ruler.py +version: 3.3 +teaser: 'Pipeline component for rule-based span and named entity recognition' +api_string_name: span_ruler +api_trainable: false +--- + +The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or +[`Doc.ents`](/api/doc#ents) using token-based rules or exact phrase matches. For +usage examples, see the docs on +[rule-based span matching](/usage/rule-based-matching#spanruler). + +## Assigned Attributes {id="assigned-attributes"} + +Matches will be saved to `Doc.spans[spans_key]` as a +[`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is +saved in the `Token.ent_type` and `Token.ent_iob` fields. + +| Location | Value | +| ---------------------- | ----------------------------------------------------------------- | +| `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | +| `Doc.ents` | The annotated spans. ~~Tuple[Span]~~ | +| `Token.ent_iob` | An enum encoding of the IOB part of the named entity tag. ~~int~~ | +| `Token.ent_iob_` | The IOB part of the named entity tag. ~~str~~ | +| `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | +| `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg`](/usage/training#config). + +> #### Example +> +> ```python +> config = { +> "spans_key": "my_spans", +> "validate": True, +> "overwrite": False, +> } +> nlp.add_pipe("span_ruler", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal `PhraseMatcher` as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to `Matcher` and `PhraseMatcher` as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/span_ruler.py +``` + +## SpanRuler.\_\_init\_\_ {id="init",tag="method"} + +Initialize the span ruler. If patterns are supplied here, they need to be a list +of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a +token pattern (list) or a phrase pattern (string). For example: +`{"label": "ORG", "pattern": "Apple"}`. + +> #### Example +> +> ```python +> # Construction via add_pipe +> ruler = nlp.add_pipe("span_ruler") +> +> # Construction from class +> from spacy.pipeline import SpanRuler +> ruler = SpanRuler(nlp, overwrite=True) +> ``` + +| Name | Description | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | + +## SpanRuler.initialize {id="initialize",tag="method"} + +Initialize the component with data and used before training to load in rules +from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method +is typically called by [`Language.initialize`](/api/language#initialize) and +lets you customize arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. Any existing patterns are removed on initialization. + +> #### Example +> +> ```python +> span_ruler = nlp.add_pipe("span_ruler") +> span_ruler.initialize(lambda: [], nlp=nlp, patterns=patterns) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.span_ruler] +> +> [initialize.components.span_ruler.patterns] +> @readers = "srsly.read_jsonl.v1" +> path = "corpus/span_ruler_patterns.jsonl" +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Not used by the `SpanRuler`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | + +## SpanRuler.\_\_len\_\_ {id="len",tag="method"} + +The number of all patterns added to the span ruler. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> assert len(ruler) == 0 +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert len(ruler) == 1 +> ``` + +| Name | Description | +| ----------- | ------------------------------- | +| **RETURNS** | The number of patterns. ~~int~~ | + +## SpanRuler.\_\_contains\_\_ {id="contains",tag="method"} + +Whether a label is present in the patterns. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> assert "ORG" in ruler +> assert not "PERSON" in ruler +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `label` | The label to check. ~~str~~ | +| **RETURNS** | Whether the span ruler contains the label. ~~bool~~ | + +## SpanRuler.\_\_call\_\_ {id="call",tag="method"} + +Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or +`doc.ents`. Typically, this happens automatically after the component has been +added to the pipeline using [`nlp.add_pipe`](/api/language#add_pipe). If the +span ruler was initialized with `overwrite=True`, existing spans and entities +will be removed. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> +> doc = nlp("A text about Apple.") +> spans = [(span.text, span.label_) for span in doc.spans["ruler"]] +> assert spans == [("Apple", "ORG")] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | +| **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ | + +## SpanRuler.add_patterns {id="add_patterns",tag="method"} + +Add patterns to the span ruler. A pattern can either be a token pattern (list of +dicts) or a phrase pattern (string). For more details, see the usage guide on +[rule-based matching](/usage/rule-based-matching). + +> #### Example +> +> ```python +> patterns = [ +> {"label": "ORG", "pattern": "Apple"}, +> {"label": "GPE", "pattern": [{"lower": "san"}, {"lower": "francisco"}]} +> ] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------- | +| `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | + +## SpanRuler.remove {id="remove",tag="method"} + +Remove patterns by label from the span ruler. A `ValueError` is raised if the +label does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove("ORG") +> ``` + +| Name | Description | +| ------- | -------------------------------------- | +| `label` | The label of the pattern rule. ~~str~~ | + +## SpanRuler.remove_by_id {id="remove_by_id",tag="method"} + +Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID +does not exist in any patterns. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.remove_by_id("apple") +> ``` + +| Name | Description | +| ------------ | ----------------------------------- | +| `pattern_id` | The ID of the pattern rule. ~~str~~ | + +## SpanRuler.clear {id="clear",tag="method"} + +Remove all patterns the span ruler. + +> #### Example +> +> ```python +> patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}] +> ruler = nlp.add_pipe("span_ruler") +> ruler.add_patterns(patterns) +> ruler.clear() +> ``` + +## SpanRuler.to_disk {id="to_disk",tag="method"} + +Save the span ruler patterns to a directory. The patterns will be saved as +newline-delimited JSON (JSONL). + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.to_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## SpanRuler.from_disk {id="from_disk",tag="method"} + +Load the span ruler from a path. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_disk("/path/to/span_ruler") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.to_bytes {id="to_bytes",tag="method"} + +Serialize the span ruler to a bytestring. + +> #### Example +> +> ```python +> ruler = nlp.add_pipe("span_ruler") +> ruler_bytes = ruler.to_bytes() +> ``` + +| Name | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The serialized patterns. ~~bytes~~ | + +## SpanRuler.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> ruler_bytes = ruler.to_bytes() +> ruler = nlp.add_pipe("span_ruler") +> ruler.from_bytes(ruler_bytes) +> ``` + +| Name | Description | +| ------------ | ---------------------------------------------- | +| `bytes_data` | The bytestring to load. ~~bytes~~ | +| **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | + +## SpanRuler.labels {id="labels",tag="property"} + +All labels present in the match patterns. + +| Name | Description | +| ----------- | -------------------------------------- | +| **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | + +## SpanRuler.ids {id="ids",tag="property"} + +All IDs present in the `id` property of the match patterns. + +| Name | Description | +| ----------- | ----------------------------------- | +| **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | + +## SpanRuler.patterns {id="patterns",tag="property"} + +All patterns that were added to the span ruler. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------- | +| **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------- | +| `key` | The spans key that spans are saved under. ~~Optional[str]~~ | +| `matcher` | The underlying matcher used to process token patterns. ~~Matcher~~ | +| `phrase_matcher` | The underlying phrase matcher used to process phrase patterns. ~~PhraseMatcher~~ | diff --git a/website/docs/api/stringstore.mdx b/website/docs/api/stringstore.mdx new file mode 100644 index 0000000..6a3e9d6 --- /dev/null +++ b/website/docs/api/stringstore.mdx @@ -0,0 +1,197 @@ +--- +title: StringStore +tag: class +source: spacy/strings.pyx +--- + +Look up strings by 64-bit hashes. As of v2.0, spaCy uses hash values instead of +integer IDs. This ensures that strings always map to the same ID, even from +different `StringStores`. + + + +Note that a `StringStore` instance is not static. It increases in size as texts +with new tokens are processed. + + + +## StringStore.\_\_init\_\_ {id="init",tag="method"} + +Create the `StringStore`. + +> #### Example +> +> ```python +> from spacy.strings import StringStore +> stringstore = StringStore(["apple", "orange"]) +> ``` + +| Name | Description | +| --------- | ---------------------------------------------------------------------- | +| `strings` | A sequence of strings to add to the store. ~~Optional[Iterable[str]]~~ | + +## StringStore.\_\_len\_\_ {id="len",tag="method"} + +Get the number of strings in the store. + +> #### Example +> +> ```python +> stringstore = StringStore(["apple", "orange"]) +> assert len(stringstore) == 2 +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| **RETURNS** | The number of strings in the store. ~~int~~ | + +## StringStore.\_\_getitem\_\_ {id="getitem",tag="method"} + +Retrieve a string from a given hash, or vice versa. + +> #### Example +> +> ```python +> stringstore = StringStore(["apple", "orange"]) +> apple_hash = stringstore["apple"] +> assert apple_hash == 8566208034543834098 +> assert stringstore[apple_hash] == "apple" +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------- | +| `string_or_id` | The value to encode. ~~Union[bytes, str, int]~~ | +| **RETURNS** | The value to be retrieved. ~~Union[str, int]~~ | + +## StringStore.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether a string is in the store. + +> #### Example +> +> ```python +> stringstore = StringStore(["apple", "orange"]) +> assert "apple" in stringstore +> assert not "cherry" in stringstore +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------- | +| `string` | The string to check. ~~str~~ | +| **RETURNS** | Whether the store contains the string. ~~bool~~ | + +## StringStore.\_\_iter\_\_ {id="iter",tag="method"} + +Iterate over the strings in the store, in order. Note that a newly initialized +store will always include an empty string `""` at position `0`. + +> #### Example +> +> ```python +> stringstore = StringStore(["apple", "orange"]) +> all_strings = [s for s in stringstore] +> assert all_strings == ["apple", "orange"] +> ``` + +| Name | Description | +| ---------- | ------------------------------ | +| **YIELDS** | A string in the store. ~~str~~ | + +## StringStore.add {id="add",tag="method",version="2"} + +Add a string to the `StringStore`. + +> #### Example +> +> ```python +> stringstore = StringStore(["apple", "orange"]) +> banana_hash = stringstore.add("banana") +> assert len(stringstore) == 3 +> assert banana_hash == 2525716904149915114 +> assert stringstore[banana_hash] == "banana" +> assert stringstore["banana"] == banana_hash +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `string` | The string to add. ~~str~~ | +| **RETURNS** | The string's hash value. ~~int~~ | + +## StringStore.to_disk {id="to_disk",tag="method",version="2"} + +Save the current state to a directory. + +> #### Example +> +> ```python +> stringstore.to_disk("/path/to/strings") +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## StringStore.from_disk {id="from_disk",tag="method",version="2"} + +Loads state from a directory. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> from spacy.strings import StringStore +> stringstore = StringStore().from_disk("/path/to/strings") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `StringStore` object. ~~StringStore~~ | + +## StringStore.to_bytes {id="to_bytes",tag="method"} + +Serialize the current state to a binary string. + +> #### Example +> +> ```python +> store_bytes = stringstore.to_bytes() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The serialized form of the `StringStore` object. ~~bytes~~ | + +## StringStore.from_bytes {id="from_bytes",tag="method"} + +Load state from a binary string. + +> #### Example +> +> ```python +> from spacy.strings import StringStore +> store_bytes = stringstore.to_bytes() +> new_store = StringStore().from_bytes(store_bytes) +> ``` + +| Name | Description | +| ------------ | ----------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The `StringStore` object. ~~StringStore~~ | + +## Utilities {id="util"} + +### strings.hash_string {id="hash_string",tag="function"} + +Get a 64-bit hash for a given string. + +> #### Example +> +> ```python +> from spacy.strings import hash_string +> assert hash_string("apple") == 8566208034543834098 +> ``` + +| Name | Description | +| ----------- | --------------------------- | +| `string` | The string to hash. ~~str~~ | +| **RETURNS** | The hash. ~~int~~ | diff --git a/website/docs/api/tagger.mdx b/website/docs/api/tagger.mdx new file mode 100644 index 0000000..d9b0506 --- /dev/null +++ b/website/docs/api/tagger.mdx @@ -0,0 +1,448 @@ +--- +title: Tagger +tag: class +source: spacy/pipeline/tagger.pyx +teaser: 'Pipeline component for part-of-speech tagging' +api_base_class: /api/pipe +api_string_name: tagger +api_trainable: true +--- + +A trainable pipeline component to predict part-of-speech tags for any +part-of-speech tag set. + +In the pre-trained pipelines, the tag schemas vary by language; see the +[individual model pages](/models) for details. + +## Assigned Attributes {id="assigned-attributes"} + +Predictions are assigned to `Token.tag`. + +| Location | Value | +| ------------ | ---------------------------------- | +| `Token.tag` | The part of speech (hash). ~~int~~ | +| `Token.tag_` | The part of speech. ~~str~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.tagger import DEFAULT_TAGGER_MODEL +> config = {"model": DEFAULT_TAGGER_MODEL} +> nlp.add_pipe("tagger", config=config) +> ``` + +| Setting | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | A model instance that predicts the tag probabilities. The output vectors should match the number of tags in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). Defaults to [Tagger](/api/architectures#Tagger). ~~Model[List[Doc], List[Floats2d]]~~ | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | +| `neg_prefix` 3.2.1 | The prefix used to specify incorrect tags while training. The tagger will learn not to predict exactly this tag. Defaults to `!`. ~~str~~ | +| `label_smoothing` 3.6 | [Label smoothing](https://arxiv.org/abs/1906.02629) factor. Defaults to `0.0`. ~~float~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/tagger.pyx +``` + +## Tagger.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> tagger = nlp.add_pipe("tagger") +> +> # Construction via create_pipe with custom model +> config = {"model": {"@architectures": "my_tagger"}} +> tagger = nlp.add_pipe("tagger", config=config) +> +> # Construction from class +> from spacy.pipeline import Tagger +> tagger = Tagger(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | A model instance that predicts the tag probabilities. The output vectors should match the number of tags in size, and be normalized as probabilities (all scores between 0 and 1, with the rows summing to `1`). ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | +| `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | + +## Tagger.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/tagger#call) and [`pipe`](/api/tagger#pipe) delegate to the +[`predict`](/api/tagger#predict) and +[`set_annotations`](/api/tagger#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> tagger = nlp.add_pipe("tagger") +> # This usually happens under the hood +> processed = tagger(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## Tagger.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/tagger#call) and +[`pipe`](/api/tagger#pipe) delegate to the [`predict`](/api/tagger#predict) and +[`set_annotations`](/api/tagger#set_annotations) methods. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> for doc in tagger.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Tagger.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> tagger.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.tagger] +> +> [initialize.components.tagger.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/tagger.json +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | + +## Tagger.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> scores = tagger.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## Tagger.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> scores = tagger.predict([doc1, doc2]) +> tagger.set_annotations([doc1, doc2], scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------ | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `Tagger.predict`. | + +## Tagger.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/tagger#predict) and +[`get_loss`](/api/tagger#get_loss). + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> optimizer = nlp.initialize() +> losses = tagger.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Tagger.rehearse {id="rehearse",tag="method,experimental",version="3"} + +Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the +current model to make predictions similar to an initial model, to try to address +the "catastrophic forgetting" problem. This feature is experimental. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> optimizer = nlp.resume_training() +> losses = tagger.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Tagger.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> scores = tagger.predict([eg.predicted for eg in examples]) +> loss, d_loss = tagger.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## Tagger.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> optimizer = tagger.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Tagger.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> with tagger.use_params(optimizer.averages): +> tagger.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## Tagger.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. Raises an error if the output dimension is already +set, or if the model has already been fully [initialized](#initialize). Note +that you don't have to call this method if you provide a **representative data +sample** to the [`initialize`](#initialize) method. In this case, all labels +found in the sample will be automatically added to the model, and the output +dimension will be [inferred](/usage/layers-architectures#thinc-shape-inference) +automatically. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> tagger.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## Tagger.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> tagger.to_disk("/path/to/tagger") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Tagger.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> tagger.from_disk("/path/to/tagger") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Tagger` object. ~~Tagger~~ | + +## Tagger.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> tagger = nlp.add_pipe("tagger") +> tagger_bytes = tagger.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Tagger` object. ~~bytes~~ | + +## Tagger.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> tagger_bytes = tagger.to_bytes() +> tagger = nlp.add_pipe("tagger") +> tagger.from_bytes(tagger_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Tagger` object. ~~Tagger~~ | + +## Tagger.labels {id="labels",tag="property"} + +The labels currently added to the component. + +> #### Example +> +> ```python +> tagger.add_label("MY_LABEL") +> assert "MY_LABEL" in tagger.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## Tagger.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`Tagger.initialize`](/api/tagger#initialize) to initialize the model with a +pre-defined label set. + +> #### Example +> +> ```python +> labels = tagger.label_data +> tagger.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = tagger.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/textcategorizer.mdx b/website/docs/api/textcategorizer.mdx new file mode 100644 index 0000000..a259b7b --- /dev/null +++ b/website/docs/api/textcategorizer.mdx @@ -0,0 +1,509 @@ +--- +title: TextCategorizer +tag: class +source: spacy/pipeline/textcat.py +version: 2 +teaser: 'Pipeline component for text classification' +api_base_class: /api/pipe +api_string_name: textcat +api_trainable: true +--- + +The text categorizer predicts **categories over a whole document**. and comes in +two flavors: `textcat` and `textcat_multilabel`. When you need to predict +exactly one true label per document, use the `textcat` which has mutually +exclusive labels. If you want to perform multi-label classification and predict +zero, one or more true labels per document, use the `textcat_multilabel` +component instead. For a binary classification task, you can use `textcat` with +**two** labels or `textcat_multilabel` with **one** label. + +Both components are documented on this page. + + + +In spaCy v2, the `textcat` component could also perform **multi-label +classification**, and even used this setting by default. Since v3.0, the +component `textcat_multilabel` should be used for multi-label classification +instead. The `textcat` component is now used for mutually exclusive classes +only. + + + +## Assigned Attributes {id="assigned-attributes"} + +Predictions will be saved to `doc.cats` as a dictionary, where the key is the +name of the category and the value is a score between 0 and 1 (inclusive). For +`textcat` (exclusive categories), the scores will sum to 1, while for +`textcat_multilabel` there is no particular guarantee about their sum. This also +means that for `textcat`, missing values are equated to a value of 0 (i.e. +`False`) and are counted as such towards the loss and scoring metrics. This is +not the case for `textcat_multilabel`, where missing values in the gold standard +data do not influence the loss or accuracy calculations. + +Note that when assigning values to create training data, the score of each +category must be 0 or 1. Using other values, for example to create a document +that is a little bit in category A and a little bit in category B, is not +supported. + +| Location | Value | +| ---------- | ------------------------------------- | +| `Doc.cats` | Category scores. ~~Dict[str, float]~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example (textcat) +> +> ```python +> from spacy.pipeline.textcat import DEFAULT_SINGLE_TEXTCAT_MODEL +> config = { +> "model": DEFAULT_SINGLE_TEXTCAT_MODEL, +> } +> nlp.add_pipe("textcat", config=config) +> ``` + +> #### Example (textcat_multilabel) +> +> ```python +> from spacy.pipeline.textcat_multilabel import DEFAULT_MULTI_TEXTCAT_MODEL +> config = { +> "threshold": 0.5, +> "model": DEFAULT_MULTI_TEXTCAT_MODEL, +> } +> nlp.add_pipe("textcat_multilabel", config=config) +> ``` + +| Setting | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | +| `model` | A model instance that predicts scores for each category. Defaults to [TextCatEnsemble](/api/architectures#TextCatEnsemble). ~~Model[List[Doc], List[Floats2d]]~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/textcat.py +``` + +```python +%%GITHUB_SPACY/spacy/pipeline/textcat_multilabel.py +``` + +## TextCategorizer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> # Use 'textcat_multilabel' for multi-label classification +> textcat = nlp.add_pipe("textcat") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_textcat"}} +> parser = nlp.add_pipe("textcat", config=config) +> +> # Construction from class +> # Use 'MultiLabel_TextCategorizer' for multi-label classification +> from spacy.pipeline import TextCategorizer +> textcat = TextCategorizer(nlp.vocab, model, threshold=0.5) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | + +## TextCategorizer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/textcategorizer#call) and [`pipe`](/api/textcategorizer#pipe) +delegate to the [`predict`](/api/textcategorizer#predict) and +[`set_annotations`](/api/textcategorizer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> textcat = nlp.add_pipe("textcat") +> # This usually happens under the hood +> processed = textcat(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## TextCategorizer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/textcategorizer#call) and +[`pipe`](/api/textcategorizer#pipe) delegate to the +[`predict`](/api/textcategorizer#predict) and +[`set_annotations`](/api/textcategorizer#set_annotations) methods. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> for doc in textcat.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## TextCategorizer.initialize {id="initialize",tag="method",version="3"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize) and lets you customize +arguments it receives via the +[`[initialize.components]`](/api/data-formats#config-initialize) block in the +config. + + + +This method was previously called `begin_training`. + + + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> textcat.initialize(lambda: examples, nlp=nlp) +> ``` +> +> ```ini +> ### config.cfg +> [initialize.components.textcat] +> positive_label = "POS" +> +> [initialize.components.textcat.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/textcat.json +> ``` + +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | +| `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | +| `positive_label` | The positive label for a binary task with exclusive classes, `None` otherwise and by default. This parameter is only used during scoring. It is not available when using the `textcat_multilabel` component. ~~Optional[str]~~ | + +## TextCategorizer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> scores = textcat.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## TextCategorizer.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> scores = textcat.predict(docs) +> textcat.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | --------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `TextCategorizer.predict`. | + +## TextCategorizer.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/textcategorizer#predict) and +[`get_loss`](/api/textcategorizer#get_loss). + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> optimizer = nlp.initialize() +> losses = textcat.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## TextCategorizer.rehearse {id="rehearse",tag="method,experimental",version="3"} + +Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the +current model to make predictions similar to an initial model to try to address +the "catastrophic forgetting" problem. This feature is experimental. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> optimizer = nlp.resume_training() +> losses = textcat.rehearse(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## TextCategorizer.get_loss {id="get_loss",tag="method"} + +Find the loss and gradient of loss for the batch of documents and their +predicted scores. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> scores = textcat.predict([eg.predicted for eg in examples]) +> loss, d_loss = textcat.get_loss(examples, scores) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------- | +| `examples` | The batch of examples. ~~Iterable[Example]~~ | +| `scores` | Scores representing the model's predictions. | +| **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | + +## TextCategorizer.score {id="score",tag="method",version="3"} + +Score a batch of examples. + +> #### Example +> +> ```python +> scores = textcat.score(examples) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------- | +| `examples` | The examples to score. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| **RETURNS** | The scores, produced by [`Scorer.score_cats`](/api/scorer#score_cats). ~~Dict[str, Union[float, Dict[str, float]]]~~ | + +## TextCategorizer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> optimizer = textcat.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## TextCategorizer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> with textcat.use_params(optimizer.averages): +> textcat.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## TextCategorizer.add_label {id="add_label",tag="method"} + +Add a new label to the pipe. Raises an error if the output dimension is already +set, or if the model has already been fully [initialized](#initialize). Note +that you don't have to call this method if you provide a **representative data +sample** to the [`initialize`](#initialize) method. In this case, all labels +found in the sample will be automatically added to the model, and the output +dimension will be [inferred](/usage/layers-architectures#thinc-shape-inference) +automatically. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> textcat.add_label("MY_LABEL") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `label` | The label to add. ~~str~~ | +| **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | + +## TextCategorizer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> textcat.to_disk("/path/to/textcat") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## TextCategorizer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> textcat.from_disk("/path/to/textcat") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `TextCategorizer` object. ~~TextCategorizer~~ | + +## TextCategorizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> textcat = nlp.add_pipe("textcat") +> textcat_bytes = textcat.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `TextCategorizer` object. ~~bytes~~ | + +## TextCategorizer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> textcat_bytes = textcat.to_bytes() +> textcat = nlp.add_pipe("textcat") +> textcat.from_bytes(textcat_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `TextCategorizer` object. ~~TextCategorizer~~ | + +## TextCategorizer.labels {id="labels",tag="property"} + +The labels currently added to the component. + +> #### Example +> +> ```python +> textcat.add_label("MY_LABEL") +> assert "MY_LABEL" in textcat.labels +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | + +## TextCategorizer.label_data {id="label_data",tag="property",version="3"} + +The labels currently added to the component and their internal meta information. +This is the data generated by [`init labels`](/api/cli#init-labels) and used by +[`TextCategorizer.initialize`](/api/textcategorizer#initialize) to initialize +the model with a pre-defined label set. + +> #### Example +> +> ```python +> labels = textcat.label_data +> textcat.initialize(lambda: [], nlp=nlp, labels=labels) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------- | +| **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = textcat.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/tok2vec.mdx b/website/docs/api/tok2vec.mdx new file mode 100644 index 0000000..a1bb126 --- /dev/null +++ b/website/docs/api/tok2vec.mdx @@ -0,0 +1,327 @@ +--- +title: Tok2Vec +source: spacy/pipeline/tok2vec.py +version: 3 +teaser: null +api_base_class: /api/pipe +api_string_name: tok2vec +api_trainable: true +--- + +Apply a "token-to-vector" model and set its outputs in the `Doc.tensor` +attribute. This is mostly useful to **share a single subnetwork** between +multiple components, e.g. to have one embedding and CNN network shared between a +[`DependencyParser`](/api/dependencyparser), [`Tagger`](/api/tagger) and +[`EntityRecognizer`](/api/entityrecognizer). + +In order to use the `Tok2Vec` predictions, subsequent components should use the +[Tok2VecListener](/api/architectures#Tok2VecListener) layer as the `tok2vec` +subnetwork of their model. This layer will read data from the `doc.tensor` +attribute during prediction. During training, the `Tok2Vec` component will save +its prediction and backprop callback for each batch, so that the subsequent +components can backpropagate to the shared weights. This implementation is used +because it allows us to avoid relying on object identity within the models to +achieve the parameter sharing. + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures) documentation for details on the +architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL +> config = {"model": DEFAULT_TOK2VEC_MODEL} +> nlp.add_pipe("tok2vec", config=config) +> ``` + +| Setting | Description | +| ------- | ------------------------------------------------------------------------------------------------------------------ | +| `model` | The model to use. Defaults to [HashEmbedCNN](/api/architectures#HashEmbedCNN). ~~Model[List[Doc], List[Floats2d]~~ | + +```python +%%GITHUB_SPACY/spacy/pipeline/tok2vec.py +``` + +## Tok2Vec.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> tok2vec = nlp.add_pipe("tok2vec") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_tok2vec"}} +> parser = nlp.add_pipe("tok2vec", config=config) +> +> # Construction from class +> from spacy.pipeline import Tok2Vec +> tok2vec = Tok2Vec(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| ------- | ------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | + +## Tok2Vec.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document and add context-sensitive embeddings to the +`Doc.tensor` attribute, allowing them to be used as features by downstream +components. The document is modified in place, and returned. This usually +happens under the hood when the `nlp` object is called on a text and all +pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/tok2vec#call) and [`pipe`](/api/tok2vec#pipe) delegate to the +[`predict`](/api/tok2vec#predict) and +[`set_annotations`](/api/tok2vec#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> tok2vec = nlp.add_pipe("tok2vec") +> # This usually happens under the hood +> processed = tok2vec(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## Tok2Vec.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/tok2vec#call) and +[`pipe`](/api/tok2vec#pipe) delegate to the [`predict`](/api/tok2vec#predict) +and [`set_annotations`](/api/tok2vec#set_annotations) methods. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> for doc in tok2vec.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Tok2Vec.initialize {id="initialize",tag="method"} + +Initialize the component for training and return an +[`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a +function that returns an iterable of [`Example`](/api/example) objects. **At +least one example should be supplied.** The data examples are used to +**initialize the model** of the component and can either be the full training +data or a representative sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> tok2vec.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## Tok2Vec.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> scores = tok2vec.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## Tok2Vec.set_annotations {id="set_annotations",tag="method"} + +Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> scores = tok2vec.predict(docs) +> tok2vec.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | ------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `Tok2Vec.predict`. | + +## Tok2Vec.update {id="update",tag="method"} + +Learn from a batch of [`Example`](/api/example) objects containing the +predictions and gold-standard annotations, and update the component's model. +Delegates to [`predict`](/api/tok2vec#predict). + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> optimizer = nlp.initialize() +> losses = tok2vec.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Tok2Vec.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> optimizer = tok2vec.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Tok2Vec.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> with tok2vec.use_params(optimizer.averages): +> tok2vec.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## Tok2Vec.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> tok2vec.to_disk("/path/to/tok2vec") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Tok2Vec.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> tok2vec.from_disk("/path/to/tok2vec") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Tok2Vec` object. ~~Tok2Vec~~ | + +## Tok2Vec.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> tok2vec = nlp.add_pipe("tok2vec") +> tok2vec_bytes = tok2vec.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Tok2Vec` object. ~~bytes~~ | + +## Tok2Vec.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> tok2vec_bytes = tok2vec.to_bytes() +> tok2vec = nlp.add_pipe("tok2vec") +> tok2vec.from_bytes(tok2vec_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Tok2Vec` object. ~~Tok2Vec~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = tok2vec.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/token.mdx b/website/docs/api/token.mdx new file mode 100644 index 0000000..63ee108 --- /dev/null +++ b/website/docs/api/token.mdx @@ -0,0 +1,477 @@ +--- +title: Token +teaser: An individual token — i.e. a word, punctuation symbol, whitespace, etc. +tag: class +source: spacy/tokens/token.pyx +--- + +## Token.\_\_init\_\_ {id="init",tag="method"} + +Construct a `Token` object. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> token = doc[0] +> assert token.text == "Give" +> ``` + +| Name | Description | +| -------- | --------------------------------------------------- | +| `vocab` | A storage container for lexical types. ~~Vocab~~ | +| `doc` | The parent document. ~~Doc~~ | +| `offset` | The index of the token within the document. ~~int~~ | + +## Token.\_\_len\_\_ {id="len",tag="method"} + +The number of unicode characters in the token, i.e. `token.text`. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> token = doc[0] +> assert len(token) == 4 +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The number of unicode characters in the token. ~~int~~ | + +## Token.set_extension {id="set_extension",tag="classmethod",version="2"} + +Define a custom attribute on the `Token` which becomes available via `Token._`. +For details, see the documentation on +[custom attributes](/usage/processing-pipelines#custom-components-attributes). + +> #### Example +> +> ```python +> from spacy.tokens import Token +> fruit_getter = lambda token: token.text in ("apple", "pear", "banana") +> Token.set_extension("is_fruit", getter=fruit_getter) +> doc = nlp("I have an apple") +> assert doc[3]._.is_fruit +> ``` + +| Name | Description | +| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the attribute to set by the extension. For example, `"my_attr"` will be available as `token._.my_attr`. ~~str~~ | +| `default` | Optional default value of the attribute if no getter or method is defined. ~~Optional[Any]~~ | +| `method` | Set a custom method on the object, for example `token._.compare(other_token)`. ~~Optional[Callable[[Token, ...], Any]]~~ | +| `getter` | Getter function that takes the object and returns an attribute value. Is called when the user accesses the `._` attribute. ~~Optional[Callable[[Token], Any]]~~ | +| `setter` | Setter function that takes the `Token` and a value, and modifies the object. Is called when the user writes to the `Token._` attribute. ~~Optional[Callable[[Token, Any], None]]~~ | +| `force` | Force overwriting existing attribute. ~~bool~~ | + +## Token.get_extension {id="get_extension",tag="classmethod",version="2"} + +Look up a previously registered extension by name. Returns a 4-tuple +`(default, method, getter, setter)` if the extension is registered. Raises a +`KeyError` otherwise. + +> #### Example +> +> ```python +> from spacy.tokens import Token +> Token.set_extension("is_fruit", default=False) +> extension = Token.get_extension("is_fruit") +> assert extension == (False, None, None, None) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Token.has_extension {id="has_extension",tag="classmethod",version="2"} + +Check whether an extension has been registered on the `Token` class. + +> #### Example +> +> ```python +> from spacy.tokens import Token +> Token.set_extension("is_fruit", default=False) +> assert Token.has_extension("is_fruit") +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| `name` | Name of the extension to check. ~~str~~ | +| **RETURNS** | Whether the extension has been registered. ~~bool~~ | + +## Token.remove_extension {id="remove_extension",tag="classmethod",version="2.0.11"} + +Remove a previously registered extension. + +> #### Example +> +> ```python +> from spacy.tokens import Token +> Token.set_extension("is_fruit", default=False) +> removed = Token.remove_extension("is_fruit") +> assert not Token.has_extension("is_fruit") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Name of the extension. ~~str~~ | +| **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | + +## Token.check_flag {id="check_flag",tag="method"} + +Check the value of a boolean flag. + +> #### Example +> +> ```python +> from spacy.attrs import IS_TITLE +> doc = nlp("Give it back! He pleaded.") +> token = doc[0] +> assert token.check_flag(IS_TITLE) == True +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------- | +| `flag_id` | The attribute ID of the flag to check. ~~int~~ | +| **RETURNS** | Whether the flag is set. ~~bool~~ | + +## Token.similarity {id="similarity",tag="method",model="vectors"} + +Compute a semantic similarity estimate. Defaults to cosine over vectors. + +> #### Example +> +> ```python +> apples, _, oranges = nlp("apples and oranges") +> apples_oranges = apples.similarity(oranges) +> oranges_apples = oranges.similarity(apples) +> assert apples_oranges == oranges_apples +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------- | +| other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | +| **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | + +## Token.nbor {id="nbor",tag="method"} + +Get a neighboring token. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> give_nbor = doc[0].nbor() +> assert give_nbor.text == "it" +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------- | +| `i` | The relative position of the token to get. Defaults to `1`. ~~int~~ | +| **RETURNS** | The token at position `self.doc[self.i+i]`. ~~Token~~ | + +## Token.set_morph {id="set_morph",tag="method"} + +Set the morphological analysis from a UD FEATS string, hash value of a UD FEATS +string, features dict or `MorphAnalysis`. The value `None` can be used to reset +the morph to an unset state. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> doc[0].set_morph("Mood=Imp|VerbForm=Fin") +> assert "Mood=Imp" in doc[0].morph +> assert doc[0].morph.get("Mood") == ["Imp"] +> ``` + +| Name | Description | +| -------- | --------------------------------------------------------------------------------- | +| features | The morphological features to set. ~~Union[int, dict, str, MorphAnalysis, None]~~ | + +## Token.has_morph {id="has_morph",tag="method"} + +Check whether the token has annotated morph information. Return `False` when the +morph annotation is unset/missing. + +| Name | Description | +| ----------- | --------------------------------------------- | +| **RETURNS** | Whether the morph annotation is set. ~~bool~~ | + +## Token.is_ancestor {id="is_ancestor",tag="method",model="parser"} + +Check whether this token is a parent, grandparent, etc. of another in the +dependency tree. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> give = doc[0] +> it = doc[1] +> assert give.is_ancestor(it) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------- | +| descendant | Another token. ~~Token~~ | +| **RETURNS** | Whether this token is the ancestor of the descendant. ~~bool~~ | + +## Token.ancestors {id="ancestors",tag="property",model="parser"} + +A sequence of the token's syntactic ancestors (parents, grandparents, etc). + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> it_ancestors = doc[1].ancestors +> assert [t.text for t in it_ancestors] == ["Give"] +> he_ancestors = doc[4].ancestors +> assert [t.text for t in he_ancestors] == ["pleaded"] +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------- | +| **YIELDS** | A sequence of ancestor tokens such that `ancestor.is_ancestor(self)`. ~~Token~~ | + +## Token.conjuncts {id="conjuncts",tag="property",model="parser"} + +A tuple of coordinated tokens, not including the token itself. + +> #### Example +> +> ```python +> doc = nlp("I like apples and oranges") +> apples_conjuncts = doc[2].conjuncts +> assert [t.text for t in apples_conjuncts] == ["oranges"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------- | +| **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | + +## Token.children {id="children",tag="property",model="parser"} + +A sequence of the token's immediate syntactic children. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> give_children = doc[0].children +> assert [t.text for t in give_children] == ["it", "back", "!"] +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------- | +| **YIELDS** | A child token such that `child.head == self`. ~~Token~~ | + +## Token.lefts {id="lefts",tag="property",model="parser"} + +The leftward immediate children of the word in the syntactic dependency parse. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> lefts = [t.text for t in doc[3].lefts] +> assert lefts == ["New"] +> ``` + +| Name | Description | +| ---------- | ------------------------------------ | +| **YIELDS** | A left-child of the token. ~~Token~~ | + +## Token.rights {id="rights",tag="property",model="parser"} + +The rightward immediate children of the word in the syntactic dependency parse. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> rights = [t.text for t in doc[3].rights] +> assert rights == ["in"] +> ``` + +| Name | Description | +| ---------- | ------------------------------------- | +| **YIELDS** | A right-child of the token. ~~Token~~ | + +## Token.n_lefts {id="n_lefts",tag="property",model="parser"} + +The number of leftward immediate children of the word in the syntactic +dependency parse. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> assert doc[3].n_lefts == 1 +> ``` + +| Name | Description | +| ----------- | ---------------------------------------- | +| **RETURNS** | The number of left-child tokens. ~~int~~ | + +## Token.n_rights {id="n_rights",tag="property",model="parser"} + +The number of rightward immediate children of the word in the syntactic +dependency parse. + +> #### Example +> +> ```python +> doc = nlp("I like New York in Autumn.") +> assert doc[3].n_rights == 1 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------- | +| **RETURNS** | The number of right-child tokens. ~~int~~ | + +## Token.subtree {id="subtree",tag="property",model="parser"} + +A sequence containing the token and all the token's syntactic descendants. + +> #### Example +> +> ```python +> doc = nlp("Give it back! He pleaded.") +> give_subtree = doc[0].subtree +> assert [t.text for t in give_subtree] == ["Give", "it", "back", "!"] +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------ | +| **YIELDS** | A descendant token such that `self.is_ancestor(token)` or `token == self`. ~~Token~~ | + +## Token.has_vector {id="has_vector",tag="property",model="vectors"} + +A boolean value indicating whether a word vector is associated with the token. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> apples = doc[2] +> assert apples.has_vector +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | Whether the token has a vector data attached. ~~bool~~ | + +## Token.vector {id="vector",tag="property",model="vectors"} + +A real-valued meaning representation. + +> #### Example +> +> ```python +> doc = nlp("I like apples") +> apples = doc[2] +> assert apples.vector.dtype == "float32" +> assert apples.vector.shape == (300,) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| **RETURNS** | A 1-dimensional array representing the token's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Token.vector_norm {id="vector_norm",tag="property",model="vectors"} + +The L2 norm of the token's vector representation. + +> #### Example +> +> ```python +> doc = nlp("I like apples and pasta") +> apples = doc[2] +> pasta = doc[4] +> apples.vector_norm # 6.89589786529541 +> pasta.vector_norm # 7.759851932525635 +> assert apples.vector_norm != pasta.vector_norm +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------- | +| **RETURNS** | The L2 norm of the vector representation. ~~float~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `lex` 3 | The underlying lexeme. ~~Lexeme~~ | +| `sent` | The sentence span that this token is a part of. ~~Span~~ | +| `text` | Verbatim text content. ~~str~~ | +| `text_with_ws` | Text content, with trailing space character if present. ~~str~~ | +| `whitespace_` | Trailing space character if present. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Token.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `vocab` | The vocab object of the parent `Doc`. ~~vocab~~ | +| `tensor` | The token's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | +| `head` | The syntactic parent, or "governor", of this token. ~~Token~~ | +| `left_edge` | The leftmost token of this token's syntactic descendants. ~~Token~~ | +| `right_edge` | The rightmost token of this token's syntactic descendants. ~~Token~~ | +| `i` | The index of the token within the parent document. ~~int~~ | +| `ent_type` | Named entity type. ~~int~~ | +| `ent_type_` | Named entity type. ~~str~~ | +| `ent_iob` | IOB code of named entity tag. `3` means the token begins an entity, `2` means it is outside an entity, `1` means it is inside an entity, and `0` means no entity tag is set. ~~int~~ | +| `ent_iob_` | IOB code of named entity tag. "B" means the token begins an entity, "I" means it is inside an entity, "O" means it is outside an entity, and "" means no entity tag is set. ~~str~~ | +| `ent_kb_id` | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~int~~ | +| `ent_kb_id_` | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~str~~ | +| `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~int~~ | +| `ent_id_` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~str~~ | +| `lemma` | Base form of the token, with no inflectional suffixes. ~~int~~ | +| `lemma_` | Base form of the token, with no inflectional suffixes. ~~str~~ | +| `norm` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~int~~ | +| `norm_` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~str~~ | +| `lower` | Lowercase form of the token. ~~int~~ | +| `lower_` | Lowercase form of the token text. Equivalent to `Token.text.lower()`. ~~str~~ | +| `shape` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | +| `shape_` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | +| `prefix` | Hash value of a length-N substring from the start of the token. Defaults to `N=1`. ~~int~~ | +| `prefix_` | A length-N substring from the start of the token. Defaults to `N=1`. ~~str~~ | +| `suffix` | Hash value of a length-N substring from the end of the token. Defaults to `N=3`. ~~int~~ | +| `suffix_` | Length-N substring from the end of the token. Defaults to `N=3`. ~~str~~ | +| `is_alpha` | Does the token consist of alphabetic characters? Equivalent to `token.text.isalpha()`. ~~bool~~ | +| `is_ascii` | Does the token consist of ASCII characters? Equivalent to `all(ord(c) < 128 for c in token.text)`. ~~bool~~ | +| `is_digit` | Does the token consist of digits? Equivalent to `token.text.isdigit()`. ~~bool~~ | +| `is_lower` | Is the token in lowercase? Equivalent to `token.text.islower()`. ~~bool~~ | +| `is_upper` | Is the token in uppercase? Equivalent to `token.text.isupper()`. ~~bool~~ | +| `is_title` | Is the token in titlecase? Equivalent to `token.text.istitle()`. ~~bool~~ | +| `is_punct` | Is the token punctuation? ~~bool~~ | +| `is_left_punct` | Is the token a left punctuation mark, e.g. `"("` ? ~~bool~~ | +| `is_right_punct` | Is the token a right punctuation mark, e.g. `")"` ? ~~bool~~ | +| `is_sent_start` | Does the token start a sentence? ~~bool~~ or `None` if unknown. Defaults to `True` for the first token in the `Doc`. | +| `is_sent_end` | Does the token end a sentence? ~~bool~~ or `None` if unknown. | +| `is_space` | Does the token consist of whitespace characters? Equivalent to `token.text.isspace()`. ~~bool~~ | +| `is_bracket` | Is the token a bracket? ~~bool~~ | +| `is_quote` | Is the token a quotation mark? ~~bool~~ | +| `is_currency` | Is the token a currency symbol? ~~bool~~ | +| `like_url` | Does the token resemble a URL? ~~bool~~ | +| `like_num` | Does the token represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | +| `like_email` | Does the token resemble an email address? ~~bool~~ | +| `is_oov` | Is the token out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | +| `is_stop` | Is the token part of a "stop list"? ~~bool~~ | +| `pos` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~int~~ | +| `pos_` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~str~~ | +| `tag` | Fine-grained part-of-speech. ~~int~~ | +| `tag_` | Fine-grained part-of-speech. ~~str~~ | +| `morph` 3 | Morphological analysis. ~~MorphAnalysis~~ | +| `dep` | Syntactic dependency relation. ~~int~~ | +| `dep_` | Syntactic dependency relation. ~~str~~ | +| `lang` | Language of the parent document's vocabulary. ~~int~~ | +| `lang_` | Language of the parent document's vocabulary. ~~str~~ | +| `prob` | Smoothed log probability estimate of token's word type (context-independent entry in the vocabulary). ~~float~~ | +| `idx` | The character offset of the token within the parent document. ~~int~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the token. ~~float~~ | +| `lex_id` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `rank` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `cluster` | Brown cluster ID. ~~int~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/tokenizer.mdx b/website/docs/api/tokenizer.mdx new file mode 100644 index 0000000..0a579ab --- /dev/null +++ b/website/docs/api/tokenizer.mdx @@ -0,0 +1,264 @@ +--- +title: Tokenizer +teaser: Segment text into words, punctuations marks, etc. +tag: class +source: spacy/tokenizer.pyx +--- + +> #### Default config +> +> ```ini +> [nlp.tokenizer] +> @tokenizers = "spacy.Tokenizer.v1" +> ``` + +Segment text, and create `Doc` objects with the discovered segment boundaries. +For a deeper understanding, see the docs on +[how spaCy's tokenizer works](/usage/linguistic-features#how-tokenizer-works). +The tokenizer is typically created automatically when a +[`Language`](/api/language) subclass is initialized and it reads its settings +like punctuation and special case rules from the +[`Language.Defaults`](/api/language#defaults) provided by the language subclass. + +## Tokenizer.\_\_init\_\_ {id="init",tag="method"} + +Create a `Tokenizer` to create `Doc` objects given unicode text. For examples of +how to construct a custom tokenizer with different tokenization rules, see the +[usage documentation](https://spacy.io/usage/linguistic-features#native-tokenizers). + +> #### Example +> +> ```python +> # Construction 1 +> from spacy.tokenizer import Tokenizer +> from spacy.lang.en import English +> nlp = English() +> # Create a blank Tokenizer with just the English vocab +> tokenizer = Tokenizer(nlp.vocab) +> +> # Construction 2 +> from spacy.lang.en import English +> nlp = English() +> # Create a Tokenizer with the default settings for English +> # including punctuation rules and exceptions +> tokenizer = nlp.tokenizer +> ``` + +| Name | Description | +| -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | A storage container for lexical types. ~~Vocab~~ | +| `rules` | Exceptions and special-cases for the tokenizer. ~~Optional[Dict[str, List[Dict[int, str]]]]~~ | +| `prefix_search` | A function matching the signature of `re.compile(string).search` to match prefixes. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `suffix_search` | A function matching the signature of `re.compile(string).search` to match suffixes. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `infix_finditer` | A function matching the signature of `re.compile(string).finditer` to find infixes. ~~Optional[Callable[[str], Iterator[Match]]]~~ | +| `token_match` | A function matching the signature of `re.compile(string).match` to find token matches. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `url_match` | A function matching the signature of `re.compile(string).match` to find token matches after considering prefixes and suffixes. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `faster_heuristics` 3.3.0 | Whether to restrict the final `Matcher`-based pass for rules to those containing affixes or space. Defaults to `True`. ~~bool~~ | + +## Tokenizer.\_\_call\_\_ {id="call",tag="method"} + +Tokenize a string. + +> #### Example +> +> ```python +> tokens = tokenizer("This is a sentence") +> assert len(tokens) == 4 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------- | +| `string` | The string to tokenize. ~~str~~ | +| **RETURNS** | A container for linguistic annotations. ~~Doc~~ | + +## Tokenizer.pipe {id="pipe",tag="method"} + +Tokenize a stream of texts. + +> #### Example +> +> ```python +> texts = ["One document.", "...", "Lots of documents"] +> for doc in tokenizer.pipe(texts, batch_size=50): +> pass +> ``` + +| Name | Description | +| ------------ | ------------------------------------------------------------------------------------ | +| `texts` | A sequence of unicode texts. ~~Iterable[str]~~ | +| `batch_size` | The number of texts to accumulate in an internal buffer. Defaults to `1000`. ~~int~~ | +| **YIELDS** | The tokenized `Doc` objects, in order. ~~Doc~~ | + +## Tokenizer.find_infix {id="find_infix",tag="method"} + +Find internal split points of the string. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `string` | The string to split. ~~str~~ | +| **RETURNS** | A list of `re.MatchObject` objects that have `.start()` and `.end()` methods, denoting the placement of internal segment separators, e.g. hyphens. ~~List[Match]~~ | + +## Tokenizer.find_prefix {id="find_prefix",tag="method"} + +Find the length of a prefix that should be segmented from the string, or `None` +if no prefix rules match. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------ | +| `string` | The string to segment. ~~str~~ | +| **RETURNS** | The length of the prefix if present, otherwise `None`. ~~Optional[int]~~ | + +## Tokenizer.find_suffix {id="find_suffix",tag="method"} + +Find the length of a suffix that should be segmented from the string, or `None` +if no suffix rules match. + +| Name | Description | +| ----------- | ------------------------------------------------------------------------ | +| `string` | The string to segment. ~~str~~ | +| **RETURNS** | The length of the suffix if present, otherwise `None`. ~~Optional[int]~~ | + +## Tokenizer.add_special_case {id="add_special_case",tag="method"} + +Add a special-case tokenization rule. This mechanism is also used to add custom +tokenizer exceptions to the language data. See the usage guide on the +[languages data](/usage/linguistic-features#language-data) and +[tokenizer special cases](/usage/linguistic-features#special-cases) for more +details and examples. + +> #### Example +> +> ```python +> from spacy.attrs import ORTH, NORM +> case = [{ORTH: "do"}, {ORTH: "n't", NORM: "not"}] +> tokenizer.add_special_case("don't", case) +> ``` + +| Name | Description | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `string` | The string to specially tokenize. ~~str~~ | +| `token_attrs` | A sequence of dicts, where each dict describes a token and its attributes. The `ORTH` fields of the attributes must exactly match the string when they are concatenated. ~~Iterable[Dict[int, str]]~~ | + +## Tokenizer.explain {id="explain",tag="method"} + +Tokenize a string with a slow debugging tokenizer that provides information +about which tokenizer rule or pattern was matched for each token. The tokens +produced are identical to `Tokenizer.__call__` except for whitespace tokens. + +> #### Example +> +> ```python +> tok_exp = nlp.tokenizer.explain("(don't)") +> assert [t[0] for t in tok_exp] == ["PREFIX", "SPECIAL-1", "SPECIAL-2", "SUFFIX"] +> assert [t[1] for t in tok_exp] == ["(", "do", "n't", ")"] +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| `string` | The string to tokenize with the debugging tokenizer. ~~str~~ | +| **RETURNS** | A list of `(pattern_string, token_string)` tuples. ~~List[Tuple[str, str]]~~ | + +## Tokenizer.to_disk {id="to_disk",tag="method"} + +Serialize the tokenizer to disk. + +> #### Example +> +> ```python +> tokenizer = Tokenizer(nlp.vocab) +> tokenizer.to_disk("/path/to/tokenizer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Tokenizer.from_disk {id="from_disk",tag="method"} + +Load the tokenizer from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> tokenizer = Tokenizer(nlp.vocab) +> tokenizer.from_disk("/path/to/tokenizer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Tokenizer` object. ~~Tokenizer~~ | + +## Tokenizer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> tokenizer = tokenizer(nlp.vocab) +> tokenizer_bytes = tokenizer.to_bytes() +> ``` + +Serialize the tokenizer to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Tokenizer` object. ~~bytes~~ | + +## Tokenizer.from_bytes {id="from_bytes",tag="method"} + +Load the tokenizer from a bytestring. Modifies the object in place and returns +it. + +> #### Example +> +> ```python +> tokenizer_bytes = tokenizer.to_bytes() +> tokenizer = Tokenizer(nlp.vocab) +> tokenizer.from_bytes(tokenizer_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Tokenizer` object. ~~Tokenizer~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The vocab object of the parent `Doc`. ~~Vocab~~ | +| `prefix_search` | A function to find segment boundaries from the start of a string. Returns the length of the segment, or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `suffix_search` | A function to find segment boundaries from the end of a string. Returns the length of the segment, or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `infix_finditer` | A function to find internal segment separators, e.g. hyphens. Returns a (possibly empty) sequence of `re.MatchObject` objects. ~~Optional[Callable[[str], Iterator[Match]]]~~ | +| `token_match` | A function matching the signature of `re.compile(string).match` to find token matches. Returns an `re.MatchObject` or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ | +| `rules` | A dictionary of tokenizer exceptions and special cases. ~~Optional[Dict[str, List[Dict[int, str]]]]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = tokenizer.to_bytes(exclude=["vocab", "exceptions"]) +> tokenizer.from_disk("./data", exclude=["token_match"]) +> ``` + +| Name | Description | +| ---------------- | --------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `prefix_search` | The prefix rules. | +| `suffix_search` | The suffix rules. | +| `infix_finditer` | The infix rules. | +| `token_match` | The token match expression. | +| `exceptions` | The tokenizer exception rules. | diff --git a/website/docs/api/top-level.mdx b/website/docs/api/top-level.mdx new file mode 100644 index 0000000..cb4bd0a --- /dev/null +++ b/website/docs/api/top-level.mdx @@ -0,0 +1,1575 @@ +--- +title: Top-level Functions +menu: + - ['spacy', 'spacy'] + - ['displacy', 'displacy'] + - ['registry', 'registry'] + - ['Loggers', 'loggers'] + - ['Readers', 'readers'] + - ['Batchers', 'batchers'] + - ['Augmenters', 'augmenters'] + - ['Callbacks', 'callbacks'] + - ['Training & Alignment', 'gold'] + - ['Utility Functions', 'util'] +--- + +## spaCy {id="spacy",hidden="true"} + +### spacy.load {id="spacy.load",tag="function"} + +Load a pipeline using the name of an installed +[package](/usage/saving-loading#models), a string path or a `Path`-like object. +spaCy will try resolving the load argument in this order. If a pipeline is +loaded from a string name, spaCy will assume it's a Python package and import it +and call the package's own `load()` method. If a pipeline is loaded from a path, +spaCy will assume it's a data directory, load its +[`config.cfg`](/api/data-formats#config) and use the language and pipeline +information to construct the `Language` class. The data will be loaded in via +[`Language.from_disk`](/api/language#from_disk). Loading a pipeline from a +package will also import any custom code, if present, whereas loading from a +directory does not. For these cases, you need to manually import your custom +code. + + + +As of v3.0, the `disable` keyword argument specifies components to load but +disable, instead of components to not load at all. Those components can now be +specified separately using the new `exclude` keyword argument. + + + +> #### Example +> +> ```python +> nlp = spacy.load("en_core_web_sm") # package +> nlp = spacy.load("/path/to/pipeline") # string path +> nlp = spacy.load(Path("/path/to/pipeline")) # pathlib Path +> +> nlp = spacy.load("en_core_web_sm", exclude=["parser", "tagger"]) +> ``` + +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | + +Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's +[`config.cfg`](/api/data-formats#config), uses the language and pipeline +information to construct a `Language` object, loads in the model data and +weights, and returns it. + +```python {title="Abstract example"} +cls = spacy.util.get_lang_class(lang) # 1. Get Language class, e.g. English +nlp = cls() # 2. Initialize it +for name in pipeline: + nlp.add_pipe(name, config={...}) # 3. Add the component to the pipeline +nlp.from_disk(data_path) # 4. Load in the binary data +``` + +### spacy.blank {id="spacy.blank",tag="function",version="2"} + +Create a blank pipeline of a given language class. This function is the twin of +`spacy.load()`. + +> #### Example +> +> ```python +> nlp_en = spacy.blank("en") # equivalent to English() +> nlp_de = spacy.blank("de") # equivalent to German() +> ``` + +| Name | Description | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Two-letter [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) or three-letter [ISO 639-3](https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes) language codes, such as 'en' and 'eng', of the language class to load. ~~str~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | +| `meta` | Optional meta overrides for [`nlp.meta`](/api/language#meta). ~~Dict[str, Any]~~ | +| **RETURNS** | An empty `Language` object of the appropriate subclass. ~~Language~~ | + +### spacy.info {id="spacy.info",tag="function"} + +The same as the [`info` command](/api/cli#info). Pretty-print information about +your installation, installed pipelines and local setup from within spaCy. + +> #### Example +> +> ```python +> spacy.info() +> spacy.info("en_core_web_sm") +> markdown = spacy.info(markdown=True, silent=True) +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------- | +| `model` | Optional pipeline, i.e. a package name or path (optional). ~~Optional[str]~~ | +| _keyword-only_ | | +| `markdown` | Print information as Markdown. ~~bool~~ | +| `silent` | Don't print anything, just return. ~~bool~~ | + +### spacy.explain {id="spacy.explain",tag="function"} + +Get a description for a given POS tag, dependency label or entity type. For a +list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py). + +> #### Example +> +> ```python +> spacy.explain("NORP") +> # Nationalities or religious or political groups +> +> doc = nlp("Hello world") +> for word in doc: +> print(word.text, word.tag_, spacy.explain(word.tag_)) +> # Hello UH interjection +> # world NN noun, singular or mass +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------- | +| `term` | Term to explain. ~~str~~ | +| **RETURNS** | The explanation, or `None` if not found in the glossary. ~~Optional[str]~~ | + +### spacy.prefer_gpu {id="spacy.prefer_gpu",tag="function",version="2.0.14"} + +Allocate data and perform operations on [GPU](/usage/#gpu), if available. If +data has already been allocated on CPU, it will not be moved. Ideally, this +function should be called right after importing spaCy and _before_ loading any +pipelines. + + + +In a Jupyter notebook, run `prefer_gpu()` in the same cell as `spacy.load()` to +ensure that the model is loaded on the correct device. See +[more details](/usage/v3#jupyter-notebook-gpu). + + + +> #### Example +> +> ```python +> import spacy +> activated = spacy.prefer_gpu() +> nlp = spacy.load("en_core_web_sm") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------ | +| `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | +| **RETURNS** | Whether the GPU was activated. ~~bool~~ | + +### spacy.require_gpu {id="spacy.require_gpu",tag="function",version="2.0.14"} + +Allocate data and perform operations on [GPU](/usage/#gpu). Will raise an error +if no GPU is available. If data has already been allocated on CPU, it will not +be moved. Ideally, this function should be called right after importing spaCy +and _before_ loading any pipelines. + + + +In a Jupyter notebook, run `require_gpu()` in the same cell as `spacy.load()` to +ensure that the model is loaded on the correct device. See +[more details](/usage/v3#jupyter-notebook-gpu). + + + +> #### Example +> +> ```python +> import spacy +> spacy.require_gpu() +> nlp = spacy.load("en_core_web_sm") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------ | +| `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | +| **RETURNS** | `True` ~~bool~~ | + +### spacy.require_cpu {id="spacy.require_cpu",tag="function",version="3.0.0"} + +Allocate data and perform operations on CPU. If data has already been allocated +on GPU, it will not be moved. Ideally, this function should be called right +after importing spaCy and _before_ loading any pipelines. + + + +In a Jupyter notebook, run `require_cpu()` in the same cell as `spacy.load()` to +ensure that the model is loaded on the correct device. See +[more details](/usage/v3#jupyter-notebook-gpu). + + + +> #### Example +> +> ```python +> import spacy +> spacy.require_cpu() +> nlp = spacy.load("en_core_web_sm") +> ``` + +| Name | Description | +| ----------- | --------------- | +| **RETURNS** | `True` ~~bool~~ | + +## displaCy {id="displacy",source="spacy/displacy"} + +As of v2.0, spaCy comes with a built-in visualization suite. For more info and +examples, see the usage guide on [visualizing spaCy](/usage/visualizers). + +### displacy.serve {id="displacy.serve",tag="method",version="2"} + +Serve a dependency parse tree or named entity visualization to view it in your +browser. Will run a simple web server. + +> #### Example +> +> ```python +> import spacy +> from spacy import displacy +> nlp = spacy.load("en_core_web_sm") +> doc1 = nlp("This is a sentence.") +> doc2 = nlp("This is another sentence.") +> displacy.serve([doc1, doc2], style="dep") +> ``` + +| Name | Description | +| ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | +| `style` 3.3 | Visualization style, `"dep"`, `"ent"` or `"span"`. Defaults to `"dep"`. ~~str~~ | +| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | +| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | +| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | +| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | +| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ | +| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | +| `auto_select_port` 3.5 | If `True`, automatically switch to a different port if the specified port is already in use. Defaults to `False`. ~~bool~~ | + +### displacy.render {id="displacy.render",tag="method",version="2"} + +Render a dependency parse tree or named entity visualization. + +> #### Example +> +> ```python +> import spacy +> from spacy import displacy +> nlp = spacy.load("en_core_web_sm") +> doc = nlp("This is a sentence.") +> html = displacy.render(doc, style="dep") +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ | +| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | +| `page` | Render markup as full HTML page. Defaults to `False`. ~~bool~~ | +| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | +| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | +| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | +| `jupyter` | Explicitly enable or disable "[Jupyter](http://jupyter.org/) mode" to return markup ready to be rendered in a notebook. Detected automatically if `None` (default). ~~Optional[bool]~~ | +| **RETURNS** | The rendered HTML markup. ~~str~~ | + +### displacy.parse_deps {id="displacy.parse_deps",tag="method",version="2"} + +Generate dependency parse in `{'words': [], 'arcs': []}` format. For use with +the `manual=True` argument in `displacy.render`. + +> #### Example +> +> ```python +> import spacy +> from spacy import displacy +> nlp = spacy.load("en_core_web_sm") +> doc = nlp("This is a sentence.") +> deps_parse = displacy.parse_deps(doc) +> html = displacy.render(deps_parse, style="dep", manual=True) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------- | +| `orig_doc` | Doc or span to parse dependencies. ~~Union[Doc, Span]~~ | +| `options` | Dependency parse specific visualisation options. ~~Dict[str, Any]~~ | +| **RETURNS** | Generated dependency parse keyed by words and arcs. ~~dict~~ | + +### displacy.parse_ents {id="displacy.parse_ents",tag="method",version="2"} + +Generate named entities in `[{start: i, end: i, label: 'label'}]` format. For +use with the `manual=True` argument in `displacy.render`. + +> #### Example +> +> ```python +> import spacy +> from spacy import displacy +> nlp = spacy.load("en_core_web_sm") +> doc = nlp("But Google is starting from behind.") +> ents_parse = displacy.parse_ents(doc) +> html = displacy.render(ents_parse, style="ent", manual=True) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------- | +| `doc` | Doc to parse entities. ~~Doc~~ | +| `options` | NER-specific visualisation options. ~~Dict[str, Any]~~ | +| **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | + +### displacy.parse_spans {id="displacy.parse_spans",tag="method",version="2"} + +Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format. For +use with the `manual=True` argument in `displacy.render`. + +> #### Example +> +> ```python +> import spacy +> from spacy import displacy +> nlp = spacy.load("en_core_web_sm") +> doc = nlp("But Google is starting from behind.") +> doc.spans['orgs'] = [doc[1:2]] +> ents_parse = displacy.parse_spans(doc, options={"spans_key" : "orgs"}) +> html = displacy.render(ents_parse, style="span", manual=True) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------- | +| `doc` | Doc to parse entities. ~~Doc~~ | +| `options` | Span-specific visualisation options. ~~Dict[str, Any]~~ | +| **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | + +### Visualizer data structures {id="displacy_structures"} + +You can use displaCy's data format to manually render data. This can be useful +if you want to visualize output from other libraries. You can find examples of +displaCy's different data formats below. + +> #### DEP example data structure +> +> ```json +> { +> "words": [ +> { "text": "This", "tag": "DT" }, +> { "text": "is", "tag": "VBZ" }, +> { "text": "a", "tag": "DT" }, +> { "text": "sentence", "tag": "NN" } +> ], +> "arcs": [ +> { "start": 0, "end": 1, "label": "nsubj", "dir": "left" }, +> { "start": 2, "end": 3, "label": "det", "dir": "left" }, +> { "start": 1, "end": 3, "label": "attr", "dir": "right" } +> ] +> } +> ``` + +#### Dependency Visualizer data structure {id="structure-dep"} + +| Dictionary Key | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------- | +| `words` | List of dictionaries describing a word token (see structure below). ~~List[Dict[str, Any]]~~ | +| `arcs` | List of dictionaries describing the relations between words (see structure below). ~~List[Dict[str, Any]]~~ | +| _Optional_ | | +| `title` | Title of the visualization. ~~Optional[str]~~ | +| `settings` | Dependency Visualizer options (see [here](/api/top-level#displacy_options)). ~~Dict[str, Any]~~ | + + + +| Dictionary Key | Description | +| -------------- | ---------------------------------------- | +| `text` | Text content of the word. ~~str~~ | +| `tag` | Fine-grained part-of-speech. ~~str~~ | +| `lemma` | Base form of the word. ~~Optional[str]~~ | + + + + + +| Dictionary Key | Description | +| -------------- | ---------------------------------------------------- | +| `start` | The index of the starting token. ~~int~~ | +| `end` | The index of the ending token. ~~int~~ | +| `label` | The type of dependency relation. ~~str~~ | +| `dir` | Direction of the relation (`left`, `right`). ~~str~~ | + + + +> #### ENT example data structure +> +> ```json +> { +> "text": "But Google is starting from behind.", +> "ents": [{ "start": 4, "end": 10, "label": "ORG" }] +> } +> ``` + +#### Named Entity Recognition data structure {id="structure-ent"} + +| Dictionary Key | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `text` | String representation of the document text. ~~str~~ | +| `ents` | List of dictionaries describing entities (see structure below). ~~List[Dict[str, Any]]~~ | +| _Optional_ | | +| `title` | Title of the visualization. ~~Optional[str]~~ | +| `settings` | Entity Visualizer options (see [here](/api/top-level#displacy_options)). ~~Dict[str, Any]~~ | + + + +| Dictionary Key | Description | +| -------------- | ---------------------------------------------------------------------- | +| `start` | The index of the first character of the entity. ~~int~~ | +| `end` | The index of the last character of the entity. (not inclusive) ~~int~~ | +| `label` | Label attached to the entity. ~~str~~ | +| _Optional_ | | +| `kb_id` | `KnowledgeBase` ID. ~~str~~ | +| `kb_url` | `KnowledgeBase` URL. ~~str~~ | + + + +> #### SPAN example data structure +> +> ```json +> { +> "text": "Welcome to the Bank of China.", +> "spans": [ +> { "start_token": 3, "end_token": 6, "label": "ORG" }, +> { "start_token": 5, "end_token": 6, "label": "GPE" } +> ], +> "tokens": ["Welcome", "to", "the", "Bank", "of", "China", "."] +> } +> ``` + +#### Span Classification data structure {id="structure-span"} + +| Dictionary Key | Description | +| -------------- | ----------------------------------------------------------------------------------------- | +| `text` | String representation of the document text. ~~str~~ | +| `spans` | List of dictionaries describing spans (see structure below). ~~List[Dict[str, Any]]~~ | +| `tokens` | List of word tokens. ~~List[str]~~ | +| _Optional_ | | +| `title` | Title of the visualization. ~~Optional[str]~~ | +| `settings` | Span Visualizer options (see [here](/api/top-level#displacy_options)). ~~Dict[str, Any]~~ | + + + +| Dictionary Key | Description | +| -------------- | ------------------------------------------------------------- | +| `start_token` | The index of the first token of the span in `tokens`. ~~int~~ | +| `end_token` | The index of the last token of the span in `tokens`. ~~int~~ | +| `label` | Label attached to the span. ~~str~~ | +| _Optional_ | | +| `kb_id` | `KnowledgeBase` ID. ~~str~~ | +| `kb_url` | `KnowledgeBase` URL. ~~str~~ | + + + +### Visualizer options {id="displacy_options"} + +The `options` argument lets you specify additional settings for each visualizer. +If a setting is not present in the options, the default value will be used. + +#### Dependency Visualizer options {id="options-dep"} + +> #### Example +> +> ```python +> options = {"compact": True, "color": "blue"} +> displacy.serve(doc, style="dep", options=options) +> ``` + +| Name | Description | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `fine_grained` | Use fine-grained part-of-speech tags (`Token.tag_`) instead of coarse-grained tags (`Token.pos_`). Defaults to `False`. ~~bool~~ | +| `add_lemma` | Print the lemmas in a separate row below the token texts. Defaults to `False`. ~~bool~~ | +| `collapse_punct` | Attach punctuation to tokens. Can make the parse more readable, as it prevents long arcs to attach punctuation. Defaults to `True`. ~~bool~~ | +| `collapse_phrases` | Merge noun phrases into one token. Defaults to `False`. ~~bool~~ | +| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | +| `color` | Text color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#000000"`. ~~str~~ | +| `bg` | Background color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#ffffff"`. ~~str~~ | +| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | +| `offset_x` | Spacing on left side of the SVG in px. Defaults to `50`. ~~int~~ | +| `arrow_stroke` | Width of arrow path in px. Defaults to `2`. ~~int~~ | +| `arrow_width` | Width of arrow head in px. Defaults to `10` in regular mode and `8` in compact mode. ~~int~~ | +| `arrow_spacing` | Spacing between arrows in px to avoid overlaps. Defaults to `20` in regular mode and `12` in compact mode. ~~int~~ | +| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | +| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | + +#### Named Entity Visualizer options {id="displacy_options-ent"} + +> #### Example +> +> ```python +> options = {"ents": ["PERSON", "ORG", "PRODUCT"], +> "colors": {"ORG": "yellow"}} +> displacy.serve(doc, style="ent", options=options) +> ``` + +| Name | Description | +| ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ents` | Entity types to highlight or `None` for all types (default). ~~Optional[List[str]]~~ | +| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | +| `template` | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | +| `kb_url_template` 3.2.1 | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ | + +#### Span Visualizer options {id="displacy_options-span"} + +> #### Example +> +> ```python +> options = {"spans_key": "sc"} +> displacy.serve(doc, style="span", options=options) +> ``` + +| Name | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ | +| `templates` | Dictionary containing the keys `"span"`, `"slice"`, and `"start"`. These dictate how the overall span, a span slice, and the starting token will be rendered. ~~Optional[Dict[str, str]~~ | +| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ | +| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | + +By default, displaCy comes with colors for all entity types used by +[spaCy's trained pipelines](/models) for both entity and span visualizer. If +you're using custom entity types, you can use the `colors` setting to add your +own colors for them. Your application or pipeline package can also expose a +[`spacy_displacy_colors` entry point](/usage/saving-loading#entry-points-displacy) +to add custom labels and their colors automatically. + +By default, displaCy links to `#` for entities without a `kb_id` set on their +span. If you wish to link an entity to their URL then consider using the +`kb_url_template` option from above. For example if the `kb_id` on a span is +`Q95` and this is a Wikidata identifier then this option can be set to +`https://www.wikidata.org/wiki/{}`. Clicking on your entity in the rendered HTML +should redirect you to their Wikidata page, in this case +`https://www.wikidata.org/wiki/Q95`. + +## registry {id="registry",source="spacy/util.py",version="3"} + +spaCy's function registry extends +[Thinc's `registry`](https://thinc.ai/docs/api-config#registry) and allows you +to map strings to functions. You can register functions to create architectures, +optimizers, schedules and more, and then refer to them and set their arguments +in your [config file](/usage/training#config). Python type hints are used to +validate the inputs. See the +[Thinc docs](https://thinc.ai/docs/api-config#registry) for details on the +`registry` methods and our helper library +[`catalogue`](https://github.com/explosion/catalogue) for some background on the +concept of function registries. spaCy also uses the function registry for +language subclasses, model architecture, lookups and pipeline component +factories. + +> #### Example +> +> ```python +> from typing import Iterator +> import spacy +> +> @spacy.registry.schedules("waltzing.v1") +> def waltzing() -> Iterator[float]: +> i = 0 +> while True: +> yield i % 3 + 1 +> i += 1 +> ``` + +| Registry name | Description | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `architectures` | Registry for functions that create [model architectures](/api/architectures). Can be used to register custom model architectures and reference them in the `config.cfg`. | +| `augmenters` | Registry for functions that create [data augmentation](#augmenters) callbacks for corpora and other training data iterators. | +| `batchers` | Registry for training and evaluation [data batchers](#batchers). | +| `callbacks` | Registry for custom callbacks to [modify the `nlp` object](/usage/training#custom-code-nlp-callbacks) before training. | +| `displacy_colors` | Registry for custom color scheme for the [`displacy` NER visualizer](/usage/visualizers). Automatically reads from [entry points](/usage/saving-loading#entry-points). | +| `factories` | Registry for functions that create [pipeline components](/usage/processing-pipelines#custom-components). Added automatically when you use the `@spacy.component` decorator and also reads from [entry points](/usage/saving-loading#entry-points). | +| `initializers` | Registry for functions that create [initializers](https://thinc.ai/docs/api-initializers). | +| `languages` | Registry for language-specific `Language` subclasses. Automatically reads from [entry points](/usage/saving-loading#entry-points). | +| `layers` | Registry for functions that create [layers](https://thinc.ai/docs/api-layers). | +| `loggers` | Registry for functions that log [training results](/usage/training). | +| `lookups` | Registry for large lookup tables available via `vocab.lookups`. | +| `losses` | Registry for functions that create [losses](https://thinc.ai/docs/api-loss). | +| `misc` | Registry for miscellaneous functions that return data assets, knowledge bases or anything else you may need. | +| `optimizers` | Registry for functions that create [optimizers](https://thinc.ai/docs/api-optimizers). | +| `readers` | Registry for file and data readers, including training and evaluation data readers like [`Corpus`](/api/corpus). | +| `schedules` | Registry for functions that create [schedules](https://thinc.ai/docs/api-schedules). | +| `scorers` | Registry for functions that create scoring methods for user with the [`Scorer`](/api/scorer). Scoring methods are called with `Iterable[Example]` and arbitrary `**kwargs` and return scores as `Dict[str, Any]`. | +| `tokenizers` | Registry for tokenizer factories. Registered functions should return a callback that receives the `nlp` object and returns a [`Tokenizer`](/api/tokenizer) or a custom callable. | + +### spacy-transformers registry {id="registry-transformers"} + +The following registries are added by the +[`spacy-transformers`](https://github.com/explosion/spacy-transformers) package. +See the [`Transformer`](/api/transformer) API reference and +[usage docs](/usage/embeddings-transformers) for details. + +> #### Example +> +> ```python +> import spacy_transformers +> +> @spacy_transformers.registry.annotation_setters("my_annotation_setter.v1") +> def configure_custom_annotation_setter(): +> def annotation_setter(docs, trf_data) -> None: +> # Set annotations on the docs +> +> return annotation_setter +> ``` + +| Registry name | Description | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`span_getters`](/api/transformer#span_getters) | Registry for functions that take a batch of `Doc` objects and return a list of `Span` objects to process by the transformer, e.g. sentences. | +| [`annotation_setters`](/api/transformer#annotation_setters) | Registry for functions that create annotation setters. Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set additional annotations on the `Doc`. | + +## Loggers {id="loggers",source="spacy/training/loggers.py",version="3"} + +A logger records the training results. When a logger is created, two functions +are returned: one for logging the information for each training step, and a +second function that is called to finalize the logging when the training is +finished. To log each training step, a +[dictionary](/usage/training#custom-logging) is passed on from the +[`spacy train`](/api/cli#train), including information such as the training loss +and the accuracy scores on the development set. + +The built-in, default logger is the ConsoleLogger, which prints results to the +console in tabular format and saves them to a `jsonl` file. The +[spacy-loggers](https://github.com/explosion/spacy-loggers) package, included as +a dependency of spaCy, enables other loggers, such as one that sends results to +a [Weights & Biases](https://www.wandb.com/) dashboard. + +Instead of using one of the built-in loggers, you can +[implement your own](/usage/training#custom-logging). + +#### spacy.ConsoleLogger.v2 {tag="registered function"} + +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.ConsoleLogger.v2" +> progress_bar = true +> console_output = true +> output_file = "training_log.jsonl" +> ``` + +Writes the results of a training step to the console in a tabular format and +saves them to a `jsonl` file. + + + +```bash +$ python -m spacy train config.cfg +``` + +``` +ℹ Using CPU +ℹ Loading config and nlp from: config.cfg +ℹ Pipeline: ['tok2vec', 'tagger'] +ℹ Start training +ℹ Training. Initial learn rate: 0.0 +ℹ Saving results to training_log.jsonl + +E # LOSS TOK2VEC LOSS TAGGER TAG_ACC SCORE +--- ------ ------------ ----------- ------- ------ + 0 0 0.00 86.20 0.22 0.00 + 0 200 3.08 18968.78 34.00 0.34 + 0 400 31.81 22539.06 33.64 0.34 + 0 600 92.13 22794.91 43.80 0.44 + 0 800 183.62 21541.39 56.05 0.56 + 0 1000 352.49 25461.82 65.15 0.65 + 0 1200 422.87 23708.82 71.84 0.72 + 0 1400 601.92 24994.79 76.57 0.77 + 0 1600 662.57 22268.02 80.20 0.80 + 0 1800 1101.50 28413.77 82.56 0.83 + 0 2000 1253.43 28736.36 85.00 0.85 + 0 2200 1411.02 28237.53 87.42 0.87 + 0 2400 1605.35 28439.95 88.70 0.89 +``` + +Note that the cumulative loss keeps increasing within one epoch, but should +start decreasing across epochs. + + + +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Whether the logger should print a progress bar tracking the steps till the next evaluation pass (default: `False`). ~~bool~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | + +#### spacy.ConsoleLogger.v3 {id="ConsoleLogger",tag="registered function"} + +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.ConsoleLogger.v3" +> progress_bar = "eval" +> console_output = true +> output_file = "training_log.jsonl" +> ``` + +Writes the results of a training step to the console in a tabular format and +optionally saves them to a `jsonl` file. + +| Name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Type of progress bar to show in the console: `"train"`, `"eval"` or `None`. | +| | The bar tracks the number of steps until `training.max_steps` and `training.eval_frequency` are reached respectively (default: `None`). ~~Optional[str]~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | + +## Readers {id="readers"} + +### File readers {id="file-readers",source="github.com/explosion/srsly",version="3"} + +The following file readers are provided by our serialization library +[`srsly`](https://github.com/explosion/srsly). All registered functions take one +argument `path`, pointing to the file path to load. + +> #### Example config +> +> ```ini +> [corpora.train.augmenter.orth_variants] +> @readers = "srsly.read_json.v1" +> path = "corpus/en_orth_variants.json" +> ``` + +| Name | Description | +| ----------------------- | ----------------------------------------------------- | +| `srsly.read_json.v1` | Read data from a JSON file. | +| `srsly.read_jsonl.v1` | Read data from a JSONL (newline-delimited JSON) file. | +| `srsly.read_yaml.v1` | Read data from a YAML file. | +| `srsly.read_msgpack.v1` | Read data from a binary MessagePack file. | + + + +Since the file readers expect a local path, you should only use them in config +blocks that are **not executed at runtime** – for example, in `[training]` and +`[corpora]` (to load data or resources like data augmentation tables) or in +`[initialize]` (to pass data to pipeline components). + + + +#### spacy.read_labels.v1 {id="read_labels",tag="registered function"} + +Read a JSON-formatted labels file generated with +[`init labels`](/api/cli#init-labels). Typically used in the +[`[initialize]`](/api/data-formats#config-initialize) block of the training +config to speed up the model initialization process and provide pre-generated +label sets. + +> #### Example config +> +> ```ini +> [initialize.components] +> +> [initialize.components.ner] +> +> [initialize.components.ner.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/ner.json" +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The path to the labels file generated with [`init labels`](/api/cli#init-labels). ~~Path~~ | +| `require` | Whether to require the file to exist. If set to `False` and the labels file doesn't exist, the loader will return `None` and the `initialize` method will extract the labels from the data. Defaults to `False`. ~~bool~~ | +| **CREATES** | The list of labels. ~~List[str]~~ | + +### Corpus readers {id="corpus-readers",source="spacy/training/corpus.py",version="3"} + +Corpus readers are registered functions that load data and return a function +that takes the current `nlp` object and yields [`Example`](/api/example) objects +that can be used for [training](/usage/training) and +[pretraining](/usage/embeddings-transformers#pretraining). You can replace it +with your own registered function in the +[`@readers` registry](/api/top-level#registry) to customize the data loading and +streaming. + +#### spacy.Corpus.v1 {id="corpus",tag="registered function"} + +The `Corpus` reader manages annotated corpora and can be used for training and +development datasets in the [DocBin](/api/docbin) (`.spacy`) format. Also see +the [`Corpus`](/api/corpus) class. + +> #### Example config +> +> ```ini +> [paths] +> train = "corpus/train.spacy" +> +> [corpora.train] +> @readers = "spacy.Corpus.v1" +> path = ${paths.train} +> gold_preproc = false +> max_length = 0 +> limit = 0 +> ``` + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects data in spaCy's binary [`.spacy` format](/api/data-formats#binary-training). ~~Union[str, Path]~~ | +| `gold_preproc` | Whether to set up the Example object with gold-standard sentences and tokens for the predictions. See [`Corpus`](/api/corpus#init) for details. ~~bool~~ | +| `max_length` | Maximum document length. Longer documents will be split into sentences, if sentence boundaries are available. Defaults to `0` for no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | +| **CREATES** | The corpus reader. ~~Corpus~~ | + +#### spacy.JsonlCorpus.v1 {id="jsonlcorpus",tag="registered function"} + +Create [`Example`](/api/example) objects from a JSONL (newline-delimited JSON) +file of texts keyed by `"text"`. Can be used to read the raw text corpus for +language model [pretraining](/usage/embeddings-transformers#pretraining) from a +JSONL file. Also see the [`JsonlCorpus`](/api/corpus#jsonlcorpus) class. + +> #### Example config +> +> ```ini +> [paths] +> pretrain = "corpus/raw_text.jsonl" +> +> [corpora.pretrain] +> @readers = "spacy.JsonlCorpus.v1" +> path = ${paths.pretrain} +> min_length = 0 +> max_length = 0 +> limit = 0 +> ``` + +| Name | Description | +| ------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| `path` | The directory or filename to read from. Expects newline-delimited JSON with a key `"text"` for each record. ~~Union[str, Path]~~ | +| `min_length` | Minimum document length (in tokens). Shorter documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | +| `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | +| **CREATES** | The corpus reader. ~~JsonlCorpus~~ | + +## Batchers {id="batchers",source="spacy/training/batchers.py",version="3"} + +A data batcher implements a batching strategy that essentially turns a stream of +items into a stream of batches, with each batch consisting of one item or a list +of items. During training, the models update their weights after processing one +batch at a time. Typical batching strategies include presenting the training +data as a stream of batches with similar sizes, or with increasing batch sizes. +See the Thinc documentation on +[`schedules`](https://thinc.ai/docs/api-schedules) for a few standard examples. + +Instead of using one of the built-in batchers listed here, you can also +[implement your own](/usage/training#custom-code-readers-batchers), which may or +may not use a custom schedule. + +### spacy.batch_by_words.v1 {id="batch_by_words",tag="registered function"} + +Create minibatches of roughly a given number of words. If any examples are +longer than the specified batch length, they will appear in a batch by +themselves, or be discarded if `discard_oversize` is set to `True`. The argument +`docs` can be a list of strings, [`Doc`](/api/doc) objects or +[`Example`](/api/example) objects. + +> #### Example config +> +> ```ini +> [training.batcher] +> @batchers = "spacy.batch_by_words.v1" +> size = 100 +> tolerance = 0.2 +> discard_oversize = false +> get_length = null +> ``` + +| Name | Description | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `seqs` | The sequences to minibatch. ~~Iterable[Any]~~ | +| `size` | The target number of words per batch. Can also be a block referencing a schedule, e.g. [`compounding`](https://thinc.ai/docs/api-schedules/#compounding). ~~Union[int, Sequence[int]]~~ | +| `tolerance` | What percentage of the size to allow batches to exceed. ~~float~~ | +| `discard_oversize` | Whether to discard sequences that by themselves exceed the tolerated size. ~~bool~~ | +| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | +| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | + +### spacy.batch_by_sequence.v1 {id="batch_by_sequence",tag="registered function"} + +> #### Example config +> +> ```ini +> [training.batcher] +> @batchers = "spacy.batch_by_sequence.v1" +> size = 32 +> get_length = null +> ``` + +Create a batcher that creates batches of the specified size. + +| Name | Description | +| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `size` | The target number of items per batch. Can also be a block referencing a schedule, e.g. [`compounding`](https://thinc.ai/docs/api-schedules/#compounding). ~~Union[int, Sequence[int]]~~ | +| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | +| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | + +### spacy.batch_by_padded.v1 {id="batch_by_padded",tag="registered function"} + +> #### Example config +> +> ```ini +> [training.batcher] +> @batchers = "spacy.batch_by_padded.v1" +> size = 100 +> buffer = 256 +> discard_oversize = false +> get_length = null +> ``` + +Minibatch a sequence by the size of padded batches that would result, with +sequences binned by length within a window. The padded size is defined as the +maximum length of sequences within the batch multiplied by the number of +sequences in the batch. + +| Name | Description | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `size` | The largest padded size to batch sequences into. Can also be a block referencing a schedule, e.g. [`compounding`](https://thinc.ai/docs/api-schedules/#compounding). ~~Union[int, Sequence[int]]~~ | +| `buffer` | The number of sequences to accumulate before sorting by length. A larger buffer will result in more even sizing, but if the buffer is very large, the iteration order will be less random, which can result in suboptimal training. ~~int~~ | +| `discard_oversize` | Whether to discard sequences that are by themselves longer than the largest padded batch size. ~~bool~~ | +| `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | +| **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | + +## Augmenters {id="augmenters",source="spacy/training/augment.py",version="3"} + +Data augmentation is the process of applying small modifications to the training +data. It can be especially useful for punctuation and case replacement – for +example, if your corpus only uses smart quotes and you want to include +variations using regular quotes, or to make the model less sensitive to +capitalization by including a mix of capitalized and lowercase examples. See the +[usage guide](/usage/training#data-augmentation) for details and examples. + +### spacy.orth_variants.v1 {id="orth_variants",tag="registered function"} + +> #### Example config +> +> ```ini +> [corpora.train.augmenter] +> @augmenters = "spacy.orth_variants.v1" +> level = 0.1 +> lower = 0.5 +> +> [corpora.train.augmenter.orth_variants] +> @readers = "srsly.read_json.v1" +> path = "corpus/en_orth_variants.json" +> ``` + +Create a data augmentation callback that uses orth-variant replacement. The +callback can be added to a corpus or other data iterator during training. It's +especially useful for punctuation and case replacement, to help generalize +beyond corpora that don't have smart quotes, or only have smart quotes etc. + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `level` | The percentage of texts that will be augmented. ~~float~~ | +| `lower` | The percentage of texts that will be lowercased. ~~float~~ | +| `orth_variants` | A dictionary containing the single and paired orth variants. Typically loaded from a JSON file. See [`en_orth_variants.json`](https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/en_orth_variants.json) for an example. ~~Dict[str, Dict[List[Union[str, List[str]]]]]~~ | +| **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | + +### spacy.lower_case.v1 {id="lower_case",tag="registered function"} + +> #### Example config +> +> ```ini +> [corpora.train.augmenter] +> @augmenters = "spacy.lower_case.v1" +> level = 0.3 +> ``` + +Create a data augmentation callback that lowercases documents. The callback can +be added to a corpus or other data iterator during training. It's especially +useful for making the model less sensitive to capitalization. + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `level` | The percentage of texts that will be augmented. ~~float~~ | +| **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | + +## Callbacks {id="callbacks",source="spacy/training/callbacks.py",version="3"} + +The config supports [callbacks](/usage/training#custom-code-nlp-callbacks) at +several points in the lifecycle that can be used modify the `nlp` object. + +### spacy.copy_from_base_model.v1 {id="copy_from_base_model",tag="registered function"} + +> #### Example config +> +> ```ini +> [initialize.before_init] +> @callbacks = "spacy.copy_from_base_model.v1" +> tokenizer = "en_core_sci_md" +> vocab = "en_core_sci_md" +> ``` + +Copy the tokenizer and/or vocab from the specified models. It's similar to the +v2 [base model](https://v2.spacy.io/api/cli#train) option and useful in +combination with +[sourced components](/usage/processing-pipelines#sourced-components) when +fine-tuning an existing pipeline. The vocab includes the lookups and the vectors +from the specified model. Intended for use in `[initialize.before_init]`. + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------------------- | +| `tokenizer` | The pipeline to copy the tokenizer from. Defaults to `None`. ~~Optional[str]~~ | +| `vocab` | The pipeline to copy the vocab from. The vocab includes the lookups and vectors. Defaults to `None`. ~~Optional[str]~~ | +| **CREATES** | A function that takes the current `nlp` object and modifies its `tokenizer` and `vocab`. ~~Callable[[Language], None]~~ | + +### spacy.models_with_nvtx_range.v1 {id="models_with_nvtx_range",tag="registered function"} + +> #### Example config +> +> ```ini +> [nlp] +> after_pipeline_creation = {"@callbacks":"spacy.models_with_nvtx_range.v1"} +> ``` + +Recursively wrap the models in each pipe using +[NVTX](https://nvidia.github.io/NVTX/) range markers. These markers aid in GPU +profiling by attributing specific operations to a ~~Model~~'s forward or +backprop passes. + +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `forward_color` | Color identifier for forward passes. Defaults to `-1`. ~~int~~ | +| `backprop_color` | Color identifier for backpropagation passes. Defaults to `-1`. ~~int~~ | +| **CREATES** | A function that takes the current `nlp` and wraps forward/backprop passes in NVTX ranges. ~~Callable[[Language], Language]~~ | + +### spacy.models_and_pipes_with_nvtx_range.v1 {id="models_and_pipes_with_nvtx_range",tag="registered function",version="3.4"} + +> #### Example config +> +> ```ini +> [nlp] +> after_pipeline_creation = {"@callbacks":"spacy.models_and_pipes_with_nvtx_range.v1"} +> ``` + +Recursively wrap both the models and methods of each pipe using +[NVTX](https://nvidia.github.io/NVTX/) range markers. By default, the following +methods are wrapped: `pipe`, `predict`, `set_annotations`, `update`, `rehearse`, +`get_loss`, `initialize`, `begin_update`, `finish_update`, `update`. + +| Name | Description | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `forward_color` | Color identifier for model forward passes. Defaults to `-1`. ~~int~~ | +| `backprop_color` | Color identifier for model backpropagation passes. Defaults to `-1`. ~~int~~ | +| `additional_pipe_functions` | Additional pipeline methods to wrap. Keys are pipeline names and values are lists of method identifiers. Defaults to `None`. ~~Optional[Dict[str, List[str]]]~~ | +| **CREATES** | A function that takes the current `nlp` and wraps pipe models and methods in NVTX ranges. ~~Callable[[Language], Language]~~ | + +## Training data and alignment {id="gold",source="spacy/training"} + +### training.offsets_to_biluo_tags {id="offsets_to_biluo_tags",tag="function"} + +Encode labelled spans into per-token tags, using the +[BILUO scheme](/usage/linguistic-features#accessing-ner) (Begin, In, Last, Unit, +Out). Returns a list of strings, describing the tags. Each tag string will be in +the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of +`"B"`, `"I"`, `"L"`, `"U"`. The string `"-"` is used where the entity offsets +don't align with the tokenization in the `Doc` object. The training algorithm +will view these as missing values. `O` denotes a non-entity token. `B` denotes +the beginning of a multi-token entity, `I` the inside of an entity of three or +more tokens, and `L` the end of an entity of two or more tokens. `U` denotes a +single-token entity. + + + +This method was previously available as `spacy.gold.biluo_tags_from_offsets`. + + + +> #### Example +> +> ```python +> from spacy.training import offsets_to_biluo_tags +> +> doc = nlp("I like London.") +> entities = [(7, 13, "LOC")] +> tags = offsets_to_biluo_tags(doc, entities) +> assert tags == ["O", "O", "U-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `doc` | The document that the entity offsets refer to. The output tags will refer to the token boundaries within the document. ~~Doc~~ | +| `entities` | A sequence of `(start, end, label)` triples. `start` and `end` should be character-offset integers denoting the slice into the original string. ~~List[Tuple[int, int, Union[str, int]]]~~ | +| `missing` | The label used for missing values, e.g. if tokenization doesn't align with the entity offsets. Defaults to `"O"`. ~~str~~ | +| **RETURNS** | A list of strings, describing the [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.biluo_tags_to_offsets {id="biluo_tags_to_offsets",tag="function"} + +Encode per-token tags following the +[BILUO scheme](/usage/linguistic-features#accessing-ner) into entity offsets. + + + +This method was previously available as `spacy.gold.offsets_from_biluo_tags`. + + + +> #### Example +> +> ```python +> from spacy.training import biluo_tags_to_offsets +> +> doc = nlp("I like London.") +> tags = ["O", "O", "U-LOC", "O"] +> entities = biluo_tags_to_offsets(doc, tags) +> assert entities == [(7, 13, "LOC")] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `doc` | The document that the BILUO tags refer to. ~~Doc~~ | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | +| **RETURNS** | A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. ~~List[Tuple[int, int, str]]~~ | + +### training.biluo_tags_to_spans {id="biluo_tags_to_spans",tag="function",version="2.1"} + +Encode per-token tags following the +[BILUO scheme](/usage/linguistic-features#accessing-ner) into +[`Span`](/api/span) objects. This can be used to create entity spans from +token-based tags, e.g. to overwrite the `doc.ents`. + + + +This method was previously available as `spacy.gold.spans_from_biluo_tags`. + + + +> #### Example +> +> ```python +> from spacy.training import biluo_tags_to_spans +> +> doc = nlp("I like London.") +> tags = ["O", "O", "U-LOC", "O"] +> doc.ents = biluo_tags_to_spans(doc, tags) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `doc` | The document that the BILUO tags refer to. ~~Doc~~ | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | +| **RETURNS** | A sequence of `Span` objects with added entity labels. ~~List[Span]~~ | + +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} + +Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to +[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want +use the BILUO tags with a model that only supports IOB tags. + +> #### Example +> +> ```python +> from spacy.training import biluo_to_iob +> +> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] +> iob_tags = biluo_to_iob(tags) +> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} + +Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to +[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you +want use the IOB tags with a model that only supports BILUO tags. + + + +This method was previously available as `spacy.gold.iob_to_biluo`. + + + +> #### Example +> +> ```python +> from spacy.training import iob_to_biluo +> +> tags = ["O", "O", "B-LOC", "I-LOC", "O"] +> biluo_tags = iob_to_biluo(tags) +> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} + +Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to +[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want +use the BILUO tags with a model that only supports IOB tags. + +> #### Example +> +> ```python +> from spacy.training import biluo_to_iob +> +> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] +> iob_tags = biluo_to_iob(tags) +> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} + +Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to +[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you +want use the IOB tags with a model that only supports BILUO tags. + + + +This method was previously available as `spacy.gold.iob_to_biluo`. + + + +> #### Example +> +> ```python +> from spacy.training import iob_to_biluo +> +> tags = ["O", "O", "B-LOC", "I-LOC", "O"] +> biluo_tags = iob_to_biluo(tags) +> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +## Utility functions {id="util",source="spacy/util.py"} + +spaCy comes with a small collection of utility functions located in +[`spacy/util.py`](%%GITHUB_SPACY/spacy/util.py). Because utility functions are +mostly intended for **internal use within spaCy**, their behavior may change +with future releases. The functions documented on this page should be safe to +use and we'll try to ensure backwards compatibility. However, we recommend +having additional tests in place if your application depends on any of spaCy's +utilities. + +### util.get_lang_class {id="util.get_lang_class",tag="function"} + +Import and load a `Language` class. Allows lazy-loading +[language data](/usage/linguistic-features#language-data) and importing +languages using the two-letter language code. To add a language code for a +custom language class, you can register it using the +[`@registry.languages`](/api/top-level#registry) decorator. + +> #### Example +> +> ```python +> for lang_id in ["en", "de"]: +> lang_class = util.get_lang_class(lang_id) +> lang = lang_class() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------- | +| `lang` | Two-letter language code, e.g. `"en"`. ~~str~~ | +| **RETURNS** | The respective subclass. ~~Language~~ | + +### util.lang_class_is_loaded {id="util.lang_class_is_loaded",tag="function",version="2.1"} + +Check whether a `Language` subclass is already loaded. `Language` subclasses are +loaded lazily to avoid expensive setup code associated with the language data. + +> #### Example +> +> ```python +> lang_cls = util.get_lang_class("en") +> assert util.lang_class_is_loaded("en") is True +> assert util.lang_class_is_loaded("de") is False +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------- | +| `name` | Two-letter language code, e.g. `"en"`. ~~str~~ | +| **RETURNS** | Whether the class has been loaded. ~~bool~~ | + +### util.load_model {id="util.load_model",tag="function",version="2"} + +Load a pipeline from a package or data path. If called with a string name, spaCy +will assume the pipeline is a Python package and import and call its `load()` +method. If called with a path, spaCy will assume it's a data directory, read the +language and pipeline settings from the [`config.cfg`](/api/data-formats#config) +and create a `Language` object. The model data will then be loaded in via +[`Language.from_disk`](/api/language#from_disk). + +> #### Example +> +> ```python +> nlp = util.load_model("en_core_web_sm") +> nlp = util.load_model("en_core_web_sm", exclude=["ner"]) +> nlp = util.load_model("/path/to/data") +> ``` + +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Package name or path. ~~str~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | + +### util.load_model_from_init_py {id="util.load_model_from_init_py",tag="function",version="2"} + +A helper function to use in the `load()` method of a pipeline package's +[`__init__.py`](https://github.com/explosion/spacy-models/tree/master/template/model/xx_model_name/__init__.py). + +> #### Example +> +> ```python +> from spacy.util import load_model_from_init_py +> +> def load(**overrides): +> return load_model_from_init_py(__file__, **overrides) +> ``` + +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init_file` | Path to package's `__init__.py`, i.e. `__file__`. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` 3 | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | + +### util.load_config {id="util.load_config",tag="function",version="3"} + +Load a pipeline's [`config.cfg`](/api/data-formats#config) from a file path. The +config typically includes details about the components and how they're created, +as well as all training settings and hyperparameters. + +> #### Example +> +> ```python +> config = util.load_config("/path/to/config.cfg") +> print(config.to_str()) +> ``` + +| Name | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `path` | Path to the pipeline's `config.cfg`. ~~Union[str, Path]~~ | +| `overrides` | Optional config overrides to replace in loaded config. Can be provided as nested dict, or as flat dict with keys in dot notation, e.g. `"nlp.pipeline"`. ~~Dict[str, Any]~~ | +| `interpolate` | Whether to interpolate the config and replace variables like `${paths.train}` with their values. Defaults to `False`. ~~bool~~ | +| **RETURNS** | The pipeline's config. ~~Config~~ | + +### util.load_meta {id="util.load_meta",tag="function",version="3"} + +Get a pipeline's [`meta.json`](/api/data-formats#meta) from a file path and +validate its contents. The meta typically includes details about author, +licensing, data sources and version. + +> #### Example +> +> ```python +> meta = util.load_meta("/path/to/meta.json") +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| `path` | Path to the pipeline's `meta.json`. ~~Union[str, Path]~~ | +| **RETURNS** | The pipeline's meta data. ~~Dict[str, Any]~~ | + +### util.get_installed_models {id="util.get_installed_models",tag="function",version="3"} + +List all pipeline packages installed in the current environment. This will +include any spaCy pipeline that was packaged with +[`spacy package`](/api/cli#package). Under the hood, pipeline packages expose a +Python entry point that spaCy can check, without having to load the `nlp` +object. + +> #### Example +> +> ```python +> names = util.get_installed_models() +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| **RETURNS** | The string names of the pipelines installed in the current environment. ~~List[str]~~ | + +### util.is_package {id="util.is_package",tag="function"} + +Check if string maps to a package installed via pip. Mainly used to validate +[pipeline packages](/usage/models). + +> #### Example +> +> ```python +> util.is_package("en_core_web_sm") # True +> util.is_package("xyz") # False +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| `name` | Name of package. ~~str~~ | +| **RETURNS** | `True` if installed package, `False` if not. ~~bool~~ | + +### util.get_package_path {id="util.get_package_path",tag="function",version="2"} + +Get path to an installed package. Mainly used to resolve the location of +[pipeline packages](/usage/models). Currently imports the package to find its +path. + +> #### Example +> +> ```python +> util.get_package_path("en_core_web_sm") +> # /usr/lib/python3.6/site-packages/en_core_web_sm +> ``` + +| Name | Description | +| -------------- | -------------------------------------------- | +| `package_name` | Name of installed package. ~~str~~ | +| **RETURNS** | Path to pipeline package directory. ~~Path~~ | + +### util.is_in_jupyter {id="util.is_in_jupyter",tag="function",version="2"} + +Check if user is running spaCy from a [Jupyter](https://jupyter.org) notebook by +detecting the IPython kernel. Mainly used for the +[`displacy`](/api/top-level#displacy) visualizer. + +> #### Example +> +> ```python +> html = "

Hello world!

" +> if util.is_in_jupyter(): +> from IPython.display import display, HTML +> display(HTML(html)) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------- | +| **RETURNS** | `True` if in Jupyter, `False` if not. ~~bool~~ | + +### util.compile_prefix_regex {id="util.compile_prefix_regex",tag="function"} + +Compile a sequence of prefix rules into a regex object. + +> #### Example +> +> ```python +> prefixes = ("§", "%", "=", r"\+") +> prefix_regex = util.compile_prefix_regex(prefixes) +> nlp.tokenizer.prefix_search = prefix_regex.search +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `entries` | The prefix rules, e.g. [`lang.punctuation.TOKENIZER_PREFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | +| **RETURNS** | The regex object to be used for [`Tokenizer.prefix_search`](/api/tokenizer#attributes). ~~Pattern~~ | + +### util.compile_suffix_regex {id="util.compile_suffix_regex",tag="function"} + +Compile a sequence of suffix rules into a regex object. + +> #### Example +> +> ```python +> suffixes = ("'s", "'S", r"(?<=[0-9])\+") +> suffix_regex = util.compile_suffix_regex(suffixes) +> nlp.tokenizer.suffix_search = suffix_regex.search +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `entries` | The suffix rules, e.g. [`lang.punctuation.TOKENIZER_SUFFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | +| **RETURNS** | The regex object to be used for [`Tokenizer.suffix_search`](/api/tokenizer#attributes). ~~Pattern~~ | + +### util.compile_infix_regex {id="util.compile_infix_regex",tag="function"} + +Compile a sequence of infix rules into a regex object. + +> #### Example +> +> ```python +> infixes = ("…", "-", "—", r"(?<=[0-9])[+\-\*^](?=[0-9-])") +> infix_regex = util.compile_infix_regex(infixes) +> nlp.tokenizer.infix_finditer = infix_regex.finditer +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `entries` | The infix rules, e.g. [`lang.punctuation.TOKENIZER_INFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | +| **RETURNS** | The regex object to be used for [`Tokenizer.infix_finditer`](/api/tokenizer#attributes). ~~Pattern~~ | + +### util.minibatch {id="util.minibatch",tag="function",version="2"} + +Iterate over batches of items. `size` may be an iterator, so that batch-size can +vary on each step. + +> #### Example +> +> ```python +> batches = minibatch(train_data) +> for batch in batches: +> nlp.update(batch) +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------ | +| `items` | The items to batch up. ~~Iterable[Any]~~ | +| `size` | The batch size(s). ~~Union[int, Sequence[int]]~~ | +| **YIELDS** | The batches. | + +### util.filter_spans {id="util.filter_spans",tag="function",version="2.1.4"} + +Filter a sequence of [`Span`](/api/span) objects and remove duplicates or +overlaps. Useful for creating named entities (where one token can only be part +of one entity) or when merging spans with +[`Retokenizer.merge`](/api/doc#retokenizer.merge). When spans overlap, the +(first) longest span is preferred over shorter spans. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> spans = [doc[0:2], doc[0:2], doc[0:4]] +> filtered = filter_spans(spans) +> ``` + +| Name | Description | +| ----------- | --------------------------------------- | +| `spans` | The spans to filter. ~~Iterable[Span]~~ | +| **RETURNS** | The filtered spans. ~~List[Span]~~ | + +### util.get_words_and_spaces {id="get_words_and_spaces",tag="function",version="3"} + +Given a list of words and a text, reconstruct the original tokens and return a +list of words and spaces that can be used to create a [`Doc`](/api/doc#init). +This can help recover destructive tokenization that didn't preserve any +whitespace information. + +> #### Example +> +> ```python +> orig_words = ["Hey", ",", "what", "'s", "up", "?"] +> orig_text = "Hey, what's up?" +> words, spaces = get_words_and_spaces(orig_words, orig_text) +> # ['Hey', ',', 'what', "'s", 'up', '?'] +> # [False, True, False, True, False, False] +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `words` | The list of words. ~~Iterable[str]~~ | +| `text` | The original text. ~~str~~ | +| **RETURNS** | A list of words and a list of boolean values indicating whether the word at this position is followed by a space. ~~Tuple[List[str], List[bool]]~~ | diff --git a/website/docs/api/transformer.mdx b/website/docs/api/transformer.mdx new file mode 100644 index 0000000..9dcafb5 --- /dev/null +++ b/website/docs/api/transformer.mdx @@ -0,0 +1,595 @@ +--- +title: Transformer +teaser: Pipeline component for multi-task learning with transformer models +tag: class +source: github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py +version: 3 +api_base_class: /api/pipe +api_string_name: transformer +--- + +> #### Installation +> +> ```bash +> $ pip install -U %%SPACY_PKG_NAME[transformers] %%SPACY_PKG_FLAGS +> ``` + + + +This component is available via the extension package +[`spacy-transformers`](https://github.com/explosion/spacy-transformers). It +exposes the component via entry points, so if you have the package installed, +using `factory = "transformer"` in your +[training config](/usage/training#config) or `nlp.add_pipe("transformer")` will +work out-of-the-box. + + + +This pipeline component lets you use transformer models in your pipeline. It +supports all models that are available via the +[HuggingFace `transformers`](https://huggingface.co/transformers) library. +Usually you will connect subsequent components to the shared transformer using +the [TransformerListener](/api/architectures#TransformerListener) layer. This +works similarly to spaCy's [Tok2Vec](/api/tok2vec) component and +[Tok2VecListener](/api/architectures/#Tok2VecListener) sublayer. + +The component assigns the output of the transformer to the `Doc`'s extension +attributes. We also calculate an alignment between the word-piece tokens and the +spaCy tokenization, so that we can use the last hidden states to set the +`Doc.tensor` attribute. When multiple word-piece tokens align to the same spaCy +token, the spaCy token receives the sum of their values. To access the values, +you can use the custom [`Doc._.trf_data`](#assigned-attributes) attribute. The +package also adds the function registries [`@span_getters`](#span_getters) and +[`@annotation_setters`](#annotation_setters) with several built-in registered +functions. For more details, see the +[usage documentation](/usage/embeddings-transformers). + +## Assigned Attributes {id="assigned-attributes"} + +The component sets the following +[custom extension attribute](/usage/processing-pipeline#custom-components-attributes): + +| Location | Value | +| ---------------- | ------------------------------------------------------------------------ | +| `Doc._.trf_data` | Transformer tokens and outputs for the `Doc` object. ~~TransformerData~~ | + +## Config and implementation {id="config"} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#transformers) documentation for details +on the transformer architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy_transformers import Transformer +> from spacy_transformers.pipeline_component import DEFAULT_CONFIG +> +> nlp.add_pipe("transformer", config=DEFAULT_CONFIG["transformer"]) +> ``` + +| Setting | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `max_batch_items` | Maximum size of a padded batch. Defaults to `4096`. ~~int~~ | +| `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs to set additional annotations on the `Doc`. The `Doc._.trf_data` attribute is set prior to calling the callback. Defaults to `null_annotation_setter` (no additional annotations). ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. Defaults to [TransformerModel](/api/architectures#TransformerModel). ~~Model[List[Doc], FullTransformerBatch]~~ | + +```python +https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py +``` + +## Transformer.\_\_init\_\_ {id="init",tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> trf = nlp.add_pipe("transformer") +> +> # Construction via add_pipe with custom config +> config = { +> "model": { +> "@architectures": "spacy-transformers.TransformerModel.v3", +> "name": "bert-base-uncased", +> "tokenizer_config": {"use_fast": True}, +> "transformer_config": {"output_attentions": True}, +> "mixed_precision": True, +> "grad_scaler_config": {"init_scale": 32768} +> } +> } +> trf = nlp.add_pipe("transformer", config=config) +> +> # Construction from class +> from spacy_transformers import Transformer +> trf = Transformer(nlp.vocab, model) +> ``` + +Construct a `Transformer` component. One or more subsequent spaCy components can +use the transformer outputs as features in its model, with gradients +backpropagated to the single shared weights. The activations from the +transformer are saved in the [`Doc._.trf_data`](#assigned-attributes) extension +attribute. You can also provide a callback to set additional annotations. In +your application, you would normally use a shortcut for this and instantiate the +component using its string name and [`nlp.add_pipe`](/api/language#create_pipe). + +| Name | Description | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. Usually you will want to use the [TransformerModel](/api/architectures#TransformerModel) layer for this. ~~Model[List[Doc], FullTransformerBatch]~~ | +| `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and stores the annotations on the `Doc`. The `Doc._.trf_data` attribute is set prior to calling the callback. By default, no additional annotations are set. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | +| _keyword-only_ | | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| `max_batch_items` | Maximum size of a padded batch. Defaults to `128*32`. ~~int~~ | + +## Transformer.\_\_call\_\_ {id="call",tag="method"} + +Apply the pipe to one document. The document is modified in place, and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/transformer#call) and [`pipe`](/api/transformer#pipe) delegate +to the [`predict`](/api/transformer#predict) and +[`set_annotations`](/api/transformer#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> trf = nlp.add_pipe("transformer") +> # This usually happens under the hood +> processed = transformer(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## Transformer.pipe {id="pipe",tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/transformer#call) and +[`pipe`](/api/transformer#pipe) delegate to the +[`predict`](/api/transformer#predict) and +[`set_annotations`](/api/transformer#set_annotations) methods. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> for doc in trf.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## Transformer.initialize {id="initialize",tag="method"} + +Initialize the component for training and return an +[`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a +function that returns an iterable of [`Example`](/api/example) objects. **At +least one example should be supplied.** The data examples are used to +**initialize the model** of the component and can either be the full training +data or a representative sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> trf.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## Transformer.predict {id="predict",tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects without +modifying them. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> scores = trf.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The model's prediction for each document. | + +## Transformer.set_annotations {id="set_annotations",tag="method"} + +Assign the extracted features to the `Doc` objects. By default, the +[`TransformerData`](/api/transformer#transformerdata) object is written to the +[`Doc._.trf_data`](#assigned-attributes) attribute. Your `set_extra_annotations` +callback is then called, if provided. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> scores = trf.predict(docs) +> trf.set_annotations(docs, scores) +> ``` + +| Name | Description | +| -------- | ----------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `scores` | The scores to set, produced by `Transformer.predict`. | + +## Transformer.update {id="update",tag="method"} + +Prepare for an update to the transformer. Like the [`Tok2Vec`](/api/tok2vec) +component, the `Transformer` component is unusual in that it does not receive +"gold standard" annotations to calculate a weight update. The optimal output of +the transformer data is unknown – it's a hidden layer inside the network that is +updated by backpropagating from output layers. + +The `Transformer` component therefore does **not** perform a weight update +during its own `update` method. Instead, it runs its transformer model and +communicates the output and the backpropagation callback to any **downstream +components** that have been connected to it via the +[TransformerListener](/api/architectures#TransformerListener) sublayer. If there +are multiple listeners, the last layer will actually backprop to the transformer +and call the optimizer, while the others simply increment the gradients. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> optimizer = nlp.initialize() +> losses = trf.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | A batch of [`Example`](/api/example) objects. Only the [`Example.predicted`](/api/example#predicted) `Doc` object is used, the reference `Doc` is ignored. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## Transformer.create_optimizer {id="create_optimizer",tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> optimizer = trf.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## Transformer.use_params {id="use_params",tag="method, contextmanager"} + +Modify the pipe's model to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> with trf.use_params(optimizer.averages): +> trf.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## Transformer.to_disk {id="to_disk",tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> trf.to_disk("/path/to/transformer") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Transformer.from_disk {id="from_disk",tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> trf.from_disk("/path/to/transformer") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Transformer` object. ~~Transformer~~ | + +## Transformer.to_bytes {id="to_bytes",tag="method"} + +> #### Example +> +> ```python +> trf = nlp.add_pipe("transformer") +> trf_bytes = trf.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Transformer` object. ~~bytes~~ | + +## Transformer.from_bytes {id="from_bytes",tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> trf_bytes = trf.to_bytes() +> trf = nlp.add_pipe("transformer") +> trf.from_bytes(trf_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Transformer` object. ~~Transformer~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = trf.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | + +## TransformerData {id="transformerdata",tag="dataclass"} + +Transformer tokens and outputs for one `Doc` object. The transformer models +return tensors that refer to a whole padded batch of documents. These tensors +are wrapped into the +[FullTransformerBatch](/api/transformer#fulltransformerbatch) object. The +`FullTransformerBatch` then splits out the per-document data, which is handled +by this class. Instances of this class are typically assigned to the +[`Doc._.trf_data`](/api/transformer#assigned-attributes) extension attribute. + +> #### Example +> +> ```python +> # Get the last hidden layer output for "is" (token index 1) +> doc = nlp("This is a text.") +> indices = doc._.trf_data.align[1].data.flatten() +> last_hidden_state = doc._.trf_data.model_output.last_hidden_state +> dim = last_hidden_state.shape[-1] +> tensors = last_hidden_state.reshape(-1, dim)[indices] +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `tokens` | A slice of the tokens data produced by the tokenizer. This may have several fields, including the token IDs, the texts and the attention mask. See the [`transformers.BatchEncoding`](https://huggingface.co/transformers/main_classes/tokenizer.html#transformers.BatchEncoding) object for details. ~~dict~~ | +| `model_output` | The model output from the transformer model, determined by the model and transformer config. New in `spacy-transformers` v1.1.0. ~~transformers.file_utils.ModelOutput~~ | +| `tensors` | The `model_output` in the earlier `transformers` tuple format converted using [`ModelOutput.to_tuple()`](https://huggingface.co/transformers/main_classes/output.html#transformers.file_utils.ModelOutput.to_tuple). Returns `Tuple` instead of `List` as of `spacy-transformers` v1.1.0. ~~Tuple[Union[FloatsXd, List[FloatsXd]]]~~ | +| `align` | Alignment from the `Doc`'s tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | +| `width` | The width of the last hidden layer. ~~int~~ | + +### TransformerData.empty {id="transformerdata-empty",tag="classmethod"} + +Create an empty `TransformerData` container. + +| Name | Description | +| ----------- | ---------------------------------- | +| **RETURNS** | The container. ~~TransformerData~~ | + + + +In `spacy-transformers` v1.0, the model output is stored in +`TransformerData.tensors` as `List[Union[FloatsXd]]` and only includes the +activations for the `Doc` from the transformer. Usually the last tensor that is +3-dimensional will be the most important, as that will provide the final hidden +state. Generally activations that are 2-dimensional will be attention weights. +Details of this variable will differ depending on the underlying transformer +model. + + + +## FullTransformerBatch {id="fulltransformerbatch",tag="dataclass"} + +Holds a batch of input and output objects for a transformer model. The data can +then be split to a list of [`TransformerData`](/api/transformer#transformerdata) +objects to associate the outputs to each [`Doc`](/api/doc) in the batch. + +| Name | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans` | The batch of input spans. The outer list refers to the Doc objects in the batch, and the inner list are the spans for that `Doc`. Note that spans are allowed to overlap or exclude tokens, but each `Span` can only refer to one `Doc` (by definition). This means that within a `Doc`, the regions of the output tensors that correspond to each `Span` may overlap or have gaps, but for each `Doc`, there is a non-overlapping contiguous slice of the outputs. ~~List[List[Span]]~~ | +| `tokens` | The output of the tokenizer. ~~transformers.BatchEncoding~~ | +| `model_output` | The model output from the transformer model, determined by the model and transformer config. New in `spacy-transformers` v1.1.0. ~~transformers.file_utils.ModelOutput~~ | +| `tensors` | The `model_output` in the earlier `transformers` tuple format converted using [`ModelOutput.to_tuple()`](https://huggingface.co/transformers/main_classes/output.html#transformers.file_utils.ModelOutput.to_tuple). Returns `Tuple` instead of `List` as of `spacy-transformers` v1.1.0. ~~Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]~~ | +| `align` | Alignment from the spaCy tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | +| `doc_data` | The outputs, split per `Doc` object. ~~List[TransformerData]~~ | + +### FullTransformerBatch.unsplit_by_doc {id="fulltransformerbatch-unsplit_by_doc",tag="method"} + +Return a new `FullTransformerBatch` from a split batch of activations, using the +current object's spans, tokens and alignment. This is used during the backward +pass, in order to construct the gradients to pass back into the transformer +model. + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| `arrays` | The split batch of activations. ~~List[List[Floats3d]]~~ | +| **RETURNS** | The transformer batch. ~~FullTransformerBatch~~ | + +### FullTransformerBatch.split_by_doc {id="fulltransformerbatch-split_by_doc",tag="method"} + +Split a `TransformerData` object that represents a batch into a list with one +`TransformerData` per `Doc`. + +| Name | Description | +| ----------- | ------------------------------------------ | +| **RETURNS** | The split batch. ~~List[TransformerData]~~ | + + + +In `spacy-transformers` v1.0, the model output is stored in +`FullTransformerBatch.tensors` as `List[torch.Tensor]`. + + + +## Span getters {id="span_getters",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"} + +Span getters are functions that take a batch of [`Doc`](/api/doc) objects and +return a lists of [`Span`](/api/span) objects for each doc to be processed by +the transformer. This is used to manage long documents by cutting them into +smaller sequences before running the transformer. The spans are allowed to +overlap, and you can also omit sections of the `Doc` if they are not relevant. + +Span getters can be referenced in the `[components.transformer.model.get_spans]` +block of the config to customize the sequences processed by the transformer. You +can also register +[custom span getters](/usage/embeddings-transformers#transformers-training-custom-settings) +using the `@spacy.registry.span_getters` decorator. + +> #### Example +> +> ```python +> @spacy.registry.span_getters("custom_sent_spans") +> def configure_get_sent_spans() -> Callable: +> def get_sent_spans(docs: Iterable[Doc]) -> List[List[Span]]: +> return [list(doc.sents) for doc in docs] +> +> return get_sent_spans +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | A batch of `Doc` objects. ~~Iterable[Doc]~~ | +| **RETURNS** | The spans to process by the transformer. ~~List[List[Span]]~~ | + +### doc_spans.v1 {id="doc_spans",tag="registered function"} + +> #### Example config +> +> ```ini +> [transformer.model.get_spans] +> @span_getters = "spacy-transformers.doc_spans.v1" +> ``` + +Create a span getter that uses the whole document as its spans. This is the best +approach if your [`Doc`](/api/doc) objects already refer to relatively short +texts. + +### sent_spans.v1 {id="sent_spans",tag="registered function"} + +> #### Example config +> +> ```ini +> [transformer.model.get_spans] +> @span_getters = "spacy-transformers.sent_spans.v1" +> ``` + +Create a span getter that uses sentence boundary markers to extract the spans. +This requires sentence boundaries to be set (e.g. by the +[`Sentencizer`](/api/sentencizer)), and may result in somewhat uneven batches, +depending on the sentence lengths. However, it does provide the transformer with +more meaningful windows to attend over. + +To set sentence boundaries with the `sentencizer` during training, add a +`sentencizer` to the beginning of the pipeline and include it in +[`[training.annotating_components]`](/usage/training#annotating-components) to +have it set the sentence boundaries before the `transformer` component runs. + +### strided_spans.v1 {id="strided_spans",tag="registered function"} + +> #### Example config +> +> ```ini +> [transformer.model.get_spans] +> @span_getters = "spacy-transformers.strided_spans.v1" +> window = 128 +> stride = 96 +> ``` + +Create a span getter for strided spans. If you set the `window` and `stride` to +the same value, the spans will cover each token once. Setting `stride` lower +than `window` will allow for an overlap, so that some tokens are counted twice. +This can be desirable, because it allows all tokens to have both a left and +right context. + +| Name | Description | +| -------- | ------------------------ | +| `window` | The window size. ~~int~~ | +| `stride` | The stride size. ~~int~~ | + +## Annotation setters {id="annotation_setters",tag="registered functions",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"} + +Annotation setters are functions that take a batch of `Doc` objects and a +[`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set +additional annotations on the `Doc`, e.g. to set custom or built-in attributes. +You can register custom annotation setters using the +`@registry.annotation_setters` decorator. + +> #### Example +> +> ```python +> @registry.annotation_setters("spacy-transformers.null_annotation_setter.v1") +> def configure_null_annotation_setter() -> Callable: +> def setter(docs: List[Doc], trf_data: FullTransformerBatch) -> None: +> pass +> +> return setter +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------- | +| `docs` | A batch of `Doc` objects. ~~List[Doc]~~ | +| `trf_data` | The transformers data for the batch. ~~FullTransformerBatch~~ | + +The following built-in functions are available: + +| Name | Description | +| ---------------------------------------------- | ------------------------------------- | +| `spacy-transformers.null_annotation_setter.v1` | Don't set any additional annotations. | diff --git a/website/docs/api/vectors.mdx b/website/docs/api/vectors.mdx new file mode 100644 index 0000000..da2d783 --- /dev/null +++ b/website/docs/api/vectors.mdx @@ -0,0 +1,461 @@ +--- +title: Vectors +teaser: Store, save and load word vectors +tag: class +source: spacy/vectors.pyx +version: 2 +--- + +Vectors data is kept in the `Vectors.data` attribute, which should be an +instance of `numpy.ndarray` (for CPU vectors) or `cupy.ndarray` (for GPU +vectors). + +As of spaCy v3.2, `Vectors` supports two types of vector tables: + +- `default`: A standard vector table (as in spaCy v3.1 and earlier) where each + key is mapped to one row in the vector table. Multiple keys can be mapped to + the same vector, and not all of the rows in the table need to be assigned – so + `vectors.n_keys` may be greater or smaller than `vectors.shape[0]`. +- `floret`: Only supports vectors trained with + [floret](https://github.com/explosion/floret), an extended version of + [fastText](https://fasttext.cc) that produces compact vector tables by + combining fastText's subword ngrams with Bloom embeddings. The compact tables + are similar to the [`HashEmbed`](https://thinc.ai/docs/api-layers#hashembed) + embeddings already used in many spaCy components. Each word is represented as + the sum of one or more rows as determined by the settings related to character + ngrams and the hash table. + +## Vectors.\_\_init\_\_ {id="init",tag="method"} + +Create a new vector store. With the default mode, you can set the vector values +and keys directly on initialization, or supply a `shape` keyword argument to +create an empty table you can add vectors to later. In floret mode, the complete +vector data and settings must be provided on initialization and cannot be +modified later. + +> #### Example +> +> ```python +> from spacy.vectors import Vectors +> +> empty_vectors = Vectors(shape=(10000, 300)) +> +> data = numpy.zeros((3, 300), dtype='f') +> keys = ["cat", "dog", "rat"] +> vectors = Vectors(data=data, keys=keys) +> ``` + +| Name | Description | +| ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `strings` | The string store. A new string store is created if one is not provided. Defaults to `None`. ~~Optional[StringStore]~~ | +| `shape` | Size of the table as `(n_entries, n_columns)`, the number of entries and number of columns. Not required if you're initializing the object with `data` and `keys`. ~~Tuple[int, int]~~ | +| `data` | The vector data. ~~numpy.ndarray[ndim=2, dtype=float32]~~ | +| `keys` | An iterable of keys aligned with the data. ~~Iterable[Union[str, int]]~~ | +| `name` | A name to identify the vectors table. ~~str~~ | +| `mode` 3.2 | Vectors mode: `"default"` or [`"floret"`](https://github.com/explosion/floret) (default: `"default"`). ~~str~~ | +| `minn` 3.2 | The floret char ngram minn (default: `0`). ~~int~~ | +| `maxn` 3.2 | The floret char ngram maxn (default: `0`). ~~int~~ | +| `hash_count` 3.2 | The floret hash count. Supported values: 1--4 (default: `1`). ~~int~~ | +| `hash_seed` 3.2 | The floret hash seed (default: `0`). ~~int~~ | +| `bow` 3.2 | The floret BOW string (default: `"<"`). ~~str~~ | +| `eow` 3.2 | The floret EOW string (default: `">"`). ~~str~~ | +| `attr` 3.6 | The token attribute for the vector keys (default: `"ORTH"`). ~~Union[int, str]~~ | + +## Vectors.\_\_getitem\_\_ {id="getitem",tag="method"} + +Get a vector by key. If the key is not found in the table, a `KeyError` is +raised. + +> #### Example +> +> ```python +> cat_id = nlp.vocab.strings["cat"] +> cat_vector = nlp.vocab.vectors[cat_id] +> assert cat_vector == nlp.vocab["cat"].vector +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------- | +| `key` | The key to get the vector for. ~~Union[int, str]~~ | +| **RETURNS** | The vector for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Vectors.\_\_setitem\_\_ {id="setitem",tag="method"} + +Set a vector for the given key. Not supported for `floret` mode. + +> #### Example +> +> ```python +> cat_id = nlp.vocab.strings["cat"] +> vector = numpy.random.uniform(-1, 1, (300,)) +> nlp.vocab.vectors[cat_id] = vector +> ``` + +| Name | Description | +| -------- | ----------------------------------------------------------- | +| `key` | The key to set the vector for. ~~int~~ | +| `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Vectors.\_\_iter\_\_ {id="iter",tag="method"} + +Iterate over the keys in the table. In `floret` mode, the keys table is not +used. + +> #### Example +> +> ```python +> for key in nlp.vocab.vectors: +> print(key, nlp.vocab.strings[key]) +> ``` + +| Name | Description | +| ---------- | --------------------------- | +| **YIELDS** | A key in the table. ~~int~~ | + +## Vectors.\_\_len\_\_ {id="len",tag="method"} + +Return the number of vectors in the table. + +> #### Example +> +> ```python +> vectors = Vectors(shape=(3, 300)) +> assert len(vectors) == 3 +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| **RETURNS** | The number of vectors in the table. ~~int~~ | + +## Vectors.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether a key has been mapped to a vector entry in the table. In `floret` +mode, returns `True` for all keys. + +> #### Example +> +> ```python +> cat_id = nlp.vocab.strings["cat"] +> nlp.vocab.vectors.add(cat_id, numpy.random.uniform(-1, 1, (300,))) +> assert cat_id in vectors +> ``` + +| Name | Description | +| ----------- | -------------------------------------------- | +| `key` | The key to check. ~~int~~ | +| **RETURNS** | Whether the key has a vector entry. ~~bool~~ | + +## Vectors.add {id="add",tag="method"} + +Add a key to the table, optionally setting a vector value as well. Keys can be +mapped to an existing vector by setting `row`, or a new vector can be added. Not +supported for `floret` mode. + +> #### Example +> +> ```python +> vector = numpy.random.uniform(-1, 1, (300,)) +> cat_id = nlp.vocab.strings["cat"] +> nlp.vocab.vectors.add(cat_id, vector=vector) +> nlp.vocab.vectors.add("dog", row=0) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------- | +| `key` | The key to add. ~~Union[str, int]~~ | +| _keyword-only_ | | +| `vector` | An optional vector to add for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `row` | An optional row number of a vector to map the key to. ~~int~~ | +| **RETURNS** | The row the vector was added to. ~~int~~ | + +## Vectors.resize {id="resize",tag="method"} + +Resize the underlying vectors array. If `inplace=True`, the memory is +reallocated. This may cause other references to the data to become invalid, so +only use `inplace=True` if you're sure that's what you want. If the number of +vectors is reduced, keys mapped to rows that have been deleted are removed. +These removed items are returned as a list of `(key, row)` tuples. Not supported +for `floret` mode. + +> #### Example +> +> ```python +> removed = nlp.vocab.vectors.resize((10000, 300)) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------- | +| `shape` | A `(rows, dims)` tuple describing the number of rows and dimensions. ~~Tuple[int, int]~~ | +| `inplace` | Reallocate the memory. ~~bool~~ | +| **RETURNS** | The removed items as a list of `(key, row)` tuples. ~~List[Tuple[int, int]]~~ | + +## Vectors.keys {id="keys",tag="method"} + +A sequence of the keys in the table. In `floret` mode, the keys table is not +used. + +> #### Example +> +> ```python +> for key in nlp.vocab.vectors.keys(): +> print(key, nlp.vocab.strings[key]) +> ``` + +| Name | Description | +| ----------- | --------------------------- | +| **RETURNS** | The keys. ~~Iterable[int]~~ | + +## Vectors.values {id="values",tag="method"} + +Iterate over vectors that have been assigned to at least one key. Note that some +vectors may be unassigned, so the number of vectors returned may be less than +the length of the vectors table. In `floret` mode, the keys table is not used. + +> #### Example +> +> ```python +> for vector in nlp.vocab.vectors.values(): +> print(vector) +> ``` + +| Name | Description | +| ---------- | --------------------------------------------------------------- | +| **YIELDS** | A vector in the table. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Vectors.items {id="items",tag="method"} + +Iterate over `(key, vector)` pairs, in order. In `floret` mode, the keys table +is empty. + +> #### Example +> +> ```python +> for key, vector in nlp.vocab.vectors.items(): +> print(key, nlp.vocab.strings[key], vector) +> ``` + +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------- | +| **YIELDS** | `(key, vector)` pairs, in order. ~~Tuple[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | + +## Vectors.find {id="find",tag="method"} + +Look up one or more keys by row, or vice versa. Not supported for `floret` mode. + +> #### Example +> +> ```python +> row = nlp.vocab.vectors.find(key="cat") +> rows = nlp.vocab.vectors.find(keys=["cat", "dog"]) +> key = nlp.vocab.vectors.find(row=256) +> keys = nlp.vocab.vectors.find(rows=[18, 256, 985]) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `key` | Find the row that the given key points to. Returns int, `-1` if missing. ~~Union[str, int]~~ | +| `keys` | Find rows that the keys point to. Returns `numpy.ndarray`. ~~Iterable[Union[str, int]]~~ | +| `row` | Find the first key that points to the row. Returns integer. ~~int~~ | +| `rows` | Find the keys that point to the rows. Returns `numpy.ndarray`. ~~Iterable[int]~~ | +| **RETURNS** | The requested key, keys, row or rows. ~~Union[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | + +## Vectors.shape {id="shape",tag="property"} + +Get `(rows, dims)` tuples of number of rows and number of dimensions in the +vector table. + +> #### Example +> +> ```python +> vectors = Vectors(shape(1, 300)) +> vectors.add("cat", numpy.random.uniform(-1, 1, (300,))) +> rows, dims = vectors.shape +> assert rows == 1 +> assert dims == 300 +> ``` + +| Name | Description | +| ----------- | ------------------------------------------ | +| **RETURNS** | A `(rows, dims)` pair. ~~Tuple[int, int]~~ | + +## Vectors.size {id="size",tag="property"} + +The vector size, i.e. `rows * dims`. + +> #### Example +> +> ```python +> vectors = Vectors(shape=(500, 300)) +> assert vectors.size == 150000 +> ``` + +| Name | Description | +| ----------- | ------------------------ | +| **RETURNS** | The vector size. ~~int~~ | + +## Vectors.is_full {id="is_full",tag="property"} + +Whether the vectors table is full and no slots are available for new keys. If a +table is full, it can be resized using [`Vectors.resize`](/api/vectors#resize). +In `floret` mode, the table is always full and cannot be resized. + +> #### Example +> +> ```python +> vectors = Vectors(shape=(1, 300)) +> vectors.add("cat", numpy.random.uniform(-1, 1, (300,))) +> assert vectors.is_full +> ``` + +| Name | Description | +| ----------- | ------------------------------------------- | +| **RETURNS** | Whether the vectors table is full. ~~bool~~ | + +## Vectors.n_keys {id="n_keys",tag="property"} + +Get the number of keys in the table. Note that this is the number of _all_ keys, +not just unique vectors. If several keys are mapped to the same vectors, they +will be counted individually. In `floret` mode, the keys table is not used. + +> #### Example +> +> ```python +> vectors = Vectors(shape=(10, 300)) +> assert len(vectors) == 10 +> assert vectors.n_keys == 0 +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------- | +| **RETURNS** | The number of all keys in the table. Returns `-1` for floret vectors. ~~int~~ | + +## Vectors.most_similar {id="most_similar",tag="method"} + +For each of the given vectors, find the `n` most similar entries to it by +cosine. Queries are by vector. Results are returned as a +`(keys, best_rows, scores)` tuple. If `queries` is large, the calculations are +performed in chunks to avoid consuming too much memory. You can set the +`batch_size` to control the size/space trade-off during the calculations. Not +supported for `floret` mode. + +> #### Example +> +> ```python +> queries = numpy.asarray([numpy.random.uniform(-1, 1, (300,))]) +> most_similar = nlp.vocab.vectors.most_similar(queries, n=10) +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `queries` | An array with one or more vectors. ~~numpy.ndarray~~ | +| _keyword-only_ | | +| `batch_size` | The batch size to use. Default to `1024`. ~~int~~ | +| `n` | The number of entries to return for each query. Defaults to `1`. ~~int~~ | +| `sort` | Whether to sort the entries returned by score. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The most similar entries as a `(keys, best_rows, scores)` tuple. ~~Tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]~~ | + +## Vectors.get_batch {id="get_batch",tag="method",version="3.2"} + +Get the vectors for the provided keys efficiently as a batch. + +> #### Example +> +> ```python +> words = ["cat", "dog"] +> vectors = nlp.vocab.vectors.get_batch(words) +> ``` + +| Name | Description | +| ------ | --------------------------------------- | +| `keys` | The keys. ~~Iterable[Union[int, str]]~~ | + +## Vectors.to_ops {id="to_ops",tag="method"} + +Change the embedding matrix to use different Thinc ops. + +> #### Example +> +> ```python +> from thinc.api import NumpyOps +> +> vectors.to_ops(NumpyOps()) +> +> ``` + +| Name | Description | +| ----- | -------------------------------------------------------- | +| `ops` | The Thinc ops to switch the embedding matrix to. ~~Ops~~ | + +## Vectors.to_disk {id="to_disk",tag="method"} + +Save the current state to a directory. + +> #### Example +> +> ```python +> vectors.to_disk("/path/to/vectors") +> +> ``` + +| Name | Description | +| ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | + +## Vectors.from_disk {id="from_disk",tag="method"} + +Loads state from a directory. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> vectors = Vectors(StringStore()) +> vectors.from_disk("/path/to/vectors") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| **RETURNS** | The modified `Vectors` object. ~~Vectors~~ | + +## Vectors.to_bytes {id="to_bytes",tag="method"} + +Serialize the current state to a binary string. + +> #### Example +> +> ```python +> vectors_bytes = vectors.to_bytes() +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------ | +| **RETURNS** | The serialized form of the `Vectors` object. ~~bytes~~ | + +## Vectors.from_bytes {id="from_bytes",tag="method"} + +Load state from a binary string. + +> #### Example +> +> ```python +> from spacy.vectors import Vectors +> vectors_bytes = vectors.to_bytes() +> new_vectors = Vectors(StringStore()) +> new_vectors.from_bytes(vectors_bytes) +> ``` + +| Name | Description | +| ----------- | --------------------------------- | +| `data` | The data to load from. ~~bytes~~ | +| **RETURNS** | The `Vectors` object. ~~Vectors~~ | + +## Attributes {id="attributes"} + +| Name | Description | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `data` | Stored vectors data. `numpy` is used for CPU vectors, `cupy` for GPU vectors. ~~Union[numpy.ndarray[ndim=1, dtype=float32], cupy.ndarray[ndim=1, dtype=float32]]~~ | +| `key2row` | Dictionary mapping word hashes to rows in the `Vectors.data` table. ~~Dict[int, int]~~ | +| `keys` | Array keeping the keys in order, such that `keys[vectors.key2row[key]] == key`. ~~Union[numpy.ndarray[ndim=1, dtype=float32], cupy.ndarray[ndim=1, dtype=float32]]~~ | +| `attr` 3.6 | The token attribute for the vector keys. ~~int~~ | diff --git a/website/docs/api/vocab.mdx b/website/docs/api/vocab.mdx new file mode 100644 index 0000000..2466f56 --- /dev/null +++ b/website/docs/api/vocab.mdx @@ -0,0 +1,345 @@ +--- +title: Vocab +teaser: A storage class for vocabulary and other data shared across a language +tag: class +source: spacy/vocab.pyx +--- + +The `Vocab` object provides a lookup table that allows you to access +[`Lexeme`](/api/lexeme) objects, as well as the +[`StringStore`](/api/stringstore). It also owns underlying C-data that is shared +between `Doc` objects. + + + +Note that a `Vocab` instance is not static. It increases in size as texts with +new tokens are processed. Some models may have an empty vocab at initialization. + + + +## Vocab.\_\_init\_\_ {id="init",tag="method"} + +Create the vocabulary. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab(strings=["hello", "world"]) +> ``` + +| Name | Description | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lex_attr_getters` | A dictionary mapping attribute IDs to functions to compute them. Defaults to `None`. ~~Optional[Dict[str, Callable[[str], Any]]]~~ | +| `strings` | A [`StringStore`](/api/stringstore) that maps strings to hash values, and vice versa, or a list of strings. ~~Union[List[str], StringStore]~~ | +| `lookups` | A [`Lookups`](/api/lookups) that stores the `lexeme_norm` and other large lookup tables. Defaults to `None`. ~~Optional[Lookups]~~ | +| `oov_prob` | The default OOV probability. Defaults to `-20.0`. ~~float~~ | +| `vectors_name` | A name to identify the vectors table. ~~str~~ | +| `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ | +| `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | + +## Vocab.\_\_len\_\_ {id="len",tag="method"} + +Get the current number of lexemes in the vocabulary. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> assert len(nlp.vocab) > 0 +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------ | +| **RETURNS** | The number of lexemes in the vocabulary. ~~int~~ | + +## Vocab.\_\_getitem\_\_ {id="getitem",tag="method"} + +Retrieve a lexeme, given an int ID or a string. If a previously unseen string is +given, a new lexeme is created and stored. + +> #### Example +> +> ```python +> apple = nlp.vocab.strings["apple"] +> assert nlp.vocab[apple] == nlp.vocab["apple"] +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------ | +| `id_or_string` | The hash value of a word, or its string. ~~Union[int, str]~~ | +| **RETURNS** | The lexeme indicated by the given ID. ~~Lexeme~~ | + +## Vocab.\_\_iter\_\_ {id="iter",tag="method"} + +Iterate over the lexemes in the vocabulary. + +> #### Example +> +> ```python +> stop_words = (lex for lex in nlp.vocab if lex.is_stop) +> ``` + +| Name | Description | +| ---------- | -------------------------------------- | +| **YIELDS** | An entry in the vocabulary. ~~Lexeme~~ | + +## Vocab.\_\_contains\_\_ {id="contains",tag="method"} + +Check whether the string has an entry in the vocabulary. To get the ID for a +given string, you need to look it up in +[`vocab.strings`](/api/vocab#attributes). + +> #### Example +> +> ```python +> nlp("I'm eating an apple") +> apple = nlp.vocab.strings["apple"] +> oov = nlp.vocab.strings["dskfodkfos"] +> assert apple in nlp.vocab +> assert oov not in nlp.vocab +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------- | +| `string` | The ID string. ~~str~~ | +| **RETURNS** | Whether the string has an entry in the vocabulary. ~~bool~~ | + +## Vocab.add_flag {id="add_flag",tag="method"} + +Set a new boolean flag to words in the vocabulary. The `flag_getter` function +will be called over the words currently in the vocab, and then applied to new +words as they occur. You'll then be able to access the flag value on each token, +using `token.check_flag(flag_id)`. + +> #### Example +> +> ```python +> def is_my_product(text): +> products = ["spaCy", "Thinc", "displaCy"] +> return text in products +> +> MY_PRODUCT = nlp.vocab.add_flag(is_my_product) +> doc = nlp("I like spaCy") +> assert doc[2].check_flag(MY_PRODUCT) == True +> ``` + +| Name | Description | +| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `flag_getter` | A function that takes the lexeme text and returns the boolean flag value. ~~Callable[[str], bool]~~ | +| `flag_id` | An integer between `1` and `63` (inclusive), specifying the bit at which the flag will be stored. If `-1`, the lowest available bit will be chosen. ~~int~~ | +| **RETURNS** | The integer ID by which the flag value can be checked. ~~int~~ | + +## Vocab.reset_vectors {id="reset_vectors",tag="method",version="2"} + +Drop the current vector table. Because all vectors must be the same width, you +have to call this to change the size of the vectors. Only one of the `width` and +`shape` keyword arguments can be specified. + +> #### Example +> +> ```python +> nlp.vocab.reset_vectors(width=300) +> ``` + +| Name | Description | +| -------------- | ---------------------- | +| _keyword-only_ | | +| `width` | The new width. ~~int~~ | +| `shape` | The new shape. ~~int~~ | + +## Vocab.prune_vectors {id="prune_vectors",tag="method",version="2"} + +Reduce the current vector table to `nr_row` unique entries. Words mapped to the +discarded vectors will be remapped to the closest vector among those remaining. +For example, suppose the original table had vectors for the words: +`['sat', 'cat', 'feline', 'reclined']`. If we prune the vector table to, two +rows, we would discard the vectors for "feline" and "reclined". These words +would then be remapped to the closest remaining vector – so "feline" would have +the same vector as "cat", and "reclined" would have the same vector as "sat". +The similarities are judged by cosine. The original vectors may be large, so the +cosines are calculated in minibatches to reduce memory usage. + +> #### Example +> +> ```python +> nlp.vocab.prune_vectors(10000) +> assert len(nlp.vocab.vectors) <= 10000 +> ``` + +| Name | Description | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nr_row` | The number of rows to keep in the vector table. ~~int~~ | +| `batch_size` | Batch of vectors for calculating the similarities. Larger batch sizes might be faster, while temporarily requiring more memory. ~~int~~ | +| **RETURNS** | A dictionary keyed by removed words mapped to `(string, score)` tuples, where `string` is the entry the removed word was mapped to, and `score` the similarity score between the two words. ~~Dict[str, Tuple[str, float]]~~ | + +## Vocab.deduplicate_vectors {id="deduplicate_vectors",tag="method",version="3.3"} + +> #### Example +> +> ```python +> nlp.vocab.deduplicate_vectors() +> ``` + +Remove any duplicate rows from the current vector table, maintaining the +mappings for all words in the vectors. + +## Vocab.get_vector {id="get_vector",tag="method",version="2"} + +Retrieve a vector for a word in the vocabulary. Words can be looked up by string +or hash value. If the current vectors do not contain an entry for the word, a +0-vector with the same number of dimensions +([`Vocab.vectors_length`](#attributes)) as the current vectors is returned. + +> #### Example +> +> ```python +> nlp.vocab.get_vector("apple") +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------------------------------------------------- | +| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | +| **RETURNS** | A word vector. Size and shape are determined by the `Vocab.vectors` instance. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Vocab.set_vector {id="set_vector",tag="method",version="2"} + +Set a vector for a word in the vocabulary. Words can be referenced by string or +hash value. + +> #### Example +> +> ```python +> nlp.vocab.set_vector("apple", array([...])) +> ``` + +| Name | Description | +| -------- | -------------------------------------------------------------------- | +| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | +| `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | + +## Vocab.has_vector {id="has_vector",tag="method",version="2"} + +Check whether a word has a vector. Returns `False` if no vectors are loaded. +Words can be looked up by string or hash value. + +> #### Example +> +> ```python +> if nlp.vocab.has_vector("apple"): +> vector = nlp.vocab.get_vector("apple") +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | +| **RETURNS** | Whether the word has a vector. ~~bool~~ | + +## Vocab.to_disk {id="to_disk",tag="method",version="2"} + +Save the current state to a directory. + +> #### Example +> +> ```python +> nlp.vocab.to_disk("/path/to/vocab") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## Vocab.from_disk {id="from_disk",tag="method",version="2"} + +Loads state from a directory. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `Vocab` object. ~~Vocab~~ | + +## Vocab.to_bytes {id="to_bytes",tag="method"} + +Serialize the current state to a binary string. + +> #### Example +> +> ```python +> vocab_bytes = nlp.vocab.to_bytes() +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `Vocab` object. ~~bytes~~ | + +## Vocab.from_bytes {id="from_bytes",tag="method"} + +Load state from a binary string. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab_bytes = nlp.vocab.to_bytes() +> vocab = Vocab() +> vocab.from_bytes(vocab_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `Vocab` object. ~~Vocab~~ | + +## Attributes {id="attributes"} + +> #### Example +> +> ```python +> apple_id = nlp.vocab.strings["apple"] +> assert type(apple_id) == int +> PERSON = nlp.vocab.strings["PERSON"] +> assert type(PERSON) == int +> ``` + +| Name | Description | +| ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `strings` | A table managing the string-to-int mapping. ~~StringStore~~ | +| `vectors` | A table associating word IDs to word vectors. ~~Vectors~~ | +| `vectors_length` | Number of dimensions for each word vector. ~~int~~ | +| `lookups` | The available lookup tables in this vocab. ~~Lookups~~ | +| `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | +| `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | + +## Serialization fields {id="serialization-fields"} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = vocab.to_bytes(exclude=["strings", "vectors"]) +> vocab.from_disk("./vocab", exclude=["strings"]) +> ``` + +| Name | Description | +| --------- | ----------------------------------------------------- | +| `strings` | The strings in the [`StringStore`](/api/stringstore). | +| `vectors` | The word vectors, if available. | +| `lookups` | The lookup tables, if available. | diff --git a/website/docs/images/displacy-long2.html b/website/docs/images/displacy-long2.html new file mode 100644 index 0000000..c428bd2 --- /dev/null +++ b/website/docs/images/displacy-long2.html @@ -0,0 +1,212 @@ + + + Autonomous + ADJ + + + + cars + NOUN + + + + shift + VERB + + + + insurance + NOUN + + + + liability + NOUN + + + + toward + ADP + + + + manufacturers + NOUN + + + + + + + amod + + + + + + + + + + nsubj + + + + + + + + + + compound + + + + + + + + + + dobj + + + + + + + + + + prep + + + + + + + + + + pobj + + + + + diff --git a/website/docs/models/index.mdx b/website/docs/models/index.mdx new file mode 100644 index 0000000..54f3c49 --- /dev/null +++ b/website/docs/models/index.mdx @@ -0,0 +1,271 @@ +--- +title: Trained Models & Pipelines +teaser: Downloadable trained pipelines and weights for spaCy +menu: + - ['Quickstart', 'quickstart'] + - ['Conventions', 'conventions'] + - ['Pipeline Design', 'design'] +--- + +{/* TODO: include interactive demo */} + +### Quickstart {hidden="true"} + +> #### 📖 Installation and usage +> +> For more details on how to use trained pipelines with spaCy, see the +> [usage guide](/usage/models). + + + +## Package naming conventions {id="conventions"} + +In general, spaCy expects all pipeline packages to follow the naming convention +of `[lang]_[name]`. For spaCy's pipelines, we also chose to divide the name into +three components: + +1. **Type:** Capabilities (e.g. `core` for general-purpose pipeline with + tagging, parsing, lemmatization and named entity recognition, or `dep` for + only tagging, parsing and lemmatization). +2. **Genre:** Type of text the pipeline is trained on, e.g. `web` or `news`. +3. **Size:** Package size indicator, `sm`, `md`, `lg` or `trf`. + + `sm` and `trf` pipelines have no static word vectors. + + For pipelines with default vectors, `md` has a reduced word vector table with + 20k unique vectors for ~500k words and `lg` has a large word vector table + with ~500k entries. + + For pipelines with floret vectors, `md` vector tables have 50k entries and + `lg` vector tables have 200k entries. + +For example, [`en_core_web_sm`](/models/en#en_core_web_sm) is a small English +pipeline trained on written web text (blogs, news, comments), that includes +vocabulary, syntax and entities. + +### Package versioning {id="model-versioning"} + +Additionally, the pipeline package versioning reflects both the compatibility +with spaCy, as well as the model version. A package version `a.b.c` translates +to: + +- `a`: **spaCy major version**. For example, `2` for spaCy v2.x. +- `b`: **spaCy minor version**. For example, `3` for spaCy v2.3.x. +- `c`: **Model version**. Different model config: e.g. from being trained on + different data, with different parameters, for different numbers of + iterations, with different vectors, etc. + +For a detailed compatibility overview, see the +[`compatibility.json`](https://github.com/explosion/spacy-models/tree/master/compatibility.json). +This is also the source of spaCy's internal compatibility check, performed when +you run the [`download`](/api/cli#download) command. + +## Trained pipeline design {id="design"} + +The spaCy v3 trained pipelines are designed to be efficient and configurable. +For example, multiple components can share a common "token-to-vector" model and +it's easy to swap out or disable the lemmatizer. The pipelines are designed to +be efficient in terms of speed and size and work well when the pipeline is run +in full. + +When modifying a trained pipeline, it's important to understand how the +components **depend on** each other. Unlike spaCy v2, where the `tagger`, +`parser` and `ner` components were all independent, some v3 components depend on +earlier components in the pipeline. As a result, disabling or reordering +components can affect the annotation quality or lead to warnings and errors. + +Main changes from spaCy v2 models: + +- The [`Tok2Vec`](/api/tok2vec) component may be a separate, shared component. A + component like a tagger or parser can + [listen](/api/architectures#Tok2VecListener) to an earlier `tok2vec` or + `transformer` rather than having its own separate tok2vec layer. +- Rule-based exceptions move from individual components to the + `attribute_ruler`. Lemma and POS exceptions move from the tokenizer exceptions + to the attribute ruler and the tag map and morph rules move from the tagger to + the attribute ruler. +- The lemmatizer tables and processing move from the vocab and tagger to a + separate `lemmatizer` component. + +### CNN/CPU pipeline design {id="design-cnn"} + +![Components and their dependencies in the CNN pipelines](/images/pipeline-design.svg) + +In the `sm`/`md`/`lg` models: + +- The `tagger`, `morphologizer` and `parser` components listen to the `tok2vec` + component. If the lemmatizer is trainable (v3.3+), `lemmatizer` also listens + to `tok2vec`. +- The `attribute_ruler` maps `token.tag` to `token.pos` if there is no + `morphologizer`. The `attribute_ruler` additionally makes sure whitespace is + tagged consistently and copies `token.pos` to `token.tag` if there is no + tagger. For English, the attribute ruler can improve its mapping from + `token.tag` to `token.pos` if dependency parses from a `parser` are present, + but the parser is not required. +- The `lemmatizer` component for many languages requires `token.pos` annotation + from either `tagger`+`attribute_ruler` or `morphologizer`. +- The `ner` component is independent with its own internal tok2vec layer. + +#### CNN/CPU pipelines with floret vectors + +The Croatian, Finnish, Korean, Slovenian, Swedish and Ukrainian `md` and `lg` +pipelines use [floret vectors](/usage/v3-2#vectors) instead of default vectors. +If you're running a trained pipeline on texts and working with [`Doc`](/api/doc) +objects, you shouldn't notice any difference with floret vectors. With floret +vectors no tokens are out-of-vocabulary, so +[`Token.is_oov`](/api/token#attributes) will return `False` for all tokens. + +If you access vectors directly for similarity comparisons, there are a few +differences because floret vectors don't include a fixed word list like the +vector keys for default vectors. + +- If your workflow iterates over the vector keys, you need to use an external + word list instead: + + ```diff + - lexemes = [nlp.vocab[orth] for orth in nlp.vocab.vectors] + + lexemes = [nlp.vocab[word] for word in external_word_list] + ``` + +- [`Vectors.most_similar`](/api/vectors#most_similar) is not supported because + there's no fixed list of vectors to compare your vectors to. + +### Transformer pipeline design {id="design-trf"} + +In the transformer (`trf`) pipelines, the `tagger`, `parser` and `ner` (if +present) all listen to the `transformer` component. The `attribute_ruler` and +`lemmatizer` have the same configuration as in the CNN models. + +For spaCy v3.0-v3.6, `trf` pipelines use +[`spacy-transformers`](https://github.com/explosion/spacy-transformers) and the +transformer output in `doc._.trf_data` is a +[`TransformerData`](/api/transformer#transformerdata) object. + +For spaCy v3.7+, `trf` pipelines use +[`spacy-curated-transformers`](https://github.com/explosion/spacy-curated-transformers) +and `doc._.trf_data` is a +[`DocTransformerOutput`](/api/curatedtransformer#doctransformeroutput) object. + +### Modifying the default pipeline {id="design-modify"} + +For faster processing, you may only want to run a subset of the components in a +trained pipeline. The `disable` and `exclude` arguments to +[`spacy.load`](/api/top-level#spacy.load) let you control which components are +loaded and run. Disabled components are loaded in the background so it's +possible to reenable them in the same pipeline in the future with +[`nlp.enable_pipe`](/api/language/#enable_pipe). To skip loading a component +completely, use `exclude` instead of `disable`. + +#### Disable part-of-speech tagging and lemmatization + +To disable part-of-speech tagging and lemmatization, disable the `tagger`, +`morphologizer`, `attribute_ruler` and `lemmatizer` components. + +```python +# Note: English doesn't include a morphologizer +nlp = spacy.load("en_core_web_sm", disable=["tagger", "attribute_ruler", "lemmatizer"]) +nlp = spacy.load("en_core_web_trf", disable=["tagger", "attribute_ruler", "lemmatizer"]) +``` + + + +The lemmatizer depends on `tagger`+`attribute_ruler` or `morphologizer` for a +number of languages. If you disable any of these components, you'll see +lemmatizer warnings unless the lemmatizer is also disabled. + +**v3.3**: Catalan, English, French, Russian and Spanish + +**v3.0-v3.2**: Catalan, Dutch, English, French, Greek, Italian, Macedonian, +Norwegian, Polish, Russian and Spanish + + + +#### Use senter rather than parser for fast sentence segmentation + +If you need fast sentence segmentation without dependency parses, disable the +`parser` use the `senter` component instead: + +```python +nlp = spacy.load("en_core_web_sm") +nlp.disable_pipe("parser") +nlp.enable_pipe("senter") +``` + +The `senter` component is ~10× faster than the parser and more accurate +than the rule-based `sentencizer`. + +#### Switch from trainable lemmatizer to default lemmatizer + +Since v3.3, a number of pipelines use a trainable lemmatizer. You can check +whether the lemmatizer is trainable: + +```python +nlp = spacy.load("de_core_web_sm") +assert nlp.get_pipe("lemmatizer").is_trainable +``` + +If you'd like to switch to a non-trainable lemmatizer that's similar to v3.2 or +earlier, you can replace the trainable lemmatizer with the default non-trainable +lemmatizer: + +```python +# Requirements: pip install spacy-lookups-data +nlp = spacy.load("de_core_web_sm") +# Remove existing lemmatizer +nlp.remove_pipe("lemmatizer") +# Add non-trainable lemmatizer from language defaults +# and load lemmatizer tables from spacy-lookups-data +nlp.add_pipe("lemmatizer").initialize() +``` + +#### Switch from rule-based to lookup lemmatization + +For the Dutch, English, French, Greek, Macedonian, Norwegian and Spanish +pipelines, you can swap out a trainable or rule-based lemmatizer for a lookup +lemmatizer: + +```python +# Requirements: pip install spacy-lookups-data +nlp = spacy.load("en_core_web_sm") +nlp.remove_pipe("lemmatizer") +nlp.add_pipe("lemmatizer", config={"mode": "lookup"}).initialize() +``` + +#### Disable everything except NER + +For the non-transformer models, the `ner` component is independent, so you can +disable everything else: + +```python +nlp = spacy.load("en_core_web_sm", disable=["tok2vec", "tagger", "parser", "attribute_ruler", "lemmatizer"]) +``` + +In the transformer models, `ner` listens to the `transformer` component, so you +can disable all components related tagging, parsing, and lemmatization. + +```python +nlp = spacy.load("en_core_web_trf", disable=["tagger", "parser", "attribute_ruler", "lemmatizer"]) +``` + +#### Move NER to the end of the pipeline + + + +As of v3.1, the NER component is at the end of the pipeline by default. + + + +For access to `POS` and `LEMMA` features in an `entity_ruler`, move `ner` to the +end of the pipeline after `attribute_ruler` and `lemmatizer`: + +```python +# load without NER +nlp = spacy.load("en_core_web_sm", exclude=["ner"]) + +# source NER from the same pipeline package as the last component +nlp.add_pipe("ner", source=spacy.load("en_core_web_sm")) + +# insert the entity ruler +nlp.add_pipe("entity_ruler", before="ner") +``` diff --git a/website/docs/styleguide.mdx b/website/docs/styleguide.mdx new file mode 100644 index 0000000..276137a --- /dev/null +++ b/website/docs/styleguide.mdx @@ -0,0 +1,615 @@ +--- +title: Styleguide +section: styleguide +search_exclude: true +menu: + - ['Logo', 'logo'] + - ['Colors', 'colors'] + - ['Typography', 'typography'] + - ['Elements', 'elements'] + - ['Components', 'components'] + - ['Markdown Reference', 'markdown'] + - ['Editorial', 'editorial'] +sidebar: + - label: Styleguide + items: + - text: '' + url: '/styleguide' + - label: Resources + items: + - text: Website Source + url: https://github.com/explosion/spacy/tree/master/website + - text: Contributing Guide + url: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md +--- + +The [spacy.io](https://spacy.io) website is implemented using +[Gatsby](https://www.gatsbyjs.org) with +[Remark](https://github.com/remarkjs/remark) and [MDX](https://mdxjs.com/). This +allows authoring content in **straightforward Markdown** without the usual +limitations. Standard elements can be overwritten with powerful +[React](http://reactjs.org/) components and wherever Markdown syntax isn't +enough, JSX components can be used. + +> #### Contributing to the site +> +> The docs can always use another example or more detail, and they should always +> be up to date and not misleading. We always appreciate a +> [pull request](https://github.com/explosion/spaCy/pulls). To quickly find the +> correct file to edit, simply click on the "Suggest edits" button at the bottom +> of a page. +> +> For more details on editing the site locally, see the installation +> instructions and markdown reference below. + +## Logo {id="logo",source="website/src/images/logo.svg"} + +If you would like to use the spaCy logo on your site, please get in touch and +ask us first. However, if you want to show support and tell others that your +project is using spaCy, you can grab one of our +[spaCy badges](/usage/spacy-101#faq-project-with-spacy). + + + +## Colors {id="colors"} + + + +### Patterns + + + +## Typography {id="typography"} + +> #### Markdown +> +> ```markdown +> ## Headline 2 +> +> ## Headline 2 {id="some_id"} +> +> ## Headline 2 {id="some_id" tag="method"} +> ``` +> +> #### JSX +> +> ```jsx +>

Headline 2

+>

Headline 2

+>

Headline 2

+> ``` + +Headlines are set in +[HK Grotesk](http://cargocollective.com/hanken/HK-Grotesk-Open-Source-Font) by +Hanken Design. All other body text and code uses the best-matching default +system font to provide a "native" reading experience. All code uses the +[JetBrains Mono](https://www.jetbrains.com/lp/mono/) typeface by JetBrains. + + + +Level 2 headings are automatically wrapped in `
` elements at compile +time, using a custom +[Markdown transformer](https://github.com/explosion/spaCy/tree/master/website/plugins/remark-wrap-section.js). +This makes it easier to highlight the section that's currently in the viewpoint +in the sidebar menu. + + + +
+

Headline 2

+

Headline 3

+

Headline 4

+
Headline 5
+ +
+ +--- + +The following optional attributes can be set on the headline to modify it. For +example, to add a tag for the documented type or mark features that have been +introduced in a specific version or require statistical models to be loaded. +Tags are also available as standalone `` components. + +| Argument | Example | Result | +| --------- | -------------------------- | ----------------------------------------- | +| `tag` | `{tag="method"}` | method | +| `version` | `{version="3"}` | 3 | +| `model` | `{model="tagger, parser"}` | tagger, parser | +| `hidden` | `{hidden="true"}` | | + +## Elements {id="elements"} + +### Links {id="links"} + +> #### Markdown +> +> ```markdown +> [I am a link](https://spacy.io) +> ``` +> +> #### JSX +> +> ```jsx +> I am a link +> ``` + +Special link styles are used depending on the link URL. + +- [I am a regular external link](https://explosion.ai) +- [I am a link to the documentation](/api/doc) +- [I am a link to an architecture](/api/architectures#HashEmbedCNN) +- [I am a link to a model](/models/en#en_core_web_sm) +- [I am a link to GitHub](https://github.com/explosion/spaCy) + +### Abbreviations {id="abbr"} + +> #### JSX +> +> ```jsx +> Abbreviation +> ``` + +Some text with an abbreviation. On small +screens, I collapse and the explanation text is displayed next to the +abbreviation. + +### Tags {id="tags"} + +> ```jsx +> method +> 4 +> tagger, parser +> ``` + +Tags can be used together with headlines, or next to properties across the +documentation, and combined with tooltips to provide additional information. An +optional `variant` argument can be used for special tags. `variant="new"` makes +the tag take a version number to mark new features. Using the component, +visibility of this tag can later be toggled once the feature isn't considered +new anymore. Setting `variant="model"` takes a description of model capabilities +and can be used to mark features that require a respective model to be +installed. + +

+ method + 4 + tagger, parser +

+ +### Buttons {id="buttons"} + +> ```jsx +> +> +> ``` + +Link buttons come in two variants, `primary` and `secondary` and two sizes, with +an optional `large` size modifier. Since they're mostly used as enhanced links, +the buttons are implemented as styled links instead of native button elements. + +

+ + +{' '} + + +

+ +

+ + +{' '} + + +

+ +## Components + +### Table {id="table"} + +> #### Markdown +> +> ```markdown +> | Header 1 | Header 2 | +> | -------- | -------- | +> | Column 1 | Column 2 | +> ``` +> +> #### JSX +> +> ```markup +> +> +> +>
Header 1Header 2
Column 1Column 2
+> ``` + +Tables are used to present data and API documentation. Certain keywords can be +used to mark a footer row with a distinct style, for example to visualize the +return values of a documented function. + +| Header 1 | Header 2 | Header 3 | Header 4 | +| ----------- | -------- | :------: | -------: | +| Column 1 | Column 2 | Column 3 | Column 4 | +| Column 1 | Column 2 | Column 3 | Column 4 | +| Column 1 | Column 2 | Column 3 | Column 4 | +| Column 1 | Column 2 | Column 3 | Column 4 | +| **RETURNS** | Column 2 | Column 3 | Column 4 | + +Tables also support optional "divider" rows that are typically used to denote +keyword-only arguments in API documentation. To turn a row into a dividing +headline, it should only include content in its first cell, and its value should +be italicized: + +> #### Markdown +> +> ```markdown +> | Header 1 | Header 2 | Header 3 | +> | -------- | -------- | -------- | +> | Column 1 | Column 2 | Column 3 | +> | _Hello_ | | | +> | Column 1 | Column 2 | Column 3 | +> ``` + +| Header 1 | Header 2 | Header 3 | +| -------- | -------- | -------- | +| Column 1 | Column 2 | Column 3 | +| _Hello_ | | | +| Column 1 | Column 2 | Column 3 | + +### Type Annotations {id="type-annotations"} + +> #### Markdown +> +> ```markdown +> ~~Model[List[Doc], Floats2d]~~ +> ``` +> +> #### JSX +> +> ```markup +> Model[List[Doc], Floats2d] +> ``` + +Type annotations are special inline code blocks are used to describe Python +types in the [type hints](https://docs.python.org/3/library/typing.html) format. +The special component will split the type, apply syntax highlighting and link +all types that specify links in `meta/type-annotations.json`. Types can link to +internal or external documentation pages. To make it easy to represent the type +annotations in Markdown, the rendering "hijacks" the `~~` tags that would +typically be converted to a `` element – but in this case, text surrounded +by `~~` becomes a type annotation. + +- ~~Dict[str, List[Union[Doc, Span]]]~~ +- ~~Model[List[Doc], List[numpy.ndarray]]~~ + +Type annotations support a special visual style in tables and will render as a +separate row, under the cell text. This allows the API docs to display complex +types without taking up too much space in the cell. The type annotation should +always be the **last element** in the row. + +> #### Markdown +> +> ```markdown +> | Header 1 | Header 2 | +> | -------- | ---------------------- | +> | Column 1 | Column 2 ~~List[Doc]~~ | +> ``` + +| Name | Description | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. ~~Model[List[Doc], FullTransformerBatch]~~ | +| `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and can set additional annotations on the `Doc`. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | + +### List {id="list"} + +> #### Markdown +> +> ```markdown +> 1. One +> 2. Two +> ``` +> +> #### JSX +> +> ```markup +>
    +>
  1. One
    2. +>
  2. Two
    3. +>
+> ``` + +Lists are available as bulleted and numbered. Markdown lists are transformed +automatically. + +- I am a bulleted list +- I have nice bullets +- Lorem ipsum dolor +- consectetur adipiscing elit + +1. I am an ordered list +2. I have nice numbers +3. Lorem ipsum dolor +4. consectetur adipiscing elit + +### Aside {id="aside"} + +> #### Markdown +> +> ```markdown +> > #### Aside title +> > +> > This is aside text. +> ``` +> +> #### JSX +> +> ```jsx +> +> ``` + +Asides can be used to display additional notes and content in the right-hand +column. Asides can contain text, code and other elements if needed. Visually, +asides are moved to the side on the X-axis, and displayed at the same level they +were inserted. On small screens, they collapse and are rendered in their +original position, in between the text. + +To make them easier to use in Markdown, paragraphs formatted as blockquotes will +turn into asides by default. Level 4 headlines (with a leading `####`) will +become aside titles. + +### Code Block {id="code-block"} + +> #### Markdown +> +> ````markdown +> ```python +> ### This is a title +> import spacy +> ``` +> ```` +> +> #### JSX +> +> ```jsx +> +> import spacy +> +> ``` + +Code blocks use the [Prism](http://prismjs.com/) syntax highlighter with a +custom theme. The language can be set individually on each block, and defaults +to raw text with no highlighting. An optional label can be added as the first +line with the prefix `####` (Python-like) and `///` (JavaScript-like). the +indented block as plain text and preserve whitespace. + +```python {title="Using spaCy"} +import spacy +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence.") +for token in doc: + print(token.text, token.pos_) +``` + +Code blocks and also specify an optional range of line numbers to highlight by +adding `{highlight="..."}` to the headline. Acceptable ranges are spans like +`5-7`, but also `5-7,10` or `5-7,10,13-14`. + +> #### Markdown +> +> ````markdown +> ```python +> ### This is a title {highlight="1-2"} +> import spacy +> nlp = spacy.load("en_core_web_sm") +> ``` +> ```` + +```python {title="Using the matcher",highlight="5-7"} +import spacy +from spacy.matcher import Matcher + +nlp = spacy.load('en_core_web_sm') +matcher = Matcher(nlp.vocab) +pattern = [{"LOWER": "hello"}, {"IS_PUNCT": True}, {"LOWER": "world"}] +matcher.add("HelloWorld", None, pattern) +doc = nlp("Hello, world! Hello world!") +matches = matcher(doc) +``` + +Adding `{executable="true"}` to the title turns the code into an executable +block, powered by [Binder](https://mybinder.org) and +[Juniper](https://github.com/ines/juniper). If JavaScript is disabled, the +interactive widget defaults to a regular code block. + +> #### Markdown +> +> ````markdown +> ```python +> ### {executable="true"} +> import spacy +> nlp = spacy.load("en_core_web_sm") +> ``` +> ```` + +```python {executable="true"} +import spacy +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence.") +for token in doc: + print(token.text, token.pos_) +``` + +If a code block only contains a URL to a GitHub file, the raw file contents are +embedded automatically and syntax highlighting is applied. The link to the +original file is shown at the top of the widget. + +> #### Markdown +> +> ````markdown +> ```python +> https://github.com/... +> ``` +> ```` +> +> #### JSX +> +> ```jsx +> +> ``` + +```python +https://github.com/explosion/spaCy/tree/master/spacy/language.py +``` + +### Infobox {id="infobox"} + +> #### JSX +> +> ```jsx +> Regular infobox +> This is a warning. +> This is dangerous. +> ``` + +Infoboxes can be used to add notes, updates, warnings or additional information +to a page or section. Semantically, they're implemented and interpreted as an +`aside` element. Infoboxes can take an optional `title` argument, as well as an +optional `variant` (either `"warning"` or `"danger"`). + + + +If needed, an infobox can contain regular text, `inline code`, lists and other +blocks. + + + + + +If needed, an infobox can contain regular text, `inline code`, lists and other +blocks. + + + + + +If needed, an infobox can contain regular text, `inline code`, lists and other +blocks. + + + +### Accordion {id="accordion"} + +> #### JSX +> +> ```jsx +> +> Accordion content goes here. +> +> ``` + +Accordions are collapsible sections that are mostly used for lengthy tables, +like the tag and label annotation schemes for different languages. They all need +to be presented – but chances are the user doesn't actually care about _all_ of +them, especially not at the same time. So it's fairly reasonable to hide them +begin a click. This particular implementation was inspired by the amazing +[Inclusive Components blog](https://inclusive-components.design/collapsible-sections/). + + + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque enim ante, +pretium a orci eget, varius dignissim augue. Nam eu dictum mauris, id tincidunt +nisi. Integer commodo pellentesque tincidunt. Nam at turpis finibus tortor +gravida sodales tincidunt sit amet est. Nullam euismod arcu in tortor auctor, +sit amet dignissim justo congue. + + + +## Markdown reference {id="markdown"} + +All page content and page meta lives in the `.mdx` files in the `/docs` +directory. The frontmatter block at the top of each file defines the page title +and other settings like the sidebar menu. + +````markdown +--- +title: Page title +--- + +## Headline starting a section {id="some_id"} + +This is a regular paragraph with a [link](https://spacy.io) and **bold text**. + +> #### This is an aside title +> +> This is aside text. + +### Subheadline + +| Header 1 | Header 2 | +| -------- | -------- | +| Column 1 | Column 2 | + +```python {title="Code block title",highlight="2-3"} +import spacy +nlp = spacy.load("en_core_web_sm") +doc = nlp("Hello world") +``` + + + +This is content in the infobox. + + +```` + +In addition to the native markdown elements, you can use the components +[``][infobox], [``][accordion], [``][abbr] and +[``][tag] via their JSX syntax. + +[infobox]: https://spacy.io/styleguide#infobox +[accordion]: https://spacy.io/styleguide#accordion +[abbr]: https://spacy.io/styleguide#abbr +[tag]: https://spacy.io/styleguide#tag + +## Editorial {id="editorial"} + +- "spaCy" should always be spelled with a lowercase "s" and a capital "C", + unless it specifically refers to the Python package or Python import `spacy` + (in which case it should be formatted as code). + - ✅ spaCy is a library for advanced NLP in Python. + - ❌ Spacy is a library for advanced NLP in Python. + - ✅ First, you need to install the `spacy` package from pip. +- Mentions of code, like function names, classes, variable names etc. in inline + text should be formatted as `code`. + - ✅ "Calling the `nlp` object on a text returns a `Doc`." +- Objects that have pages in the [API docs](/api) should be linked – for + example, [`Doc`](/api/doc) or [`Language.to_disk`](/api/language#to_disk). The + mentions should still be formatted as code within the link. Links pointing to + the API docs will automatically receive a little icon. However, if a paragraph + includes many references to the API, the links can easily get messy. In that + case, we typically only link the first mention of an object and not any + subsequent ones. + - ✅ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a + [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a `Doc` object + from a `Span`. + - ❌ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a + [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a + [`Doc`](/api/doc) object from a [`Span`](/api/span). +- Other things we format as code are: references to trained pipeline packages + like `en_core_web_sm` or file names like `code.py` or `meta.json`. + - ✅ After training, the `config.cfg` is saved to disk. +- [Type annotations](#type-annotations) are a special type of code formatting, + expressed by wrapping the text in `~~` instead of backticks. The result looks + like this: ~~List[Doc]~~. All references to known types will be linked + automatically. + - ✅ The model has the input type ~~List[Doc]~~ and it outputs a + ~~List[Array2d]~~. +- We try to keep links meaningful but short. + - ✅ For details, see the usage guide on + [training with custom code](/usage/training#custom-code). + - ❌ For details, see + [the usage guide on training with custom code](/usage/training#custom-code). + - ❌ For details, see the usage guide on training with custom code + [here](/usage/training#custom-code). diff --git a/website/docs/usage/101/_architecture.mdx b/website/docs/usage/101/_architecture.mdx new file mode 100644 index 0000000..2a63a37 --- /dev/null +++ b/website/docs/usage/101/_architecture.mdx @@ -0,0 +1,90 @@ +The central data structures in spaCy are the [`Language`](/api/language) class, +the [`Vocab`](/api/vocab) and the [`Doc`](/api/doc) object. The `Language` class +is used to process a text and turn it into a `Doc` object. It's typically stored +as a variable called `nlp`. The `Doc` object owns the **sequence of tokens** and +all their annotations. By centralizing strings, word vectors and lexical +attributes in the `Vocab`, we avoid storing multiple copies of this data. This +saves memory, and ensures there's a **single source of truth**. + +Text annotations are also designed to allow a single source of truth: the `Doc` +object owns the data, and [`Span`](/api/span) and [`Token`](/api/token) are +**views that point into it**. The `Doc` object is constructed by the +[`Tokenizer`](/api/tokenizer), and then **modified in place** by the components +of the pipeline. The `Language` object coordinates these components. It takes +raw text and sends it through the pipeline, returning an **annotated document**. +It also orchestrates training and serialization. + +![Library architecture {{w:1080, h:1254}}](/images/architecture.svg) + +### Container objects {id="architecture-containers"} + +| Name | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`Doc`](/api/doc) | A container for accessing linguistic annotations. | +| [`DocBin`](/api/docbin) | A collection of `Doc` objects for efficient binary serialization. Also used for [training data](/api/data-formats#binary-training). | +| [`Example`](/api/example) | A collection of training annotations, containing two `Doc` objects: the reference data and the predictions. | +| [`Language`](/api/language) | Processing class that turns text into `Doc` objects. Different languages implement their own subclasses of it. The variable is typically called `nlp`. | +| [`Lexeme`](/api/lexeme) | An entry in the vocabulary. It's a word type with no context, as opposed to a word token. It therefore has no part-of-speech tag, dependency parse etc. | +| [`Span`](/api/span) | A slice from a `Doc` object. | +| [`SpanGroup`](/api/spangroup) | A named collection of spans belonging to a `Doc`. | +| [`Token`](/api/token) | An individual token — i.e. a word, punctuation symbol, whitespace, etc. | + +### Processing pipeline {id="architecture-pipeline"} + +The processing pipeline consists of one or more **pipeline components** that are +called on the `Doc` in order. The tokenizer runs before the components. Pipeline +components can be added using [`Language.add_pipe`](/api/language#add_pipe). +They can contain a statistical model and trained weights, or only make +rule-based modifications to the `Doc`. spaCy provides a range of built-in +components for different language processing tasks and also allows adding +[custom components](/usage/processing-pipelines#custom-components). + +![The processing pipeline](/images/pipeline.svg) + +| Name | Description | +| ----------------------------------------------- | ------------------------------------------------------------------------------------------- | +| [`AttributeRuler`](/api/attributeruler) | Set token attributes using matcher rules. | +| [`DependencyParser`](/api/dependencyparser) | Predict syntactic dependencies. | +| [`EditTreeLemmatizer`](/api/edittreelemmatizer) | Predict base forms of words. | +| [`EntityLinker`](/api/entitylinker) | Disambiguate named entities to nodes in a knowledge base. | +| [`EntityRecognizer`](/api/entityrecognizer) | Predict named entities, e.g. persons or products. | +| [`EntityRuler`](/api/entityruler) | Add entity spans to the `Doc` using token-based rules or exact phrase matches. | +| [`Lemmatizer`](/api/lemmatizer) | Determine the base forms of words using rules and lookups. | +| [`Morphologizer`](/api/morphologizer) | Predict morphological features and coarse-grained part-of-speech tags. | +| [`SentenceRecognizer`](/api/sentencerecognizer) | Predict sentence boundaries. | +| [`Sentencizer`](/api/sentencizer) | Implement rule-based sentence boundary detection that doesn't require the dependency parse. | +| [`Tagger`](/api/tagger) | Predict part-of-speech tags. | +| [`TextCategorizer`](/api/textcategorizer) | Predict categories or labels over the whole document. | +| [`Tok2Vec`](/api/tok2vec) | Apply a "token-to-vector" model and set its outputs. | +| [`Tokenizer`](/api/tokenizer) | Segment raw text and create `Doc` objects from the words. | +| [`TrainablePipe`](/api/pipe) | Class that all trainable pipeline components inherit from. | +| [`Transformer`](/api/transformer) | Use a transformer model and set its outputs. | +| [Other functions](/api/pipeline-functions) | Automatically apply something to the `Doc`, e.g. to merge spans of tokens. | + +### Matchers {id="architecture-matchers"} + +Matchers help you find and extract information from [`Doc`](/api/doc) objects +based on match patterns describing the sequences you're looking for. A matcher +operates on a `Doc` and gives you access to the matched tokens **in context**. + +| Name | Description | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`DependencyMatcher`](/api/dependencymatcher) | Match sequences of tokens based on dependency trees using [Semgrex operators](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html). | +| [`Matcher`](/api/matcher) | Match sequences of tokens, based on pattern rules, similar to regular expressions. | +| [`PhraseMatcher`](/api/phrasematcher) | Match sequences of tokens based on phrases. | + +### Other classes {id="architecture-other"} + +| Name | Description | +| ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | +| [`Corpus`](/api/corpus) | Class for managing annotated corpora for training and evaluation data. | +| [`KnowledgeBase`](/api/kb) | Abstract base class for storage and retrieval of data for entity linking. | +| [`InMemoryLookupKB`](/api/inmemorylookupkb) | Implementation of `KnowledgeBase` storing all data in memory. | +| [`Candidate`](/api/kb#candidate) | Object associating a textual mention with a specific entity contained in a `KnowledgeBase`. | +| [`Lookups`](/api/lookups) | Container for convenient access to large lookup tables and dictionaries. | +| [`MorphAnalysis`](/api/morphology#morphanalysis) | A morphological analysis. | +| [`Morphology`](/api/morphology) | Store morphological analyses and map them to and from hash values. | +| [`Scorer`](/api/scorer) | Compute evaluation scores. | +| [`StringStore`](/api/stringstore) | Map strings to and from hash values. | +| [`Vectors`](/api/vectors) | Container class for vector data keyed by string. | +| [`Vocab`](/api/vocab) | The shared vocabulary that stores strings and gives you access to [`Lexeme`](/api/lexeme) objects. | diff --git a/website/docs/usage/101/_language-data.mdx b/website/docs/usage/101/_language-data.mdx new file mode 100644 index 0000000..29f7bf4 --- /dev/null +++ b/website/docs/usage/101/_language-data.mdx @@ -0,0 +1,32 @@ +Every language is different – and usually full of **exceptions and special +cases**, especially amongst the most common words. Some of these exceptions are +shared across languages, while others are **entirely specific** – usually so +specific that they need to be hard-coded. The +[`lang`](%%GITHUB_SPACY/spacy/lang) module contains all language-specific data, +organized in simple Python files. This makes the data easy to update and extend. + +The **shared language data** in the directory root includes rules that can be +generalized across languages – for example, rules for basic punctuation, emoji, +emoticons and single-letter abbreviations. The **individual language data** in a +submodule contains rules that are only relevant to a particular language. It +also takes care of putting together all components and creating the +[`Language`](/api/language) subclass – for example, `English` or `German`. The +values are defined in the [`Language.Defaults`](/api/language#defaults). + +> ```python +> from spacy.lang.en import English +> from spacy.lang.de import German +> +> nlp_en = English() # Includes English data +> nlp_de = German() # Includes German data +> ``` + +| Name | Description | +| --------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Stop words**
[`stop_words.py`](%%GITHUB_SPACY/spacy/lang/en/stop_words.py) | List of most common words of a language that are often useful to filter out, for example "and" or "I". Matching tokens will return `True` for `is_stop`. | +| **Tokenizer exceptions**
[`tokenizer_exceptions.py`](%%GITHUB_SPACY/spacy/lang/de/tokenizer_exceptions.py) | Special-case rules for the tokenizer, for example, contractions like "can't" and abbreviations with punctuation, like "U.K.". | +| **Punctuation rules**
[`punctuation.py`](%%GITHUB_SPACY/spacy/lang/punctuation.py) | Regular expressions for splitting tokens, e.g. on punctuation or special characters like emoji. Includes rules for prefixes, suffixes and infixes. | +| **Character classes**
[`char_classes.py`](%%GITHUB_SPACY/spacy/lang/char_classes.py) | Character classes to be used in regular expressions, for example, Latin characters, quotes, hyphens or icons. | +| **Lexical attributes**
[`lex_attrs.py`](%%GITHUB_SPACY/spacy/lang/en/lex_attrs.py) | Custom functions for setting lexical attributes on tokens, e.g. `like_num`, which includes language-specific words like "ten" or "hundred". | +| **Syntax iterators**
[`syntax_iterators.py`](%%GITHUB_SPACY/spacy/lang/en/syntax_iterators.py) | Functions that compute views of a `Doc` object based on its syntax. At the moment, only used for [noun chunks](/usage/linguistic-features#noun-chunks). | +| **Lemmatizer**
[`lemmatizer.py`](%%GITHUB_SPACY/spacy/lang/fr/lemmatizer.py) [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) | Custom lemmatizer implementation and lemmatization tables. | diff --git a/website/docs/usage/101/_named-entities.mdx b/website/docs/usage/101/_named-entities.mdx new file mode 100644 index 0000000..da43c0d --- /dev/null +++ b/website/docs/usage/101/_named-entities.mdx @@ -0,0 +1,36 @@ +A named entity is a "real-world object" that's assigned a name – for example, a +person, a country, a product or a book title. spaCy can **recognize various +types of named entities in a document, by asking the model for a prediction**. +Because models are statistical and strongly depend on the examples they were +trained on, this doesn't always work _perfectly_ and might need some tuning +later, depending on your use case. + +Named entities are available as the `ents` property of a `Doc`: + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Apple is looking at buying U.K. startup for $1 billion") + +for ent in doc.ents: + print(ent.text, ent.start_char, ent.end_char, ent.label_) +``` + +> - **Text:** The original entity text. +> - **Start:** Index of start of entity in the `Doc`. +> - **End:** Index of end of entity in the `Doc`. +> - **Label:** Entity label, i.e. type. + +| Text | Start | End | Label | Description | +| ----------- | :---: | :-: | ------- | ---------------------------------------------------- | +| Apple | 0 | 5 | `ORG` | Companies, agencies, institutions. | +| U.K. | 27 | 31 | `GPE` | Geopolitical entity, i.e. countries, cities, states. | +| \$1 billion | 44 | 54 | `MONEY` | Monetary values, including unit. | + +Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what +our example sentence and its named entities look like: + + +
Apple ORG is looking at buying U.K. GPE startup for $1 billion MONEY
+ diff --git a/website/docs/usage/101/_pipelines.mdx b/website/docs/usage/101/_pipelines.mdx new file mode 100644 index 0000000..3152917 --- /dev/null +++ b/website/docs/usage/101/_pipelines.mdx @@ -0,0 +1,77 @@ +When you call `nlp` on a text, spaCy first tokenizes the text to produce a `Doc` +object. The `Doc` is then processed in several different steps – this is also +referred to as the **processing pipeline**. The pipeline used by the +[trained pipelines](/models) typically include a tagger, a lemmatizer, a parser +and an entity recognizer. Each pipeline component returns the processed `Doc`, +which is then passed on to the next component. + +![The processing pipeline](/images/pipeline.svg) + +> - **Name**: ID of the pipeline component. +> - **Component:** spaCy's implementation of the component. +> - **Creates:** Objects, attributes and properties modified and set by the +> component. + +| Name | Component | Creates | Description | +| --------------------- | ------------------------------------------------------------------ | --------------------------------------------------------- | ------------------------------------------------ | +| **tokenizer** | [`Tokenizer`](/api/tokenizer) | `Doc` | Segment text into tokens. | +| _processing pipeline_ | | | +| **tagger** | [`Tagger`](/api/tagger) | `Token.tag` | Assign part-of-speech tags. | +| **parser** | [`DependencyParser`](/api/dependencyparser) | `Token.head`, `Token.dep`, `Doc.sents`, `Doc.noun_chunks` | Assign dependency labels. | +| **ner** | [`EntityRecognizer`](/api/entityrecognizer) | `Doc.ents`, `Token.ent_iob`, `Token.ent_type` | Detect and label named entities. | +| **lemmatizer** | [`Lemmatizer`](/api/lemmatizer) | `Token.lemma` | Assign base forms. | +| **textcat** | [`TextCategorizer`](/api/textcategorizer) | `Doc.cats` | Assign document labels. | +| **custom** | [custom components](/usage/processing-pipelines#custom-components) | `Doc._.xxx`, `Token._.xxx`, `Span._.xxx` | Assign custom attributes, methods or properties. | + +The capabilities of a processing pipeline always depend on the components, their +models and how they were trained. For example, a pipeline for named entity +recognition needs to include a trained named entity recognizer component with a +statistical model and weights that enable it to **make predictions** of entity +labels. This is why each pipeline specifies its components and their settings in +the [config](/usage/training#config): + +```ini +[nlp] +pipeline = ["tok2vec", "tagger", "parser", "ner"] +``` + + + +The statistical components like the tagger or parser are typically independent +and don't share any data between each other. For example, the named entity +recognizer doesn't use any features set by the tagger and parser, and so on. +This means that you can swap them, or remove single components from the pipeline +without affecting the others. However, components may share a "token-to-vector" +component like [`Tok2Vec`](/api/tok2vec) or [`Transformer`](/api/transformer). +You can read more about this in the docs on +[embedding layers](/usage/embeddings-transformers#embedding-layers). + +Custom components may also depend on annotations set by other components. For +example, a custom lemmatizer may need the part-of-speech tags assigned, so it'll +only work if it's added after the tagger. The parser will respect pre-defined +sentence boundaries, so if a previous component in the pipeline sets them, its +dependency predictions may be different. Similarly, it matters if you add the +[`EntityRuler`](/api/entityruler) before or after the statistical entity +recognizer: if it's added before, the entity recognizer will take the existing +entities into account when making predictions. The +[`EntityLinker`](/api/entitylinker), which resolves named entities to knowledge +base IDs, should be preceded by a pipeline component that recognizes entities +such as the [`EntityRecognizer`](/api/entityrecognizer). + + + + + +The tokenizer is a "special" component and isn't part of the regular pipeline. +It also doesn't show up in `nlp.pipe_names`. The reason is that there can only +really be one tokenizer, and while all other pipeline components take a `Doc` +and return it, the tokenizer takes a **string of text** and turns it into a +`Doc`. You can still customize the tokenizer, though. `nlp.tokenizer` is +writable, so you can either create your own +[`Tokenizer` class from scratch](/usage/linguistic-features#native-tokenizers), +or even replace it with an +[entirely custom function](/usage/linguistic-features#custom-tokenizer). + + + +--- diff --git a/website/docs/usage/101/_pos-deps.mdx b/website/docs/usage/101/_pos-deps.mdx new file mode 100644 index 0000000..db1e128 --- /dev/null +++ b/website/docs/usage/101/_pos-deps.mdx @@ -0,0 +1,62 @@ +After tokenization, spaCy can **parse** and **tag** a given `Doc`. This is where +the trained pipeline and its statistical models come in, which enable spaCy to +**make predictions** of which tag or label most likely applies in this context. +A trained component includes binary data that is produced by showing a system +enough examples for it to make predictions that generalize across the language – +for example, a word following "the" in English is most likely a noun. + +Linguistic annotations are available as +[`Token` attributes](/api/token#attributes). Like many NLP libraries, spaCy +**encodes all strings to hash values** to reduce memory usage and improve +efficiency. So to get the readable string representation of an attribute, we +need to add an underscore `_` to its name: + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Apple is looking at buying U.K. startup for $1 billion") + +for token in doc: + print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_, + token.shape_, token.is_alpha, token.is_stop) +``` + +> - **Text:** The original word text. +> - **Lemma:** The base form of the word. +> - **POS:** The simple [UPOS](https://universaldependencies.org/u/pos/) +> part-of-speech tag. +> - **Tag:** The detailed part-of-speech tag. +> - **Dep:** Syntactic dependency, i.e. the relation between tokens. +> - **Shape:** The word shape – capitalization, punctuation, digits. +> - **is alpha:** Is the token an alpha character? +> - **is stop:** Is the token part of a stop list, i.e. the most common words of +> the language? + +| Text | Lemma | POS | Tag | Dep | Shape | alpha | stop | +| ------- | ------- | ------- | ----- | ---------- | ------- | ------- | ------- | +| Apple | apple | `PROPN` | `NNP` | `nsubj` | `Xxxxx` | `True` | `False` | +| is | be | `AUX` | `VBZ` | `aux` | `xx` | `True` | `True` | +| looking | look | `VERB` | `VBG` | `ROOT` | `xxxx` | `True` | `False` | +| at | at | `ADP` | `IN` | `prep` | `xx` | `True` | `True` | +| buying | buy | `VERB` | `VBG` | `pcomp` | `xxxx` | `True` | `False` | +| U.K. | u.k. | `PROPN` | `NNP` | `compound` | `X.X.` | `False` | `False` | +| startup | startup | `NOUN` | `NN` | `dobj` | `xxxx` | `True` | `False` | +| for | for | `ADP` | `IN` | `prep` | `xxx` | `True` | `True` | +| \$ | \$ | `SYM` | `$` | `quantmod` | `$` | `False` | `False` | +| 1 | 1 | `NUM` | `CD` | `compound` | `d` | `False` | `False` | +| billion | billion | `NUM` | `CD` | `pobj` | `xxxx` | `True` | `False` | + +> #### Tip: Understanding tags and labels +> +> Most of the tags and labels look pretty abstract, and they vary between +> languages. `spacy.explain` will show you a short description – for example, +> `spacy.explain("VBZ")` returns "verb, 3rd person singular present". + +Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what +our example sentence and its dependencies look like: + + diff --git a/website/docs/usage/101/_serialization.mdx b/website/docs/usage/101/_serialization.mdx new file mode 100644 index 0000000..ce34ea6 --- /dev/null +++ b/website/docs/usage/101/_serialization.mdx @@ -0,0 +1,29 @@ +If you've been modifying the pipeline, vocabulary, vectors and entities, or made +updates to the component models, you'll eventually want to **save your +progress** – for example, everything that's in your `nlp` object. This means +you'll have to translate its contents and structure into a format that can be +saved, like a file or a byte string. This process is called serialization. spaCy +comes with **built-in serialization methods** and supports the +[Pickle protocol](https://www.diveinto.org/python3/serializing.html#dump). + +> #### What's pickle? +> +> Pickle is Python's built-in object persistence system. It lets you transfer +> arbitrary Python objects between processes. This is usually used to load an +> object to and from disk, but it's also used for distributed computing, e.g. +> with +> [PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) +> or [Dask](https://dask.org). When you unpickle an object, you're agreeing to +> execute whatever code it contains. It's like calling `eval()` on a string – so +> don't unpickle objects from untrusted sources. + +All container classes, i.e. [`Language`](/api/language) (`nlp`), +[`Doc`](/api/doc), [`Vocab`](/api/vocab) and [`StringStore`](/api/stringstore) +have the following methods available: + +| Method | Returns | Example | +| ------------ | ------- | ------------------------ | +| `to_bytes` | bytes | `data = nlp.to_bytes()` | +| `from_bytes` | object | `nlp.from_bytes(data)` | +| `to_disk` | - | `nlp.to_disk("/path")` | +| `from_disk` | object | `nlp.from_disk("/path")` | diff --git a/website/docs/usage/101/_tokenization.mdx b/website/docs/usage/101/_tokenization.mdx new file mode 100644 index 0000000..4315ab4 --- /dev/null +++ b/website/docs/usage/101/_tokenization.mdx @@ -0,0 +1,49 @@ +During processing, spaCy first **tokenizes** the text, i.e. segments it into +words, punctuation and so on. This is done by applying rules specific to each +language. For example, punctuation at the end of a sentence should be split off +– whereas "U.K." should remain one token. Each `Doc` consists of individual +tokens, and we can iterate over them: + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Apple is looking at buying U.K. startup for $1 billion") +for token in doc: + print(token.text) +``` + +| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | +| :---: | :-: | :-----: | :-: | :----: | :--: | :-----: | :-: | :-: | :-: | :-----: | +| Apple | is | looking | at | buying | U.K. | startup | for | \$ | 1 | billion | + +First, the raw text is split on whitespace characters, similar to +`text.split(' ')`. Then, the tokenizer processes the text from left to right. On +each substring, it performs two checks: + +1. **Does the substring match a tokenizer exception rule?** For example, "don't" + does not contain whitespace, but should be split into two tokens, "do" and + "n't", while "U.K." should always remain one token. + +2. **Can a prefix, suffix or infix be split off?** For example punctuation like + commas, periods, hyphens or quotes. + +If there's a match, the rule is applied and the tokenizer continues its loop, +starting with the newly split substrings. This way, spaCy can split **complex, +nested tokens** like combinations of abbreviations and multiple punctuation +marks. + +> - **Tokenizer exception:** Special-case rule to split a string into several +> tokens or prevent a token from being split when punctuation rules are +> applied. +> - **Prefix:** Character(s) at the beginning, e.g. `$`, `(`, `“`, `¿`. +> - **Suffix:** Character(s) at the end, e.g. `km`, `)`, `”`, `!`. +> - **Infix:** Character(s) in between, e.g. `-`, `--`, `/`, `…`. + +![Example of the tokenization process](/images/tokenization.svg) + +While punctuation rules are usually pretty general, tokenizer exceptions +strongly depend on the specifics of the individual language. This is why each +[available language](/usage/models#languages) has its own subclass, like +`English` or `German`, that loads in lists of hard-coded data and exception +rules. diff --git a/website/docs/usage/101/_training.mdx b/website/docs/usage/101/_training.mdx new file mode 100644 index 0000000..6587ea3 --- /dev/null +++ b/website/docs/usage/101/_training.mdx @@ -0,0 +1,41 @@ +spaCy's tagger, parser, text categorizer and many other components are powered +by **statistical models**. Every "decision" these components make – for example, +which part-of-speech tag to assign, or whether a word is a named entity – is a +**prediction** based on the model's current **weight values**. The weight values +are estimated based on examples the model has seen during **training**. To train +a model, you first need training data – examples of text, and the labels you +want the model to predict. This could be a part-of-speech tag, a named entity or +any other information. + +Training is an iterative process in which the model's predictions are compared +against the reference annotations in order to estimate the **gradient of the +loss**. The gradient of the loss is then used to calculate the gradient of the +weights through [backpropagation](https://thinc.ai/docs/backprop101). The +gradients indicate how the weight values should be changed so that the model's +predictions become more similar to the reference labels over time. + +> - **Training data:** Examples and their annotations. +> - **Text:** The input text the model should predict a label for. +> - **Label:** The label the model should predict. +> - **Gradient:** The direction and rate of change for a numeric value. +> Minimising the gradient of the weights should result in predictions that are +> closer to the reference labels on the training data. + +![The training process](/images/training.svg) + +When training a model, we don't just want it to memorize our examples – we want +it to come up with a theory that can be **generalized across unseen data**. +After all, we don't just want the model to learn that this one instance of +"Amazon" right here is a company – we want it to learn that "Amazon", in +contexts _like this_, is most likely a company. That's why the training data +should always be representative of the data we want to process. A model trained +on Wikipedia, where sentences in the first person are extremely rare, will +likely perform badly on Twitter. Similarly, a model trained on romantic novels +will likely perform badly on legal text. + +This also means that in order to know how the model is performing, and whether +it's learning the right things, you don't only need **training data** – you'll +also need **evaluation data**. If you only test the model with the data it was +trained on, you'll have no idea how well it's generalizing. If you want to train +a model from scratch, you usually need at least a few hundred examples for both +training and evaluation. diff --git a/website/docs/usage/101/_vectors-similarity.mdx b/website/docs/usage/101/_vectors-similarity.mdx new file mode 100644 index 0000000..6deab92 --- /dev/null +++ b/website/docs/usage/101/_vectors-similarity.mdx @@ -0,0 +1,150 @@ +Similarity is determined by comparing **word vectors** or "word embeddings", +multi-dimensional meaning representations of a word. Word vectors can be +generated using an algorithm like +[word2vec](https://en.wikipedia.org/wiki/Word2vec) and usually look like this: + +```python {title="banana.vector"} +array([2.02280000e-01, -7.66180009e-02, 3.70319992e-01, + 3.28450017e-02, -4.19569999e-01, 7.20689967e-02, + -3.74760002e-01, 5.74599989e-02, -1.24009997e-02, + 5.29489994e-01, -5.23800015e-01, -1.97710007e-01, + -3.41470003e-01, 5.33169985e-01, -2.53309999e-02, + 1.73800007e-01, 1.67720005e-01, 8.39839995e-01, + 5.51070012e-02, 1.05470002e-01, 3.78719985e-01, + 2.42750004e-01, 1.47449998e-02, 5.59509993e-01, + 1.25210002e-01, -6.75960004e-01, 3.58420014e-01, + # ... and so on ... + 3.66849989e-01, 2.52470002e-03, -6.40089989e-01, + -2.97650009e-01, 7.89430022e-01, 3.31680000e-01, + -1.19659996e+00, -4.71559986e-02, 5.31750023e-01], dtype=float32) +``` + + + +To make them compact and fast, spaCy's small [pipeline packages](/models) (all +packages that end in `sm`) **don't ship with word vectors**, and only include +context-sensitive **tensors**. This means you can still use the `similarity()` +methods to compare documents, spans and tokens – but the result won't be as +good, and individual tokens won't have any vectors assigned. So in order to use +_real_ word vectors, you need to download a larger pipeline package: + +```diff +- python -m spacy download en_core_web_sm ++ python -m spacy download en_core_web_lg +``` + + + +Pipeline packages that come with built-in word vectors make them available as +the [`Token.vector`](/api/token#vector) attribute. +[`Doc.vector`](/api/doc#vector) and [`Span.vector`](/api/span#vector) will +default to an average of their token vectors. You can also check if a token has +a vector assigned, and get the L2 norm, which can be used to normalize vectors. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_md") +tokens = nlp("dog cat banana afskfsd") + +for token in tokens: + print(token.text, token.has_vector, token.vector_norm, token.is_oov) +``` + +> - **Text**: The original token text. +> - **has vector**: Does the token have a vector representation? +> - **Vector norm**: The L2 norm of the token's vector (the square root of the +> sum of the values squared) +> - **OOV**: Out-of-vocabulary + +The words "dog", "cat" and "banana" are all pretty common in English, so they're +part of the pipeline's vocabulary, and come with a vector. The word "afskfsd" on +the other hand is a lot less common and out-of-vocabulary – so its vector +representation consists of 300 dimensions of `0`, which means it's practically +nonexistent. If your application will benefit from a **large vocabulary** with +more vectors, you should consider using one of the larger pipeline packages or +loading in a full vector package, for example, +[`en_core_web_lg`](/models/en#en_core_web_lg), which includes **685k unique +vectors**. + +spaCy is able to compare two objects, and make a prediction of **how similar +they are**. Predicting similarity is useful for building recommendation systems +or flagging duplicates. For example, you can suggest a user content that's +similar to what they're currently looking at, or label a support ticket as a +duplicate if it's very similar to an already existing one. + +Each [`Doc`](/api/doc), [`Span`](/api/span), [`Token`](/api/token) and +[`Lexeme`](/api/lexeme) comes with a [`.similarity`](/api/token#similarity) +method that lets you compare it with another object, and determine the +similarity. Of course similarity is always subjective – whether two words, spans +or documents are similar really depends on how you're looking at it. spaCy's +similarity implementation usually assumes a pretty general-purpose definition of +similarity. + +> #### 📝 Things to try +> +> 1. Compare two different tokens and try to find the two most _dissimilar_ +> tokens in the texts with the lowest similarity score (according to the +> vectors). +> 2. Compare the similarity of two [`Lexeme`](/api/lexeme) objects, entries in +> the vocabulary. You can get a lexeme via the `.lex` attribute of a token. +> You should see that the similarity results are identical to the token +> similarity. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_md") # make sure to use larger package! +doc1 = nlp("I like salty fries and hamburgers.") +doc2 = nlp("Fast food tastes very good.") + +# Similarity of two documents +print(doc1, "<->", doc2, doc1.similarity(doc2)) +# Similarity of tokens and spans +french_fries = doc1[2:4] +burgers = doc1[5] +print(french_fries, "<->", burgers, french_fries.similarity(burgers)) +``` + +### What to expect from similarity results {id="similarity-expectations"} + +Computing similarity scores can be helpful in many situations, but it's also +important to maintain **realistic expectations** about what information it can +provide. Words can be related to each other in many ways, so a single +"similarity" score will always be a **mix of different signals**, and vectors +trained on different data can produce very different results that may not be +useful for your purpose. Here are some important considerations to keep in mind: + +- There's no objective definition of similarity. Whether "I like burgers" and "I + like pasta" is similar **depends on your application**. Both talk about food + preferences, which makes them very similar – but if you're analyzing mentions + of food, those sentences are pretty dissimilar, because they talk about very + different foods. +- The similarity of [`Doc`](/api/doc) and [`Span`](/api/span) objects defaults + to the **average** of the token vectors. This means that the vector for "fast + food" is the average of the vectors for "fast" and "food", which isn't + necessarily representative of the phrase "fast food". +- Vector averaging means that the vector of multiple tokens is **insensitive to + the order** of the words. Two documents expressing the same meaning with + dissimilar wording will return a lower similarity score than two documents + that happen to contain the same words while expressing different meanings. + + + +sense2vec Screenshot + +[`sense2vec`](https://github.com/explosion/sense2vec) is a library developed by +us that builds on top of spaCy and lets you train and query more interesting and +detailed word vectors. It combines noun phrases like "fast food" or "fair game" +and includes the part-of-speech tags and entity labels. The library also +includes annotation recipes for our annotation tool [Prodigy](https://prodi.gy) +that let you evaluate vectors and create terminology lists. For more details, +check out [our blog post](https://explosion.ai/blog/sense2vec-reloaded). To +explore the semantic similarities across all Reddit comments of 2015 and 2019, +see the [interactive demo](https://explosion.ai/demos/sense2vec). + + diff --git a/website/docs/usage/_benchmarks-models.mdx b/website/docs/usage/_benchmarks-models.mdx new file mode 100644 index 0000000..c85a119 --- /dev/null +++ b/website/docs/usage/_benchmarks-models.mdx @@ -0,0 +1,40 @@ +
+ +| Pipeline | Parser | Tagger | NER | +| ---------------------------------------------------------- | -----: | -----: | ---: | +| [`en_core_web_trf`](/models/en#en_core_web_trf) (spaCy v3) | 95.1 | 97.8 | 89.8 | +| [`en_core_web_lg`](/models/en#en_core_web_lg) (spaCy v3) | 92.0 | 97.4 | 85.5 | +| `en_core_web_lg` (spaCy v2) | 91.9 | 97.2 | 85.5 | + +
+ +**Full pipeline accuracy** on the +[OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) corpus (reported on +the development set). + +
+ +
+ +
+ +| Named Entity Recognition System | OntoNotes | CoNLL '03 | +| -------------------------------- | --------: | --------: | +| spaCy RoBERTa (2020) | 89.8 | 91.6 | +| Stanza (StanfordNLP)1 | 88.8 | 92.1 | +| Flair2 | 89.7 | 93.1 | + +
+ +**Named entity recognition accuracy** on the +[OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19) and +[CoNLL-2003](https://www.aclweb.org/anthology/W03-0419.pdf) corpora. See +[NLP-progress](http://nlpprogress.com/english/named_entity_recognition.html) for +more results. Project template: +[`benchmarks/ner_conll03`](%%GITHUB_PROJECTS/benchmarks/ner_conll03). **1.** +[Qi et al. (2020)](https://arxiv.org/pdf/2003.07082.pdf). **2.** +[Akbik et al. (2018)](https://www.aclweb.org/anthology/C18-1139/). + +
+ +
diff --git a/website/docs/usage/embeddings-transformers.mdx b/website/docs/usage/embeddings-transformers.mdx new file mode 100644 index 0000000..a400762 --- /dev/null +++ b/website/docs/usage/embeddings-transformers.mdx @@ -0,0 +1,987 @@ +--- +title: Embeddings, Transformers and Transfer Learning +teaser: Using transformer embeddings like BERT in spaCy +menu: + - ['Embedding Layers', 'embedding-layers'] + - ['Transformers', 'transformers'] + - ['Static Vectors', 'static-vectors'] + - ['Pretraining', 'pretraining'] +next: /usage/training +--- + +spaCy supports a number of **transfer and multi-task learning** workflows that +can often help improve your pipeline's efficiency or accuracy. Transfer learning +refers to techniques such as word vector tables and language model pretraining. +These techniques can be used to import knowledge from raw text into your +pipeline, so that your models are able to generalize better from your annotated +examples. + +You can convert **word vectors** from popular tools like +[FastText](https://fasttext.cc) and [Gensim](https://radimrehurek.com/gensim), +or you can load in any pretrained **transformer model** if you install +[`spacy-transformers`](https://github.com/explosion/spacy-transformers). You can +also do your own language model pretraining via the +[`spacy pretrain`](/api/cli#pretrain) command. You can even **share** your +transformer or other contextual embedding model across multiple components, +which can make long pipelines several times more efficient. To use transfer +learning, you'll need at least a few annotated examples for what you're trying +to predict. Otherwise, you could try using a "one-shot learning" approach using +[vectors and similarity](/usage/linguistic-features#vectors-similarity). + + + +[Transformers](#transformers) are large and powerful neural networks that give +you better accuracy, but are harder to deploy in production, as they require a +GPU to run effectively. [Word vectors](#word-vectors) are a slightly older +technique that can give your models a smaller improvement in accuracy, and can +also provide some additional capabilities. + +The key difference between word-vectors and contextual language models such as +transformers is that word vectors model **lexical types**, rather than _tokens_. +If you have a list of terms with no context around them, a transformer model +like BERT can't really help you. BERT is designed to understand language **in +context**, which isn't what you have. A word vectors table will be a much better +fit for your task. However, if you do have words in context – whole sentences or +paragraphs of running text – word vectors will only provide a very rough +approximation of what the text is about. + +Word vectors are also very computationally efficient, as they map a word to a +vector with a single indexing operation. Word vectors are therefore useful as a +way to **improve the accuracy** of neural network models, especially models that +are small or have received little or no pretraining. In spaCy, word vector +tables are only used as **static features**. spaCy does not backpropagate +gradients to the pretrained word vectors table. The static vectors table is +usually used in combination with a smaller table of learned task-specific +embeddings. + + + + + +Word vectors are not compatible with most [transformer models](#transformers), +but if you're training another type of NLP network, it's almost always worth +adding word vectors to your model. As well as improving your final accuracy, +word vectors often make experiments more consistent, as the accuracy you reach +will be less sensitive to how the network is randomly initialized. High variance +due to random chance can slow down your progress significantly, as you need to +run many experiments to filter the signal from the noise. + +Word vector features need to be enabled prior to training, and the same word +vectors table will need to be available at runtime as well. You cannot add word +vector features once the model has already been trained, and you usually cannot +replace one word vectors table with another without causing a significant loss +of performance. + + + +## Shared embedding layers {id="embedding-layers"} + +spaCy lets you share a single transformer or other token-to-vector ("tok2vec") +embedding layer between multiple components. You can even update the shared +layer, performing **multi-task learning**. Reusing the tok2vec layer between +components can make your pipeline run a lot faster and result in much smaller +models. However, it can make the pipeline less modular and make it more +difficult to swap components or retrain parts of the pipeline. Multi-task +learning can affect your accuracy (either positively or negatively), and may +require some retuning of your hyper-parameters. + +![Pipeline components using a shared embedding component vs. independent embedding layers](/images/tok2vec.svg) + +| Shared | Independent | +| ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| ✅ **smaller:** models only need to include a single copy of the embeddings | ❌ **larger:** models need to include the embeddings for each component | +| ✅ **faster:** embed the documents once for your whole pipeline | ❌ **slower:** rerun the embedding for each component | +| ❌ **less composable:** all components require the same embedding component in the pipeline | ✅ **modular:** components can be moved and swapped freely | + +You can share a single transformer or other tok2vec model between multiple +components by adding a [`Transformer`](/api/transformer) or +[`Tok2Vec`](/api/tok2vec) component near the start of your pipeline. Components +later in the pipeline can "connect" to it by including a **listener layer** like +[Tok2VecListener](/api/architectures#Tok2VecListener) within their model. + +![Pipeline components listening to shared embedding component](/images/tok2vec-listener.svg) + +At the beginning of training, the [`Tok2Vec`](/api/tok2vec) component will grab +a reference to the relevant listener layers in the rest of your pipeline. When +it processes a batch of documents, it will pass forward its predictions to the +listeners, allowing the listeners to **reuse the predictions** when they are +eventually called. A similar mechanism is used to pass gradients from the +listeners back to the model. The [`Transformer`](/api/transformer) component and +[TransformerListener](/api/architectures#TransformerListener) layer do the same +thing for transformer models, but the `Transformer` component will also save the +transformer outputs to the +[`Doc._.trf_data`](/api/transformer#custom_attributes) extension attribute, +giving you access to them after the pipeline has finished running. + +### Example: Shared vs. independent config {id="embedding-layers-config"} + +The [config system](/usage/training#config) lets you express model configuration +for both shared and independent embedding layers. The shared setup uses a single +[`Tok2Vec`](/api/tok2vec) component with the +[Tok2Vec](/api/architectures#Tok2Vec) architecture. All other components, like +the entity recognizer, use a +[Tok2VecListener](/api/architectures#Tok2VecListener) layer as their model's +`tok2vec` argument, which connects to the `tok2vec` component model. + +```ini {title="Shared",highlight="1-2,4-5,19-20"} +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v2" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v2" + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" + +[components.ner] +factory = "ner" + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v1" + +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2VecListener.v1" +``` + +In the independent setup, the entity recognizer component defines its own +[Tok2Vec](/api/architectures#Tok2Vec) instance. Other components will do the +same. This makes them fully independent and doesn't require an upstream +[`Tok2Vec`](/api/tok2vec) component to be present in the pipeline. + +```ini {title="Independent", highlight="7-8"} +[components.ner] +factory = "ner" + +[components.ner.model] +@architectures = "spacy.TransitionBasedParser.v1" + +[components.ner.model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[components.ner.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" + +[components.ner.model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +``` + +{/* TODO: Once rehearsal is tested, mention it here. */} + +## Using transformer models {id="transformers"} + +Transformers are a family of neural network architectures that compute **dense, +context-sensitive representations** for the tokens in your documents. Downstream +models in your pipeline can then use these representations as input features to +**improve their predictions**. You can connect multiple components to a single +transformer model, with any or all of those components giving feedback to the +transformer to fine-tune it to your tasks. spaCy's transformer support +interoperates with [PyTorch](https://pytorch.org) and the +[HuggingFace `transformers`](https://huggingface.co/transformers/) library, +giving you access to thousands of pretrained models for your pipelines. There +are many [great guides](http://jalammar.github.io/illustrated-transformer/) to +transformer models, but for practical purposes, you can simply think of them as +drop-in replacements that let you achieve **higher accuracy** in exchange for +**higher training and runtime costs**. + +### Setup and installation {id="transformers-installation"} + +> #### System requirements +> +> We recommend an NVIDIA **GPU** with at least **10GB of memory** in order to +> work with transformer models. Make sure your GPU drivers are up to date and +> you have **CUDA v9+** installed. + +> The exact requirements will depend on the transformer model. Training a +> transformer-based model without a GPU will be too slow for most practical +> purposes. +> +> Provisioning a new machine will require about **5GB** of data to be +> downloaded: 3GB CUDA runtime, 800MB PyTorch, 400MB CuPy, 500MB weights, 200MB +> spaCy and dependencies. + +Once you have CUDA installed, we recommend installing PyTorch following the +[PyTorch installation guidelines](https://pytorch.org/get-started/locally/) for +your package manager and CUDA version. If you skip this step, pip will install +PyTorch as a dependency below, but it may not find the best version for your +setup. + +```bash {title="Example: Install PyTorch 1.11.0 for CUDA 11.3 with pip"} +# See: https://pytorch.org/get-started/locally/ +$ pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html +``` + +Next, install spaCy with the extras for your CUDA version and transformers. The +CUDA extra (e.g., `cuda102`, `cuda113`) installs the correct version of +[`cupy`](https://docs.cupy.dev/en/stable/install.html#installing-cupy), which is +just like `numpy`, but for GPU. You may also need to set the `CUDA_PATH` +environment variable if your CUDA runtime is installed in a non-standard +location. Putting it all together, if you had installed CUDA 11.3 in +`/opt/nvidia/cuda`, you would run: + +```bash {title="Installation with CUDA"} +$ export CUDA_PATH="/opt/nvidia/cuda" +$ pip install -U %%SPACY_PKG_NAME[cuda113,transformers]%%SPACY_PKG_FLAGS +``` + +For [`transformers`](https://huggingface.co/transformers/) v4.0.0+ and models +that require [`SentencePiece`](https://github.com/google/sentencepiece) (e.g., +ALBERT, CamemBERT, XLNet, Marian, and T5), install the additional dependencies +with: + +```bash {title="Install sentencepiece"} +$ pip install transformers[sentencepiece] +``` + +### Runtime usage {id="transformers-runtime"} + +Transformer models can be used as **drop-in replacements** for other types of +neural networks, so your spaCy pipeline can include them in a way that's +completely invisible to the user. Users will download, load and use the model in +the standard way, like any other spaCy pipeline. Instead of using the +transformers as subnetworks directly, you can also use them via the +[`Transformer`](/api/transformer) pipeline component. + +![The processing pipeline with the transformer component](/images/pipeline_transformer.svg) + +The `Transformer` component sets the +[`Doc._.trf_data`](/api/transformer#custom_attributes) extension attribute, +which lets you access the transformers outputs at runtime. The trained +transformer-based [pipelines](/models) provided by spaCy end on `_trf`, e.g. +[`en_core_web_trf`](/models/en#en_core_web_trf). + +```bash +$ python -m spacy download en_core_web_trf +``` + +```python {title="Example"} +import spacy +from thinc.api import set_gpu_allocator, require_gpu + +# Use the GPU, with memory allocations directed via PyTorch. +# This prevents out-of-memory errors that would otherwise occur from competing +# memory pools. +set_gpu_allocator("pytorch") +require_gpu(0) + +nlp = spacy.load("en_core_web_trf") +for doc in nlp.pipe(["some text", "some other text"]): + # For spaCy v3.0-v3.6, trf pipelines use spacy-transformers and the transformer output in doc._.trf_data is a TransformerData object. + try: + tokvecs = doc._.trf_data.tensors[-1] + # For spaCy v3.7+, trf pipelines use spacy-curated-transformers and doc._.trf_data is a DocTransformerOutput object. + except AttributeError as e: + tokvecs = doc._.trf_data.last_hidden_layer_state # or doc._.trf_data.all_outputs[-1] +``` + +You can also customize how the [`Transformer`](/api/transformer) component sets +annotations onto the [`Doc`](/api/doc) by specifying a custom +`set_extra_annotations` function. This callback will be called with the raw +input and output data for the whole batch, along with the batch of `Doc` +objects, allowing you to implement whatever you need. The annotation setter is +called with a batch of [`Doc`](/api/doc) objects and a +[`FullTransformerBatch`](/api/transformer#fulltransformerbatch) containing the +transformers data for the batch. + +```python +def custom_annotation_setter(docs, trf_data): + doc_data = list(trf_data.doc_data) + for doc, data in zip(docs, doc_data): + doc._.custom_attr = data + +nlp = spacy.load("en_core_web_trf") +nlp.get_pipe("transformer").set_extra_annotations = custom_annotation_setter +doc = nlp("This is a text") +assert isinstance(doc._.custom_attr, TransformerData) +print(doc._.custom_attr.tensors) +``` + +### Training usage {id="transformers-training"} + +The recommended workflow for training is to use spaCy's +[config system](/usage/training#config), usually via the +[`spacy train`](/api/cli#train) command. The training config defines all +component settings and hyperparameters in one place and lets you describe a tree +of objects by referring to creation functions, including functions you register +yourself. For details on how to get started with training your own model, check +out the [training quickstart](/usage/training#quickstart). + +{/* TODO: */} + +{/* The easiest way to get started is to clone a transformers-based project */} +{/* template. Swap in your data, edit the settings and hyperparameters and train, */} +{/* evaluate, package and visualize your model. */} + +{/* */} + +The `[components]` section in the [`config.cfg`](/api/data-formats#config) +describes the pipeline components and the settings used to construct them, +including their model implementation. Here's a config snippet for the +[`Transformer`](/api/transformer) component, along with matching Python code. In +this case, the `[components.transformer]` block describes the `transformer` +component: + +> #### Python equivalent +> +> ```python +> from spacy_transformers import Transformer, TransformerModel +> from spacy_transformers.annotation_setters import null_annotation_setter +> from spacy_transformers.span_getters import get_doc_spans +> +> trf = Transformer( +> nlp.vocab, +> TransformerModel( +> "bert-base-cased", +> get_spans=get_doc_spans, +> tokenizer_config={"use_fast": True}, +> ), +> set_extra_annotations=null_annotation_setter, +> max_batch_items=4096, +> ) +> ``` + +```ini {title="config.cfg",excerpt="true"} +[components.transformer] +factory = "transformer" +max_batch_items = 4096 + +[components.transformer.model] +@architectures = "spacy-transformers.TransformerModel.v3" +name = "bert-base-cased" +tokenizer_config = {"use_fast": true} + +[components.transformer.model.get_spans] +@span_getters = "spacy-transformers.doc_spans.v1" + +[components.transformer.set_extra_annotations] +@annotation_setters = "spacy-transformers.null_annotation_setter.v1" + +``` + +The `[components.transformer.model]` block describes the `model` argument passed +to the transformer component. It's a Thinc +[`Model`](https://thinc.ai/docs/api-model) object that will be passed into the +component. Here, it references the function +[spacy-transformers.TransformerModel.v3](/api/architectures#TransformerModel) +registered in the [`architectures` registry](/api/top-level#registry). If a key +in a block starts with `@`, it's **resolved to a function** and all other +settings are passed to the function as arguments. In this case, `name`, +`tokenizer_config` and `get_spans`. + +`get_spans` is a function that takes a batch of `Doc` objects and returns lists +of potentially overlapping `Span` objects to process by the transformer. Several +[built-in functions](/api/transformer#span_getters) are available – for example, +to process the whole document or individual sentences. When the config is +resolved, the function is created and passed into the model as an argument. + +The `name` value is the name of any [HuggingFace model](huggingface-models), +which will be downloaded automatically the first time it's used. You can also +use a local file path. For full details, see the +[`TransformerModel` docs](/api/architectures#TransformerModel). + +[huggingface-models]: + https://huggingface.co/models?library=pytorch&sort=downloads + +A wide variety of PyTorch models are supported, but some might not work. If a +model doesn't seem to work feel free to open an +[issue](https://github.com/explosion/spacy/issues). Additionally note that +Transformers loaded in spaCy can only be used for tensors, and pretrained +task-specific heads or text generation features cannot be used as part of the +`transformer` pipeline component. + + + +Remember that the `config.cfg` used for training should contain **no missing +values** and requires all settings to be defined. You don't want any hidden +defaults creeping in and changing your results! spaCy will tell you if settings +are missing, and you can run +[`spacy init fill-config`](/api/cli#init-fill-config) to automatically fill in +all defaults. + + + +### Customizing the settings {id="transformers-training-custom-settings"} + +To change any of the settings, you can edit the `config.cfg` and re-run the +training. To change any of the functions, like the span getter, you can replace +the name of the referenced function – e.g. +`@span_getters = "spacy-transformers.sent_spans.v1"` to process sentences. You +can also register your own functions using the +[`span_getters` registry](/api/top-level#registry). For instance, the following +custom function returns [`Span`](/api/span) objects following sentence +boundaries, unless a sentence succeeds a certain amount of tokens, in which case +subsentences of at most `max_length` tokens are returned. + +> #### config.cfg +> +> ```ini +> [components.transformer.model.get_spans] +> @span_getters = "custom_sent_spans" +> max_length = 25 +> ``` + +```python {title="code.py"} +import spacy_transformers + +@spacy_transformers.registry.span_getters("custom_sent_spans") +def configure_custom_sent_spans(max_length: int): + def get_custom_sent_spans(docs): + spans = [] + for doc in docs: + spans.append([]) + for sent in doc.sents: + start = 0 + end = max_length + while end <= len(sent): + spans[-1].append(sent[start:end]) + start += max_length + end += max_length + if start < len(sent): + spans[-1].append(sent[start:len(sent)]) + return spans + + return get_custom_sent_spans +``` + +To resolve the config during training, spaCy needs to know about your custom +function. You can make it available via the `--code` argument that can point to +a Python file. For more details on training with custom code, see the +[training documentation](/usage/training#custom-functions). + +```bash +python -m spacy train ./config.cfg --code ./code.py +``` + +### Customizing the model implementations {id="training-custom-model"} + +The [`Transformer`](/api/transformer) component expects a Thinc +[`Model`](https://thinc.ai/docs/api-model) object to be passed in as its `model` +argument. You're not limited to the implementation provided by +`spacy-transformers` – the only requirement is that your registered function +must return an object of type ~~Model[List[Doc], FullTransformerBatch]~~: that +is, a Thinc model that takes a list of [`Doc`](/api/doc) objects, and returns a +[`FullTransformerBatch`](/api/transformer#fulltransformerbatch) object with the +transformer data. + +The same idea applies to task models that power the **downstream components**. +Most of spaCy's built-in model creation functions support a `tok2vec` argument, +which should be a Thinc layer of type ~~Model[List[Doc], List[Floats2d]]~~. This +is where we'll plug in our transformer model, using the +[TransformerListener](/api/architectures#TransformerListener) layer, which +sneakily delegates to the `Transformer` pipeline component. + +```ini {title="config.cfg (excerpt)",highlight="12"} +[components.ner] +factory = "ner" + +[nlp.pipeline.ner.model] +@architectures = "spacy.TransitionBasedParser.v1" +state_type = "ner" +extra_state_tokens = false +hidden_width = 128 +maxout_pieces = 3 +use_upper = false + +[nlp.pipeline.ner.model.tok2vec] +@architectures = "spacy-transformers.TransformerListener.v1" +grad_factor = 1.0 + +[nlp.pipeline.ner.model.tok2vec.pooling] +@layers = "reduce_mean.v1" +``` + +The [TransformerListener](/api/architectures#TransformerListener) layer expects +a [pooling layer](https://thinc.ai/docs/api-layers#reduction-ops) as the +argument `pooling`, which needs to be of type ~~Model[Ragged, Floats2d]~~. This +layer determines how the vector for each spaCy token will be computed from the +zero or more source rows the token is aligned against. Here we use the +[`reduce_mean`](https://thinc.ai/docs/api-layers#reduce_mean) layer, which +averages the wordpiece rows. We could instead use +[`reduce_max`](https://thinc.ai/docs/api-layers#reduce_max), or a custom +function you write yourself. + +You can have multiple components all listening to the same transformer model, +and all passing gradients back to it. By default, all of the gradients will be +**equally weighted**. You can control this with the `grad_factor` setting, which +lets you reweight the gradients from the different listeners. For instance, +setting `grad_factor = 0` would disable gradients from one of the listeners, +while `grad_factor = 2.0` would multiply them by 2. This is similar to having a +custom learning rate for each component. Instead of a constant, you can also +provide a schedule, allowing you to freeze the shared parameters at the start of +training. + +## Static vectors {id="static-vectors"} + +If your pipeline includes a **word vectors table**, you'll be able to use the +`.similarity()` method on the [`Doc`](/api/doc), [`Span`](/api/span), +[`Token`](/api/token) and [`Lexeme`](/api/lexeme) objects. You'll also be able +to access the vectors using the `.vector` attribute, or you can look up one or +more vectors directly using the [`Vocab`](/api/vocab) object. Pipelines with +word vectors can also **use the vectors as features** for the statistical +models, which can **improve the accuracy** of your components. + +Word vectors in spaCy are "static" in the sense that they are not learned +parameters of the statistical models, and spaCy itself does not feature any +algorithms for learning word vector tables. You can train a word vectors table +using tools such as [floret](https://github.com/explosion/floret), +[Gensim](https://radimrehurek.com/gensim/), [FastText](https://fasttext.cc/) or +[GloVe](https://nlp.stanford.edu/projects/glove/), or download existing +pretrained vectors. The [`init vectors`](/api/cli#init-vectors) command lets you +convert vectors for use with spaCy and will give you a directory you can load or +refer to in your [training configs](/usage/training#config). + + + +For more details on loading word vectors into spaCy, using them for similarity +and improving word vector coverage by truncating and pruning the vectors, see +the usage guide on +[word vectors and similarity](/usage/linguistic-features#vectors-similarity). + + + +### Using word vectors in your models {id="word-vectors-models"} + +Many neural network models are able to use word vector tables as additional +features, which sometimes results in significant improvements in accuracy. +spaCy's built-in embedding layer, +[MultiHashEmbed](/api/architectures#MultiHashEmbed), can be configured to use +word vector tables using the `include_static_vectors` flag. + +```ini +[tagger.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 128 +attrs = ["LOWER","PREFIX","SUFFIX","SHAPE"] +rows = [5000,2500,2500,2500] +include_static_vectors = true +``` + + + +The configuration system will look up the string `"spacy.MultiHashEmbed.v2"` in +the `architectures` [registry](/api/top-level#registry), and call the returned +object with the rest of the arguments from the block. This will result in a call +to the +[`MultiHashEmbed`](https://github.com/explosion/spacy/tree/develop/spacy/ml/models/tok2vec.py) +function, which will return a [Thinc](https://thinc.ai) model object with the +type signature ~~Model[List[Doc], List[Floats2d]]~~. Because the embedding layer +takes a list of `Doc` objects as input, it does not need to store a copy of the +vectors table. The vectors will be retrieved from the `Doc` objects that are +passed in, via the `doc.vocab.vectors` attribute. This part of the process is +handled by the [StaticVectors](/api/architectures#StaticVectors) layer. + + + +#### Creating a custom embedding layer {id="custom-embedding-layer"} + +The [MultiHashEmbed](/api/architectures#StaticVectors) layer is spaCy's +recommended strategy for constructing initial word representations for your +neural network models, but you can also implement your own. You can register any +function to a string name, and then reference that function within your config +(see the [training docs](/usage/training) for more details). To try this out, +you can save the following little example to a new Python file: + +```python +from spacy.ml.staticvectors import StaticVectors +from spacy.util import registry + +print("I was imported!") + +@registry.architectures("my_example.MyEmbedding.v1") +def MyEmbedding(output_width: int) -> Model[List[Doc], List[Floats2d]]: + print("I was called!") + return StaticVectors(nO=output_width) +``` + +If you pass the path to your file to the [`spacy train`](/api/cli#train) command +using the `--code` argument, your file will be imported, which means the +decorator registering the function will be run. Your function is now on equal +footing with any of spaCy's built-ins, so you can drop it in instead of any +other model with the same input and output signature. For instance, you could +use it in the tagger model as follows: + +```ini +[tagger.model.tok2vec.embed] +@architectures = "my_example.MyEmbedding.v1" +output_width = 128 +``` + +Now that you have a custom function wired into the network, you can start +implementing the logic you're interested in. For example, let's say you want to +try a relatively simple embedding strategy that makes use of static word +vectors, but combines them via summation with a smaller table of learned +embeddings. + +```python +from thinc.api import add, chain, remap_ids, Embed +from spacy.ml.staticvectors import StaticVectors +from spacy.ml.featureextractor import FeatureExtractor +from spacy.util import registry + +@registry.architectures("my_example.MyEmbedding.v1") +def MyCustomVectors( + output_width: int, + vector_width: int, + embed_rows: int, + key2row: Dict[int, int] +) -> Model[List[Doc], List[Floats2d]]: + return add( + StaticVectors(nO=output_width), + chain( + FeatureExtractor(["ORTH"]), + remap_ids(key2row), + Embed(nO=output_width, nV=embed_rows) + ) + ) +``` + +#### Creating a custom vectors implementation {id="custom-vectors",version="3.7"} + +You can specify a custom registered vectors class under `[nlp.vectors]` in order +to use static vectors in formats other than the ones supported by +[`Vectors`](/api/vectors). Extend the abstract [`BaseVectors`](/api/basevectors) +class to implement your custom vectors. + +As an example, the following `BPEmbVectors` class implements support for +[BPEmb subword embeddings](https://bpemb.h-its.org/): + +```python +# requires: pip install bpemb +import warnings +from pathlib import Path +from typing import Callable, Optional, cast + +from bpemb import BPEmb +from thinc.api import Ops, get_current_ops +from thinc.backends import get_array_ops +from thinc.types import Floats2d + +from spacy.strings import StringStore +from spacy.util import registry +from spacy.vectors import BaseVectors +from spacy.vocab import Vocab + + +class BPEmbVectors(BaseVectors): + def __init__( + self, + *, + strings: Optional[StringStore] = None, + lang: Optional[str] = None, + vs: Optional[int] = None, + dim: Optional[int] = None, + cache_dir: Optional[Path] = None, + encode_extra_options: Optional[str] = None, + model_file: Optional[Path] = None, + emb_file: Optional[Path] = None, + ): + kwargs = {} + if lang is not None: + kwargs["lang"] = lang + if vs is not None: + kwargs["vs"] = vs + if dim is not None: + kwargs["dim"] = dim + if cache_dir is not None: + kwargs["cache_dir"] = cache_dir + if encode_extra_options is not None: + kwargs["encode_extra_options"] = encode_extra_options + if model_file is not None: + kwargs["model_file"] = model_file + if emb_file is not None: + kwargs["emb_file"] = emb_file + self.bpemb = BPEmb(**kwargs) + self.strings = strings + self.name = repr(self.bpemb) + self.n_keys = -1 + self.mode = "BPEmb" + self.to_ops(get_current_ops()) + + def __contains__(self, key): + return True + + def is_full(self): + return True + + def add(self, key, *, vector=None, row=None): + warnings.warn( + ( + "Skipping BPEmbVectors.add: the bpemb vector table cannot be " + "modified. Vectors are calculated from bytepieces." + ) + ) + return -1 + + def __getitem__(self, key): + return self.get_batch([key])[0] + + def get_batch(self, keys): + keys = [self.strings.as_string(key) for key in keys] + bp_ids = self.bpemb.encode_ids(keys) + ops = get_array_ops(self.bpemb.emb.vectors) + indices = ops.asarray(ops.xp.hstack(bp_ids), dtype="int32") + lengths = ops.asarray([len(x) for x in bp_ids], dtype="int32") + vecs = ops.reduce_mean(cast(Floats2d, self.bpemb.emb.vectors[indices]), lengths) + return vecs + + @property + def shape(self): + return self.bpemb.vectors.shape + + def __len__(self): + return self.shape[0] + + @property + def vectors_length(self): + return self.shape[1] + + @property + def size(self): + return self.bpemb.vectors.size + + def to_ops(self, ops: Ops): + self.bpemb.emb.vectors = ops.asarray(self.bpemb.emb.vectors) + + +@registry.vectors("BPEmbVectors.v1") +def create_bpemb_vectors( + lang: Optional[str] = "multi", + vs: Optional[int] = None, + dim: Optional[int] = None, + cache_dir: Optional[Path] = None, + encode_extra_options: Optional[str] = None, + model_file: Optional[Path] = None, + emb_file: Optional[Path] = None, +) -> Callable[[Vocab], BPEmbVectors]: + def bpemb_vectors_factory(vocab: Vocab) -> BPEmbVectors: + return BPEmbVectors( + strings=vocab.strings, + lang=lang, + vs=vs, + dim=dim, + cache_dir=cache_dir, + encode_extra_options=encode_extra_options, + model_file=model_file, + emb_file=emb_file, + ) + + return bpemb_vectors_factory +``` + + + +Note that the serialization methods are not implemented, so the embeddings are +loaded from your local cache or downloaded by `BPEmb` each time the pipeline is +loaded. + + + +To use this in your pipeline, specify this registered function under +`[nlp.vectors]` in your config: + +```ini +[nlp.vectors] +@vectors = "BPEmbVectors.v1" +lang = "en" +``` + +Or specify it when creating a blank pipeline: + +```python +nlp = spacy.blank("en", config={"nlp.vectors": {"@vectors": "BPEmbVectors.v1", "lang": "en"}}) +``` + +Remember to include this code with `--code` when using +[`spacy train`](/api/cli#train) and [`spacy package`](/api/cli#package). + +## Pretraining {id="pretraining"} + +The [`spacy pretrain`](/api/cli#pretrain) command lets you initialize your +models with **information from raw text**. Without pretraining, the models for +your components will usually be initialized randomly. The idea behind +pretraining is simple: random probably isn't optimal, so if we have some text to +learn from, we can probably find a way to get the model off to a better start. + +Pretraining uses the same [`config.cfg`](/usage/training#config) file as the +regular training, which helps keep the settings and hyperparameters consistent. +The additional `[pretraining]` section has several configuration subsections +that are familiar from the training block: the `[pretraining.batcher]`, +`[pretraining.optimizer]` and `[pretraining.corpus]` all work the same way and +expect the same types of objects, although for pretraining your corpus does not +need to have any annotations, so you will often use a different reader, such as +the [`JsonlCorpus`](/api/top-level#jsonlcorpus). + +> #### Raw text format +> +> The raw text can be provided in spaCy's +> [binary `.spacy` format](/api/data-formats#training) consisting of serialized +> `Doc` objects or as a JSONL (newline-delimited JSON) with a key `"text"` per +> entry. This allows the data to be read in line by line, while also allowing +> you to include newlines in the texts. +> +> ```json +> {"text": "Can I ask where you work now and what you do, and if you enjoy it?"} +> {"text": "They may just pull out of the Seattle market completely, at least until they have autonomous vehicles."} +> ``` +> +> You can also use your own custom corpus loader instead. + +You can add a `[pretraining]` block to your config by setting the +`--pretraining` flag on [`init config`](/api/cli#init-config) or +[`init fill-config`](/api/cli#init-fill-config): + +```bash +$ python -m spacy init fill-config config.cfg config_pretrain.cfg --pretraining +``` + +You can then run [`spacy pretrain`](/api/cli#pretrain) with the updated config +and pass in optional config overrides, like the path to the raw text file: + +```bash +$ python -m spacy pretrain config_pretrain.cfg ./output --paths.raw_text text.jsonl +``` + +The following defaults are used for the `[pretraining]` block and merged into +your existing config when you run [`init config`](/api/cli#init-config) or +[`init fill-config`](/api/cli#init-fill-config) with `--pretraining`. If needed, +you can [configure](#pretraining-configure) the settings and hyperparameters or +change the [objective](#pretraining-objectives). + +```ini +%%GITHUB_SPACY/spacy/default_config_pretraining.cfg +``` + +### How pretraining works {id="pretraining-details"} + +The impact of [`spacy pretrain`](/api/cli#pretrain) varies, but it will usually +be worth trying if you're **not using a transformer** model and you have +**relatively little training data** (for instance, fewer than 5,000 sentences). +A good rule of thumb is that pretraining will generally give you a similar +accuracy improvement to using word vectors in your model. If word vectors have +given you a 10% error reduction, pretraining with spaCy might give you another +10%, for a 20% error reduction in total. + +The [`spacy pretrain`](/api/cli#pretrain) command will take a **specific +subnetwork** within one of your components, and add additional layers to build a +network for a temporary task that forces the model to learn something about +sentence structure and word cooccurrence statistics. + +Pretraining produces a **binary weights file** that can be loaded back in at the +start of training, using the configuration option `initialize.init_tok2vec`. The +weights file specifies an initial set of weights. Training then proceeds as +normal. + +You can only pretrain one subnetwork from your pipeline at a time, and the +subnetwork must be typed ~~Model[List[Doc], List[Floats2d]]~~ (i.e. it has to be +a "tok2vec" layer). The most common workflow is to use the +[`Tok2Vec`](/api/tok2vec) component to create a shared token-to-vector layer for +several components of your pipeline, and apply pretraining to its whole model. + +#### Configuring the pretraining {id="pretraining-configure"} + +The [`spacy pretrain`](/api/cli#pretrain) command is configured using the +`[pretraining]` section of your [config file](/usage/training#config). The +`component` and `layer` settings tell spaCy how to **find the subnetwork** to +pretrain. The `layer` setting should be either the empty string (to use the +whole model), or a +[node reference](https://thinc.ai/docs/usage-models#model-state). Most of +spaCy's built-in model architectures have a reference named `"tok2vec"` that +will refer to the right layer. + +```ini {title="config.cfg"} +# 1. Use the whole model of the "tok2vec" component +[pretraining] +component = "tok2vec" +layer = "" + +# 2. Pretrain the "tok2vec" node of the "textcat" component +[pretraining] +component = "textcat" +layer = "tok2vec" +``` + +#### Connecting pretraining to training {id="pretraining-training"} + +To benefit from pretraining, your training step needs to know to initialize its +`tok2vec` component with the weights learned from the pretraining step. You do +this by setting `initialize.init_tok2vec` to the filename of the `.bin` file +that you want to use from pretraining. + +A pretraining step that runs for 5 epochs with an output path of `pretrain/`, as +an example, produces `pretrain/model0.bin` through `pretrain/model4.bin` plus a +copy of the last iteration as `pretrain/model-last.bin`. Additionally, you can +configure `n_save_epoch` to tell pretraining in which epoch interval it should +save the current training progress. To use the final output to initialize your +`tok2vec` layer, you could fill in this value in your config file: + +```ini {title="config.cfg"} + +[paths] +init_tok2vec = "pretrain/model-last.bin" + +[initialize] +init_tok2vec = ${paths.init_tok2vec} +``` + + + +The outputs of `spacy pretrain` are not the same data format as the pre-packaged +static word vectors that would go into +[`initialize.vectors`](/api/data-formats#config-initialize). The pretraining +output consists of the weights that the `tok2vec` component should start with in +an existing pipeline, so it goes in `initialize.init_tok2vec`. + + + +#### Pretraining objectives {id="pretraining-objectives"} + +> ```ini +> ### Characters objective +> [pretraining.objective] +> @architectures = "spacy.PretrainCharacters.v1" +> maxout_pieces = 3 +> hidden_size = 300 +> n_characters = 4 +> ``` +> +> ```ini +> ### Vectors objective +> [pretraining.objective] +> @architectures = "spacy.PretrainVectors.v1" +> maxout_pieces = 3 +> hidden_size = 300 +> loss = "cosine" +> ``` + +Two pretraining objectives are available, both of which are variants of the +cloze task [Devlin et al. (2018)](https://arxiv.org/abs/1810.04805) introduced +for BERT. The objective can be defined and configured via the +`[pretraining.objective]` config block. + +- [`PretrainCharacters`](/api/architectures#pretrain_chars): The `"characters"` + objective asks the model to predict some number of leading and trailing UTF-8 + bytes for the words. For instance, setting `n_characters = 2`, the model will + try to predict the first two and last two characters of the word. + +- [`PretrainVectors`](/api/architectures#pretrain_vectors): The `"vectors"` + objective asks the model to predict the word's vector, from a static + embeddings table. This requires a word vectors model to be trained and loaded. + The vectors objective can optimize either a cosine or an L2 loss. We've + generally found cosine loss to perform better. + +These pretraining objectives use a trick that we term **language modelling with +approximate outputs (LMAO)**. The motivation for the trick is that predicting an +exact word ID introduces a lot of incidental complexity. You need a large output +layer, and even then, the vocabulary is too large, which motivates tokenization +schemes that do not align to actual word boundaries. At the end of training, the +output layer will be thrown away regardless: we just want a task that forces the +network to model something about word cooccurrence statistics. Predicting +leading and trailing characters does that more than adequately, as the exact +word sequence could be recovered with high accuracy if the initial and trailing +characters are predicted accurately. With the vectors objective, the pretraining +uses the embedding space learned by an algorithm such as +[GloVe](https://nlp.stanford.edu/projects/glove/) or +[Word2vec](https://code.google.com/archive/p/word2vec/), allowing the model to +focus on the contextual modelling we actual care about. diff --git a/website/docs/usage/facts-figures.mdx b/website/docs/usage/facts-figures.mdx new file mode 100644 index 0000000..75ef7e4 --- /dev/null +++ b/website/docs/usage/facts-figures.mdx @@ -0,0 +1,116 @@ +--- +title: Facts & Figures +teaser: The hard numbers for spaCy and how it compares to other tools +next: /usage/spacy-101 +menu: + - ['Feature Comparison', 'comparison'] + - ['Benchmarks', 'benchmarks'] + # TODO: - ['Citing spaCy', 'citation'] +--- + +## Comparison {id="comparison",hidden="true"} + +spaCy is a **free, open-source library** for advanced **Natural Language +Processing** (NLP) in Python. It's designed specifically for **production use** +and helps you build applications that process and "understand" large volumes of +text. It can be used to build information extraction or natural language +understanding systems. + +### Feature overview {id="comparison-features"} + + + +### When should I use spaCy? {id="comparison-usage"} + +- ✅ **I'm a beginner and just getting started with NLP.** – spaCy makes it easy + to get started and comes with extensive documentation, including a + beginner-friendly [101 guide](/usage/spacy-101), a free interactive + [online course](https://course.spacy.io) and a range of + [video tutorials](https://www.youtube.com/c/ExplosionAI). +- ✅ **I want to build an end-to-end production application.** – spaCy is + specifically designed for production use and lets you build and train powerful + NLP pipelines and package them for easy deployment. +- ✅ **I want my application to be efficient on GPU _and_ CPU.** – While spaCy + lets you train modern NLP models that are best run on GPU, it also offers + CPU-optimized pipelines, which are less accurate but much cheaper to run. +- ✅ **I want to try out different neural network architectures for NLP.** – + spaCy lets you customize and swap out the model architectures powering its + components, and implement your own using a framework like PyTorch or + TensorFlow. The declarative configuration system makes it easy to mix and + match functions and keep track of your hyperparameters to make sure your + experiments are reproducible. +- ❌ **I want to build a language generation application.** – spaCy's focus is + natural language _processing_ and extracting information from large volumes of + text. While you can use it to help you re-write existing text, it doesn't + include any specific functionality for language generation tasks. +- ❌ **I want to research machine learning algorithms.** spaCy is built on the + latest research, but it's not a research library. If your goal is to write + papers and run benchmarks, spaCy is probably not a good choice. However, you + can use it to make the results of your research easily available for others to + use, e.g. via a custom spaCy component. + +## Benchmarks {id="benchmarks"} + +spaCy v3.0 introduces transformer-based pipelines that bring spaCy's accuracy +right up to **current state-of-the-art**. You can also use a CPU-optimized +pipeline, which is less accurate but much cheaper to run. + +{/* TODO: update benchmarks and intro */} + +> #### Evaluation details +> +> - **OntoNotes 5.0:** spaCy's English models are trained on this corpus, as +> it's several times larger than other English treebanks. However, most +> systems do not report accuracies on it. +> - **Penn Treebank:** The "classic" parsing evaluation for research. However, +> it's quite far removed from actual usage: it uses sentences with +> gold-standard segmentation and tokenization, from a pretty specific type of +> text (articles from a single newspaper, 1984-1989). + + + +
+ +| Dependency Parsing System | UAS | LAS | +| ------------------------------------------------------------------------------ | ---: | ---: | +| spaCy RoBERTa (2020) | 95.1 | 93.7 | +| [Mrini et al.](https://khalilmrini.github.io/Label_Attention_Layer.pdf) (2019) | 97.4 | 96.3 | +| [Zhou and Zhao](https://www.aclweb.org/anthology/P19-1230/) (2019) | 97.2 | 95.7 | + +
+ +**Dependency parsing accuracy** on the Penn Treebank. See +[NLP-progress](http://nlpprogress.com/english/dependency_parsing.html) for more +results. Project template: +[`benchmarks/parsing_penn_treebank`](%%GITHUB_PROJECTS/benchmarks/parsing_penn_treebank). + +
+ +
+ +### Speed comparison {id="benchmarks-speed"} + +We compare the speed of different NLP libraries, measured in words per second +(WPS) - higher is better. The evaluation was performed on 10,000 Reddit +comments. + +
+ +| Library | Pipeline | WPS CPU words per second on CPU, higher is better | WPS GPU words per second on GPU, higher is better | +| ------- | ----------------------------------------------- | -------------------------------------------------------------: | -------------------------------------------------------------: | +| spaCy | [`en_core_web_lg`](/models/en#en_core_web_lg) | 10,014 | 14,954 | +| spaCy | [`en_core_web_trf`](/models/en#en_core_web_trf) | 684 | 3,768 | +| Stanza | `en_ewt` | 878 | 2,180 | +| Flair | `pos`(`-fast`) & `ner`(`-fast`) | 323 | 1,184 | +| UDPipe | `english-ewt-ud-2.5` | 1,101 | _n/a_ | + +
+ +**End-to-end processing speed** on raw unannotated text. Project template: +[`benchmarks/speed`](%%GITHUB_PROJECTS/benchmarks/speed). + +
+ +
+ +{/* TODO: ## Citing spaCy {id="citation"} */} diff --git a/website/docs/usage/index.mdx b/website/docs/usage/index.mdx new file mode 100644 index 0000000..c50e9db --- /dev/null +++ b/website/docs/usage/index.mdx @@ -0,0 +1,470 @@ +--- +title: Install spaCy +next: /usage/models +menu: + - ['Quickstart', 'quickstart'] + - ['Instructions', 'installation'] + - ['Troubleshooting', 'troubleshooting'] + - ['Changelog', 'changelog'] +--- + +## Quickstart {hidden="true"} + +> #### 📖 Looking for the old docs? +> +> To help you make the transition from v2.x to v3.0, we've uploaded the old +> website to [**v2.spacy.io**](https://v2.spacy.io/docs). To see what's changed +> and how to migrate, see the [v3.0 guide](/usage/v3). + + + +## Installation instructions {id="installation"} + +spaCy is compatible with **64-bit CPython 3.7+** and runs on **Unix/Linux**, +**macOS/OS X** and **Windows**. The latest spaCy releases are available over +[pip](https://pypi.python.org/pypi/spacy) and +[conda](https://anaconda.org/conda-forge/spacy). + +### pip {id="pip"} + +Using pip, spaCy releases are available as source packages and binary wheels. +Before you install spaCy and its dependencies, make sure that your `pip`, +`setuptools` and `wheel` are up to date. + +> #### Download pipelines +> +> After installation you typically want to download a trained pipeline. For more +> info and available packages, see the [models directory](/models). +> +> ```bash +> $ python -m spacy download en_core_web_sm +> +> >>> import spacy +> >>> nlp = spacy.load("en_core_web_sm") +> ``` + +```bash +$ pip install -U pip setuptools wheel +$ pip install -U %%SPACY_PKG_NAME%%SPACY_PKG_FLAGS +``` + +When using pip it is generally recommended to install packages in a virtual +environment to avoid modifying system state: + +```bash +$ python -m venv .env +$ source .env/bin/activate +$ pip install -U pip setuptools wheel +$ pip install -U %%SPACY_PKG_NAME%%SPACY_PKG_FLAGS +``` + +spaCy also lets you install extra dependencies by specifying the following +keywords in brackets, e.g. `spacy[ja]` or `spacy[lookups,transformers]` (with +multiple comma-separated extras). See the `[options.extras_require]` section in +spaCy's [`setup.cfg`](%%GITHUB_SPACY/setup.cfg) for details on what's included. + +> #### Example +> +> ```bash +> $ pip install %%SPACY_PKG_NAME[lookups,transformers]%%SPACY_PKG_FLAGS +> ``` + +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lookups` | Install [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) for data tables for lemmatization and lexeme normalization. The data is serialized with trained pipelines, so you only need this package if you want to train your own models. | +| `transformers` | Install [`spacy-transformers`](https://github.com/explosion/spacy-transformers). The package will be installed automatically when you install a transformer-based pipeline. | +| `cuda`, ... | Install spaCy with GPU support provided by [CuPy](https://cupy.chainer.org) for your given CUDA version. See the GPU [installation instructions](#gpu) for details and options. | +| `apple` | Install [`thinc-apple-ops`](https://github.com/explosion/thinc-apple-ops) to improve performance on an Apple M1. | +| `ja`, `ko`, `th` | Install additional dependencies required for tokenization for the [languages](/usage/models#languages). | + +### conda {id="conda"} + +Thanks to our great community, we've been able to re-add conda support. You can +also install spaCy via `conda-forge`: + +```bash +$ conda install -c conda-forge spacy +``` + +For the feedstock including the build recipe and configuration, check out +[this repository](https://github.com/conda-forge/spacy-feedstock). Note that we +currently don't publish any [pre-releases](#changelog-pre) on conda. + +### Upgrading spaCy {id="upgrading"} + +> #### Upgrading from v2 to v3 +> +> Although we've tried to keep breaking changes to a minimum, upgrading from +> spaCy v2.x to v3.x may still require some changes to your code base. For +> details see the sections on [backwards incompatibilities](/usage/v3#incompat) +> and [migrating](/usage/v3#migrating). Also remember to download the new +> trained pipelines, and retrain your own pipelines. + +When updating to a newer version of spaCy, it's generally recommended to start +with a clean virtual environment. If you're upgrading to a new major version, +make sure you have the latest **compatible trained pipelines** installed, and +that there are no old and incompatible packages left over in your environment, +as this can often lead to unexpected results and errors. If you've trained your +own models, keep in mind that your train and runtime inputs must match. This +means you'll have to **retrain your pipelines** with the new version. + +spaCy also provides a [`validate`](/api/cli#validate) command, which lets you +verify that all installed pipeline packages are compatible with your spaCy +version. If incompatible packages are found, tips and installation instructions +are printed. It's recommended to run the command with `python -m` to make sure +you're executing the correct version of spaCy. + +```bash +$ pip install -U %%SPACY_PKG_NAME%%SPACY_PKG_FLAGS +$ python -m spacy validate +``` + +### Run spaCy with GPU {id="gpu",version="2.0.14"} + +As of v2.0, spaCy comes with neural network models that are implemented in our +machine learning library, [Thinc](https://thinc.ai). For GPU support, we've been +grateful to use the work of Chainer's [CuPy](https://cupy.chainer.org) module, +which provides a numpy-compatible interface for GPU arrays. + +spaCy can be installed for a CUDA-compatible GPU by specifying `spacy[cuda]`, +`spacy[cuda102]`, `spacy[cuda112]`, `spacy[cuda113]`, etc. If you know your CUDA +version, using the more explicit specifier allows CuPy to be installed via +wheel, saving some compilation time. The specifiers should install +[`cupy`](https://cupy.chainer.org). + +```bash +$ pip install -U %%SPACY_PKG_NAME[cuda113]%%SPACY_PKG_FLAGS +``` + +Once you have a GPU-enabled installation, the best way to activate it is to call +[`spacy.prefer_gpu`](/api/top-level#spacy.prefer_gpu) or +[`spacy.require_gpu()`](/api/top-level#spacy.require_gpu) somewhere in your +script before any pipelines have been loaded. `require_gpu` will raise an error +if no GPU is available. + +```python +import spacy + +spacy.prefer_gpu() +nlp = spacy.load("en_core_web_sm") +``` + +### Compile from source {id="source"} + +The other way to install spaCy is to clone its +[GitHub repository](https://github.com/explosion/spaCy) and build it from +source. That is the common way if you want to make changes to the code base. +You'll need to make sure that you have a development environment consisting of a +Python distribution including header files, a compiler, +[pip](https://pip.pypa.io/en/stable/) and [git](https://git-scm.com) installed. +The compiler part is the trickiest. How to do that depends on your system. See +notes on [Ubuntu](#source-ubuntu), [macOS / OS X](#source-osx) and +[Windows](#source-windows) for details. + +```bash +$ python -m pip install -U pip setuptools wheel # install/update build tools +$ git clone https://github.com/explosion/spaCy # clone spaCy +$ cd spaCy # navigate into dir +$ python -m venv .env # create environment in .env +$ source .env/bin/activate # activate virtual env +$ pip install -r requirements.txt # install requirements +$ pip install --no-build-isolation --editable . # compile and install spaCy +``` + +To install with extras: + +```bash +$ pip install --no-build-isolation --editable .[lookups,cuda102] +``` + +How to install compilers and related build tools: + +- Ubuntu: Install system-level dependencies via + `apt-get`: `sudo apt-get install build-essential python-dev git` +- macOS / OS X: Install a recent version of [XCode](https://developer.apple.com/xcode/), + including the so-called "Command Line Tools". macOS and OS X ship with Python and + Git preinstalled. +- Windows: Install a version of the [Visual + C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) or + [Visual Studio Express](https://www.visualstudio.com/vs/visual-studio-express/) + that matches the version that was used to compile your Python interpreter. + +#### Using build constraints when compiling from source + +If you install spaCy from source or with `pip` for platforms where there are not +binary wheels on PyPI, you may need to use build constraints if any package in +your environment requires an older version of `numpy`. + +If `numpy` gets downgraded from the most recent release at any point after +you've compiled `spacy`, you might see an error that looks like this: + +``` +numpy.ndarray size changed, may indicate binary incompatibility. +``` + +To fix this, create a new virtual environment and install `spacy` and all of its +dependencies using build constraints. +[Build constraints](https://pip.pypa.io/en/stable/user_guide/#constraints-files) +specify an older version of `numpy` that is only used while compiling `spacy`, +and then your runtime environment can use any newer version of `numpy` and still +be compatible. In addition, use `--no-cache-dir` to ignore any previously cached +wheels so that all relevant packages are recompiled from scratch: + +```shell +PIP_CONSTRAINT=https://raw.githubusercontent.com/explosion/spacy/master/build-constraints.txt \ +pip install spacy --no-cache-dir +``` + +Our build constraints currently specify the oldest supported `numpy` available +on PyPI for `x86_64` and `aarch64`. Depending on your platform and environment, +you may want to customize the specific versions of `numpy`. For other platforms, +you can have a look at SciPy's +[`oldest-supported-numpy`](https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg) +package to see what the oldest recommended versions of `numpy` are. + +(_Warning_: don't use `pip install -c constraints.txt` instead of +`PIP_CONSTRAINT`, since this isn't applied to the isolated build environments.) + +#### Additional options for developers {id="source-developers"} + +Some additional options may be useful for spaCy developers who are editing the +source code and recompiling frequently. + +- Install in editable mode. Changes to `.py` files will be reflected as soon as + the files are saved, but edits to Cython files (`.pxd`, `.pyx`) will require + the `pip install` command below to be run again. Before installing in editable + mode, be sure you have removed any previous installs with + `pip uninstall spacy`, which you may need to run multiple times to remove all + traces of earlier installs. + + ```bash + $ pip install -r requirements.txt + $ pip install --no-build-isolation --editable . + ``` + +- Build in parallel. Starting in v3.4.0, you can specify the number of build + jobs with the environment variable `SPACY_NUM_BUILD_JOBS`: + + ```bash + $ pip install -r requirements.txt + $ SPACY_NUM_BUILD_JOBS=4 pip install --no-build-isolation --editable . + ``` + +- For editable mode and parallel builds with `python setup.py` instead of `pip` + (no longer recommended): + + ```bash + $ pip install -r requirements.txt + $ python setup.py build_ext --inplace -j 4 + $ python setup.py develop + ``` + +#### Visual Studio Code extension + +![spaCy extension demo](/images/spacy-extension-demo.gif) + +The [spaCy VSCode Extension](https://github.com/explosion/spacy-vscode) provides +additional tooling and features for working with spaCy's config files. Version +1.0.0 includes hover descriptions for registry functions, variables, and section +names within the config as an installable extension. + +1. Install a supported version of Python on your system (`>=3.7`) +2. Install the + [Python Extension for Visual Studio Code](https://code.visualstudio.com/docs/python/python-tutorial) +3. Create a + [virtual python environment](https://docs.python.org/3/library/venv.html) +4. Install all python requirements (`spaCy >= 3.4.0` & `pygls >= 1.0.0`) +5. Install + [spaCy extension for Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=Explosion.spacy-extension) +6. Select your python environment +7. You are ready to work with `.cfg` files in spaCy! + +### Building an executable {id="executable"} + +The spaCy repository includes a [`Makefile`](%%GITHUB_SPACY/Makefile) that +builds an executable zip file using [`pex`](https://github.com/pantsbuild/pex) +(**P**ython **Ex**ecutable). The executable includes spaCy and all its package +dependencies and only requires the system Python at runtime. Building an +executable `.pex` file is often the most convenient way to deploy spaCy, as it +lets you separate the build from the deployment process. + +> #### Usage +> +> To use a `.pex` file, just replace `python` with the path to the file when you +> execute your code or CLI commands. This is equivalent to running Python in a +> virtual environment with spaCy installed. +> +> ```bash +> $ ./spacy.pex my_script.py +> $ ./spacy.pex -m spacy info +> ``` + +```bash +$ git clone https://github.com/explosion/spaCy +$ cd spaCy +$ make +``` + +You can configure the build process with the following environment variables: + +| Variable | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SPACY_EXTRAS` | Additional Python packages to install alongside spaCy with optional version specifications. Should be a string that can be passed to `pip install`. See [`Makefile`](%%GITHUB_SPACY/Makefile) for defaults. | +| `PYVER` | The Python version to build against. This version needs to be available on your build and runtime machines. Defaults to `3.8`. | +| `WHEELHOUSE` | Directory to store the wheel files during compilation. Defaults to `./wheelhouse`. | + +### Run tests {id="run-tests"} + +spaCy comes with an [extensive test suite](%%GITHUB_SPACY/spacy/tests). In order +to run the tests, you'll usually want to clone the [repository](%%GITHUB_SPACY) +and [build spaCy from source](#source). This will also install the required +development dependencies and test utilities defined in the `requirements.txt`. + +Alternatively, you can find out where spaCy is installed and run `pytest` on +that directory. Don't forget to also install the test utilities via spaCy's +[`requirements.txt`](%%GITHUB_SPACY/requirements.txt): + +```bash +$ python -c "import os; import spacy; print(os.path.dirname(spacy.__file__))" +$ pip install -r path/to/requirements.txt +$ python -m pytest --pyargs %%SPACY_PKG_NAME +``` + +Calling `pytest` on the spaCy directory will run only the basic tests. The flag +`--slow` is optional and enables additional tests that take longer. + +```bash +$ python -m pip install -U pytest # update pytest +$ python -m pytest --pyargs %%SPACY_PKG_NAME # basic tests +$ python -m pytest --pyargs %%SPACY_PKG_NAME --slow # basic and slow tests +``` + +## Troubleshooting guide {id="troubleshooting"} + +This section collects some of the most common errors you may come across when +installing, loading and using spaCy, as well as their solutions. Also see the +[Discussions FAQ Thread](https://github.com/explosion/spaCy/discussions/8226), +which is updated more frequently and covers more transitory issues. + +> #### Help us improve this guide +> +> Did you come across a problem like the ones listed here and want to share the +> solution? You can find the "Suggest edits" button at the bottom of this page +> that points you to the source. We always appreciate +> [pull requests](https://github.com/explosion/spaCy/pulls)! + + + +``` +No compatible package found for [lang] (spaCy vX.X.X). +``` + +This usually means that the trained pipeline you're trying to download does not +exist, or isn't available for your version of spaCy. Check the +[compatibility table](https://github.com/explosion/spacy-models/tree/master/compatibility.json) +to see which packages are available for your spaCy version. If you're using an +old version, consider upgrading to the latest release. Note that while spaCy +supports tokenization for [a variety of languages](/usage/models#languages), not +all of them come with trained pipelines. To only use the tokenizer, import the +language's `Language` class instead, for example +`from spacy.lang.fr import French`. + + + + + +``` +Import Error: No module named spacy +``` + +This error means that the spaCy module can't be located on your system, or in +your environment. Make sure you have spaCy installed. If you're using a virtual +environment, make sure it's activated and check that spaCy is installed in that +environment – otherwise, you're trying to load a system installation. You can +also run `which python` to find out where your Python executable is located. + + + + + +``` +ImportError: No module named 'en_core_web_sm' +``` + +As of spaCy v1.7, all trained pipelines can be installed as Python packages. +This means that they'll become importable modules of your application. If this +fails, it's usually a sign that the package is not installed in the current +environment. Run `pip list` or `pip freeze` to check which pipeline packages you +have installed, and install the [correct package](/models) if necessary. If +you're importing a package manually at the top of a file, make sure to use the +full name of the package. + + + + + +``` +command not found: spacy +``` + +This error may occur when running the `spacy` command from the command line. +spaCy does not currently add an entry to your `PATH` environment variable, as +this can lead to unexpected results, especially when using a virtual +environment. Instead, spaCy adds an auto-alias that maps `spacy` to +`python -m spacy`. If this is not working as expected, run the command with +`python -m`, yourself – for example `python -m spacy download en_core_web_sm`. +For more info on this, see the [`download`](/api/cli#download) command. + + + + + +``` +AttributeError: 'module' object has no attribute 'load' +``` + +While this could technically have many causes, including spaCy being broken, the +most likely one is that your script's file or directory name is "shadowing" the +module – e.g. your file is called `spacy.py`, or a directory you're importing +from is called `spacy`. So, when using spaCy, never call anything else `spacy`. + + + + + +If your training data only contained new entities and you didn't mix in any +examples the model previously recognized, it can cause the model to "forget" +what it had previously learned. This is also referred to as the "catastrophic +forgetting problem". A solution is to pre-label some text, and mix it with the +new text in your updates. You can also do this by running spaCy over some text, +extracting a bunch of entities the model previously recognized correctly, and +adding them to your training examples. + + + + + +``` +TypeError: unhashable type: 'list' +``` + +If you're training models, writing them to disk, and versioning them with git, +you might encounter this error when trying to load them in a Windows +environment. This happens because a default install of Git for Windows is +configured to automatically convert Unix-style end-of-line characters (LF) to +Windows-style ones (CRLF) during file checkout (and the reverse when +committing). While that's mostly fine for text files, a trained model written to +disk has some binary files that should not go through this conversion. When they +do, you get the error above. You can fix it by either changing your +[`core.autocrlf`](https://git-scm.com/book/en/v2/Customizing-Git-Git-Configuration) +setting to `"false"`, or by committing a +[`.gitattributes`](https://git-scm.com/docs/gitattributes) file to your +repository to tell Git on which files or folders it shouldn't do LF-to-CRLF +conversion, with an entry like `path/to/spacy/model/** -text`. After you've done +either of these, clone your repository again. + + + +## Changelog {id="changelog"} + + diff --git a/website/docs/usage/large-language-models.mdx b/website/docs/usage/large-language-models.mdx new file mode 100644 index 0000000..c799e91 --- /dev/null +++ b/website/docs/usage/large-language-models.mdx @@ -0,0 +1,534 @@ +--- +title: Large Language Models +teaser: Integrating LLMs into structured NLP pipelines +menu: + - ['Motivation', 'motivation'] + - ['Install', 'install'] + - ['Usage', 'usage'] + - ['Logging', 'logging'] + - ['API', 'api'] + - ['Tasks', 'tasks'] + - ['Models', 'models'] +--- + +[The spacy-llm package](https://github.com/explosion/spacy-llm) integrates Large +Language Models (LLMs) into spaCy pipelines, featuring a modular system for +**fast prototyping** and **prompting**, and turning unstructured responses into +**robust outputs** for various NLP tasks, **no training data** required. + +- Serializable `llm` **component** to integrate prompts into your pipeline +- **Modular functions** to define the [**task**](#tasks) (prompting and parsing) + and [**model**](#models) (model to use) +- Support for **hosted APIs** and self-hosted **open-source models** +- Integration with [`LangChain`](https://github.com/hwchase17/langchain) +- Access to + **[OpenAI API](https://platform.openai.com/docs/api-reference/introduction)**, + including GPT-4 and various GPT-3 models +- Built-in support for various **open-source** models hosted on + [Hugging Face](https://huggingface.co/) +- Usage examples for standard NLP tasks such as **Named Entity Recognition** and + **Text Classification** +- Easy implementation of **your own functions** via the + [registry](/api/top-level#registry) for custom prompting, parsing and model + integrations + +## Motivation {id="motivation"} + +Large Language Models (LLMs) feature powerful natural language understanding +capabilities. With only a few (and sometimes no) examples, an LLM can be +prompted to perform custom NLP tasks such as text categorization, named entity +recognition, coreference resolution, information extraction and more. + +Supervised learning is much worse than LLM prompting for prototyping, but for +many tasks it's much better for production. A transformer model that runs +comfortably on a single GPU is extremely powerful, and it's likely to be a +better choice for any task for which you have a well-defined output. You train +the model with anything from a few hundred to a few thousand labelled examples, +and it will learn to do exactly that. Efficiency, reliability and control are +all better with supervised learning, and accuracy will generally be higher than +LLM prompting as well. + +`spacy-llm` lets you have **the best of both worlds**. You can quickly +initialize a pipeline with components powered by LLM prompts, and freely mix in +components powered by other approaches. As your project progresses, you can look +at replacing some or all of the LLM-powered components as you require. + +Of course, there can be components in your system for which the power of an LLM +is fully justified. If you want a system that can synthesize information from +multiple documents in subtle ways and generate a nuanced summary for you, bigger +is better. However, even if your production system needs an LLM for some of the +task, that doesn't mean you need an LLM for all of it. Maybe you want to use a +cheap text classification model to help you find the texts to summarize, or +maybe you want to add a rule-based system to sanity check the output of the +summary. These before-and-after tasks are much easier with a mature and +well-thought-out library, which is exactly what spaCy provides. + +## Install {id="install"} + +`spacy-llm` will be installed automatically in future spaCy versions. For now, +you can run the following in the same virtual environment where you already have +`spacy` [installed](/usage). + +> ⚠️ This package is still experimental and it is possible that changes made to +> the interface will be breaking in minor version updates. + +```bash +python -m pip install spacy-llm +``` + +## Usage {id="usage"} + +The task and the model have to be supplied to the `llm` pipeline component using +the [config system](/api/data-formats#config). This package provides various +built-in functionality, as detailed in the [API](#-api) documentation. + +### Example 1: Add a text classifier using a GPT-3 model from OpenAI {id="example-1"} + +Create a new API key from openai.com or fetch an existing one, and ensure the +keys are set as environmental variables. For more background information, see +the [OpenAI](/api/large-language-models#gpt-3-5) section. + +Create a config file `config.cfg` containing at least the following (or see the +full example +[here](https://github.com/explosion/spacy-llm/tree/main/usage_examples/textcat_openai)): + +```ini +[nlp] +lang = "en" +pipeline = ["llm"] + +[components] + +[components.llm] +factory = "llm" + +[components.llm.task] +@llm_tasks = "spacy.TextCat.v2" +labels = ["COMPLIMENT", "INSULT"] + +[components.llm.model] +@llm_models = "spacy.GPT-3-5.v1" +config = {"temperature": 0.0} +``` + +Now run: + +```python +from spacy_llm.util import assemble + +nlp = assemble("config.cfg") +doc = nlp("You look gorgeous!") +print(doc.cats) +``` + +### Example 2: Add NER using an open-source model through Hugging Face {id="example-2"} + +To run this example, ensure that you have a GPU enabled, and `transformers`, +`torch` and CUDA installed. For more background information, see the +[DollyHF](/api/large-language-models#dolly) section. + +Create a config file `config.cfg` containing at least the following (or see the +full example +[here](https://github.com/explosion/spacy-llm/tree/main/usage_examples/ner_dolly)): + +```ini +[nlp] +lang = "en" +pipeline = ["llm"] + +[components] + +[components.llm] +factory = "llm" + +[components.llm.task] +@llm_tasks = "spacy.NER.v3" +labels = ["PERSON", "ORGANISATION", "LOCATION"] + +[components.llm.model] +@llm_models = "spacy.Dolly.v1" +# For better performance, use dolly-v2-12b instead +name = "dolly-v2-3b" +``` + +Now run: + +```python +from spacy_llm.util import assemble + +nlp = assemble("config.cfg") +doc = nlp("Jack and Jill rode up the hill in Les Deux Alpes") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +Note that Hugging Face will download the `"databricks/dolly-v2-3b"` model the +first time you use it. You can +[define the cached directory](https://huggingface.co/docs/huggingface_hub/main/en/guides/manage-cache) +by setting the environmental variable `HF_HOME`. Also, you can upgrade the model +to be `"databricks/dolly-v2-12b"` for better performance. + +### Example 3: Create the component directly in Python {id="example-3"} + +The `llm` component behaves as any other component does, and there are +[task-specific components](/api/large-language-models#config) defined to help +you hit the ground running with a reasonable built-in task implementation. + +```python +import spacy + +nlp = spacy.blank("en") +llm_ner = nlp.add_pipe("llm_ner") +llm_ner.add_label("PERSON") +llm_ner.add_label("LOCATION") +nlp.initialize() +doc = nlp("Jack and Jill rode up the hill in Les Deux Alpes") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +Note that for efficient usage of resources, typically you would use +[`nlp.pipe(docs)`](/api/language#pipe) with a batch, instead of calling +`nlp(doc)` with a single document. + +### Example 4: Implement your own custom task {id="example-4"} + +To write a [`task`](#tasks), you need to implement two functions: +`generate_prompts` that takes a list of [`Doc`](/api/doc) objects and transforms +them into a list of prompts, and `parse_responses` that transforms the LLM +outputs into annotations on the [`Doc`](/api/doc), e.g. entity spans, text +categories and more. + +To register your custom task, decorate a factory function using the +`spacy_llm.registry.llm_tasks` decorator with a custom name that you can refer +to in your config. + +> 📖 For more details, see the +> [**usage example on writing your own task**](https://github.com/explosion/spacy-llm/tree/main/usage_examples#writing-your-own-task) + +```python +from typing import Iterable, List +from spacy.tokens import Doc +from spacy_llm.registry import registry +from spacy_llm.util import split_labels + + +@registry.llm_tasks("my_namespace.MyTask.v1") +def make_my_task(labels: str, my_other_config_val: float) -> "MyTask": + labels_list = split_labels(labels) + return MyTask(labels=labels_list, my_other_config_val=my_other_config_val) + + +class MyTask: + def __init__(self, labels: List[str], my_other_config_val: float): + ... + + def generate_prompts(self, docs: Iterable[Doc]) -> Iterable[str]: + ... + + def parse_responses( + self, docs: Iterable[Doc], responses: Iterable[str] + ) -> Iterable[Doc]: + ... +``` + +```ini +# config.cfg (excerpt) +[components.llm.task] +@llm_tasks = "my_namespace.MyTask.v1" +labels = LABEL1,LABEL2,LABEL3 +my_other_config_val = 0.3 +``` + +## Logging {id="logging"} + +spacy-llm has a built-in logger that can log the prompt sent to the LLM as well +as its raw response. This logger uses the debug level and by default has a +`logging.NullHandler()` configured. + +In order to use this logger, you can setup a simple handler like this: + +```python +import logging +import spacy_llm + + +spacy_llm.logger.addHandler(logging.StreamHandler()) +spacy_llm.logger.setLevel(logging.DEBUG) +``` + +> NOTE: Any `logging` handler will work here so you probably want to use some +> sort of rotating `FileHandler` as the generated prompts can be quite long, +> especially for tasks with few-shot examples. + +Then when using the pipeline you'll be able to view the prompt and response. + +E.g. with the config and code from [Example 1](#example-1) above: + +```python +from spacy_llm.util import assemble + + +nlp = assemble("config.cfg") +doc = nlp("You look gorgeous!") +print(doc.cats) +``` + +You will see `logging` output similar to: + +``` +Generated prompt for doc: You look gorgeous! + +You are an expert Text Classification system. Your task is to accept Text as input +and provide a category for the text based on the predefined labels. + +Classify the text below to any of the following labels: COMPLIMENT, INSULT +The task is non-exclusive, so you can provide more than one label as long as +they're comma-delimited. For example: Label1, Label2, Label3. +Do not put any other text in your answer, only one or more of the provided labels with nothing before or after. +If the text cannot be classified into any of the provided labels, answer `==NONE==`. + +Here is the text that needs classification + + +Text: +''' +You look gorgeous! +''' + +Model response for doc: You look gorgeous! +COMPLIMENT +``` + +`print(doc.cats)` to standard output should look like: + +``` +{'COMPLIMENT': 1.0, 'INSULT': 0.0} +``` + +## API {id="api"} + +`spacy-llm` exposes an `llm` factory with +[configurable settings](/api/large-language-models#config). + +An `llm` component is defined by two main settings: + +- A [**task**](#tasks), defining the prompt to send to the LLM as well as the + functionality to parse the resulting response back into structured fields on + the [Doc](/api/doc) objects. +- A [**model**](#models) defining the model to use and how to connect to it. + Note that `spacy-llm` supports both access to external APIs (such as OpenAI) + as well as access to self-hosted open-source LLMs (such as using Dolly through + Hugging Face). + +Moreover, `spacy-llm` exposes a customizable [**caching**](#cache) functionality +to avoid running the same document through an LLM service (be it local or +through a REST API) more than once. + +Finally, you can choose to save a stringified version of LLM prompts/responses +within the `Doc.user_data["llm_io"]` attribute by setting `save_io` to `True`. +`Doc.user_data["llm_io"]` is a dictionary containing one entry for every LLM +component within the `nlp` pipeline. Each entry is itself a dictionary, with two +keys: `prompt` and `response`. + +A note on `validate_types`: by default, `spacy-llm` checks whether the +signatures of the `model` and `task` callables are consistent with each other +and emits a warning if they don't. `validate_types` can be set to `False` if you +want to disable this behavior. + +### Tasks {id="tasks"} + +A _task_ defines an NLP problem or question, that will be sent to the LLM via a +prompt. Further, the task defines how to parse the LLM's responses back into +structured information. All tasks are registered in the `llm_tasks` registry. + +Practically speaking, a task should adhere to the `Protocol` named `LLMTask` +defined in +[`ty.py`](https://github.com/explosion/spacy-llm/blob/main/spacy_llm/ty.py). It +needs to define a `generate_prompts` function and a `parse_responses` function. + +Tasks may support prompt sharding (for more info see the API docs on +[sharding](/api/large-language-models#task-sharding) and +[non-sharding](/api/large-language-models#task-nonsharding) tasks). The function +signatures for `generate_prompts` and `parse_responses` depend on whether they +do. + +For tasks **not supporting** sharding: + +| Task | Description | | +| --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | --- | +| [`task.generate_prompts`](/api/large-language-models#task-nonsharding-generate-prompts) | Takes a collection of documents, and returns a collection of prompts, which can be of type `Any`. | +| [`task.parse_responses`](/api/large-language-models#task-nonsharding-parse-responses) | Takes a collection of LLM responses and the original documents, parses the responses into structured information, and sets the annotations on the documents. | + +For tasks **supporting** sharding: + +| Task | Description | | +| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --- | +| [`task.generate_prompts`](/api/large-language-models#task-sharding-generate-prompts) | Takes a collection of documents, and returns a collection of collections of prompt shards, which can be of type `Any`. | +| [`task.parse_responses`](/api/large-language-models#task-sharding-parse-responses) | Takes a collection of collections of LLM responses (one per prompt shard) and the original documents, parses the responses into structured information, sets the annotations on the doc shards, and merges those doc shards back into a single doc instance. | + +Moreover, the task may define an optional [`scorer` method](/api/scorer#score). +It should accept an iterable of `Example` objects as input and return a score +dictionary. If the `scorer` method is defined, `spacy-llm` will call it to +evaluate the component. + +| Component | Description | +| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| [`spacy.EntityLinker.v1`](/api/large-language-models#el-v1) | The entity linking task prompts the model to link all entities in a given text to entries in a knowledge base. | +| [`spacy.Summarization.v1`](/api/large-language-models#summarization-v1) | The summarization task prompts the model for a concise summary of the provided text. | +| [`spacy.NER.v3`](/api/large-language-models#ner-v3) | Implements Chain-of-Thought reasoning for NER extraction - obtains higher accuracy than v1 or v2. | +| [`spacy.NER.v2`](/api/large-language-models#ner-v2) | Builds on v1 and additionally supports defining the provided labels with explicit descriptions. | +| [`spacy.NER.v1`](/api/large-language-models#ner-v1) | The original version of the built-in NER task supports both zero-shot and few-shot prompting. | +| [`spacy.SpanCat.v3`](/api/large-language-models#spancat-v3) | Adaptation of the v3 NER task to support overlapping entities and store its annotations in `doc.spans`. | +| [`spacy.SpanCat.v2`](/api/large-language-models#spancat-v2) | Adaptation of the v2 NER task to support overlapping entities and store its annotations in `doc.spans`. | +| [`spacy.SpanCat.v1`](/api/large-language-models#spancat-v1) | Adaptation of the v1 NER task to support overlapping entities and store its annotations in `doc.spans`. | +| [`spacy.REL.v1`](/api/large-language-models#rel-v1) | Relation Extraction task supporting both zero-shot and few-shot prompting. | +| [`spacy.TextCat.v3`](/api/large-language-models#textcat-v3) | Version 3 builds on v2 and allows setting definitions of labels. | +| [`spacy.TextCat.v2`](/api/large-language-models#textcat-v2) | Version 2 builds on v1 and includes an improved prompt template. | +| [`spacy.TextCat.v1`](/api/large-language-models#textcat-v1) | Version 1 of the built-in TextCat task supports both zero-shot and few-shot prompting. | +| [`spacy.Lemma.v1`](/api/large-language-models#lemma-v1) | Lemmatizes the provided text and updates the `lemma_` attribute of the tokens accordingly. | +| [`spacy.Raw.v1`](/api/large-language-models#raw-v1) | Executes raw doc content as prompt to LLM. | +| [`spacy.Sentiment.v1`](/api/large-language-models#sentiment-v1) | Performs sentiment analysis on provided texts. | +| [`spacy.Translation.v1`](/api/large-language-models#translation-v1) | Translates doc content into the specified target language. | +| [`spacy.NoOp.v1`](/api/large-language-models#noop-v1) | This task is only useful for testing - it tells the LLM to do nothing, and does not set any fields on the `docs`. | + +#### Providing examples for few-shot prompts {id="few-shot-prompts"} + +All built-in tasks support few-shot prompts, i. e. including examples in a +prompt. Examples can be supplied in two ways: (1) as a separate file containing +only examples or (2) by initializing `llm` with a `get_examples()` callback +(like any other pipeline component). + +##### (1) Few-shot example file + +A file containing examples for few-shot prompting can be configured like this: + +```ini +[components.llm.task] +@llm_tasks = "spacy.NER.v2" +labels = PERSON,ORGANISATION,LOCATION +[components.llm.task.examples] +@misc = "spacy.FewShotReader.v1" +path = "ner_examples.yml" +``` + +The supplied file has to conform to the format expected by the required task +(see the task documentation further down). + +##### (2) Initializing the `llm` component with a `get_examples()` callback + +Alternatively, you can initialize your `nlp` pipeline by providing a +`get_examples` callback for [`nlp.initialize`](/api/language#initialize) and +setting `n_prompt_examples` to a positive number to automatically fetch a few +examples for few-shot learning. Set `n_prompt_examples` to `-1` to use all +examples as part of the few-shot learning prompt. + +```ini +[initialize.components.llm] +n_prompt_examples = 3 +``` + +### Model {id="models"} + +A _model_ defines which LLM model to query, and how to query it. It can be a +simple function taking a collection of prompts (consistent with the output type +of `task.generate_prompts()`) and returning a collection of responses +(consistent with the expected input of `parse_responses`). Generally speaking, +it's a function of type `Callable[[Iterable[Any]], Iterable[Any]]`, but specific +implementations can have other signatures, like +`Callable[[Iterable[str]], Iterable[str]]`. + +All built-in models are registered in `llm_models`. If no model is specified, +the repo currently connects to the `OpenAI` API by default using REST, and +accesses the `"gpt-3.5-turbo"` model. + +Currently three different approaches to use LLMs are supported: + +1. `spacy-llm`s native REST interface. This is the default for all hosted models + (e. g. OpenAI, Cohere, Anthropic, ...). +2. A HuggingFace integration that allows to run a limited set of HF models + locally. +3. A LangChain integration that allows to run any model supported by LangChain + (hosted or locally). + +Approaches 1. and 2 are the default for hosted model and local models, +respectively. Alternatively you can use LangChain to access hosted or local +models by specifying one of the models registered with the `langchain.` prefix. + + +_Why LangChain if there are also are native REST and HuggingFace interfaces? When should I use what?_ + +Third-party libraries like `langchain` focus on prompt management, integration +of many different LLM APIs, and other related features such as conversational +memory or agents. `spacy-llm` on the other hand emphasizes features we consider +useful in the context of NLP pipelines utilizing LLMs to process documents +(mostly) independent from each other. It makes sense that the feature sets of +such third-party libraries and `spacy-llm` aren't identical - and users might +want to take advantage of features not available in `spacy-llm`. + +The advantage of implementing our own REST and HuggingFace integrations is that +we can ensure a larger degree of stability and robustness, as we can guarantee +backwards-compatibility and more smoothly integrated error handling. + +If however there are features or APIs not natively covered by `spacy-llm`, it's +trivial to utilize LangChain to cover this - and easy to customize the prompting +mechanism, if so required. + + + + +Note that when using hosted services, you have to ensure that the [proper API +keys](/api/large-language-models#api-keys) are set as environment variables as described by the corresponding +provider's documentation. + + + +| Model | Description | +| ----------------------------------------------------------------------- | ---------------------------------------------- | +| [`spacy.GPT-4.v2`](/api/large-language-models#models-rest) | OpenAI’s `gpt-4` model family. | +| [`spacy.GPT-3-5.v2`](/api/large-language-models#models-rest) | OpenAI’s `gpt-3-5` model family. | +| [`spacy.Text-Davinci.v2`](/api/large-language-models#models-rest) | OpenAI’s `text-davinci` model family. | +| [`spacy.Code-Davinci.v2`](/api/large-language-models#models-rest) | OpenAI’s `code-davinci` model family. | +| [`spacy.Text-Curie.v2`](/api/large-language-models#models-rest) | OpenAI’s `text-curie` model family. | +| [`spacy.Text-Babbage.v2`](/api/large-language-models#models-rest) | OpenAI’s `text-babbage` model family. | +| [`spacy.Text-Ada.v2`](/api/large-language-models#models-rest) | OpenAI’s `text-ada` model family. | +| [`spacy.Davinci.v2`](/api/large-language-models#models-rest) | OpenAI’s `davinci` model family. | +| [`spacy.Curie.v2`](/api/large-language-models#models-rest) | OpenAI’s `curie` model family. | +| [`spacy.Babbage.v2`](/api/large-language-models#models-rest) | OpenAI’s `babbage` model family. | +| [`spacy.Ada.v2`](/api/large-language-models#models-rest) | OpenAI’s `ada` model family. | +| [`spacy.Azure.v1`](/api/large-language-models#models-rest) | Azure's OpenAI models. | +| [`spacy.Command.v1`](/api/large-language-models#models-rest) | Cohere’s `command` model family. | +| [`spacy.Claude-2.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-2` model family. | +| [`spacy.Claude-1.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-1` model family. | +| [`spacy.Claude-instant-1.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-instant-1` model family. | +| [`spacy.Claude-instant-1-1.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-instant-1.1` model family. | +| [`spacy.Claude-1-0.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-1.0` model family. | +| [`spacy.Claude-1-2.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-1.2` model family. | +| [`spacy.Claude-1-3.v1`](/api/large-language-models#models-rest) | Anthropic’s `claude-1.3` model family. | +| [`spacy.PaLM.v1`](/api/large-language-models#models-rest) | Google’s `PaLM` model family. | +| [`spacy.Dolly.v1`](/api/large-language-models#models-hf) | Dolly models through HuggingFace. | +| [`spacy.Falcon.v1`](/api/large-language-models#models-hf) | Falcon models through HuggingFace. | +| [`spacy.Mistral.v1`](/api/large-language-models#models-hf) | Mistral models through HuggingFace. | +| [`spacy.Llama2.v1`](/api/large-language-models#models-hf) | Llama2 models through HuggingFace. | +| [`spacy.StableLM.v1`](/api/large-language-models#models-hf) | StableLM models through HuggingFace. | +| [`spacy.OpenLLaMA.v1`](/api/large-language-models#models-hf) | OpenLLaMA models through HuggingFace. | +| [LangChain models](/api/large-language-models#langchain-models) | LangChain models for API retrieval. | + +Note that the chat models variants of Llama 2 are currently not supported. This +is because they need a particular prompting setup and don't add any discernible +benefits in the use case of `spacy-llm` (i. e. no interactive chat) compared to +the completion model variants. + +### Cache {id="cache"} + +Interacting with LLMs, either through an external API or a local instance, is +costly. Since developing an NLP pipeline generally means a lot of exploration +and prototyping, `spacy-llm` implements a built-in +[cache](/api/large-language-models#cache) to avoid reprocessing the same +documents at each run that keeps batches of documents stored on disk. + +### Various functions {id="various-functions"} + +| Function | Description | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`spacy.FewShotReader.v1`](/api/large-language-models#fewshotreader-v1) | This function is registered in spaCy's `misc` registry, and reads in examples from a `.yml`, `.yaml`, `.json` or `.jsonl` file. It uses [`srsly`](https://github.com/explosion/srsly) to read in these files and parses them depending on the file extension. | +| [`spacy.FileReader.v1`](/api/large-language-models#filereader-v1) | This function is registered in spaCy's `misc` registry, and reads a file provided to the `path` to return a `str` representation of its contents. This function is typically used to read [Jinja](https://jinja.palletsprojects.com/en/3.1.x/) files containing the prompt template. | +| [Normalizer functions](/api/large-language-models#normalizer-functions) | These functions provide simple normalizations for string comparisons, e.g. between a list of specified labels and a label given in the raw text of the LLM response. | diff --git a/website/docs/usage/layers-architectures.mdx b/website/docs/usage/layers-architectures.mdx new file mode 100644 index 0000000..344c66e --- /dev/null +++ b/website/docs/usage/layers-architectures.mdx @@ -0,0 +1,1032 @@ +--- +title: Layers and Model Architectures +teaser: Power spaCy components with custom neural networks +menu: + - ['Type Signatures', 'type-sigs'] + - ['Swapping Architectures', 'swap-architectures'] + - ['PyTorch & TensorFlow', 'frameworks'] + - ['Custom Thinc Models', 'thinc'] + - ['Trainable Components', 'components'] +next: /usage/projects +--- + +> #### Example +> +> ```python +> from thinc.api import Model, chain +> +> @spacy.registry.architectures("model.v1") +> def build_model(width: int, classes: int) -> Model: +> tok2vec = build_tok2vec(width) +> output_layer = build_output_layer(width, classes) +> model = chain(tok2vec, output_layer) +> return model +> ``` + +A **model architecture** is a function that wires up a +[Thinc `Model`](https://thinc.ai/docs/api-model) instance. It describes the +neural network that is run internally as part of a component in a spaCy +pipeline. To define the actual architecture, you can implement your logic in +Thinc directly, or you can use Thinc as a thin wrapper around frameworks such as +PyTorch, TensorFlow and MXNet. Each `Model` can also be used as a sublayer of a +larger network, allowing you to freely combine implementations from different +frameworks into a single model. + +spaCy's built-in components require a `Model` instance to be passed to them via +the config system. To change the model architecture of an existing component, +you just need to [**update the config**](#swap-architectures) so that it refers +to a different registered function. Once the component has been created from +this config, you won't be able to change it anymore. The architecture is like a +recipe for the network, and you can't change the recipe once the dish has +already been prepared. You have to make a new one. + +```ini {title="config.cfg (excerpt)"} +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "model.v1" +width = 512 +classes = 16 +``` + +## Type signatures {id="type-sigs"} + +> #### Example +> +> ```python +> from typing import List +> from thinc.api import Model, chain +> from thinc.types import Floats2d +> def chain_model( +> tok2vec: Model[List[Doc], List[Floats2d]], +> layer1: Model[List[Floats2d], Floats2d], +> layer2: Model[Floats2d, Floats2d] +> ) -> Model[List[Doc], Floats2d]: +> model = chain(tok2vec, layer1, layer2) +> return model +> ``` + +The Thinc `Model` class is a **generic type** that can specify its input and +output types. Python uses a square-bracket notation for this, so the type +~~Model[List, Dict]~~ says that each batch of inputs to the model will be a +list, and the outputs will be a dictionary. You can be even more specific and +write for instance~~Model[List[Doc], Dict[str, float]]~~ to specify that the +model expects a list of [`Doc`](/api/doc) objects as input, and returns a +dictionary mapping of strings to floats. Some of the most common types you'll +see are: ​ + +| Type | Description | +| ------------------ | ---------------------------------------------------------------------------------------------------- | +| ~~List[Doc]~~ | A batch of [`Doc`](/api/doc) objects. Most components expect their models to take this as input. | +| ~~Floats2d~~ | A two-dimensional `numpy` or `cupy` array of floats. Usually 32-bit. | +| ~~Ints2d~~ | A two-dimensional `numpy` or `cupy` array of integers. Common dtypes include uint64, int32 and int8. | +| ~~List[Floats2d]~~ | A list of two-dimensional arrays, generally with one array per `Doc` and one row per token. | +| ~~Ragged~~ | A container to handle variable-length sequence data in an unpadded contiguous array. | +| ~~Padded~~ | A container to handle variable-length sequence data in a padded contiguous array. | + +See the [Thinc type reference](https://thinc.ai/docs/api-types) for details. The +model type signatures help you figure out which model architectures and +components can **fit together**. For instance, the +[`TextCategorizer`](/api/textcategorizer) class expects a model typed +~~Model[List[Doc], Floats2d]~~, because the model will predict one row of +category probabilities per [`Doc`](/api/doc). In contrast, the +[`Tagger`](/api/tagger) class expects a model typed ~~Model[List[Doc], +List[Floats2d]]~~, because it needs to predict one row of probabilities per +token. + +There's no guarantee that two models with the same type signature can be used +interchangeably. There are many other ways they could be incompatible. However, +if the types don't match, they almost surely _won't_ be compatible. This little +bit of validation goes a long way, especially if you +[configure your editor](https://thinc.ai/docs/usage-type-checking) or other +tools to highlight these errors early. The config file is also validated at the +beginning of training, to verify that all the types match correctly. + + + +If you're using a modern editor like Visual Studio Code, you can +[set up `mypy`](https://thinc.ai/docs/usage-type-checking#install) with the +custom Thinc plugin and get live feedback about mismatched types as you write +code. + +Screenshot of Thinc type checking in VSCode with mypy + + + +## Swapping model architectures {id="swap-architectures"} + +If no model is specified for the [`TextCategorizer`](/api/textcategorizer), the +[TextCatEnsemble](/api/architectures#TextCatEnsemble) architecture is used by +default. This architecture combines a simple bag-of-words model with a neural +network, usually resulting in the most accurate results, but at the cost of +speed. The config file for this model would look something like this: + +```ini {title="config.cfg (excerpt)"} +[components.textcat] +factory = "textcat" +labels = [] + +[components.textcat.model] +@architectures = "spacy.TextCatEnsemble.v2" +nO = null + +[components.textcat.model.tok2vec] +@architectures = "spacy.Tok2Vec.v2" + +[components.textcat.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v2" +width = 64 +rows = [2000, 2000, 1000, 1000, 1000, 1000] +attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] +include_static_vectors = false + +[components.textcat.model.tok2vec.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +width = ${components.textcat.model.tok2vec.embed.width} +window_size = 1 +maxout_pieces = 3 +depth = 2 + +[components.textcat.model.linear_model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false +``` + +spaCy has two additional built-in `textcat` architectures, and you can easily +use those by swapping out the definition of the textcat's model. For instance, +to use the simple and fast bag-of-words model +[TextCatBOW](/api/architectures#TextCatBOW), you can change the config to: + +```ini {title="config.cfg (excerpt)",highlight="6-10"} +[components.textcat] +factory = "textcat" +labels = [] + +[components.textcat.model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false +nO = null +``` + +For details on all pre-defined architectures shipped with spaCy and how to +configure them, check out the [model architectures](/api/architectures) +documentation. + +### Defining sublayers {id="sublayers"} + +Model architecture functions often accept **sublayers as arguments**, so that +you can try **substituting a different layer** into the network. Depending on +how the architecture function is structured, you might be able to define your +network structure entirely through the [config system](/usage/training#config), +using layers that have already been defined. ​ + +In most neural network models for NLP, the most important parts of the network +are what we refer to as the +[embed and encode](https://explosion.ai/blog/deep-learning-formula-nlp) steps. +These steps together compute dense, context-sensitive representations of the +tokens, and their combination forms a typical +[`Tok2Vec`](/api/architectures#Tok2Vec) layer: + +```ini {title="config.cfg (excerpt)"} +[components.tok2vec] +factory = "tok2vec" + +[components.tok2vec.model] +@architectures = "spacy.Tok2Vec.v2" + +[components.tok2vec.model.embed] +@architectures = "spacy.MultiHashEmbed.v2" +# ... + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +# ... +``` + +By defining these sublayers specifically, it becomes straightforward to swap out +a sublayer for another one, for instance changing the first sublayer to a +character embedding with the [CharacterEmbed](/api/architectures#CharacterEmbed) +architecture: + +```ini {title="config.cfg (excerpt)"} +[components.tok2vec.model.embed] +@architectures = "spacy.CharacterEmbed.v2" +# ... + +[components.tok2vec.model.encode] +@architectures = "spacy.MaxoutWindowEncoder.v2" +# ... +``` + +Most of spaCy's default architectures accept a `tok2vec` layer as a sublayer +within the larger task-specific neural network. This makes it easy to **switch +between** transformer, CNN, BiLSTM or other feature extraction approaches. The +[transformers documentation](/usage/embeddings-transformers#training-custom-model) +section shows an example of swapping out a model's standard `tok2vec` layer with +a transformer. And if you want to define your own solution, all you need to do +is register a ~~Model[List[Doc], List[Floats2d]]~~ architecture function, and +you'll be able to try it out in any of the spaCy components. ​ + +## Wrapping PyTorch, TensorFlow and other frameworks {id="frameworks"} + +Thinc allows you to [wrap models](https://thinc.ai/docs/usage-frameworks) +written in other machine learning frameworks like PyTorch, TensorFlow and MXNet +using a unified [`Model`](https://thinc.ai/docs/api-model) API. This makes it +easy to use a model implemented in a different framework to power a component in +your spaCy pipeline. For example, to wrap a PyTorch model as a Thinc `Model`, +you can use Thinc's +[`PyTorchWrapper`](https://thinc.ai/docs/api-layers#pytorchwrapper): + +```python +from thinc.api import PyTorchWrapper + +wrapped_pt_model = PyTorchWrapper(torch_model) +``` + +Let's use PyTorch to define a very simple neural network consisting of two +hidden `Linear` layers with `ReLU` activation and dropout, and a +softmax-activated output layer: + +```python {title="PyTorch model"} +from torch import nn + +torch_model = nn.Sequential( + nn.Linear(width, hidden_width), + nn.ReLU(), + nn.Dropout2d(dropout), + nn.Linear(hidden_width, nO), + nn.ReLU(), + nn.Dropout2d(dropout), + nn.Softmax(dim=1) +) +``` + +The resulting wrapped `Model` can be used as a **custom architecture** as such, +or can be a **subcomponent of a larger model**. For instance, we can use Thinc's +[`chain`](https://thinc.ai/docs/api-layers#chain) combinator, which works like +`Sequential` in PyTorch, to combine the wrapped model with other components in a +larger network. This effectively means that you can easily wrap different +components from different frameworks, and "glue" them together with Thinc: + +```python +from thinc.api import chain, with_array, PyTorchWrapper +from spacy.ml import CharacterEmbed + +wrapped_pt_model = PyTorchWrapper(torch_model) +char_embed = CharacterEmbed(width, embed_size, nM, nC) +model = chain(char_embed, with_array(wrapped_pt_model)) +``` + +In the above example, we have combined our custom PyTorch model with a character +embedding layer defined by spaCy. +[CharacterEmbed](/api/architectures#CharacterEmbed) returns a `Model` that takes +a ~~List[Doc]~~ as input, and outputs a ~~List[Floats2d]~~. To make sure that +the wrapped PyTorch model receives valid inputs, we use Thinc's +[`with_array`](https://thinc.ai/docs/api-layers#with_array) helper. + +You could also implement a model that only uses PyTorch for the transformer +layers, and "native" Thinc layers to do fiddly input and output transformations +and add on task-specific "heads", as efficiency is less of a consideration for +those parts of the network. + +### Using wrapped models {id="frameworks-usage"} + +To use our custom model including the PyTorch subnetwork, all we need to do is +register the architecture using the +[`architectures` registry](/api/top-level#registry). This assigns the +architecture a name so spaCy knows how to find it, and allows passing in +arguments like hyperparameters via the [config](/usage/training#config). The +full example then becomes: + +```python {title="Registering the architecture",highlight="9"} +from typing import List +from thinc.types import Floats2d +from thinc.api import Model, PyTorchWrapper, chain, with_array +import spacy +from spacy.tokens.doc import Doc +from spacy.ml import CharacterEmbed +from torch import nn + +@spacy.registry.architectures("CustomTorchModel.v1") +def create_torch_model( + nO: int, + width: int, + hidden_width: int, + embed_size: int, + nM: int, + nC: int, + dropout: float, +) -> Model[List[Doc], List[Floats2d]]: + char_embed = CharacterEmbed(width, embed_size, nM, nC) + torch_model = nn.Sequential( + nn.Linear(width, hidden_width), + nn.ReLU(), + nn.Dropout2d(dropout), + nn.Linear(hidden_width, nO), + nn.ReLU(), + nn.Dropout2d(dropout), + nn.Softmax(dim=1) + ) + wrapped_pt_model = PyTorchWrapper(torch_model) + model = chain(char_embed, with_array(wrapped_pt_model)) + return model +``` + +The model definition can now be used in any existing trainable spaCy component, +by specifying it in the config file. In this configuration, all required +parameters for the various subcomponents of the custom architecture are passed +in as settings via the config. + +```ini {title="config.cfg (excerpt)",highlight="5-5"} +[components.tagger] +factory = "tagger" + +[components.tagger.model] +@architectures = "CustomTorchModel.v1" +nO = 50 +width = 96 +hidden_width = 48 +embed_size = 2000 +nM = 64 +nC = 8 +dropout = 0.2 +``` + + + +Remember that it is best not to rely on any (hidden) default values to ensure +that training configs are complete and experiments fully reproducible. + + + +Note that when using a PyTorch or Tensorflow model, it is recommended to set the +GPU memory allocator accordingly. When `gpu_allocator` is set to "pytorch" or +"tensorflow" in the training config, cupy will allocate memory via those +respective libraries, preventing OOM errors when there's available memory +sitting in the other library's pool. + +```ini {title="config.cfg (excerpt)"} +[training] +gpu_allocator = "pytorch" +``` + +## Custom models with Thinc {id="thinc"} + +Of course it's also possible to define the `Model` from the previous section +entirely in Thinc. The Thinc documentation provides details on the +[various layers](https://thinc.ai/docs/api-layers) and helper functions +available. Combinators can be used to +[overload operators](https://thinc.ai/docs/usage-models#operators) and a common +usage pattern is to bind `chain` to `>>`. The "native" Thinc version of our +simple neural network would then become: + +```python +from thinc.api import chain, with_array, Model, Relu, Dropout, Softmax +from spacy.ml import CharacterEmbed + +char_embed = CharacterEmbed(width, embed_size, nM, nC) +with Model.define_operators({">>": chain}): + layers = ( + Relu(hidden_width, width) + >> Dropout(dropout) + >> Relu(hidden_width, hidden_width) + >> Dropout(dropout) + >> Softmax(nO, hidden_width) + ) + model = char_embed >> with_array(layers) +``` + + + +Note that Thinc layers define the output dimension (`nO`) as the first argument, +followed (optionally) by the input dimension (`nI`). This is in contrast to how +the PyTorch layers are defined, where `in_features` precedes `out_features`. + + + +### Shape inference in Thinc {id="thinc-shape-inference"} + +It is **not** strictly necessary to define all the input and output dimensions +for each layer, as Thinc can perform +[shape inference](https://thinc.ai/docs/usage-models#validation) between +sequential layers by matching up the output dimensionality of one layer to the +input dimensionality of the next. This means that we can simplify the `layers` +definition: + +> #### Diff +> +> ```diff +> layers = ( +> Relu(hidden_width, width) +> >> Dropout(dropout) +> - >> Relu(hidden_width, hidden_width) +> + >> Relu(hidden_width) +> >> Dropout(dropout) +> - >> Softmax(nO, hidden_width) +> + >> Softmax(nO) +> ) +> ``` + +```python +with Model.define_operators({">>": chain}): + layers = ( + Relu(hidden_width, width) + >> Dropout(dropout) + >> Relu(hidden_width) + >> Dropout(dropout) + >> Softmax(nO) + ) +``` + +Thinc can even go one step further and **deduce the correct input dimension** of +the first layer, and output dimension of the last. To enable this functionality, +you have to call +[`Model.initialize`](https://thinc.ai/docs/api-model#initialize) with an **input +sample** `X` and an **output sample** `Y` with the correct dimensions: + +```python {title="Shape inference with initialization",highlight="3,7,10"} +with Model.define_operators({">>": chain}): + layers = ( + Relu(hidden_width) + >> Dropout(dropout) + >> Relu(hidden_width) + >> Dropout(dropout) + >> Softmax() + ) + model = char_embed >> with_array(layers) + model.initialize(X=input_sample, Y=output_sample) +``` + +The built-in [pipeline components](/usage/processing-pipelines) in spaCy ensure +that their internal models are **always initialized** with appropriate sample +data. In this case, `X` is typically a ~~List[Doc]~~, while `Y` is typically a +~~List[Array1d]~~ or ~~List[Array2d]~~, depending on the specific task. This +functionality is triggered when [`nlp.initialize`](/api/language#initialize) is +called. + +### Dropout and normalization in Thinc {id="thinc-dropout-norm"} + +Many of the available Thinc [layers](https://thinc.ai/docs/api-layers) allow you +to define a `dropout` argument that will result in "chaining" an additional +[`Dropout`](https://thinc.ai/docs/api-layers#dropout) layer. Optionally, you can +often specify whether or not you want to add layer normalization, which would +result in an additional +[`LayerNorm`](https://thinc.ai/docs/api-layers#layernorm) layer. That means that +the following `layers` definition is equivalent to the previous: + +```python +with Model.define_operators({">>": chain}): + layers = ( + Relu(hidden_width, dropout=dropout, normalize=False) + >> Relu(hidden_width, dropout=dropout, normalize=False) + >> Softmax() + ) + model = char_embed >> with_array(layers) + model.initialize(X=input_sample, Y=output_sample) +``` + +## Create new trainable components {id="components"} + +In addition to [swapping out](#swap-architectures) layers in existing +components, you can also implement an entirely new, +[trainable](/usage/processing-pipelines#trainable-components) pipeline component +from scratch. This can be done by creating a new class inheriting from +[`TrainablePipe`](/api/pipe), and linking it up to your custom model +implementation. + + + +For details on how to implement pipeline components, check out the usage guide +on [custom components](/usage/processing-pipelines#custom-component) and the +overview of the `TrainablePipe` methods used by +[trainable components](/usage/processing-pipelines#trainable-components). + + + +### Example: Entity relation extraction component {id="component-rel"} + +This section outlines an example use-case of implementing a **novel relation +extraction component** from scratch. We'll implement a binary relation +extraction method that determines whether or not **two entities** in a document +are related, and if so, what type of relation connects them. We allow multiple +types of relations between two such entities (a multi-label setting). There are +two major steps required: + +1. Implement a [machine learning model](#component-rel-model) specific to this + task. It will have to extract candidate relation instances from a + [`Doc`](/api/doc) and predict the corresponding scores for each relation + label. +2. Implement a custom [pipeline component](#component-rel-pipe) - powered by the + machine learning model from step 1 - that translates the predicted scores + into annotations that are stored on the [`Doc`](/api/doc) objects as they + pass through the `nlp` pipeline. + + + Run this example use-case by using our project template. It includes all the + code to create the ML model and the pipeline component from scratch. It also + contains two config files to train the model: one to run on CPU with a Tok2Vec + layer, and one for the GPU using a transformer. The project applies the + relation extraction component to identify biomolecular interactions in a + sample dataset, but you can easily swap in your own dataset for your + experiments in any other domain. + + + + +#### Step 1: Implementing the Model {id="component-rel-model"} + +We need to implement a [`Model`](https://thinc.ai/docs/api-model) that takes a +**list of documents** (~~List[Doc]~~) as input, and outputs a **two-dimensional +matrix** (~~Floats2d~~) of predictions: + +> #### Model type annotations +> +> The `Model` class is a generic type that can specify its input and output +> types, e.g. ~~Model[List[Doc], Floats2d]~~. Type hints are used for static +> type checks and validation. See the section on [type signatures](#type-sigs) +> for details. + +```python {title="The model architecture"} +@spacy.registry.architectures("rel_model.v1") +def create_relation_model(...) -> Model[List[Doc], Floats2d]: + model = ... # 👈 model will go here + return model +``` + +We adapt a **modular approach** to the definition of this relation model, and +define it as chaining two layers together: the first layer that generates an +instance tensor from a given set of documents, and the second layer that +transforms the instance tensor into a final tensor holding the predictions: + +> #### config.cfg (excerpt) +> +> ```ini +> [model] +> @architectures = "rel_model.v1" +> +> [model.create_instance_tensor] +> # ... +> +> [model.classification_layer] +> # ... +> ``` + +```python {title="The model architecture",highlight="6"} +@spacy.registry.architectures("rel_model.v1") +def create_relation_model( + create_instance_tensor: Model[List[Doc], Floats2d], + classification_layer: Model[Floats2d, Floats2d], +) -> Model[List[Doc], Floats2d]: + model = chain(create_instance_tensor, classification_layer) + return model +``` + +The `classification_layer` could be something like a +[Linear](https://thinc.ai/docs/api-layers#linear) layer followed by a +[logistic](https://thinc.ai/docs/api-layers#logistic) activation function: + +> #### config.cfg (excerpt) +> +> ```ini +> [model.classification_layer] +> @architectures = "rel_classification_layer.v1" +> nI = null +> nO = null +> ``` + +```python {title="The classification layer"} +@spacy.registry.architectures("rel_classification_layer.v1") +def create_classification_layer( + nO: int = None, nI: int = None +) -> Model[Floats2d, Floats2d]: + return chain(Linear(nO=nO, nI=nI), Logistic()) +``` + +The first layer that **creates the instance tensor** can be defined by +implementing a +[custom forward function](https://thinc.ai/docs/usage-models#weights-layers-forward) +with an appropriate backpropagation callback. We also define an +[initialization method](https://thinc.ai/docs/usage-models#weights-layers-init) +that ensures that the layer is properly set up for training. + +We omit some of the implementation details here, and refer to the +[spaCy project](https://github.com/explosion/projects/tree/v3/tutorials/rel_component) +that has the full implementation. + +> #### config.cfg (excerpt) +> +> ```ini +> [model.create_instance_tensor] +> @architectures = "rel_instance_tensor.v1" +> +> [model.create_instance_tensor.tok2vec] +> @architectures = "spacy.HashEmbedCNN.v2" +> # ... +> +> [model.create_instance_tensor.pooling] +> @layers = "reduce_mean.v1" +> +> [model.create_instance_tensor.get_instances] +> # ... +> ``` + +```python {title="The layer that creates the instance tensor"} +@spacy.registry.architectures("rel_instance_tensor.v1") +def create_tensors( + tok2vec: Model[List[Doc], List[Floats2d]], + pooling: Model[Ragged, Floats2d], + get_instances: Callable[[Doc], List[Tuple[Span, Span]]], +) -> Model[List[Doc], Floats2d]: + + return Model( + "instance_tensors", + instance_forward, + init=instance_init, + layers=[tok2vec, pooling], + refs={"tok2vec": tok2vec, "pooling": pooling}, + attrs={"get_instances": get_instances}, + ) + + +# The custom forward function +def instance_forward( + model: Model[List[Doc], Floats2d], + docs: List[Doc], + is_train: bool, +) -> Tuple[Floats2d, Callable]: + tok2vec = model.get_ref("tok2vec") + tokvecs, bp_tokvecs = tok2vec(docs, is_train) + get_instances = model.attrs["get_instances"] + all_instances = [get_instances(doc) for doc in docs] + pooling = model.get_ref("pooling") + relations = ... + + def backprop(d_relations: Floats2d) -> List[Doc]: + d_tokvecs = ... + return bp_tokvecs(d_tokvecs) + + return relations, backprop + + +# The custom initialization method +def instance_init( + model: Model, + X: List[Doc] = None, + Y: Floats2d = None, +) -> Model: + tok2vec = model.get_ref("tok2vec") + tok2vec.initialize(X) + return model + +``` + +This custom layer uses an [embedding layer](/usage/embeddings-transformers) such +as a [`Tok2Vec`](/api/tok2vec) component or a [`Transformer`](/api/transformer). +This layer is assumed to be of type ~~Model[List[Doc], List[Floats2d]]~~ as it +transforms each **document into a list of tokens**, with each token being +represented by its embedding in the vector space. + +The `pooling` layer will be applied to summarize the token vectors into **entity +vectors**, as named entities (represented by ~~Span~~ objects) can consist of +one or multiple tokens. For instance, the pooling layer could resort to +calculating the average of all token vectors in an entity. Thinc provides +several +[built-in pooling operators](https://thinc.ai/docs/api-layers#reduction-ops) for +this purpose. + +Finally, we need a `get_instances` method that **generates pairs of entities** +that we want to classify as being related or not. As these candidate pairs are +typically formed within one document, this function takes a [`Doc`](/api/doc) as +input and outputs a `List` of `Span` tuples. For instance, the following +implementation takes any two entities from the same document, as long as they +are within a **maximum distance** (in number of tokens) of each other: + +> #### config.cfg (excerpt) +> +> ```ini +> +> [model.create_instance_tensor.get_instances] +> @misc = "rel_instance_generator.v1" +> max_length = 100 +> ``` + +```python {title="Candidate generation"} +@spacy.registry.misc("rel_instance_generator.v1") +def create_instances(max_length: int) -> Callable[[Doc], List[Tuple[Span, Span]]]: + def get_candidates(doc: "Doc") -> List[Tuple[Span, Span]]: + candidates = [] + for ent1 in doc.ents: + for ent2 in doc.ents: + if ent1 != ent2: + if max_length and abs(ent2.start - ent1.start) <= max_length: + candidates.append((ent1, ent2)) + return candidates + return get_candidates +``` + +This function is added to the [`@misc` registry](/api/top-level#registry) so we +can refer to it from the config, and easily swap it out for any other candidate +generation function. + +#### Intermezzo: define how to store the relations data {id="component-rel-attribute"} + +> #### Example output +> +> ```python +> doc = nlp("Amsterdam is the capital of the Netherlands.") +> print("spans", [(e.start, e.text, e.label_) for e in doc.ents]) +> for value, rel_dict in doc._.rel.items(): +> print(f"{value}: {rel_dict}") +> +> # spans [(0, 'Amsterdam', 'LOC'), (6, 'Netherlands', 'LOC')] +> # (0, 6): {'CAPITAL_OF': 0.89, 'LOCATED_IN': 0.75, 'UNRELATED': 0.002} +> # (6, 0): {'CAPITAL_OF': 0.01, 'LOCATED_IN': 0.13, 'UNRELATED': 0.017} +> ``` + +For our new relation extraction component, we will use a custom +[extension attribute](/usage/processing-pipelines#custom-components-attributes) +`doc._.rel` in which we store relation data. The attribute refers to a +dictionary, keyed by the **start offsets of each entity** involved in the +candidate relation. The values in the dictionary refer to another dictionary +where relation labels are mapped to values between 0 and 1. We assume anything +above 0.5 to be a `True` relation. The ~~Example~~ instances that we'll use as +training data, will include their gold-standard relation annotations in +`example.reference._.rel`. + +```python {title="Registering the extension attribute"} +from spacy.tokens import Doc +Doc.set_extension("rel", default={}) +``` + +#### Step 2: Implementing the pipeline component {id="component-rel-pipe"} + +To use our new relation extraction model as part of a custom +[trainable component](/usage/processing-pipelines#trainable-components), we +create a subclass of [`TrainablePipe`](/api/pipe) that holds the model. + +![Illustration of Pipe methods](/images/trainable_component.svg) + +```python {title="Pipeline component skeleton"} +from spacy.pipeline import TrainablePipe + +class RelationExtractor(TrainablePipe): + def __init__(self, vocab, model, name="rel"): + """Create a component instance.""" + self.model = model + self.vocab = vocab + self.name = name + + def update(self, examples, drop=0.0, sgd=None, losses=None): + """Learn from a batch of Example objects.""" + ... + + def predict(self, docs): + """Apply the model to a batch of Doc objects.""" + ... + + def set_annotations(self, docs, predictions): + """Modify a batch of Doc objects using the predictions.""" + ... + + def initialize(self, get_examples, nlp=None, labels=None): + """Initialize the model before training.""" + ... + + def add_label(self, label): + """Add a label to the component.""" + ... +``` + +Typically, the **constructor** defines the vocab, the Machine Learning model, +and the name of this component. Additionally, this component, just like the +`textcat` and the `tagger`, stores an **internal list of labels**. The ML model +will predict scores for each label. We add convenience methods to easily +retrieve and add to them. + +```python {title="The constructor (continued)"} + def __init__(self, vocab, model, name="rel"): + """Create a component instance.""" + # ... + self.cfg = {"labels": []} + + @property + def labels(self) -> Tuple[str, ...]: + """Returns the labels currently added to the component.""" + return tuple(self.cfg["labels"]) + + def add_label(self, label: str): + """Add a new label to the pipe.""" + self.cfg["labels"] = list(self.labels) + [label] +``` + +After creation, the component needs to be +[initialized](/usage/training#initialization). This method can define the +relevant labels in two ways: explicitly by setting the `labels` argument in the +[`initialize` block](/api/data-formats#config-initialize) of the config, or +implicately by deducing them from the `get_examples` callback that generates the +full **training data set**, or a representative sample. + +The final number of labels defines the output dimensionality of the network, and +will be used to do +[shape inference](https://thinc.ai/docs/usage-models#validation) throughout the +layers of the neural network. This is triggered by calling +[`Model.initialize`](https://thinc.ai/api/model#initialize). + +```python {title="The initialize method",highlight="12,15,18,22"} +from itertools import islice + +def initialize( + self, + get_examples: Callable[[], Iterable[Example]], + *, + nlp: Language = None, + labels: Optional[List[str]] = None, +): + if labels is not None: + for label in labels: + self.add_label(label) + else: + for example in get_examples(): + relations = example.reference._.rel + for indices, label_dict in relations.items(): + for label in label_dict.keys(): + self.add_label(label) + subbatch = list(islice(get_examples(), 10)) + doc_sample = [eg.reference for eg in subbatch] + label_sample = self._examples_to_truth(subbatch) + self.model.initialize(X=doc_sample, Y=label_sample) +``` + +The `initialize` method is triggered whenever this component is part of an `nlp` +pipeline, and [`nlp.initialize`](/api/language#initialize) is invoked. +Typically, this happens when the pipeline is set up before training in +[`spacy train`](/api/cli#training). After initialization, the pipeline component +and its internal model can be trained and used to make predictions. + +During training, the method [`update`](/api/pipe#update) is invoked which +delegates to +[`Model.begin_update`](https://thinc.ai/docs/api-model#begin_update) and a +[`get_loss`](/api/pipe#get_loss) function that **calculates the loss** for a +batch of examples, as well as the **gradient** of loss that will be used to +update the weights of the model layers. Thinc provides several +[loss functions](https://thinc.ai/docs/api-loss) that can be used for the +implementation of the `get_loss` function. + +```python {title="The update method",highlight="12-14"} +def update( + self, + examples: Iterable[Example], + *, + drop: float = 0.0, + sgd: Optional[Optimizer] = None, + losses: Optional[Dict[str, float]] = None, +) -> Dict[str, float]: + # ... + docs = [eg.predicted for eg in examples] + predictions, backprop = self.model.begin_update(docs) + loss, gradient = self.get_loss(examples, predictions) + backprop(gradient) + losses[self.name] += loss + # ... + return losses +``` + +After training the model, the component can be used to make novel +**predictions**. The [`predict`](/api/pipe#predict) method needs to be +implemented for each subclass of `TrainablePipe`. In our case, we can simply +delegate to the internal model's +[predict](https://thinc.ai/docs/api-model#predict) function that takes a batch +of `Doc` objects and returns a ~~Floats2d~~ array: + +```python {title="The predict method"} +def predict(self, docs: Iterable[Doc]) -> Floats2d: + predictions = self.model.predict(docs) + return self.model.ops.asarray(predictions) +``` + +The final method that needs to be implemented, is +[`set_annotations`](/api/pipe#set_annotations). This function takes the +predictions, and modifies the given `Doc` object in place to store them. For our +relation extraction component, we store the data in the +[custom attribute](#component-rel-attribute)`doc._.rel`. + +To interpret the scores predicted by the relation extraction model correctly, we +need to refer to the model's `get_instances` function that defined which pairs +of entities were relevant candidates, so that the predictions can be linked to +those exact entities: + +```python {title="The set_annotations method",highlight="5-6,10"} +def set_annotations(self, docs: Iterable[Doc], predictions: Floats2d): + c = 0 + get_instances = self.model.attrs["get_instances"] + for doc in docs: + for (e1, e2) in get_instances(doc): + offset = (e1.start, e2.start) + if offset not in doc._.rel: + doc._.rel[offset] = {} + for j, label in enumerate(self.labels): + doc._.rel[offset][label] = predictions[c, j] + c += 1 +``` + +Under the hood, when the pipe is applied to a document, it delegates to the +`predict` and `set_annotations` methods: + +```python {title="The __call__ method"} +def __call__(self, doc: Doc): + predictions = self.predict([doc]) + self.set_annotations([doc], predictions) + return doc +``` + +There is one more optional method to implement: [`score`](/api/pipe#score) +calculates the performance of your component on a set of examples, and returns +the results as a dictionary: + +```python {title="The score method"} +def score(self, examples: Iterable[Example]) -> Dict[str, Any]: + prf = PRFScore() + for example in examples: + ... + + return { + "rel_micro_p": prf.precision, + "rel_micro_r": prf.recall, + "rel_micro_f": prf.fscore, + } +``` + +This is particularly useful for calculating relevant scores on the development +corpus when training the component with [`spacy train`](/api/cli#training). + +Once our `TrainablePipe` subclass is fully implemented, we can +[register](/usage/processing-pipelines#custom-components-factories) the +component with the [`@Language.factory`](/api/language#factory) decorator. This +assigns it a name and lets you create the component with +[`nlp.add_pipe`](/api/language#add_pipe) and via the +[config](/usage/training#config). + +> #### config.cfg (excerpt) +> +> ```ini +> [components.relation_extractor] +> factory = "relation_extractor" +> +> [components.relation_extractor.model] +> @architectures = "rel_model.v1" +> # ... +> +> [training.score_weights] +> rel_micro_p = 0.0 +> rel_micro_r = 0.0 +> rel_micro_f = 1.0 +> ``` + +```python {title="Registering the pipeline component"} +from spacy.language import Language + +@Language.factory("relation_extractor") +def make_relation_extractor(nlp, name, model): + return RelationExtractor(nlp.vocab, model, name) +``` + +You can extend the decorator to include information such as the type of +annotations that are required for this component to run, the type of annotations +it produces, and the scores that can be calculated: + +```python {title="Factory annotations",highlight="5-11"} +from spacy.language import Language + +@Language.factory( + "relation_extractor", + requires=["doc.ents", "token.ent_iob", "token.ent_type"], + assigns=["doc._.rel"], + default_score_weights={ + "rel_micro_p": None, + "rel_micro_r": None, + "rel_micro_f": None, + }, +) +def make_relation_extractor(nlp, name, model): + return RelationExtractor(nlp.vocab, model, name) +``` + + + Run this example use-case by using our project template. It includes all the + code to create the ML model and the pipeline component from scratch. It + contains two config files to train the model: one to run on CPU with a Tok2Vec + layer, and one for the GPU using a transformer. The project applies the + relation extraction component to identify biomolecular interactions, but you + can easily swap in your own dataset for your experiments in any other domain. + diff --git a/website/docs/usage/linguistic-features.mdx b/website/docs/usage/linguistic-features.mdx new file mode 100644 index 0000000..6ca9704 --- /dev/null +++ b/website/docs/usage/linguistic-features.mdx @@ -0,0 +1,2024 @@ +--- +title: Linguistic Features +next: /usage/rule-based-matching +menu: + - ['POS Tagging', 'pos-tagging'] + - ['Morphology', 'morphology'] + - ['Lemmatization', 'lemmatization'] + - ['Dependency Parse', 'dependency-parse'] + - ['Named Entities', 'named-entities'] + - ['Entity Linking', 'entity-linking'] + - ['Tokenization', 'tokenization'] + - ['Merging & Splitting', 'retokenization'] + - ['Sentence Segmentation', 'sbd'] + - ['Mappings & Exceptions', 'mappings-exceptions'] + - ['Vectors & Similarity', 'vectors-similarity'] + - ['Language Data', 'language-data'] +--- + +Processing raw text intelligently is difficult: most words are rare, and it's +common for words that look completely different to mean almost the same thing. +The same words in a different order can mean something completely different. +Even splitting text into useful word-like units can be difficult in many +languages. While it's possible to solve some problems starting from only the raw +characters, it's usually better to use linguistic knowledge to add useful +information. That's exactly what spaCy is designed to do: you put in raw text, +and get back a [`Doc`](/api/doc) object, that comes with a variety of +annotations. + +## Part-of-speech tagging {id="pos-tagging",model="tagger, parser"} + + + + + +For a list of the fine-grained and coarse-grained part-of-speech tags assigned +by spaCy's models across different languages, see the label schemes documented +in the [models directory](/models). + + + +## Morphology {id="morphology"} + +Inflectional morphology is the process by which a root form of a word is +modified by adding prefixes or suffixes that specify its grammatical function +but do not change its part-of-speech. We say that a **lemma** (root form) is +**inflected** (modified/combined) with one or more **morphological features** to +create a surface form. Here are some examples: + +| Context | Surface | Lemma | POS | Morphological Features | +| ---------------------------------------- | ------- | ----- | ------ | ---------------------------------------- | +| I was reading the paper | reading | read | `VERB` | `VerbForm=Ger` | +| I don't watch the news, I read the paper | read | read | `VERB` | `VerbForm=Fin`, `Mood=Ind`, `Tense=Pres` | +| I read the paper yesterday | read | read | `VERB` | `VerbForm=Fin`, `Mood=Ind`, `Tense=Past` | + +Morphological features are stored in the +[`MorphAnalysis`](/api/morphology#morphanalysis) under `Token.morph`, which +allows you to access individual morphological features. + +> #### 📝 Things to try +> +> 1. Change "I" to "She". You should see that the morphological features change +> and express that it's a pronoun in the third person. +> 2. Inspect `token.morph` for the other tokens. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +print("Pipeline:", nlp.pipe_names) +doc = nlp("I was reading the paper.") +token = doc[0] # 'I' +print(token.morph) # 'Case=Nom|Number=Sing|Person=1|PronType=Prs' +print(token.morph.get("PronType")) # ['Prs'] +``` + +### Statistical morphology {id="morphologizer",version="3",model="morphologizer"} + +spaCy's statistical [`Morphologizer`](/api/morphologizer) component assigns the +morphological features and coarse-grained part-of-speech tags as `Token.morph` +and `Token.pos`. + +```python {executable="true"} +import spacy + +nlp = spacy.load("de_core_news_sm") +doc = nlp("Wo bist du?") # English: 'Where are you?' +print(doc[2].morph) # 'Case=Nom|Number=Sing|Person=2|PronType=Prs' +print(doc[2].pos_) # 'PRON' +``` + +### Rule-based morphology {id="rule-based-morphology"} + +For languages with relatively simple morphological systems like English, spaCy +can assign morphological features through a rule-based approach, which uses the +**token text** and **fine-grained part-of-speech tags** to produce +coarse-grained part-of-speech tags and morphological features. + +1. The part-of-speech tagger assigns each token a **fine-grained part-of-speech + tag**. In the API, these tags are known as `Token.tag`. They express the + part-of-speech (e.g. verb) and some amount of morphological information, e.g. + that the verb is past tense (e.g. `VBD` for a past tense verb in the Penn + Treebank) . +2. For words whose coarse-grained POS is not set by a prior process, a + [mapping table](#mappings-exceptions) maps the fine-grained tags to a + coarse-grained POS tags and morphological features. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Where are you?") +print(doc[2].morph) # 'Case=Nom|Person=2|PronType=Prs' +print(doc[2].pos_) # 'PRON' +``` + +## Lemmatization {id="lemmatization",version="3"} + +spaCy provides two pipeline components for lemmatization: + +1. The [`Lemmatizer`](/api/lemmatizer) component provides lookup and rule-based + lemmatization methods in a configurable component. An individual language can + extend the `Lemmatizer` as part of its [language data](#language-data). +2. The [`EditTreeLemmatizer`](/api/edittreelemmatizer) + 3.3 component provides a trainable lemmatizer. + +```python {executable="true"} +import spacy + +# English pipelines include a rule-based lemmatizer +nlp = spacy.load("en_core_web_sm") +lemmatizer = nlp.get_pipe("lemmatizer") +print(lemmatizer.mode) # 'rule' + +doc = nlp("I was reading the paper.") +print([token.lemma_ for token in doc]) +# ['I', 'be', 'read', 'the', 'paper', '.'] +``` + + + +Unlike spaCy v2, spaCy v3 models do _not_ provide lemmas by default or switch +automatically between lookup and rule-based lemmas depending on whether a tagger +is in the pipeline. To have lemmas in a `Doc`, the pipeline needs to include a +[`Lemmatizer`](/api/lemmatizer) component. The lemmatizer component is +configured to use a single mode such as `"lookup"` or `"rule"` on +initialization. The `"rule"` mode requires `Token.pos` to be set by a previous +component. + + + +The data for spaCy's lemmatizers is distributed in the package +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). The +provided trained pipelines already include all the required tables, but if you +are creating new pipelines, you'll probably want to install `spacy-lookups-data` +to provide the data when the lemmatizer is initialized. + +### Lookup lemmatizer {id="lemmatizer-lookup"} + +For pipelines without a tagger or morphologizer, a lookup lemmatizer can be +added to the pipeline as long as a lookup table is provided, typically through +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). The +lookup lemmatizer looks up the token surface form in the lookup table without +reference to the token's part-of-speech or context. + +```python +# pip install -U %%SPACY_PKG_NAME[lookups]%%SPACY_PKG_FLAGS +import spacy + +nlp = spacy.blank("sv") +nlp.add_pipe("lemmatizer", config={"mode": "lookup"}) +``` + +### Rule-based lemmatizer {id="lemmatizer-rule",model="morphologizer"} + +When training pipelines that include a component that assigns part-of-speech +tags (a morphologizer or a tagger with a [POS mapping](#mappings-exceptions)), a +rule-based lemmatizer can be added using rule tables from +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data): + +```python +# pip install -U %%SPACY_PKG_NAME[lookups]%%SPACY_PKG_FLAGS +import spacy + +nlp = spacy.blank("de") +# Morphologizer (note: model is not yet trained!) +nlp.add_pipe("morphologizer") +# Rule-based lemmatizer +nlp.add_pipe("lemmatizer", config={"mode": "rule"}) +``` + +The rule-based deterministic lemmatizer maps the surface form to a lemma in +light of the previously assigned coarse-grained part-of-speech and morphological +information, without consulting the context of the token. The rule-based +lemmatizer also accepts list-based exception files. For English, these are +acquired from [WordNet](https://wordnet.princeton.edu/). + +### Trainable lemmatizer {id="lemmatizer-train",model="trainable_lemmatizer"} + +The [`EditTreeLemmatizer`](/api/edittreelemmatizer) can learn form-to-lemma +transformations from a training corpus that includes lemma annotations. This +removes the need to write language-specific rules and can (in many cases) +provide higher accuracies than lookup and rule-based lemmatizers. + +```python +import spacy + +nlp = spacy.blank("de") +nlp.add_pipe("trainable_lemmatizer", name="lemmatizer") +``` + +## Dependency Parsing {id="dependency-parse",model="parser"} + +spaCy features a fast and accurate syntactic dependency parser, and has a rich +API for navigating the tree. The parser also powers the sentence boundary +detection, and lets you iterate over base noun phrases, or "chunks". You can +check whether a [`Doc`](/api/doc) object has been parsed by calling +`doc.has_annotation("DEP")`, which checks whether the attribute `Token.dep` has +been set returns a boolean value. If the result is `False`, the default sentence +iterator will raise an exception. + + + +For a list of the syntactic dependency labels assigned by spaCy's models across +different languages, see the label schemes documented in the +[models directory](/models). + + + +### Noun chunks {id="noun-chunks"} + +Noun chunks are "base noun phrases" – flat phrases that have a noun as their +head. You can think of noun chunks as a noun plus the words describing the noun +– for example, "the lavish green grass" or "the world’s largest tech fund". To +get the noun chunks in a document, simply iterate over +[`Doc.noun_chunks`](/api/doc#noun_chunks). + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Autonomous cars shift insurance liability toward manufacturers") +for chunk in doc.noun_chunks: + print(chunk.text, chunk.root.text, chunk.root.dep_, + chunk.root.head.text) +``` + +> - **Text:** The original noun chunk text. +> - **Root text:** The original text of the word connecting the noun chunk to +> the rest of the parse. +> - **Root dep:** Dependency relation connecting the root to its head. +> - **Root head text:** The text of the root token's head. + +| Text | root.text | root.dep\_ | root.head.text | +| ------------------- | ------------- | ---------- | -------------- | +| Autonomous cars | cars | `nsubj` | shift | +| insurance liability | liability | `dobj` | shift | +| manufacturers | manufacturers | `pobj` | toward | + +### Navigating the parse tree {id="navigating"} + +spaCy uses the terms **head** and **child** to describe the words **connected by +a single arc** in the dependency tree. The term **dep** is used for the arc +label, which describes the type of syntactic relation that connects the child to +the head. As with other attributes, the value of `.dep` is a hash value. You can +get the string value with `.dep_`. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Autonomous cars shift insurance liability toward manufacturers") +for token in doc: + print(token.text, token.dep_, token.head.text, token.head.pos_, + [child for child in token.children]) +``` + +> - **Text:** The original token text. +> - **Dep:** The syntactic relation connecting child to head. +> - **Head text:** The original text of the token head. +> - **Head POS:** The part-of-speech tag of the token head. +> - **Children:** The immediate syntactic dependents of the token. + +| Text | Dep | Head text | Head POS | Children | +| ------------- | ---------- | --------- | -------- | ----------------------- | +| Autonomous | `amod` | cars | `NOUN` | | +| cars | `nsubj` | shift | `VERB` | Autonomous | +| shift | `ROOT` | shift | `VERB` | cars, liability, toward | +| insurance | `compound` | liability | `NOUN` | | +| liability | `dobj` | shift | `VERB` | insurance | +| toward | `prep` | shift | `NOUN` | manufacturers | +| manufacturers | `pobj` | toward | `ADP` | | + + + +Because the syntactic relations form a tree, every word has **exactly one +head**. You can therefore iterate over the arcs in the tree by iterating over +the words in the sentence. This is usually the best way to match an arc of +interest – from below: + +```python {executable="true"} +import spacy +from spacy.symbols import nsubj, VERB + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Autonomous cars shift insurance liability toward manufacturers") + +# Finding a verb with a subject from below — good +verbs = set() +for possible_subject in doc: + if possible_subject.dep == nsubj and possible_subject.head.pos == VERB: + verbs.add(possible_subject.head) +print(verbs) +``` + +If you try to match from above, you'll have to iterate twice. Once for the head, +and then again through the children: + +```python +# Finding a verb with a subject from above — less good +verbs = [] +for possible_verb in doc: + if possible_verb.pos == VERB: + for possible_subject in possible_verb.children: + if possible_subject.dep == nsubj: + verbs.append(possible_verb) + break +``` + +To iterate through the children, use the `token.children` attribute, which +provides a sequence of [`Token`](/api/token) objects. + +#### Iterating around the local tree {id="navigating-around"} + +A few more convenience attributes are provided for iterating around the local +tree from the token. [`Token.lefts`](/api/token#lefts) and +[`Token.rights`](/api/token#rights) attributes provide sequences of syntactic +children that occur before and after the token. Both sequences are in sentence +order. There are also two integer-typed attributes, +[`Token.n_lefts`](/api/token#n_lefts) and +[`Token.n_rights`](/api/token#n_rights) that give the number of left and right +children. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("bright red apples on the tree") +print([token.text for token in doc[2].lefts]) # ['bright', 'red'] +print([token.text for token in doc[2].rights]) # ['on'] +print(doc[2].n_lefts) # 2 +print(doc[2].n_rights) # 1 +``` + +```python {executable="true"} +import spacy + +nlp = spacy.load("de_core_news_sm") +doc = nlp("schöne rote Äpfel auf dem Baum") +print([token.text for token in doc[2].lefts]) # ['schöne', 'rote'] +print([token.text for token in doc[2].rights]) # ['auf'] +``` + +You can get a whole phrase by its syntactic head using the +[`Token.subtree`](/api/token#subtree) attribute. This returns an ordered +sequence of tokens. You can walk up the tree with the +[`Token.ancestors`](/api/token#ancestors) attribute, and check dominance with +[`Token.is_ancestor`](/api/token#is_ancestor) + +> #### Projective vs. non-projective +> +> For the [default English pipelines](/models/en), the parse tree is +> **projective**, which means that there are no crossing brackets. The tokens +> returned by `.subtree` are therefore guaranteed to be contiguous. This is not +> true for the German pipelines, which have many +> [non-projective dependencies](https://explosion.ai/blog/german-model#word-order). + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Credit and mortgage account holders must submit their requests") + +root = [token for token in doc if token.head == token][0] +subject = list(root.lefts)[0] +for descendant in subject.subtree: + assert subject is descendant or subject.is_ancestor(descendant) + print(descendant.text, descendant.dep_, descendant.n_lefts, + descendant.n_rights, + [ancestor.text for ancestor in descendant.ancestors]) +``` + +| Text | Dep | n_lefts | n_rights | ancestors | +| -------- | ---------- | ------- | -------- | -------------------------------- | +| Credit | `nmod` | `0` | `2` | holders, submit | +| and | `cc` | `0` | `0` | holders, submit | +| mortgage | `compound` | `0` | `0` | account, Credit, holders, submit | +| account | `conj` | `1` | `0` | Credit, holders, submit | +| holders | `nsubj` | `1` | `0` | submit | + +Finally, the `.left_edge` and `.right_edge` attributes can be especially useful, +because they give you the first and last token of the subtree. This is the +easiest way to create a `Span` object for a syntactic phrase. Note that +`.right_edge` gives a token **within** the subtree – so if you use it as the +end-point of a range, don't forget to `+1`! + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Credit and mortgage account holders must submit their requests") +span = doc[doc[4].left_edge.i : doc[4].right_edge.i+1] +with doc.retokenize() as retokenizer: + retokenizer.merge(span) +for token in doc: + print(token.text, token.pos_, token.dep_, token.head.text) +``` + +| Text | POS | Dep | Head text | +| ----------------------------------- | ------ | ------- | --------- | +| Credit and mortgage account holders | `NOUN` | `nsubj` | submit | +| must | `VERB` | `aux` | submit | +| submit | `VERB` | `ROOT` | submit | +| their | `ADJ` | `poss` | requests | +| requests | `NOUN` | `dobj` | submit | + +The dependency parse can be a useful tool for **information extraction**, +especially when combined with other predictions like +[named entities](#named-entities). The following example extracts money and +currency values, i.e. entities labeled as `MONEY`, and then uses the dependency +parse to find the noun phrase they are referring to – for example `"Net income"` +→ `"$9.4 million"`. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +# Merge noun phrases and entities for easier analysis +nlp.add_pipe("merge_entities") +nlp.add_pipe("merge_noun_chunks") + +TEXTS = [ + "Net income was $9.4 million compared to the prior year of $2.7 million.", + "Revenue exceeded twelve billion dollars, with a loss of $1b.", +] +for doc in nlp.pipe(TEXTS): + for token in doc: + if token.ent_type_ == "MONEY": + # We have an attribute and direct object, so check for subject + if token.dep_ in ("attr", "dobj"): + subj = [w for w in token.head.lefts if w.dep_ == "nsubj"] + if subj: + print(subj[0], "-->", token) + # We have a prepositional object with a preposition + elif token.dep_ == "pobj" and token.head.dep_ == "prep": + print(token.head.head, "-->", token) +``` + + + +For more examples of how to write rule-based information extraction logic that +takes advantage of the model's predictions produced by the different components, +see the usage guide on +[combining models and rules](/usage/rule-based-matching#models-rules). + + + +### Visualizing dependencies {id="displacy"} + +The best way to understand spaCy's dependency parser is interactively. To make +this easier, spaCy comes with a visualization module. You can pass a `Doc` or a +list of `Doc` objects to displaCy and run +[`displacy.serve`](/api/top-level#displacy.serve) to run the web server, or +[`displacy.render`](/api/top-level#displacy.render) to generate the raw markup. +If you want to know how to write rules that hook into some type of syntactic +construction, just plug the sentence into the visualizer and see how spaCy +annotates it. + +```python {executable="true"} +import spacy +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Autonomous cars shift insurance liability toward manufacturers") +# Since this is an interactive Jupyter environment, we can use displacy.render here +displacy.render(doc, style='dep') +``` + + + +For more details and examples, see the +[usage guide on visualizing spaCy](/usage/visualizers). You can also test +displaCy in our [online demo](https://explosion.ai/demos/displacy).. + + + +### Disabling the parser {id="disabling"} + +In the [trained pipelines](/models) provided by spaCy, the parser is loaded and +enabled by default as part of the +[standard processing pipeline](/usage/processing-pipelines). If you don't need +any of the syntactic information, you should disable the parser. Disabling the +parser will make spaCy load and run much faster. If you want to load the parser, +but need to disable it for specific documents, you can also control its use on +the `nlp` object. For more details, see the usage guide on +[disabling pipeline components](/usage/processing-pipelines/#disabling). + +```python +nlp = spacy.load("en_core_web_sm", disable=["parser"]) +``` + +## Named Entity Recognition {id="named-entities"} + +spaCy features an extremely fast statistical entity recognition system, that +assigns labels to contiguous spans of tokens. The default +[trained pipelines](/models) can identify a variety of named and numeric +entities, including companies, locations, organizations and products. You can +add arbitrary classes to the entity recognition system, and update the model +with new examples. + +### Named Entity Recognition 101 {id="named-entities-101"} + + + +### Accessing entity annotations and labels {id="accessing-ner"} + +The standard way to access entity annotations is the [`doc.ents`](/api/doc#ents) +property, which produces a sequence of [`Span`](/api/span) objects. The entity +type is accessible either as a hash value or as a string, using the attributes +`ent.label` and `ent.label_`. The `Span` object acts as a sequence of tokens, so +you can iterate over the entity or index into it. You can also get the text form +of the whole entity, as though it were a single token. + +You can also access token entity annotations using the +[`token.ent_iob`](/api/token#attributes) and +[`token.ent_type`](/api/token#attributes) attributes. `token.ent_iob` indicates +whether an entity starts, continues or ends on the tag. If no entity type is set +on a token, it will return an empty string. + +> #### IOB Scheme +> +> - `I` – Token is **inside** an entity. +> - `O` – Token is **outside** an entity. +> - `B` – Token is the **beginning** of an entity. +> +> #### BILUO Scheme +> +> - `B` – Token is the **beginning** of a multi-token entity. +> - `I` – Token is **inside** a multi-token entity. +> - `L` – Token is the **last** token of a multi-token entity. +> - `U` – Token is a single-token **unit** entity. +> - `O` – Token is **outside** an entity. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("San Francisco considers banning sidewalk delivery robots") + +# document level +ents = [(e.text, e.start_char, e.end_char, e.label_) for e in doc.ents] +print(ents) + +# token level +ent_san = [doc[0].text, doc[0].ent_iob_, doc[0].ent_type_] +ent_francisco = [doc[1].text, doc[1].ent_iob_, doc[1].ent_type_] +print(ent_san) # ['San', 'B', 'GPE'] +print(ent_francisco) # ['Francisco', 'I', 'GPE'] +``` + +| Text | ent_iob | ent_iob\_ | ent_type\_ | Description | +| --------- | ------- | --------- | ---------- | ---------------------- | +| San | `3` | `B` | `"GPE"` | beginning of an entity | +| Francisco | `1` | `I` | `"GPE"` | inside an entity | +| considers | `2` | `O` | `""` | outside an entity | +| banning | `2` | `O` | `""` | outside an entity | +| sidewalk | `2` | `O` | `""` | outside an entity | +| delivery | `2` | `O` | `""` | outside an entity | +| robots | `2` | `O` | `""` | outside an entity | + +### Setting entity annotations {id="setting-entities"} + +To ensure that the sequence of token annotations remains consistent, you have to +set entity annotations **at the document level**. However, you can't write +directly to the `token.ent_iob` or `token.ent_type` attributes, so the easiest +way to set entities is to use the [`doc.set_ents`](/api/doc#set_ents) function +and create the new entity as a [`Span`](/api/span). + +```python {executable="true"} +import spacy +from spacy.tokens import Span + +nlp = spacy.load("en_core_web_sm") +doc = nlp("fb is hiring a new vice president of global policy") +ents = [(e.text, e.start_char, e.end_char, e.label_) for e in doc.ents] +print('Before', ents) +# The model didn't recognize "fb" as an entity :( + +# Create a span for the new entity +fb_ent = Span(doc, 0, 1, label="ORG") +orig_ents = list(doc.ents) + +# Option 1: Modify the provided entity spans, leaving the rest unmodified +doc.set_ents([fb_ent], default="unmodified") + +# Option 2: Assign a complete list of ents to doc.ents +doc.ents = orig_ents + [fb_ent] + +ents = [(e.text, e.start, e.end, e.label_) for e in doc.ents] +print('After', ents) +# [('fb', 0, 1, 'ORG')] 🎉 +``` + +Keep in mind that `Span` is initialized with the start and end **token** +indices, not the character offsets. To create a span from character offsets, use +[`Doc.char_span`](/api/doc#char_span): + +```python +fb_ent = doc.char_span(0, 2, label="ORG") +``` + +#### Setting entity annotations from array {id="setting-from-array"} + +You can also assign entity annotations using the +[`doc.from_array`](/api/doc#from_array) method. To do this, you should include +both the `ENT_TYPE` and the `ENT_IOB` attributes in the array you're importing +from. + +```python {executable="true"} +import numpy +import spacy +from spacy.attrs import ENT_IOB, ENT_TYPE + +nlp = spacy.load("en_core_web_sm") +doc = nlp.make_doc("London is a big city in the United Kingdom.") +print("Before", doc.ents) # [] + +header = [ENT_IOB, ENT_TYPE] +attr_array = numpy.zeros((len(doc), len(header)), dtype="uint64") +attr_array[0, 0] = 3 # B +attr_array[0, 1] = doc.vocab.strings["GPE"] +doc.from_array(header, attr_array) +print("After", doc.ents) # [London] +``` + +#### Setting entity annotations in Cython {id="setting-cython"} + +Finally, you can always write to the underlying struct if you compile a +[Cython](http://cython.org/) function. This is easy to do, and allows you to +write efficient native code. + +```python +# cython: infer_types=True +from spacy.typedefs cimport attr_t +from spacy.tokens.doc cimport Doc + +cpdef set_entity(Doc doc, int start, int end, attr_t ent_type): + for i in range(start, end): + doc.c[i].ent_type = ent_type + doc.c[start].ent_iob = 3 + for i in range(start+1, end): + doc.c[i].ent_iob = 2 +``` + +Obviously, if you write directly to the array of `TokenC*` structs, you'll have +responsibility for ensuring that the data is left in a consistent state. + +### Built-in entity types {id="entity-types"} + +> #### Tip: Understanding entity types +> +> You can also use `spacy.explain()` to get the description for the string +> representation of an entity label. For example, `spacy.explain("LANGUAGE")` +> will return "any named language". + + + +For details on the entity types available in spaCy's trained pipelines, see the +"label scheme" sections of the individual models in the +[models directory](/models). + + + +### Visualizing named entities {id="displacy"} + +The +[displaCy ENT visualizer](https://explosion.ai/demos/displacy-ent) +lets you explore an entity recognition model's behavior interactively. If you're +training a model, it's very useful to run the visualization yourself. To help +you do that, spaCy comes with a visualization module. You can pass a `Doc` or a +list of `Doc` objects to displaCy and run +[`displacy.serve`](/api/top-level#displacy.serve) to run the web server, or +[`displacy.render`](/api/top-level#displacy.render) to generate the raw markup. + +For more details and examples, see the +[usage guide on visualizing spaCy](/usage/visualizers). + +```python {title="Named Entity example"} +import spacy +from spacy import displacy + +text = "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously." + +nlp = spacy.load("en_core_web_sm") +doc = nlp(text) +displacy.serve(doc, style="ent") +``` + + +
When Sebastian Thrun PERSON started working on self-driving cars at Google ORG in 2007 DATE, few people outside of the company took him seriously.
+ + +## Entity Linking {id="entity-linking"} + +To ground the named entities into the "real world", spaCy provides functionality +to perform entity linking, which resolves a textual entity to a unique +identifier from a knowledge base (KB). You can create your own +[`KnowledgeBase`](/api/kb) and [train](/usage/training) a new +[`EntityLinker`](/api/entitylinker) using that custom knowledge base. + +As an example on how to define a KnowledgeBase and train an entity linker model, +see [`this tutorial`](https://github.com/explosion/projects/blob/v3/tutorials/nel_emerson) +using [spaCy projects](/usage/projects). + +### Accessing entity identifiers {id="entity-linking-accessing",model="entity linking"} + +The annotated KB identifier is accessible as either a hash value or as a string, +using the attributes `ent.kb_id` and `ent.kb_id_` of a [`Span`](/api/span) +object, or the `ent_kb_id` and `ent_kb_id_` attributes of a +[`Token`](/api/token) object. + +```python +import spacy + +# "my_custom_el_pipeline" is assumed to be a custom NLP pipeline that was trained and serialized to disk +nlp = spacy.load("my_custom_el_pipeline") +doc = nlp("Ada Lovelace was born in London") + +# Document level +ents = [(e.text, e.label_, e.kb_id_) for e in doc.ents] +print(ents) # [('Ada Lovelace', 'PERSON', 'Q7259'), ('London', 'GPE', 'Q84')] + +# Token level +ent_ada_0 = [doc[0].text, doc[0].ent_type_, doc[0].ent_kb_id_] +ent_ada_1 = [doc[1].text, doc[1].ent_type_, doc[1].ent_kb_id_] +ent_london_5 = [doc[5].text, doc[5].ent_type_, doc[5].ent_kb_id_] +print(ent_ada_0) # ['Ada', 'PERSON', 'Q7259'] +print(ent_ada_1) # ['Lovelace', 'PERSON', 'Q7259'] +print(ent_london_5) # ['London', 'GPE', 'Q84'] +``` + +## Tokenization {id="tokenization"} + +Tokenization is the task of splitting a text into meaningful segments, called +_tokens_. The input to the tokenizer is a unicode text, and the output is a +[`Doc`](/api/doc) object. To construct a `Doc` object, you need a +[`Vocab`](/api/vocab) instance, a sequence of `word` strings, and optionally a +sequence of `spaces` booleans, which allow you to maintain alignment of the +tokens into the original string. + + + +spaCy's tokenization is **non-destructive**, which means that you'll always be +able to reconstruct the original input from the tokenized output. Whitespace +information is preserved in the tokens and no information is added or removed +during tokenization. This is kind of a core principle of spaCy's `Doc` object: +`doc.text == input_text` should always hold true. + + + + + + + +spaCy introduces a novel tokenization algorithm that gives a better balance +between performance, ease of definition and ease of alignment into the original +string. + +After consuming a prefix or suffix, we consult the special cases again. We want +the special cases to handle things like "don't" in English, and we want the same +rule to work for "(don't)!". We do this by splitting off the open bracket, then +the exclamation, then the closed bracket, and finally matching the special case. +Here's an implementation of the algorithm in Python optimized for readability +rather than performance: + +```python +def tokenizer_pseudo_code( + text, + special_cases, + prefix_search, + suffix_search, + infix_finditer, + token_match, + url_match +): + tokens = [] + for substring in text.split(): + suffixes = [] + while substring: + if substring in special_cases: + tokens.extend(special_cases[substring]) + substring = "" + continue + while prefix_search(substring) or suffix_search(substring): + if token_match(substring): + tokens.append(substring) + substring = "" + break + if substring in special_cases: + tokens.extend(special_cases[substring]) + substring = "" + break + if prefix_search(substring): + split = prefix_search(substring).end() + tokens.append(substring[:split]) + substring = substring[split:] + if substring in special_cases: + continue + if suffix_search(substring): + split = suffix_search(substring).start() + suffixes.append(substring[split:]) + substring = substring[:split] + if token_match(substring): + tokens.append(substring) + substring = "" + elif url_match(substring): + tokens.append(substring) + substring = "" + elif substring in special_cases: + tokens.extend(special_cases[substring]) + substring = "" + elif list(infix_finditer(substring)): + infixes = infix_finditer(substring) + offset = 0 + for match in infixes: + if offset == 0 and match.start() == 0: + continue + tokens.append(substring[offset : match.start()]) + tokens.append(substring[match.start() : match.end()]) + offset = match.end() + if substring[offset:]: + tokens.append(substring[offset:]) + substring = "" + elif substring: + tokens.append(substring) + substring = "" + tokens.extend(reversed(suffixes)) + for match in matcher(special_cases, text): + tokens.replace(match, special_cases[match]) + return tokens +``` + +The algorithm can be summarized as follows: + +1. Iterate over space-separated substrings. +2. Check whether we have an explicitly defined special case for this substring. + If we do, use it. +3. Look for a token match. If there is a match, stop processing and keep this + token. +4. Check whether we have an explicitly defined special case for this substring. + If we do, use it. +5. Otherwise, try to consume one prefix. If we consumed a prefix, go back to #3, + so that the token match and special cases always get priority. +6. If we didn't consume a prefix, try to consume a suffix and then go back to + #3. +7. If we can't consume a prefix or a suffix, look for a URL match. +8. If there's no URL match, then look for a special case. +9. Look for "infixes" – stuff like hyphens etc. and split the substring into + tokens on all infixes. +10. Once we can't consume any more of the string, handle it as a single token. +11. Make a final pass over the text to check for special cases that include + spaces or that were missed due to the incremental processing of affixes. + + + +**Global** and **language-specific** tokenizer data is supplied via the language +data in [`spacy/lang`](%%GITHUB_SPACY/spacy/lang). The tokenizer exceptions +define special cases like "don't" in English, which needs to be split into two +tokens: `{ORTH: "do"}` and `{ORTH: "n't", NORM: "not"}`. The prefixes, suffixes +and infixes mostly define punctuation rules – for example, when to split off +periods (at the end of a sentence), and when to leave tokens containing periods +intact (abbreviations like "U.S."). + + + +Tokenization rules that are specific to one language, but can be **generalized +across that language**, should ideally live in the language data in +[`spacy/lang`](%%GITHUB_SPACY/spacy/lang) – we always appreciate pull requests! +Anything that's specific to a domain or text type – like financial trading +abbreviations or Bavarian youth slang – should be added as a special case rule +to your tokenizer instance. If you're dealing with a lot of customizations, it +might make sense to create an entirely custom subclass. + + + +--- + +### Adding special case tokenization rules {id="special-cases"} + +Most domains have at least some idiosyncrasies that require custom tokenization +rules. This could be very certain expressions, or abbreviations only used in +this specific field. Here's how to add a special case rule to an existing +[`Tokenizer`](/api/tokenizer) instance: + +```python {executable="true"} +import spacy +from spacy.symbols import ORTH + +nlp = spacy.load("en_core_web_sm") +doc = nlp("gimme that") # phrase to tokenize +print([w.text for w in doc]) # ['gimme', 'that'] + +# Add special case rule +special_case = [{ORTH: "gim"}, {ORTH: "me"}] +nlp.tokenizer.add_special_case("gimme", special_case) + +# Check new tokenization +print([w.text for w in nlp("gimme that")]) # ['gim', 'me', 'that'] +``` + +The special case doesn't have to match an entire whitespace-delimited substring. +The tokenizer will incrementally split off punctuation, and keep looking up the +remaining substring. The special case rules also have precedence over the +punctuation splitting. + +```python +assert "gimme" not in [w.text for w in nlp("gimme!")] +assert "gimme" not in [w.text for w in nlp('("...gimme...?")')] + +nlp.tokenizer.add_special_case("...gimme...?", [{"ORTH": "...gimme...?"}]) +assert len(nlp("...gimme...?")) == 1 +``` + +#### Debugging the tokenizer {id="tokenizer-debug",version="2.2.3"} + +A working implementation of the pseudo-code above is available for debugging as +[`nlp.tokenizer.explain(text)`](/api/tokenizer#explain). It returns a list of +tuples showing which tokenizer rule or pattern was matched for each token. The +tokens produced are identical to `nlp.tokenizer()` except for whitespace tokens: + +> #### Expected output +> +> ``` +> " PREFIX +> Let SPECIAL-1 +> 's SPECIAL-2 +> go TOKEN +> ! SUFFIX +> " SUFFIX +> ``` + +```python {executable="true"} +from spacy.lang.en import English + +nlp = English() +text = '''"Let's go!"''' +doc = nlp(text) +tok_exp = nlp.tokenizer.explain(text) +assert [t.text for t in doc if not t.is_space] == [t[1] for t in tok_exp] +for t in tok_exp: + print(t[1], "\\t", t[0]) +``` + +### Customizing spaCy's Tokenizer class {id="native-tokenizers"} + +Let's imagine you wanted to create a tokenizer for a new language or specific +domain. There are six things you may need to define: + +1. A dictionary of **special cases**. This handles things like contractions, + units of measurement, emoticons, certain abbreviations, etc. +2. A function `prefix_search`, to handle **preceding punctuation**, such as open + quotes, open brackets, etc. +3. A function `suffix_search`, to handle **succeeding punctuation**, such as + commas, periods, close quotes, etc. +4. A function `infix_finditer`, to handle non-whitespace separators, such as + hyphens etc. +5. An optional boolean function `token_match` matching strings that should never + be split, overriding the infix rules. Useful for things like numbers. +6. An optional boolean function `url_match`, which is similar to `token_match` + except that prefixes and suffixes are removed before applying the match. + +You shouldn't usually need to create a `Tokenizer` subclass. Standard usage is +to use `re.compile()` to build a regular expression object, and pass its +`.search()` and `.finditer()` methods: + +```python {executable="true"} +import re +import spacy +from spacy.tokenizer import Tokenizer + +special_cases = {":)": [{"ORTH": ":)"}]} +prefix_re = re.compile(r'''^[\\[\\("']''') +suffix_re = re.compile(r'''[\\]\\)"']$''') +infix_re = re.compile(r'''[-~]''') +simple_url_re = re.compile(r'''^https?://''') + +def custom_tokenizer(nlp): + return Tokenizer(nlp.vocab, rules=special_cases, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + url_match=simple_url_re.match) + +nlp = spacy.load("en_core_web_sm") +nlp.tokenizer = custom_tokenizer(nlp) +doc = nlp("hello-world. :)") +print([t.text for t in doc]) # ['hello', '-', 'world.', ':)'] +``` + +If you need to subclass the tokenizer instead, the relevant methods to +specialize are `find_prefix`, `find_suffix` and `find_infix`. + + + +When customizing the prefix, suffix and infix handling, remember that you're +passing in **functions** for spaCy to execute, e.g. `prefix_re.search` – not +just the regular expressions. This means that your functions also need to define +how the rules should be applied. For example, if you're adding your own prefix +rules, you need to make sure they're only applied to characters at the +**beginning of a token**, e.g. by adding `^`. Similarly, suffix rules should +only be applied at the **end of a token**, so your expression should end with a +`$`. + + + +#### Modifying existing rule sets {id="native-tokenizer-additions"} + +In many situations, you don't necessarily need entirely custom rules. Sometimes +you just want to add another character to the prefixes, suffixes or infixes. The +default prefix, suffix and infix rules are available via the `nlp` object's +`Defaults` and the `Tokenizer` attributes such as +[`Tokenizer.suffix_search`](/api/tokenizer#attributes) are writable, so you can +overwrite them with compiled regular expression objects using modified default +rules. spaCy ships with utility functions to help you compile the regular +expressions – for example, +[`compile_suffix_regex`](/api/top-level#util.compile_suffix_regex): + +```python +suffixes = nlp.Defaults.suffixes + [r'''-+$''',] +suffix_regex = spacy.util.compile_suffix_regex(suffixes) +nlp.tokenizer.suffix_search = suffix_regex.search +``` + +Similarly, you can remove a character from the default suffixes: + +```python +suffixes = list(nlp.Defaults.suffixes) +suffixes.remove("\\\\[") +suffix_regex = spacy.util.compile_suffix_regex(suffixes) +nlp.tokenizer.suffix_search = suffix_regex.search +``` + +The `Tokenizer.suffix_search` attribute should be a function which takes a +unicode string and returns a **regex match object** or `None`. Usually we use +the `.search` attribute of a compiled regex object, but you can use some other +function that behaves the same way. + + + +If you've loaded a trained pipeline, writing to the +[`nlp.Defaults`](/api/language#defaults) or `English.Defaults` directly won't +work, since the regular expressions are read from the pipeline data and will be +compiled when you load it. If you modify `nlp.Defaults`, you'll only see the +effect if you call [`spacy.blank`](/api/top-level#spacy.blank). If you want to +modify the tokenizer loaded from a trained pipeline, you should modify +`nlp.tokenizer` directly. If you're training your own pipeline, you can register +[callbacks](/usage/training/#custom-code-nlp-callbacks) to modify the `nlp` +object before training. + + + +The prefix, infix and suffix rule sets include not only individual characters +but also detailed regular expressions that take the surrounding context into +account. For example, there is a regular expression that treats a hyphen between +letters as an infix. If you do not want the tokenizer to split on hyphens +between letters, you can modify the existing infix definition from +[`lang/punctuation.py`](%%GITHUB_SPACY/spacy/lang/punctuation.py): + +```python {executable="true"} +import spacy +from spacy.lang.char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER +from spacy.lang.char_classes import CONCAT_QUOTES, LIST_ELLIPSES, LIST_ICONS +from spacy.util import compile_infix_regex + +# Default tokenizer +nlp = spacy.load("en_core_web_sm") +doc = nlp("mother-in-law") +print([t.text for t in doc]) # ['mother', '-', 'in', '-', 'law'] + +# Modify tokenizer infix patterns +infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\\-\\*^](?=[0-9-])", + r"(?<=[{al}{q}])\\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + # ✅ Commented out regex that splits on hyphens between letters: + # r"(?<=[{a}])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + +infix_re = compile_infix_regex(infixes) +nlp.tokenizer.infix_finditer = infix_re.finditer +doc = nlp("mother-in-law") +print([t.text for t in doc]) # ['mother-in-law'] +``` + +For an overview of the default regular expressions, see +[`lang/punctuation.py`](%%GITHUB_SPACY/spacy/lang/punctuation.py) and +language-specific definitions such as +[`lang/de/punctuation.py`](%%GITHUB_SPACY/spacy/lang/de/punctuation.py) for +German. + +### Hooking a custom tokenizer into the pipeline {id="custom-tokenizer"} + +The tokenizer is the first component of the processing pipeline and the only one +that can't be replaced by writing to `nlp.pipeline`. This is because it has a +different signature from all the other components: it takes a text and returns a +[`Doc`](/api/doc), whereas all other components expect to already receive a +tokenized `Doc`. + +![The processing pipeline](/images/pipeline.svg) + +To overwrite the existing tokenizer, you need to replace `nlp.tokenizer` with a +custom function that takes a text and returns a [`Doc`](/api/doc). + +> #### Creating a Doc +> +> Constructing a [`Doc`](/api/doc) object manually requires at least two +> arguments: the shared `Vocab` and a list of words. Optionally, you can pass in +> a list of `spaces` values indicating whether the token at this position is +> followed by a space (default `True`). See the section on +> [pre-tokenized text](#own-annotations) for more info. +> +> ```python +> words = ["Let", "'s", "go", "!"] +> spaces = [False, True, False, False] +> doc = Doc(nlp.vocab, words=words, spaces=spaces) +> ``` + +```python +nlp = spacy.blank("en") +nlp.tokenizer = my_tokenizer +``` + +| Argument | Type | Description | +| ----------- | ----------------- | ------------------------- | +| `text` | `str` | The raw text to tokenize. | +| **RETURNS** | [`Doc`](/api/doc) | The tokenized document. | + +#### Example 1: Basic whitespace tokenizer {id="custom-tokenizer-example"} + +Here's an example of the most basic whitespace tokenizer. It takes the shared +vocab, so it can construct `Doc` objects. When it's called on a text, it returns +a `Doc` object consisting of the text split on single space characters. We can +then overwrite the `nlp.tokenizer` attribute with an instance of our custom +tokenizer. + +```python {executable="true"} +import spacy +from spacy.tokens import Doc + +class WhitespaceTokenizer: + def __init__(self, vocab): + self.vocab = vocab + + def __call__(self, text): + words = text.split(" ") + spaces = [True] * len(words) + # Avoid zero-length tokens + for i, word in enumerate(words): + if word == "": + words[i] = " " + spaces[i] = False + # Remove the final trailing space + if words[-1] == " ": + words = words[0:-1] + spaces = spaces[0:-1] + else: + spaces[-1] = False + + return Doc(self.vocab, words=words, spaces=spaces) + +nlp = spacy.blank("en") +nlp.tokenizer = WhitespaceTokenizer(nlp.vocab) +doc = nlp("What's happened to me? he thought. It wasn't a dream.") +print([token.text for token in doc]) +``` + +#### Example 2: Third-party tokenizers (BERT word pieces) {id="custom-tokenizer-example2"} + +You can use the same approach to plug in any other third-party tokenizers. Your +custom callable just needs to return a `Doc` object with the tokens produced by +your tokenizer. In this example, the wrapper uses the **BERT word piece +tokenizer**, provided by the +[`tokenizers`](https://github.com/huggingface/tokenizers) library. The tokens +available in the `Doc` object returned by spaCy now match the exact word pieces +produced by the tokenizer. + +> #### 💡 Tip: spacy-transformers +> +> If you're working with transformer models like BERT, check out the +> [`spacy-transformers`](https://github.com/explosion/spacy-transformers) +> extension package and [documentation](/usage/embeddings-transformers). It +> includes a pipeline component for using pretrained transformer weights and +> **training transformer models** in spaCy, as well as helpful utilities for +> aligning word pieces to linguistic tokenization. + +```python {title="Custom BERT word piece tokenizer"} +from tokenizers import BertWordPieceTokenizer +from spacy.tokens import Doc +import spacy + +class BertTokenizer: + def __init__(self, vocab, vocab_file, lowercase=True): + self.vocab = vocab + self._tokenizer = BertWordPieceTokenizer(vocab_file, lowercase=lowercase) + + def __call__(self, text): + tokens = self._tokenizer.encode(text) + words = [] + spaces = [] + for i, (text, (start, end)) in enumerate(zip(tokens.tokens, tokens.offsets)): + words.append(text) + if i < len(tokens.tokens) - 1: + # If next start != current end we assume a space in between + next_start, next_end = tokens.offsets[i + 1] + spaces.append(next_start > end) + else: + spaces.append(True) + return Doc(self.vocab, words=words, spaces=spaces) + +nlp = spacy.blank("en") +nlp.tokenizer = BertTokenizer(nlp.vocab, "bert-base-uncased-vocab.txt") +doc = nlp("Justin Drew Bieber is a Canadian singer, songwriter, and actor.") +print(doc.text, [token.text for token in doc]) +# [CLS]justin drew bi##eber is a canadian singer, songwriter, and actor.[SEP] +# ['[CLS]', 'justin', 'drew', 'bi', '##eber', 'is', 'a', 'canadian', 'singer', +# ',', 'songwriter', ',', 'and', 'actor', '.', '[SEP]'] +``` + + + +Keep in mind that your models' results may be less accurate if the tokenization +during training differs from the tokenization at runtime. So if you modify a +trained pipeline's tokenization afterwards, it may produce very different +predictions. You should therefore train your pipeline with the **same +tokenizer** it will be using at runtime. See the docs on +[training with custom tokenization](#custom-tokenizer-training) for details. + + + +#### Training with custom tokenization {id="custom-tokenizer-training",version="3"} + +spaCy's [training config](/usage/training#config) describes the settings, +hyperparameters, pipeline and tokenizer used for constructing and training the +pipeline. The `[nlp.tokenizer]` block refers to a **registered function** that +takes the `nlp` object and returns a tokenizer. Here, we're registering a +function called `whitespace_tokenizer` in the +[`@tokenizers` registry](/api/top-level#registry). To make sure spaCy knows how +to construct your tokenizer during training, you can pass in your Python file by +setting `--code functions.py` when you run [`spacy train`](/api/cli#train). + +> #### config.cfg +> +> ```ini +> [nlp.tokenizer] +> @tokenizers = "whitespace_tokenizer" +> ``` + +```python {title="functions.py",highlight="1"} +@spacy.registry.tokenizers("whitespace_tokenizer") +def create_whitespace_tokenizer(): + def create_tokenizer(nlp): + return WhitespaceTokenizer(nlp.vocab) + + return create_tokenizer +``` + +Registered functions can also take arguments that are then passed in from the +config. This allows you to quickly change and keep track of different settings. +Here, the registered function called `bert_word_piece_tokenizer` takes two +arguments: the path to a vocabulary file and whether to lowercase the text. The +Python type hints `str` and `bool` ensure that the received values have the +correct type. + +> #### config.cfg +> +> ```ini +> [nlp.tokenizer] +> @tokenizers = "bert_word_piece_tokenizer" +> vocab_file = "bert-base-uncased-vocab.txt" +> lowercase = true +> ``` + +```python {title="functions.py",highlight="1"} +@spacy.registry.tokenizers("bert_word_piece_tokenizer") +def create_bert_tokenizer(vocab_file: str, lowercase: bool): + def create_tokenizer(nlp): + return BertTokenizer(nlp.vocab, vocab_file, lowercase) + + return create_tokenizer +``` + +To avoid hard-coding local paths into your config file, you can also set the +vocab path on the CLI by using the `--nlp.tokenizer.vocab_file` +[override](/usage/training#config-overrides) when you run +[`spacy train`](/api/cli#train). For more details on using registered functions, +see the docs in [training with custom code](/usage/training#custom-code). + + + +Remember that a registered function should always be a function that spaCy +**calls to create something**, not the "something" itself. In this case, it +**creates a function** that takes the `nlp` object and returns a callable that +takes a text and returns a `Doc`. + + + +#### Using pre-tokenized text {id="own-annotations"} + +spaCy generally assumes by default that your data is **raw text**. However, +sometimes your data is partially annotated, e.g. with pre-existing tokenization, +part-of-speech tags, etc. The most common situation is that you have +**pre-defined tokenization**. If you have a list of strings, you can create a +[`Doc`](/api/doc) object directly. Optionally, you can also specify a list of +boolean values, indicating whether each word is followed by a space. + +> #### ✏️ Things to try +> +> 1. Change a boolean value in the list of `spaces`. You should see it reflected +> in the `doc.text` and whether the token is followed by a space. +> 2. Remove `spaces=spaces` from the `Doc`. You should see that every token is +> now followed by a space. +> 3. Copy-paste a random sentence from the internet and manually construct a +> `Doc` with `words` and `spaces` so that the `doc.text` matches the original +> input text. + +```python {executable="true"} +import spacy +from spacy.tokens import Doc + +nlp = spacy.blank("en") +words = ["Hello", ",", "world", "!"] +spaces = [False, True, False, False] +doc = Doc(nlp.vocab, words=words, spaces=spaces) +print(doc.text) +print([(t.text, t.text_with_ws, t.whitespace_) for t in doc]) +``` + +If provided, the spaces list must be the **same length** as the words list. The +spaces list affects the `doc.text`, `span.text`, `token.idx`, `span.start_char` +and `span.end_char` attributes. If you don't provide a `spaces` sequence, spaCy +will assume that all words are followed by a space. Once you have a +[`Doc`](/api/doc) object, you can write to its attributes to set the +part-of-speech tags, syntactic dependencies, named entities and other +attributes. + +#### Aligning tokenization {id="aligning-tokenization"} + +spaCy's tokenization is non-destructive and uses language-specific rules +optimized for compatibility with treebank annotations. Other tools and resources +can sometimes tokenize things differently – for example, `"I'm"` → +`["I", "'", "m"]` instead of `["I", "'m"]`. + +In situations like that, you often want to align the tokenization so that you +can merge annotations from different sources together, or take vectors predicted +by a +[pretrained BERT model](https://github.com/huggingface/pytorch-transformers) and +apply them to spaCy tokens. spaCy's [`Alignment`](/api/example#alignment-object) +object allows the one-to-one mappings of token indices in both directions as +well as taking into account indices where multiple tokens align to one single +token. + +> #### ✏️ Things to try +> +> 1. Change the capitalization in one of the token lists – for example, +> `"obama"` to `"Obama"`. You'll see that the alignment is case-insensitive. +> 2. Change `"podcasts"` in `other_tokens` to `"pod", "casts"`. You should see +> that there are now two tokens of length 2 in `y2x`, one corresponding to +> "'s", and one to "podcasts". +> 3. Make `other_tokens` and `spacy_tokens` identical. You'll see that all +> tokens now correspond 1-to-1. + +```python {executable="true"} +from spacy.training import Alignment + +other_tokens = ["i", "listened", "to", "obama", "'", "s", "podcasts", "."] +spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts", "."] +align = Alignment.from_strings(other_tokens, spacy_tokens) +print(f"a -> b, lengths: {align.x2y.lengths}") # array([1, 1, 1, 1, 1, 1, 1, 1]) +print(f"a -> b, mapping: {align.x2y.data}") # array([0, 1, 2, 3, 4, 4, 5, 6]) : two tokens both refer to "'s" +print(f"b -> a, lengths: {align.y2x.lengths}") # array([1, 1, 1, 1, 2, 1, 1]) : the token "'s" refers to two tokens +print(f"b -> a, mappings: {align.y2x.data}") # array([0, 1, 2, 3, 4, 5, 6, 7]) +``` + +Here are some insights from the alignment information generated in the example +above: + +- The one-to-one mappings for the first four tokens are identical, which means + they map to each other. This makes sense because they're also identical in the + input: `"i"`, `"listened"`, `"to"` and `"obama"`. +- The value of `x2y.data[6]` is `5`, which means that `other_tokens[6]` + (`"podcasts"`) aligns to `spacy_tokens[5]` (also `"podcasts"`). +- `x2y.data[4]` and `x2y.data[5]` are both `4`, which means that both tokens 4 + and 5 of `other_tokens` (`"'"` and `"s"`) align to token 4 of `spacy_tokens` + (`"'s"`). + + + +The current implementation of the alignment algorithm assumes that both +tokenizations add up to the same string. For example, you'll be able to align +`["I", "'", "m"]` and `["I", "'m"]`, which both add up to `"I'm"`, but not +`["I", "'m"]` and `["I", "am"]`. + + + +## Merging and splitting {id="retokenization",version="2.1"} + +The [`Doc.retokenize`](/api/doc#retokenize) context manager lets you merge and +split tokens. Modifications to the tokenization are stored and performed all at +once when the context manager exits. To merge several tokens into one single +token, pass a `Span` to [`retokenizer.merge`](/api/doc#retokenizer.merge). An +optional dictionary of `attrs` lets you set attributes that will be assigned to +the merged token – for example, the lemma, part-of-speech tag or entity type. By +default, the merged token will receive the same attributes as the merged span's +root. + +> #### ✏️ Things to try +> +> 1. Inspect the `token.lemma_` attribute with and without setting the `attrs`. +> You'll see that the lemma defaults to "New", the lemma of the span's root. +> 2. Overwrite other attributes like the `"ENT_TYPE"`. Since "New York" is also +> recognized as a named entity, this change will also be reflected in the +> `doc.ents`. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I live in New York") +print("Before:", [token.text for token in doc]) + +with doc.retokenize() as retokenizer: + retokenizer.merge(doc[3:5], attrs={"LEMMA": "new york"}) +print("After:", [token.text for token in doc]) +``` + +> #### Tip: merging entities and noun phrases +> +> If you need to merge named entities or noun chunks, check out the built-in +> [`merge_entities`](/api/pipeline-functions#merge_entities) and +> [`merge_noun_chunks`](/api/pipeline-functions#merge_noun_chunks) pipeline +> components. When added to your pipeline using `nlp.add_pipe`, they'll take +> care of merging the spans automatically. + +If an attribute in the `attrs` is a context-dependent token attribute, it will +be applied to the underlying [`Token`](/api/token). For example `LEMMA`, `POS` +or `DEP` only apply to a word in context, so they're token attributes. If an +attribute is a context-independent lexical attribute, it will be applied to the +underlying [`Lexeme`](/api/lexeme), the entry in the vocabulary. For example, +`LOWER` or `IS_STOP` apply to all words of the same spelling, regardless of the +context. + + + +If you're trying to merge spans that overlap, spaCy will raise an error because +it's unclear how the result should look. Depending on the application, you may +want to match the shortest or longest possible span, so it's up to you to filter +them. If you're looking for the longest non-overlapping span, you can use the +[`util.filter_spans`](/api/top-level#util.filter_spans) helper: + +```python +doc = nlp("I live in Berlin Kreuzberg") +spans = [doc[3:5], doc[3:4], doc[4:5]] +filtered_spans = filter_spans(spans) +``` + + + +### Splitting tokens + +The [`retokenizer.split`](/api/doc#retokenizer.split) method allows splitting +one token into two or more tokens. This can be useful for cases where +tokenization rules alone aren't sufficient. For example, you might want to split +"its" into the tokens "it" and "is" – but not the possessive pronoun "its". You +can write rule-based logic that can find only the correct "its" to split, but by +that time, the `Doc` will already be tokenized. + +This process of splitting a token requires more settings, because you need to +specify the text of the individual tokens, optional per-token attributes and how +the tokens should be attached to the existing syntax tree. This can be done by +supplying a list of `heads` – either the token to attach the newly split token +to, or a `(token, subtoken)` tuple if the newly split token should be attached +to another subtoken. In this case, "New" should be attached to "York" (the +second split subtoken) and "York" should be attached to "in". + +> #### ✏️ Things to try +> +> 1. Assign different attributes to the subtokens and compare the result. +> 2. Change the heads so that "New" is attached to "in" and "York" is attached +> to "New". +> 3. Split the token into three tokens instead of two – for example, +> `["New", "Yo", "rk"]`. + +```python {executable="true"} +import spacy +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I live in NewYork") +print("Before:", [token.text for token in doc]) +displacy.render(doc) # displacy.serve if you're not in a Jupyter environment + +with doc.retokenize() as retokenizer: + heads = [(doc[3], 1), doc[2]] + attrs = {"POS": ["PROPN", "PROPN"], "DEP": ["pobj", "compound"]} + retokenizer.split(doc[3], ["New", "York"], heads=heads, attrs=attrs) +print("After:", [token.text for token in doc]) +displacy.render(doc) # displacy.serve if you're not in a Jupyter environment +``` + +Specifying the heads as a list of `token` or `(token, subtoken)` tuples allows +attaching split subtokens to other subtokens, without having to keep track of +the token indices after splitting. + +| Token | Head | Description | +| -------- | ------------- | --------------------------------------------------------------------------------------------------- | +| `"New"` | `(doc[3], 1)` | Attach this token to the second subtoken (index `1`) that `doc[3]` will be split into, i.e. "York". | +| `"York"` | `doc[2]` | Attach this token to `doc[1]` in the original `Doc`, i.e. "in". | + +If you don't care about the heads (for example, if you're only running the +tokenizer and not the parser), you can attach each subtoken to itself: + +```python {highlight="3"} +doc = nlp("I live in NewYorkCity") +with doc.retokenize() as retokenizer: + heads = [(doc[3], 0), (doc[3], 1), (doc[3], 2)] + retokenizer.split(doc[3], ["New", "York", "City"], heads=heads) +``` + + + +When splitting tokens, the subtoken texts always have to match the original +token text – or, put differently `"".join(subtokens) == token.text` always needs +to hold true. If this wasn't the case, splitting tokens could easily end up +producing confusing and unexpected results that would contradict spaCy's +non-destructive tokenization policy. + +```diff +doc = nlp("I live in L.A.") +with doc.retokenize() as retokenizer: +- retokenizer.split(doc[3], ["Los", "Angeles"], heads=[(doc[3], 1), doc[2]]) ++ retokenizer.split(doc[3], ["L.", "A."], heads=[(doc[3], 1), doc[2]]) +``` + + + +### Overwriting custom extension attributes {id="retokenization-extensions"} + +If you've registered custom +[extension attributes](/usage/processing-pipelines#custom-components-attributes), +you can overwrite them during tokenization by providing a dictionary of +attribute names mapped to new values as the `"_"` key in the `attrs`. For +merging, you need to provide one dictionary of attributes for the resulting +merged token. For splitting, you need to provide a list of dictionaries with +custom attributes, one per split subtoken. + + + +To set extension attributes during retokenization, the attributes need to be +**registered** using the [`Token.set_extension`](/api/token#set_extension) +method and they need to be **writable**. This means that they should either have +a default value that can be overwritten, or a getter _and_ setter. Method +extensions or extensions with only a getter are computed dynamically, so their +values can't be overwritten. For more details, see the +[extension attribute docs](/usage/processing-pipelines/#custom-components-attributes). + + + +> #### ✏️ Things to try +> +> 1. Add another custom extension – maybe `"music_style"`? – and overwrite it. +> 2. Change the extension attribute to use only a `getter` function. You should +> see that spaCy raises an error, because the attribute is not writable +> anymore. +> 3. Rewrite the code to split a token with `retokenizer.split`. Remember that +> you need to provide a list of extension attribute values as the `"_"` +> property, one for each split subtoken. + +```python {executable="true"} +import spacy +from spacy.tokens import Token + +# Register a custom token attribute, token._.is_musician +Token.set_extension("is_musician", default=False) + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I like David Bowie") +print("Before:", [(token.text, token._.is_musician) for token in doc]) + +with doc.retokenize() as retokenizer: + retokenizer.merge(doc[2:4], attrs={"_": {"is_musician": True}}) +print("After:", [(token.text, token._.is_musician) for token in doc]) +``` + +## Sentence Segmentation {id="sbd"} + +A [`Doc`](/api/doc) object's sentences are available via the `Doc.sents` +property. To view a `Doc`'s sentences, you can iterate over the `Doc.sents`, a +generator that yields [`Span`](/api/span) objects. You can check whether a `Doc` +has sentence boundaries by calling +[`Doc.has_annotation`](/api/doc#has_annotation) with the attribute name +`"SENT_START"`. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence. This is another sentence.") +assert doc.has_annotation("SENT_START") +for sent in doc.sents: + print(sent.text) +``` + +spaCy provides four alternatives for sentence segmentation: + +1. [Dependency parser](#sbd-parser): the statistical + [`DependencyParser`](/api/dependencyparser) provides the most accurate + sentence boundaries based on full dependency parses. +2. [Statistical sentence segmenter](#sbd-senter): the statistical + [`SentenceRecognizer`](/api/sentencerecognizer) is a simpler and faster + alternative to the parser that only sets sentence boundaries. +3. [Rule-based pipeline component](#sbd-component): the rule-based + [`Sentencizer`](/api/sentencizer) sets sentence boundaries using a + customizable list of sentence-final punctuation. +4. [Custom function](#sbd-custom): your own custom function added to the + processing pipeline can set sentence boundaries by writing to + `Token.is_sent_start`. + +### Default: Using the dependency parse {id="sbd-parser",model="parser"} + +Unlike other libraries, spaCy uses the dependency parse to determine sentence +boundaries. This is usually the most accurate approach, but it requires a +**trained pipeline** that provides accurate predictions. If your texts are +closer to general-purpose news or web text, this should work well out-of-the-box +with spaCy's provided trained pipelines. For social media or conversational text +that doesn't follow the same rules, your application may benefit from a custom +trained or rule-based component. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence. This is another sentence.") +for sent in doc.sents: + print(sent.text) +``` + +spaCy's dependency parser respects already set boundaries, so you can preprocess +your `Doc` using custom components _before_ it's parsed. Depending on your text, +this may also improve parse accuracy, since the parser is constrained to predict +parses consistent with the sentence boundaries. + +### Statistical sentence segmenter {id="sbd-senter",model="senter",version="3"} + +The [`SentenceRecognizer`](/api/sentencerecognizer) is a simple statistical +component that only provides sentence boundaries. Along with being faster and +smaller than the parser, its primary advantage is that it's easier to train +because it only requires annotated sentence boundaries rather than full +dependency parses. spaCy's [trained pipelines](/models) include both a parser +and a trained sentence segmenter, which is +[disabled](/usage/processing-pipelines#disabling) by default. If you only need +sentence boundaries and no parser, you can use the `exclude` or `disable` +argument on [`spacy.load`](/api/top-level#spacy.load) to load the pipeline +without the parser and then enable the sentence recognizer explicitly with +[`nlp.enable_pipe`](/api/language#enable_pipe). + +> #### senter vs. parser +> +> The recall for the `senter` is typically slightly lower than for the parser, +> which is better at predicting sentence boundaries when punctuation is not +> present. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm", exclude=["parser"]) +nlp.enable_pipe("senter") +doc = nlp("This is a sentence. This is another sentence.") +for sent in doc.sents: + print(sent.text) +``` + +### Rule-based pipeline component {id="sbd-component"} + +The [`Sentencizer`](/api/sentencizer) component is a +[pipeline component](/usage/processing-pipelines) that splits sentences on +punctuation like `.`, `!` or `?`. You can plug it into your pipeline if you only +need sentence boundaries without dependency parses. + +```python {executable="true"} +import spacy +from spacy.lang.en import English + +nlp = English() # just the language with no pipeline +nlp.add_pipe("sentencizer") +doc = nlp("This is a sentence. This is another sentence.") +for sent in doc.sents: + print(sent.text) +``` + +### Custom rule-based strategy {id="sbd-custom"} + +If you want to implement your own strategy that differs from the default +rule-based approach of splitting on sentences, you can also create a +[custom pipeline component](/usage/processing-pipelines#custom-components) that +takes a `Doc` object and sets the `Token.is_sent_start` attribute on each +individual token. If set to `False`, the token is explicitly marked as _not_ the +start of a sentence. If set to `None` (default), it's treated as a missing value +and can still be overwritten by the parser. + + + +To prevent inconsistent state, you can only set boundaries **before** a document +is parsed (and `doc.has_annotation("DEP")` is `False`). To ensure that your +component is added in the right place, you can set `before='parser'` or +`first=True` when adding it to the pipeline using +[`nlp.add_pipe`](/api/language#add_pipe). + + + +Here's an example of a component that implements a pre-processing rule for +splitting on `"..."` tokens. The component is added before the parser, which is +then used to further segment the text. That's possible, because `is_sent_start` +is only set to `True` for some of the tokens – all others still specify `None` +for unset sentence boundaries. This approach can be useful if you want to +implement **additional** rules specific to your data, while still being able to +take advantage of dependency-based sentence segmentation. + +```python {executable="true"} +from spacy.language import Language +import spacy + +text = "this is a sentence...hello...and another sentence." + +nlp = spacy.load("en_core_web_sm") +doc = nlp(text) +print("Before:", [sent.text for sent in doc.sents]) + +@Language.component("set_custom_boundaries") +def set_custom_boundaries(doc): + for token in doc[:-1]: + if token.text == "...": + doc[token.i + 1].is_sent_start = True + return doc + +nlp.add_pipe("set_custom_boundaries", before="parser") +doc = nlp(text) +print("After:", [sent.text for sent in doc.sents]) +``` + +## Mappings & Exceptions {id="mappings-exceptions",version="3"} + +The [`AttributeRuler`](/api/attributeruler) manages **rule-based mappings and +exceptions** for all token-level attributes. As the number of +[pipeline components](/api/#architecture-pipeline) has grown from spaCy v2 to +v3, handling rules and exceptions in each component individually has become +impractical, so the `AttributeRuler` provides a single component with a unified +pattern format for all token attribute mappings and exceptions. + +The `AttributeRuler` uses +[`Matcher` patterns](/usage/rule-based-matching#adding-patterns) to identify +tokens and then assigns them the provided attributes. If needed, the +[`Matcher`](/api/matcher) patterns can include context around the target token. +For example, the attribute ruler can: + +- provide exceptions for any **token attributes** +- map **fine-grained tags** to **coarse-grained tags** for languages without + statistical morphologizers (replacing the v2.x `tag_map` in the + [language data](#language-data)) +- map token **surface form + fine-grained tags** to **morphological features** + (replacing the v2.x `morph_rules` in the [language data](#language-data)) +- specify the **tags for space tokens** (replacing hard-coded behavior in the + tagger) + +The following example shows how the tag and POS `NNP`/`PROPN` can be specified +for the phrase `"The Who"`, overriding the tags provided by the statistical +tagger and the POS tag map. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +text = "I saw The Who perform. Who did you see?" +doc1 = nlp(text) +print(doc1[2].tag_, doc1[2].pos_) # DT DET +print(doc1[3].tag_, doc1[3].pos_) # WP PRON + +# Add attribute ruler with exception for "The Who" as NNP/PROPN NNP/PROPN +ruler = nlp.get_pipe("attribute_ruler") +# Pattern to match "The Who" +patterns = [[{"LOWER": "the"}, {"TEXT": "Who"}]] +# The attributes to assign to the matched token +attrs = {"TAG": "NNP", "POS": "PROPN"} +# Add rules to the attribute ruler +ruler.add(patterns=patterns, attrs=attrs, index=0) # "The" in "The Who" +ruler.add(patterns=patterns, attrs=attrs, index=1) # "Who" in "The Who" + +doc2 = nlp(text) +print(doc2[2].tag_, doc2[2].pos_) # NNP PROPN +print(doc2[3].tag_, doc2[3].pos_) # NNP PROPN +# The second "Who" remains unmodified +print(doc2[5].tag_, doc2[5].pos_) # WP PRON +``` + + + +The [`AttributeRuler`](/api/attributeruler) can import a **tag map and morph +rules** in the v2.x format via its built-in methods or when the component is +initialized before training. See the +[migration guide](/usage/v3#migrating-training-mappings-exceptions) for details. + + + +## Word vectors and semantic similarity {id="vectors-similarity"} + + + +### Adding word vectors {id="adding-vectors"} + +Custom word vectors can be trained using a number of open-source libraries, such +as [Gensim](https://radimrehurek.com/gensim), [FastText](https://fasttext.cc), +or Tomas Mikolov's original +[Word2vec implementation](https://code.google.com/archive/p/word2vec/). Most +word vector libraries output an easy-to-read text-based format, where each line +consists of the word followed by its vector. For everyday use, we want to +convert the vectors into a binary format that loads faster and takes up less +space on disk. The easiest way to do this is the +[`init vectors`](/api/cli#init-vectors) command-line utility. This will output a +blank spaCy pipeline in the directory `/tmp/la_vectors_wiki_lg`, giving you +access to some nice Latin vectors. You can then pass the directory path to +[`spacy.load`](/api/top-level#spacy.load) or use it in the +[`[initialize]`](/api/data-formats#config-initialize) of your config when you +[train](/usage/training) a model. + +> #### Usage example +> +> ```python +> nlp_latin = spacy.load("/tmp/la_vectors_wiki_lg") +> doc1 = nlp_latin("Caecilius est in horto") +> doc2 = nlp_latin("servus est in atrio") +> doc1.similarity(doc2) +> ``` + +```bash +$ wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.la.300.vec.gz +$ python -m spacy init vectors en cc.la.300.vec.gz /tmp/la_vectors_wiki_lg +``` + + + +To help you strike a good balance between coverage and memory usage, spaCy's +[`Vectors`](/api/vectors) class lets you map **multiple keys** to the **same +row** of the table. If you're using the +[`spacy init vectors`](/api/cli#init-vectors) command to create a vocabulary, +pruning the vectors will be taken care of automatically if you set the `--prune` +flag. You can also do it manually in the following steps: + +1. Start with a **word vectors package** that covers a huge vocabulary. For + instance, the [`en_core_web_lg`](/models/en#en_core_web_lg) package provides + 300-dimensional GloVe vectors for 685k terms of English. +2. If your vocabulary has values set for the `Lexeme.prob` attribute, the + lexemes will be sorted by descending probability to determine which vectors + to prune. Otherwise, lexemes will be sorted by their order in the `Vocab`. +3. Call [`Vocab.prune_vectors`](/api/vocab#prune_vectors) with the number of + vectors you want to keep. + +```python +nlp = spacy.load("en_core_web_lg") +n_vectors = 105000 # number of vectors to keep +removed_words = nlp.vocab.prune_vectors(n_vectors) + +assert len(nlp.vocab.vectors) <= n_vectors # unique vectors have been pruned +assert nlp.vocab.vectors.n_keys > n_vectors # but not the total entries +``` + +[`Vocab.prune_vectors`](/api/vocab#prune_vectors) reduces the current vector +table to a given number of unique entries, and returns a dictionary containing +the removed words, mapped to `(string, score)` tuples, where `string` is the +entry the removed word was mapped to and `score` the similarity score between +the two words. + +```python {title="Removed words"} +{ + "Shore": ("coast", 0.732257), + "Precautionary": ("caution", 0.490973), + "hopelessness": ("sadness", 0.742366), + "Continuous": ("continuous", 0.732549), + "Disemboweled": ("corpse", 0.499432), + "biostatistician": ("scientist", 0.339724), + "somewheres": ("somewheres", 0.402736), + "observing": ("observe", 0.823096), + "Leaving": ("leaving", 1.0), +} +``` + +In the example above, the vector for "Shore" was removed and remapped to the +vector of "coast", which is deemed about 73% similar. "Leaving" was remapped to +the vector of "leaving", which is identical. If you're using the +[`init vectors`](/api/cli#init-vectors) command, you can set the `--prune` +option to easily reduce the size of the vectors as you add them to a spaCy +pipeline: + +```bash +$ python -m spacy init vectors en la.300d.vec.tgz /tmp/la_vectors_web_md --prune 10000 +``` + +This will create a blank spaCy pipeline with vectors for the first 10,000 words +in the vectors. All other words in the vectors are mapped to the closest vector +among those retained. + + + +### Adding vectors individually {id="adding-individual-vectors"} + +The `vector` attribute is a **read-only** numpy or cupy array (depending on +whether you've configured spaCy to use GPU memory), with dtype `float32`. The +array is read-only so that spaCy can avoid unnecessary copy operations where +possible. You can modify the vectors via the [`Vocab`](/api/vocab) or +[`Vectors`](/api/vectors) table. Using the +[`Vocab.set_vector`](/api/vocab#set_vector) method is often the easiest approach +if you have vectors in an arbitrary format, as you can read in the vectors with +your own logic, and just set them with a simple loop. This method is likely to +be slower than approaches that work with the whole vectors table at once, but +it's a great approach for once-off conversions before you save out your `nlp` +object to disk. + +```python {title="Adding vectors"} +from spacy.vocab import Vocab + +vector_data = { + "dog": numpy.random.uniform(-1, 1, (300,)), + "cat": numpy.random.uniform(-1, 1, (300,)), + "orange": numpy.random.uniform(-1, 1, (300,)) +} +vocab = Vocab() +for word, vector in vector_data.items(): + vocab.set_vector(word, vector) +``` + +## Language Data {id="language-data"} + + + +### Creating a custom language subclass {id="language-subclass"} + +If you want to customize multiple components of the language data or add support +for a custom language or domain-specific "dialect", you can also implement your +own language subclass. The subclass should define two attributes: the `lang` +(unique language code) and the `Defaults` defining the language data. For an +overview of the available attributes that can be overwritten, see the +[`Language.Defaults`](/api/language#defaults) documentation. + +```python {executable="true"} +from spacy.lang.en import English + +class CustomEnglishDefaults(English.Defaults): + stop_words = set(["custom", "stop"]) + +class CustomEnglish(English): + lang = "custom_en" + Defaults = CustomEnglishDefaults + +nlp1 = English() +nlp2 = CustomEnglish() + +print(nlp1.lang, [token.is_stop for token in nlp1("custom stop")]) +print(nlp2.lang, [token.is_stop for token in nlp2("custom stop")]) +``` + +The [`@spacy.registry.languages`](/api/top-level#registry) decorator lets you +register a custom language class and assign it a string name. This means that +you can call [`spacy.blank`](/api/top-level#spacy.blank) with your custom +language name, and even train pipelines with it and refer to it in your +[training config](/usage/training#config). + +> #### Config usage +> +> After registering your custom language class using the `languages` registry, +> you can refer to it in your [training config](/usage/training#config). This +> means spaCy will train your pipeline using the custom subclass. +> +> ```ini +> [nlp] +> lang = "custom_en" +> ``` +> +> In order to resolve `"custom_en"` to your subclass, the registered function +> needs to be available during training. You can load a Python file containing +> the code using the `--code` argument: +> +> ```bash +> python -m spacy train config.cfg --code code.py +> ``` + +```python {title="Registering a custom language",highlight="7,12-13"} +import spacy +from spacy.lang.en import English + +class CustomEnglishDefaults(English.Defaults): + stop_words = set(["custom", "stop"]) + +@spacy.registry.languages("custom_en") +class CustomEnglish(English): + lang = "custom_en" + Defaults = CustomEnglishDefaults + +# This now works! 🎉 +nlp = spacy.blank("custom_en") +``` diff --git a/website/docs/usage/memory-management.mdx b/website/docs/usage/memory-management.mdx new file mode 100644 index 0000000..330c117 --- /dev/null +++ b/website/docs/usage/memory-management.mdx @@ -0,0 +1,131 @@ +--- +title: Memory Management +teaser: Managing Memory for persistent services +version: 3.8 +menu: + - ['Memory Zones', 'memoryzones'] + - ['Clearing Doc attributes', 'doc-attrs'] +--- + +spaCy maintains a few internal caches that improve speed, +but cause memory to increase slightly over time. If you're +running a batch process that you don't need to be long-lived, +the increase in memory usage generally isn't a problem. +However, if you're running spaCy inside a web service, you'll +often want spaCy's memory usage to stay consistent. Transformer +models can also run into memory problems sometimes, especially when +used on a GPU. + +## Memory zones {id="memoryzones"} + +You can tell spaCy to free data from its internal caches (especially the +[`Vocab`](/api/vocab)) using the [`Language.memory_zone`](/api/language#memory_zone) context manager. Enter +the contextmanager and process your text within it, and spaCy will +**reset its internal caches** (freeing up the associated memory) at the +end of the block. spaCy objects created inside the memory zone must +not be accessed once the memory zone is finished. + +```python +### Using memory zones +from collections import Counter + +def count_words(nlp, texts): + counts = Counter() + with nlp.memory_zone(): + for doc in nlp.pipe(texts): + for token in doc: + counts[token.text] += 1 + return counts +``` + + + +Exiting the memory-zone invalidates all `Doc`, `Token`, `Span` and `Lexeme` +objects that were created within it. If you access these objects +after the memory zone exits, you may encounter a segmentation fault +due to invalid memory access. + + + +spaCy needs the memory zone contextmanager because the processing pipeline +can't keep track of which [`Doc`](/api/doc) objects are referring to data in the shared +[`Vocab`](/api/vocab) cache. For instance, when spaCy encounters a new word, a new [`Lexeme`](/api/lexeme) +entry is stored in the `Vocab`, and the `Doc` object points to this shared +data. When the `Doc` goes out of scope, the `Vocab` has no way of knowing that +this `Lexeme` is no longer in use. + +The memory zone solves this problem by +allowing you to tell the processing pipeline that all data created +between two points is no longer in use. It is up to the you to honor +this agreement. If you access objects that are supposed to no longer be in +use, you may encounter a segmentation fault due to invalid memory access. + +A common use case for memory zones will be **within a web service**. The processing +pipeline can be loaded once, either as a context variable or a global, and each +request can be handled within a memory zone: + +```python +### Memory zones with FastAPI {highlight="10,23"} +from fastapi import FastAPI, APIRouter, Depends, Request +import spacy +from spacy.language import Language + +router = APIRouter() + + +def make_app(): + app = FastAPI() + app.state.NLP = spacy.load("en_core_web_sm") + app.include_router(router) + return app + + +def get_nlp(request: Request) -> Language: + return request.app.state.NLP + + +@router.post("/parse") +def parse_texts( + *, text_batch: list[str], nlp: Language = Depends(get_nlp) +) -> list[dict]: + with nlp.memory_zone(): + # Put the spaCy call within a separate function, so we can't + # leak the Doc objects outside the scope of the memory zone. + output = _process_text(nlp, text_batch) + return output + + +def _process_text(nlp: Language, texts: list[str]) -> list[dict]: + # Call spaCy, and transform the output into our own data + # structures. This function is called from inside a memory + # zone, so must not return the spaCy objects. + docs = list(nlp.pipe(texts)) + return [ + { + "tokens": [{"text": t.text} for t in doc], + "entities": [ + {"start": e.start, "end": e.end, "label": e.label_} for e in doc.ents + ], + } + for doc in docs + ] + + +app = make_app() +``` + +## Clearing transformer tensors and other Doc attributes {id="doc-attrs"} + +The [`Transformer`](/api/transformer) and [`Tok2Vec`](/api/tok2vec) components set intermediate values onto the `Doc` +object during parsing. This can cause GPU memory to be exhausted if many `Doc` +objects are kept in memory together. + +To resolve this, you can add the [`doc_cleaner`](/api/pipeline-functions#doc_cleaner) component to your pipeline. By default +this will clean up the [`Doc._.trf_data`](/api/transformer#custom_attributes) extension attribute and the [`Doc.tensor`](/api/doc#attributes) attribute. +You can have it clean up other intermediate extension attributes you use in custom +pipeline components as well. + +```python +### Adding the doc_cleaner +nlp.add_pipe("doc_cleaner", config={"attrs": {"tensor": None}}) +``` diff --git a/website/docs/usage/models.mdx b/website/docs/usage/models.mdx new file mode 100644 index 0000000..7fed9f4 --- /dev/null +++ b/website/docs/usage/models.mdx @@ -0,0 +1,570 @@ +--- +title: Models & Languages +next: usage/facts-figures +menu: + - ['Quickstart', 'quickstart'] + - ['Language Support', 'languages'] + - ['Installation & Usage', 'download'] + - ['Production Use', 'production'] +--- + +spaCy's trained pipelines can be installed as **Python packages**. This means +that they're a component of your application, just like any other module. +They're versioned and can be defined as a dependency in your `requirements.txt`. +Trained pipelines can be installed from a download URL or a local directory, +manually or via [pip](https://pypi.python.org/pypi/pip). Their data can be +located anywhere on your file system. + +> #### Important note +> +> If you're upgrading to spaCy v3.x, you need to **download the new pipeline +> packages**. If you've trained your own pipelines, you need to **retrain** them +> after updating spaCy. + +## Quickstart {hidden="true"} + + + +### Usage note + +> If lemmatization rules are available for your language, make sure to install +> spaCy with the `lookups` option, or install +> [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) +> separately in the same environment: +> +> ```bash +> $ pip install -U %%SPACY_PKG_NAME[lookups]%%SPACY_PKG_FLAGS +> ``` + +If a trained pipeline is available for a language, you can download it using the +[`spacy download`](/api/cli#download) command as shown above. In order to use +languages that don't yet come with a trained pipeline, you have to import them +directly, or use [`spacy.blank`](/api/top-level#spacy.blank): + +```python +from spacy.lang.yo import Yoruba +nlp = Yoruba() # use directly +nlp = spacy.blank("yo") # blank instance +``` + +A blank pipeline is typically just a tokenizer. You might want to create a blank +pipeline when you only need a tokenizer, when you want to add more components +from scratch, or for testing purposes. Initializing the language object directly +yields the same result as generating it using `spacy.blank()`. In both cases the +default configuration for the chosen language is loaded, and no pretrained +components will be available. + +## Language support {id="languages"} + +spaCy currently provides support for the following languages. You can help by +improving the existing [language data](/usage/linguistic-features#language-data) +and extending the tokenization patterns. +[See here](https://github.com/explosion/spaCy/issues/3056) for details on how to +contribute to development. Also see the +[training documentation](/usage/training) for how to train your own pipelines on +your data. + + + +### Multi-language support {id="multi-language",version="2"} + +> ```python +> # Standard import +> from spacy.lang.xx import MultiLanguage +> nlp = MultiLanguage() +> +> # With lazy-loading +> nlp = spacy.blank("xx") +> ``` + +spaCy also supports pipelines trained on more than one language. This is +especially useful for named entity recognition. The language ID used for +multi-language or language-neutral pipelines is `xx`. The language class, a +generic subclass containing only the base language data, can be found in +[`lang/xx`](%%GITHUB_SPACY/spacy/lang/xx). + +To train a pipeline using the neutral multi-language class, you can set +`lang = "xx"` in your [training config](/usage/training#config). You can also +\import the `MultiLanguage` class directly, or call +[`spacy.blank("xx")`](/api/top-level#spacy.blank) for lazy-loading. + +### Chinese language support {id="chinese",version="2.3"} + +The Chinese language class supports three word segmentation options, `char`, +`jieba` and `pkuseg`. + +> #### Manual setup +> +> ```python +> from spacy.lang.zh import Chinese +> +> # Character segmentation (default) +> nlp = Chinese() +> # Jieba +> cfg = {"segmenter": "jieba"} +> nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}}) +> # PKUSeg with "mixed" model provided by pkuseg +> cfg = {"segmenter": "pkuseg"} +> nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}}) +> nlp.tokenizer.initialize(pkuseg_model="mixed") +> ``` + +```ini {title="config.cfg"} +[nlp.tokenizer] +@tokenizers = "spacy.zh.ChineseTokenizer" +segmenter = "char" +``` + +| Segmenter | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `char` | **Character segmentation:** Character segmentation is the default segmentation option. It's enabled when you create a new `Chinese` language class or call `spacy.blank("zh")`. | +| `jieba` | **Jieba:** to use [Jieba](https://github.com/fxsjy/jieba) for word segmentation, you can set the option `segmenter` to `"jieba"`. | +| `pkuseg` | **PKUSeg**: As of spaCy v2.3.0, support for [PKUSeg](https://github.com/explosion/spacy-pkuseg) has been added to support better segmentation for Chinese OntoNotes and the provided [Chinese pipelines](/models/zh). Enable PKUSeg by setting tokenizer option `segmenter` to `"pkuseg"`. | + + + +In v3.0, the default word segmenter has switched from Jieba to character +segmentation. Because the `pkuseg` segmenter depends on a model that can be +loaded from a file, the model is loaded on +[initialization](/usage/training#config-lifecycle) (typically before training). +This ensures that your packaged Chinese model doesn't depend on a local path at +runtime. + + + + + +The `initialize` method for the Chinese tokenizer class supports the following +config settings for loading `pkuseg` models: + +| Name | Description | +| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `pkuseg_model` | Name of a model provided by `spacy-pkuseg` or the path to a local model directory. ~~str~~ | +| `pkuseg_user_dict` | Optional path to a file with one word per line which overrides the default `pkuseg` user dictionary. Defaults to `"default"`, the default provided dictionary. ~~str~~ | + +The initialization settings are typically provided in the +[training config](/usage/training#config) and the data is loaded in before +training and serialized with the model. This allows you to load the data from a +local path and save out your pipeline and config, without requiring the same +local path at runtime. See the usage guide on the +[config lifecycle](/usage/training#config-lifecycle) for more background on +this. + +```ini {title="config.cfg"} +[initialize] + +[initialize.tokenizer] +pkuseg_model = "/path/to/model" +pkuseg_user_dict = "default" +``` + +You can also initialize the tokenizer for a blank language class by calling its +`initialize` method: + +```python {title="Examples"} +# Initialize the pkuseg tokenizer +cfg = {"segmenter": "pkuseg"} +nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}}) + +# Load spaCy's OntoNotes model +nlp.tokenizer.initialize(pkuseg_model="spacy_ontonotes") + +# Load pkuseg's "news" model +nlp.tokenizer.initialize(pkuseg_model="news") + +# Load local model +nlp.tokenizer.initialize(pkuseg_model="/path/to/pkuseg_model") + +# Override the user directory +nlp.tokenizer.initialize(pkuseg_model="spacy_ontonotes", pkuseg_user_dict="/path/to/user_dict") +``` + +You can also modify the user dictionary on-the-fly: + +```python +# Append words to user dict +nlp.tokenizer.pkuseg_update_user_dict(["中国", "ABC"]) + +# Remove all words from user dict and replace with new words +nlp.tokenizer.pkuseg_update_user_dict(["中国"], reset=True) + +# Remove all words from user dict +nlp.tokenizer.pkuseg_update_user_dict([], reset=True) +``` + + + + + +The [Chinese pipelines](/models/zh) provided by spaCy include a custom `pkuseg` +model trained only on +[Chinese OntoNotes 5.0](https://catalog.ldc.upenn.edu/LDC2013T19), since the +models provided by `pkuseg` include data restricted to research use. For +research use, `pkuseg` provides models for several different domains (`"mixed"` +(equivalent to `"default"` from `pkuseg` packages), `"news"` `"web"`, +`"medicine"`, `"tourism"`) and for other uses, `pkuseg` provides a simple +[training API](https://github.com/explosion/spacy-pkuseg/blob/master/readme/readme_english.md#usage): + +```python +import spacy_pkuseg as pkuseg +from spacy.lang.zh import Chinese + +# Train pkuseg model +pkuseg.train("train.utf8", "test.utf8", "/path/to/pkuseg_model") + +# Load pkuseg model in spaCy Chinese tokenizer +cfg = {"segmenter": "pkuseg"} +nlp = Chinese.from_config({"nlp": {"tokenizer": cfg}}) +nlp.tokenizer.initialize(pkuseg_model="/path/to/pkuseg_model") +``` + + + +### Japanese language support {id="japanese",version="2.3"} + +> #### Manual setup +> +> ```python +> from spacy.lang.ja import Japanese +> +> # Load SudachiPy with split mode A (default) +> nlp = Japanese() +> # Load SudachiPy with split mode B +> cfg = {"split_mode": "B"} +> nlp = Japanese.from_config({"nlp": {"tokenizer": cfg}}) +> ``` + +The Japanese language class uses +[SudachiPy](https://github.com/WorksApplications/SudachiPy) for word +segmentation and part-of-speech tagging. The default Japanese language class and +the provided Japanese pipelines use SudachiPy split mode `A`. The tokenizer +config can be used to configure the split mode to `A`, `B` or `C`. + +```ini {title="config.cfg"} +[nlp.tokenizer] +@tokenizers = "spacy.ja.JapaneseTokenizer" +split_mode = "A" +``` + +Extra information, such as reading, inflection form, and the SudachiPy +normalized form, is available in `Token.morph`. For `B` or `C` split modes, +subtokens are stored in `Doc.user_data["sub_tokens"]`. + + + +If you run into errors related to `sudachipy`, which is currently under active +development, we suggest downgrading to `sudachipy==0.4.9`, which is the version +used for training the current [Japanese pipelines](/models/ja). + + + +### Korean language support {id="korean"} + +> #### mecab-ko tokenizer +> +> ```python +> nlp = spacy.blank("ko") +> ``` + +The default MeCab-based Korean tokenizer requires: + +- [mecab-ko](https://bitbucket.org/eunjeon/mecab-ko/src/master/README.md) +- [mecab-ko-dic](https://bitbucket.org/eunjeon/mecab-ko-dic) +- [natto-py](https://github.com/buruzaemon/natto-py) + +For some Korean datasets and tasks, the +[rule-based tokenizer](/usage/linguistic-features#tokenization) is better-suited +than MeCab. To configure a Korean pipeline with the rule-based tokenizer: + +> #### Rule-based tokenizer +> +> ```python +> config = {"nlp": {"tokenizer": {"@tokenizers": "spacy.Tokenizer.v1"}}} +> nlp = spacy.blank("ko", config=config) +> ``` + +```ini {title="config.cfg"} +[nlp] +lang = "ko" +tokenizer = {"@tokenizers" = "spacy.Tokenizer.v1"} +``` + + + +The [Korean trained pipelines](/models/ko) use the rule-based tokenizer, so no +additional dependencies are required. + + + +## Installing and using trained pipelines {id="download"} + +The easiest way to download a trained pipeline is via spaCy's +[`download`](/api/cli#download) command. It takes care of finding the +best-matching package compatible with your spaCy installation. + +> #### Important note for v3.0 +> +> Note that as of spaCy v3.0, shortcut links like `en` that create (potentially +> brittle) symlinks in your spaCy installation are **deprecated**. To download +> and load an installed pipeline package, use its full name: +> +> ```diff +> - python -m spacy download en +> + python -m spacy download en_core_web_sm +> ``` +> +> ```diff +> - nlp = spacy.load("en") +> + nlp = spacy.load("en_core_web_sm") +> ``` + +```bash +# Download best-matching version of a package for your spaCy installation +$ python -m spacy download en_core_web_sm + +# Download exact package version +$ python -m spacy download en_core_web_sm-3.0.0 --direct +``` + +The download command will [install the package](/usage/models#download-pip) via +pip and place the package in your `site-packages` directory. + +```bash +$ pip install -U %%SPACY_PKG_NAME%%SPACY_PKG_FLAGS +$ python -m spacy download en_core_web_sm +``` + +```python +import spacy +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence.") +``` + +If you're in a **Jupyter notebook** or similar environment, you can use the `!` +prefix to +[execute commands](https://ipython.org/ipython-doc/3/interactive/tutorial.html#system-shell-commands). +Make sure to **restart your kernel** or runtime after installation (just like +you would when installing other Python packages) to make sure that the installed +pipeline package can be found. + +```bash +!python -m spacy download en_core_web_sm +``` + +### Installation via pip {id="download-pip"} + +To download a trained pipeline directly using +[pip](https://pypi.python.org/pypi/pip), point `pip install` to the URL or local +path of the wheel file or archive. Installing the wheel is usually more +efficient. + +> #### Pipeline Package URLs {id="pipeline-urls"} +> +> Pretrained pipeline distributions are hosted on +> [Github Releases](https://github.com/explosion/spacy-models/releases), and you +> can find download links there, as well as on the model page. You can also get +> URLs directly from the command line by using `spacy info` with the `--url` +> flag, which may be useful for automation. +> +> ```bash +> spacy info en_core_web_sm --url +> ``` +> +> This command will print the URL for the latest version of a pipeline +> compatible with the version of spaCy you're using. Note that in order to look +> up the compatibility information an internet connection is required. + +```bash +# With external URL +$ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl +$ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz + +# Using spacy info to get the external URL +$ pip install $(spacy info en_core_web_sm --url) + +# With local file +$ pip install /Users/you/en_core_web_sm-3.0.0-py3-none-any.whl +$ pip install /Users/you/en_core_web_sm-3.0.0.tar.gz +``` + +By default, this will install the pipeline package into your `site-packages` +directory. You can then use `spacy.load` to load it via its package name or +[import it](#usage-import) explicitly as a module. If you need to download +pipeline packages as part of an automated process, we recommend using pip with a +direct link, instead of relying on spaCy's [`download`](/api/cli#download) +command. + +You can also add the direct download link to your application's +`requirements.txt`. For more details, see the section on +[working with pipeline packages in production](#production). + +### Manual download and installation {id="download-manual"} + +In some cases, you might prefer downloading the data manually, for example to +place it into a custom directory. You can download the package via your browser +from the [latest releases](https://github.com/explosion/spacy-models/releases), +or configure your own download script using the URL of the archive file. The +archive consists of a package directory that contains another directory with the +pipeline data. + +```yaml {title="Directory structure",highlight="6"} +└── en_core_web_md-3.0.0.tar.gz # downloaded archive + ├── setup.py # setup file for pip installation + ├── meta.json # copy of pipeline meta + └── en_core_web_md # 📦 pipeline package + ├── __init__.py # init for pip installation + └── en_core_web_md-3.0.0 # pipeline data + ├── config.cfg # pipeline config + ├── meta.json # pipeline meta + └── ... # directories with component data +``` + +You can place the **pipeline package directory** anywhere on your local file +system. + +### Installation from Python {id="download-python"} + +Since the [`spacy download`](/api/cli#download) command installs the pipeline as +a **Python package**, we always recommend running it from the command line, just +like you install other Python packages with `pip install`. However, if you need +to, or if you want to integrate the download process into another CLI command, +you can also import and call the `download` function used by the CLI via Python. + + + +Keep in mind that the `download` command installs a Python package into your +environment. In order for it to be found after installation, you will need to +**restart or reload** your Python process so that new packages are recognized. + + + +```python +import spacy +spacy.cli.download("en_core_web_sm") +``` + +### Using trained pipelines with spaCy {id="usage"} + +To load a pipeline package, use [`spacy.load`](/api/top-level#spacy.load) with +the package name or a path to the data directory: + +> #### Important note for v3.0 +> +> Note that as of spaCy v3.0, shortcut links like `en` that create (potentially +> brittle) symlinks in your spaCy installation are **deprecated**. To download +> and load an installed pipeline package, use its full name: +> +> ```diff +> - python -m spacy download en +> + python -m spacy download en_core_web_sm +> ``` + +```python +import spacy +nlp = spacy.load("en_core_web_sm") # load package "en_core_web_sm" +nlp = spacy.load("/path/to/en_core_web_sm") # load package from a directory + +doc = nlp("This is a sentence.") +``` + + + +You can use the [`info`](/api/cli#info) command or +[`spacy.info()`](/api/top-level#spacy.info) method to print a pipeline package's +meta data before loading it. Each `Language` object with a loaded pipeline also +exposes the pipeline's meta data as the attribute `meta`. For example, +`nlp.meta['version']` will return the package version. + + + +### Importing pipeline packages as modules {id="usage-import"} + +If you've installed a trained pipeline via [`spacy download`](/api/cli#download) +or directly via pip, you can also `import` it and then call its `load()` method +with no arguments: + +```python {executable="true"} +import en_core_web_sm + +nlp = en_core_web_sm.load() +doc = nlp("This is a sentence.") +``` + +How you choose to load your trained pipelines ultimately depends on personal +preference. However, **for larger code bases**, we usually recommend native +imports, as this will make it easier to integrate pipeline packages with your +existing build process, continuous integration workflow and testing framework. +It'll also prevent you from ever trying to load a package that is not installed, +as your code will raise an `ImportError` immediately, instead of failing +somewhere down the line when calling `spacy.load()`. For more details, see the +section on [working with pipeline packages in production](#production). + +## Using trained pipelines in production {id="production"} + +If your application depends on one or more trained pipeline packages, you'll +usually want to integrate them into your continuous integration workflow and +build process. While spaCy provides a range of useful helpers for downloading +and loading pipeline packages, the underlying functionality is entirely based on +native Python packaging. This allows your application to handle a spaCy pipeline +like any other package dependency. + +### Downloading and requiring package dependencies {id="models-download"} + +spaCy's built-in [`download`](/api/cli#download) command is mostly intended as a +convenient, interactive wrapper. It performs compatibility checks and prints +detailed error messages and warnings. However, if you're downloading pipeline +packages as part of an automated build process, this only adds an unnecessary +layer of complexity. If you know which packages your application needs, you +should be specifying them directly. + +Because pipeline packages are valid Python packages, you can add them to your +application's `requirements.txt`. If you're running your own internal PyPi +installation, you can upload the pipeline packages there. pip's +[requirements file format](https://pip.pypa.io/en/latest/reference/requirements-file-format/) +supports both package names to download via a PyPi server, as well as +[direct URLs](#pipeline-urls). For instance, you can specify the +`en_core_web_sm` model for spaCy 3.7.x as follows: + +```text {title="requirements.txt"} +spacy>=3.0.0,<4.0.0 +en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl +``` + +See the [list of models](https://spacy.io/models) for model download links for +the current spaCy version. + +All pipeline packages are versioned and specify their spaCy dependency. This +ensures cross-compatibility and lets you specify exact version requirements for +each pipeline. If you've [trained](/usage/training) your own pipeline, you can +use the [`spacy package`](/api/cli#package) command to generate the required +meta data and turn it into a loadable package. + +### Loading and testing pipeline packages {id="models-loading"} + +Pipeline packages are regular Python packages, so you can also import them as a +package using Python's native `import` syntax, and then call the `load` method +to load the data and return an `nlp` object: + +```python +import en_core_web_sm +nlp = en_core_web_sm.load() +``` + +In general, this approach is recommended for larger code bases, as it's more +"native", and doesn't rely on spaCy's loader to resolve string names to +packages. If a package can't be imported, Python will raise an `ImportError` +immediately. And if a package is imported but not used, any linter will catch +that. + +Similarly, it'll give you more flexibility when writing tests that require +loading pipelines. For example, instead of writing your own `try` and `except` +logic around spaCy's loader, you can use +[pytest](http://pytest.readthedocs.io/en/latest/)'s +[`importorskip()`](https://docs.pytest.org/en/latest/builtin.html#_pytest.outcomes.importorskip) +method to only run a test if a specific pipeline package or version is +installed. Each pipeline package exposes a `__version__` attribute which you can +also use to perform your own version compatibility checks before loading it. diff --git a/website/docs/usage/processing-pipelines.mdx b/website/docs/usage/processing-pipelines.mdx new file mode 100644 index 0000000..605affa --- /dev/null +++ b/website/docs/usage/processing-pipelines.mdx @@ -0,0 +1,1830 @@ +--- +title: Language Processing Pipelines +next: /usage/embeddings-transformers +menu: + - ['Processing Text', 'processing'] + - ['Pipelines & Components', 'pipelines'] + - ['Custom Components', 'custom-components'] + - ['Component Data', 'component-data'] + - ['Type Hints & Validation', 'type-hints'] + - ['Trainable Components', 'trainable-components'] + - ['Extension Attributes', 'custom-components-attributes'] + - ['Plugins & Wrappers', 'plugins'] +--- + + + +## Processing text {id="processing"} + +When you call `nlp` on a text, spaCy will **tokenize** it and then **call each +component** on the `Doc`, in order. It then returns the processed `Doc` that you +can work with. + +```python +doc = nlp("This is a text") +``` + +When processing large volumes of text, the statistical models are usually more +efficient if you let them work on batches of texts. spaCy's +[`nlp.pipe`](/api/language#pipe) method takes an iterable of texts and yields +processed `Doc` objects. The batching is done internally. + +```diff +texts = ["This is a text", "These are lots of texts", "..."] +- docs = [nlp(text) for text in texts] ++ docs = list(nlp.pipe(texts)) +``` + + + +- Process the texts **as a stream** using [`nlp.pipe`](/api/language#pipe) and + buffer them in batches, instead of one-by-one. This is usually much more + efficient. +- Only apply the **pipeline components you need**. Getting predictions from the + model that you don't actually need adds up and becomes very inefficient at + scale. To prevent this, use the `disable` keyword argument to disable + components you don't need – either when loading a pipeline, or during + processing with `nlp.pipe`. See the section on + [disabling pipeline components](#disabling) for more details and examples. + + + +In this example, we're using [`nlp.pipe`](/api/language#pipe) to process a +(potentially very large) iterable of texts as a stream. Because we're only +accessing the named entities in `doc.ents` (set by the `ner` component), we'll +disable all other components during processing. `nlp.pipe` yields `Doc` objects, +so we can iterate over them and access the named entity predictions: + +> #### ✏️ Things to try +> +> 1. Also disable the `"ner"` component. You'll see that the `doc.ents` are now +> empty, because the entity recognizer didn't run. + +```python {executable="true"} +import spacy + +texts = [ + "Net income was $9.4 million compared to the prior year of $2.7 million.", + "Revenue exceeded twelve billion dollars, with a loss of $1b.", +] + +nlp = spacy.load("en_core_web_sm") +for doc in nlp.pipe(texts, disable=["tok2vec", "tagger", "parser", "attribute_ruler", "lemmatizer"]): + # Do something with the doc here + print([(ent.text, ent.label_) for ent in doc.ents]) +``` + + + +When using [`nlp.pipe`](/api/language#pipe), keep in mind that it returns a +[generator](https://realpython.com/introduction-to-python-generators/) that +yields `Doc` objects – not a list. So if you want to use it like a list, you'll +have to call `list()` on it first: + +```diff +- docs = nlp.pipe(texts)[0] # will raise an error ++ docs = list(nlp.pipe(texts))[0] # works as expected +``` + + + +You can use the `as_tuples` option to pass additional context along with each +doc when using [`nlp.pipe`](/api/language#pipe). If `as_tuples` is `True`, then +the input should be a sequence of `(text, context)` tuples and the output will +be a sequence of `(doc, context)` tuples. For example, you can pass metadata in +the context and save it in a [custom attribute](#custom-components-attributes): + +```python {executable="true"} +import spacy +from spacy.tokens import Doc + +if not Doc.has_extension("text_id"): + Doc.set_extension("text_id", default=None) + +text_tuples = [ + ("This is the first text.", {"text_id": "text1"}), + ("This is the second text.", {"text_id": "text2"}) +] + +nlp = spacy.load("en_core_web_sm") +doc_tuples = nlp.pipe(text_tuples, as_tuples=True) + +docs = [] +for doc, context in doc_tuples: + doc._.text_id = context["text_id"] + docs.append(doc) + +for doc in docs: + print(f"{doc._.text_id}: {doc.text}") +``` + +### Multiprocessing {id="multiprocessing"} + +spaCy includes built-in support for multiprocessing with +[`nlp.pipe`](/api/language#pipe) using the `n_process` option: + +```python +# Multiprocessing with 4 processes +docs = nlp.pipe(texts, n_process=4) + +# With as many processes as CPUs (use with caution!) +docs = nlp.pipe(texts, n_process=-1) +``` + +Depending on your platform, starting many processes with multiprocessing can add +a lot of overhead. In particular, the default start method `spawn` used in +macOS/OS X (as of Python 3.8) and in Windows can be slow for larger models +because the model data is copied in memory for each new process. See the +[Python docs on multiprocessing](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods) +for further details. + +For shorter tasks and in particular with `spawn`, it can be faster to use a +smaller number of processes with a larger batch size. The optimal `batch_size` +setting will depend on the pipeline components, the length of your documents, +the number of processes and how much memory is available. + +```python +# Default batch size is `nlp.batch_size` (typically 1000) +docs = nlp.pipe(texts, n_process=2, batch_size=2000) +``` + + + +Multiprocessing is not generally recommended on GPU because RAM is too limited. +If you want to try it out, be aware that it is only possible using `spawn` due +to limitations in CUDA. + + + + + +In Linux, transformer models may hang or deadlock with multiprocessing due to an +[issue in PyTorch](https://github.com/pytorch/pytorch/issues/17199). One +suggested workaround is to use `spawn` instead of `fork` and another is to limit +the number of threads before loading any models using +`torch.set_num_threads(1)`. + + + +## Pipelines and built-in components {id="pipelines"} + +spaCy makes it very easy to create your own pipelines consisting of reusable +components – this includes spaCy's default tagger, parser and entity recognizer, +but also your own custom processing functions. A pipeline component can be added +to an already existing `nlp` object, specified when initializing a +[`Language`](/api/language) class, or defined within a +[pipeline package](/usage/saving-loading#models). + +> #### config.cfg (excerpt) +> +> ```ini +> [nlp] +> lang = "en" +> pipeline = ["tok2vec", "parser"] +> +> [components] +> +> [components.tok2vec] +> factory = "tok2vec" +> # Settings for the tok2vec component +> +> [components.parser] +> factory = "parser" +> # Settings for the parser component +> ``` + +When you load a pipeline, spaCy first consults the +[`meta.json`](/usage/saving-loading#models) and +[`config.cfg`](/usage/training#config). The config tells spaCy what language +class to use, which components are in the pipeline, and how those components +should be created. spaCy will then do the following: + +1. Load the **language class and data** for the given ID via + [`get_lang_class`](/api/top-level#util.get_lang_class) and initialize it. The + `Language` class contains the shared vocabulary, tokenization rules and the + language-specific settings. +2. Iterate over the **pipeline names** and look up each component name in the + `[components]` block. The `factory` tells spaCy which + [component factory](#custom-components-factories) to use for adding the + component with [`add_pipe`](/api/language#add_pipe). The settings are passed + into the factory. +3. Make the **model data** available to the `Language` class by calling + [`from_disk`](/api/language#from_disk) with the path to the data directory. + +So when you call this... + +```python +nlp = spacy.load("en_core_web_sm") +``` + +... the pipeline's `config.cfg` tells spaCy to use the language `"en"` and the +pipeline +`["tok2vec", "tagger", "parser", "ner", "attribute_ruler", "lemmatizer"]`. spaCy +will then initialize `spacy.lang.en.English`, and create each pipeline component +and add it to the processing pipeline. It'll then load in the model data from +the data directory and return the modified `Language` class for you to use as +the `nlp` object. + + + +spaCy v3.0 introduces a `config.cfg`, which includes more detailed settings for +the pipeline, its components and the [training process](/usage/training#config). +You can export the config of your current `nlp` object by calling +[`nlp.config.to_disk`](/api/language#config). + + + +Fundamentally, a [spaCy pipeline package](/models) consists of three components: +**the weights**, i.e. binary data loaded in from a directory, a **pipeline** of +functions called in order, and **language data** like the tokenization rules and +language-specific settings. For example, a Spanish NER pipeline requires +different weights, language data and components than an English parsing and +tagging pipeline. This is also why the pipeline state is always held by the +`Language` class. [`spacy.load`](/api/top-level#spacy.load) puts this all +together and returns an instance of `Language` with a pipeline set and access to +the binary data: + +```python {title="spacy.load under the hood (abstract example)"} +lang = "en" +pipeline = ["tok2vec", "tagger", "parser", "ner", "attribute_ruler", "lemmatizer"] +data_path = "path/to/en_core_web_sm/en_core_web_sm-3.0.0" + +cls = spacy.util.get_lang_class(lang) # 1. Get Language class, e.g. English +nlp = cls() # 2. Initialize it +for name in pipeline: + nlp.add_pipe(name, config={...}) # 3. Add the component to the pipeline +nlp.from_disk(data_path) # 4. Load in the binary data +``` + +When you call `nlp` on a text, spaCy will **tokenize** it and then **call each +component** on the `Doc`, in order. Since the model data is loaded, the +components can access it to assign annotations to the `Doc` object, and +subsequently to the `Token` and `Span` which are only views of the `Doc`, and +don't own any data themselves. All components return the modified document, +which is then processed by the next component in the pipeline. + +```python {title="The pipeline under the hood"} +doc = nlp.make_doc("This is a sentence") # Create a Doc from raw text +for name, proc in nlp.pipeline: # Iterate over components in order + doc = proc(doc) # Apply each component +``` + +The current processing pipeline is available as `nlp.pipeline`, which returns a +list of `(name, component)` tuples, or `nlp.pipe_names`, which only returns a +list of human-readable component names. + +```python +print(nlp.pipeline) +# [('tok2vec', ), ('tagger', ), ('parser', ), ('ner', ), ('attribute_ruler', ), ('lemmatizer', )] +print(nlp.pipe_names) +# ['tok2vec', 'tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'] +``` + +### Built-in pipeline components {id="built-in"} + +spaCy ships with several built-in pipeline components that are registered with +string names. This means that you can initialize them by calling +[`nlp.add_pipe`](/api/language#add_pipe) with their names and spaCy will know +how to create them. See the [API documentation](/api) for a full list of +available pipeline components and component functions. + +> #### Usage +> +> ```python +> nlp = spacy.blank("en") +> nlp.add_pipe("sentencizer") +> # add_pipe returns the added component +> ruler = nlp.add_pipe("entity_ruler") +> ``` + +| String name | Component | Description | +| ---------------------- | ---------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| `tagger` | [`Tagger`](/api/tagger) | Assign part-of-speech-tags. | +| `parser` | [`DependencyParser`](/api/dependencyparser) | Assign dependency labels. | +| `ner` | [`EntityRecognizer`](/api/entityrecognizer) | Assign named entities. | +| `entity_linker` | [`EntityLinker`](/api/entitylinker) | Assign knowledge base IDs to named entities. Should be added after the entity recognizer. | +| `entity_ruler` | [`EntityRuler`](/api/entityruler) | Assign named entities based on pattern rules and dictionaries. | +| `textcat` | [`TextCategorizer`](/api/textcategorizer) | Assign text categories: exactly one category is predicted per document. | +| `textcat_multilabel` | [`MultiLabel_TextCategorizer`](/api/textcategorizer) | Assign text categories in a multi-label setting: zero, one or more labels per document. | +| `lemmatizer` | [`Lemmatizer`](/api/lemmatizer) | Assign base forms to words using rules and lookups. | +| `trainable_lemmatizer` | [`EditTreeLemmatizer`](/api/edittreelemmatizer) | Assign base forms to words. | +| `morphologizer` | [`Morphologizer`](/api/morphologizer) | Assign morphological features and coarse-grained POS tags. | +| `attribute_ruler` | [`AttributeRuler`](/api/attributeruler) | Assign token attribute mappings and rule-based exceptions. | +| `senter` | [`SentenceRecognizer`](/api/sentencerecognizer) | Assign sentence boundaries. | +| `sentencizer` | [`Sentencizer`](/api/sentencizer) | Add rule-based sentence segmentation without the dependency parse. | +| `tok2vec` | [`Tok2Vec`](/api/tok2vec) | Assign token-to-vector embeddings. | +| `transformer` | [`Transformer`](/api/transformer) | Assign the tokens and outputs of a transformer model. | + +### Disabling, excluding and modifying components {id="disabling"} + +If you don't need a particular component of the pipeline – for example, the +tagger or the parser, you can **disable or exclude** it. This can sometimes make +a big difference and improve loading and inference speed. There are two +different mechanisms you can use: + +1. **Disable:** The component and its data will be loaded with the pipeline, but + it will be disabled by default and not run as part of the processing + pipeline. To run it, you can explicitly enable it by calling + [`nlp.enable_pipe`](/api/language#enable_pipe). When you save out the `nlp` + object, the disabled component will be included but disabled by default. +2. **Exclude:** Don't load the component and its data with the pipeline. Once + the pipeline is loaded, there will be no reference to the excluded component. + +Disabled and excluded component names can be provided to +[`spacy.load`](/api/top-level#spacy.load) as a list. + +> #### 💡 Optional pipeline components +> +> The `disable` mechanism makes it easy to distribute pipeline packages with +> optional components that you can enable or disable at runtime. For instance, +> your pipeline may include a statistical _and_ a rule-based component for +> sentence segmentation, and you can choose which one to run depending on your +> use case. +> +> For example, spaCy's [trained pipelines](/models) like +> [`en_core_web_sm`](/models/en#en_core_web_sm) contain both a `parser` and +> `senter` that perform sentence segmentation, but the `senter` is disabled by +> default. + +```python +# Load the pipeline without the entity recognizer +nlp = spacy.load("en_core_web_sm", exclude=["ner"]) + +# Load the tagger and parser but don't enable them +nlp = spacy.load("en_core_web_sm", disable=["tagger", "parser"]) +# Explicitly enable the tagger later on +nlp.enable_pipe("tagger") +``` + +In addition to `disable`, `spacy.load()` also accepts `enable`. If `enable` is +set, all components except for those in `enable` are disabled. If `enable` and +`disable` conflict (i.e. the same component is included in both), an error is +raised. + +```python +# Load the complete pipeline, but disable all components except for tok2vec and tagger +nlp = spacy.load("en_core_web_sm", enable=["tok2vec", "tagger"]) +# Has the same effect, as NER is already not part of enabled set of components +nlp = spacy.load("en_core_web_sm", enable=["tok2vec", "tagger"], disable=["ner"]) +# Will raise an error, as the sets of enabled and disabled components are conflicting +nlp = spacy.load("en_core_web_sm", enable=["ner"], disable=["ner"]) +``` + + + +As of v3.0, the `disable` keyword argument specifies components to load but +disable, instead of components to not load at all. Those components can now be +specified separately using the new `exclude` keyword argument. + + + +As a shortcut, you can use the [`nlp.select_pipes`](/api/language#select_pipes) +context manager to temporarily disable certain components for a given block. At +the end of the `with` block, the disabled pipeline components will be restored +automatically. Alternatively, `select_pipes` returns an object that lets you +call its `restore()` method to restore the disabled components when needed. This +can be useful if you want to prevent unnecessary code indentation of large +blocks. + +```python {title="Disable for block"} +# 1. Use as a context manager +with nlp.select_pipes(disable=["tagger", "parser", "lemmatizer"]): + doc = nlp("I won't be tagged and parsed") +doc = nlp("I will be tagged and parsed") + +# 2. Restore manually +disabled = nlp.select_pipes(disable="ner") +doc = nlp("I won't have named entities") +disabled.restore() +``` + +If you want to disable all pipes except for one or a few, you can use the +`enable` keyword. Just like the `disable` keyword, it takes a list of pipe +names, or a string defining just one pipe. + +```python +# Enable only the parser +with nlp.select_pipes(enable="parser"): + doc = nlp("I will only be parsed") +``` + +The [`nlp.pipe`](/api/language#pipe) method also supports a `disable` keyword +argument if you only want to disable components during processing: + +```python +for doc in nlp.pipe(texts, disable=["tagger", "parser", "lemmatizer"]): + # Do something with the doc here +``` + +Finally, you can also use the [`remove_pipe`](/api/language#remove_pipe) method +to remove pipeline components from an existing pipeline, the +[`rename_pipe`](/api/language#rename_pipe) method to rename them, or the +[`replace_pipe`](/api/language#replace_pipe) method to replace them with a +custom component entirely (more details on this in the section on +[custom components](#custom-components)). + +```python +nlp.remove_pipe("parser") +nlp.rename_pipe("ner", "entityrecognizer") +nlp.replace_pipe("tagger", "my_custom_tagger") +``` + +The `Language` object exposes different [attributes](/api/language#attributes) +that let you inspect all available components and the components that currently +run as part of the pipeline. + +> #### Example +> +> ```python +> nlp = spacy.blank("en") +> nlp.add_pipe("ner") +> nlp.add_pipe("textcat") +> assert nlp.pipe_names == ["ner", "textcat"] +> nlp.disable_pipe("ner") +> assert nlp.pipe_names == ["textcat"] +> assert nlp.component_names == ["ner", "textcat"] +> assert nlp.disabled == ["ner"] +> ``` + +| Name | Description | +| --------------------- | ---------------------------------------------------------------- | +| `nlp.pipeline` | `(name, component)` tuples of the processing pipeline, in order. | +| `nlp.pipe_names` | Pipeline component names, in order. | +| `nlp.components` | All `(name, component)` tuples, including disabled components. | +| `nlp.component_names` | All component names, including disabled components. | +| `nlp.disabled` | Names of components that are currently disabled. | + +### Sourcing components from existing pipelines {id="sourced-components",version="3"} + +Pipeline components that are independent can also be reused across pipelines. +Instead of adding a new blank component, you can also copy an existing component +from a trained pipeline by setting the `source` argument on +[`nlp.add_pipe`](/api/language#add_pipe). The first argument will then be +interpreted as the name of the component in the source pipeline – for instance, +`"ner"`. This is especially useful for +[training a pipeline](/usage/training#config-components) because it lets you mix +and match components and create fully custom pipeline packages with updated +trained components and new components trained on your data. + + + +When reusing components across pipelines, keep in mind that the **vocabulary**, +**vectors** and model settings **must match**. If a trained pipeline includes +[word vectors](/usage/linguistic-features#vectors-similarity) and the component +uses them as features, the pipeline you copy it to needs to have the _same_ +vectors available – otherwise, it won't be able to make the same predictions. + + + +> #### In training config +> +> Instead of providing a `factory`, component blocks in the training +> [config](/usage/training#config) can also define a `source`. The string needs +> to be a loadable spaCy pipeline package or path. +> +> ```ini +> [components.ner] +> source = "en_core_web_sm" +> component = "ner" +> ``` +> +> By default, sourced components will be updated with your data during training. +> If you want to preserve the component as-is, you can "freeze" it if the +> pipeline is not using a shared `Tok2Vec` layer: +> +> ```ini +> [training] +> frozen_components = ["ner"] +> ``` + +```python {executable="true"} +import spacy + +# The source pipeline with different components +source_nlp = spacy.load("en_core_web_sm") +print(source_nlp.pipe_names) + +# Add only the entity recognizer to the new blank pipeline +nlp = spacy.blank("en") +nlp.add_pipe("ner", source=source_nlp) +print(nlp.pipe_names) +``` + +### Analyzing pipeline components {id="analysis",version="3"} + +The [`nlp.analyze_pipes`](/api/language#analyze_pipes) method analyzes the +components in the current pipeline and outputs information about them like the +attributes they set on the [`Doc`](/api/doc) and [`Token`](/api/token), whether +they retokenize the `Doc` and which scores they produce during training. It will +also show warnings if components require values that aren't set by previous +component – for instance, if the entity linker is used but no component that +runs before it sets named entities. Setting `pretty=True` will pretty-print a +table instead of only returning the structured data. + +> #### ✏️ Things to try +> +> 1. Add the components `"ner"` and `"sentencizer"` _before_ the +> `"entity_linker"`. The analysis should now show no problems, because +> requirements are met. + +```python {executable="true"} +import spacy + +nlp = spacy.blank("en") +nlp.add_pipe("tagger") +# This is a problem because it needs entities and sentence boundaries +nlp.add_pipe("entity_linker") +analysis = nlp.analyze_pipes(pretty=True) +``` + + + +```json {title="Structured"} +{ + "summary": { + "tagger": { + "assigns": ["token.tag"], + "requires": [], + "scores": ["tag_acc", "pos_acc", "lemma_acc"], + "retokenizes": false + }, + "entity_linker": { + "assigns": ["token.ent_kb_id"], + "requires": ["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"], + "scores": [], + "retokenizes": false + } + }, + "problems": { + "tagger": [], + "entity_linker": [ + "doc.ents", + "doc.sents", + "token.ent_iob", + "token.ent_type" + ] + }, + "attrs": { + "token.ent_iob": { "assigns": [], "requires": ["entity_linker"] }, + "doc.ents": { "assigns": [], "requires": ["entity_linker"] }, + "token.ent_kb_id": { "assigns": ["entity_linker"], "requires": [] }, + "doc.sents": { "assigns": [], "requires": ["entity_linker"] }, + "token.tag": { "assigns": ["tagger"], "requires": [] }, + "token.ent_type": { "assigns": [], "requires": ["entity_linker"] } + } +} +``` + +``` +### Pretty +============================= Pipeline Overview ============================= + +# Component Assigns Requires Scores Retokenizes +- ------------- --------------- -------------- ----------- ----------- +0 tagger token.tag tag_acc False + +1 entity_linker token.ent_kb_id doc.ents nel_micro_f False + doc.sents nel_micro_r + token.ent_iob nel_micro_p + token.ent_type + + +================================ Problems (4) ================================ +⚠ 'entity_linker' requirements not met: doc.ents, doc.sents, +token.ent_iob, token.ent_type +``` + + + + + +The pipeline analysis is static and does **not actually run the components**. +This means that it relies on the information provided by the components +themselves. If a custom component declares that it assigns an attribute but it +doesn't, the pipeline analysis won't catch that. + + + +## Creating custom pipeline components {id="custom-components"} + +A pipeline component is a function that receives a `Doc` object, modifies it and +returns it – for example, by using the current weights to make a prediction and +set some annotation on the document. By adding a component to the pipeline, +you'll get access to the `Doc` at any point **during processing** – instead of +only being able to modify it afterwards. + +> #### Example +> +> ```python +> from spacy.language import Language +> +> @Language.component("my_component") +> def my_component(doc): +> # Do something to the doc here +> return doc +> ``` + +| Argument | Type | Description | +| ----------- | ----------------- | ------------------------------------------------------ | +| `doc` | [`Doc`](/api/doc) | The `Doc` object processed by the previous component. | +| **RETURNS** | [`Doc`](/api/doc) | The `Doc` object processed by this pipeline component. | + +The [`@Language.component`](/api/language#component) decorator lets you turn a +simple function into a pipeline component. It takes at least one argument, the +**name** of the component factory. You can use this name to add an instance of +your component to the pipeline. It can also be listed in your pipeline config, +so you can save, load and train pipelines using your component. + +Custom components can be added to the pipeline using the +[`add_pipe`](/api/language#add_pipe) method. Optionally, you can either specify +a component to add it **before or after**, tell spaCy to add it **first or +last** in the pipeline, or define a **custom name**. If no name is set and no +`name` attribute is present on your component, the function name is used. + +> #### Example +> +> ```python +> nlp.add_pipe("my_component") +> nlp.add_pipe("my_component", first=True) +> nlp.add_pipe("my_component", before="parser") +> ``` + +| Argument | Description | +| -------- | --------------------------------------------------------------------------------- | +| `last` | If set to `True`, component is added **last** in the pipeline (default). ~~bool~~ | +| `first` | If set to `True`, component is added **first** in the pipeline. ~~bool~~ | +| `before` | String name or index to add the new component **before**. ~~Union[str, int]~~ | +| `after` | String name or index to add the new component **after**. ~~Union[str, int]~~ | + + + +As of v3.0, components need to be registered using the +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorator so spaCy knows that a +function is a component. [`nlp.add_pipe`](/api/language#add_pipe) now takes the +**string name** of the component factory instead of the component function. This +doesn't only save you lines of code, it also allows spaCy to validate and track +your custom components, and make sure they can be saved and loaded. + +```diff +- ruler = nlp.create_pipe("entity_ruler") +- nlp.add_pipe(ruler) ++ ruler = nlp.add_pipe("entity_ruler") +``` + + + +### Examples: Simple stateless pipeline components {id="custom-components-simple"} + +The following component receives the `Doc` in the pipeline and prints some +information about it: the number of tokens, the part-of-speech tags of the +tokens and a conditional message based on the document length. The +[`@Language.component`](/api/language#component) decorator lets you register the +component under the name `"info_component"`. + +> #### ✏️ Things to try +> +> 1. Add the component first in the pipeline by setting `first=True`. You'll see +> that the part-of-speech tags are empty, because the component now runs +> before the tagger and the tags aren't available yet. +> 2. Change the component `name` or remove the `name` argument. You should see +> this change reflected in `nlp.pipe_names`. +> 3. Print `nlp.pipeline`. You'll see a list of tuples describing the component +> name and the function that's called on the `Doc` object in the pipeline. +> 4. Change the first argument to `@Language.component`, the name, to something +> else. spaCy should now complain that it doesn't know a component of the +> name `"info_component"`. + +```python {executable="true"} +import spacy +from spacy.language import Language + +@Language.component("info_component") +def my_component(doc): + print(f"After tokenization, this doc has {len(doc)} tokens.") + print("The part-of-speech tags are:", [token.pos_ for token in doc]) + if len(doc) < 10: + print("This is a pretty short document.") + return doc + +nlp = spacy.load("en_core_web_sm") +nlp.add_pipe("info_component", name="print_info", last=True) +print(nlp.pipe_names) # ['tagger', 'parser', 'ner', 'print_info'] +doc = nlp("This is a sentence.") +``` + +Here's another example of a pipeline component that implements custom logic to +improve the sentence boundaries set by the dependency parser. The custom logic +should therefore be applied **after** tokenization, but _before_ the dependency +parsing – this way, the parser can also take advantage of the sentence +boundaries. + +> #### ✏️ Things to try +> +> 1. Print `[token.dep_ for token in doc]` with and without the custom pipeline +> component. You'll see that the predicted dependency parse changes to match +> the sentence boundaries. +> 2. Remove the `else` block. All other tokens will now have `is_sent_start` set +> to `None` (missing value), the parser will assign sentence boundaries in +> between. + +```python {executable="true"} +import spacy +from spacy.language import Language + +@Language.component("custom_sentencizer") +def custom_sentencizer(doc): + for i, token in enumerate(doc[:-2]): + # Define sentence start if pipe + titlecase token + if token.text == "|" and doc[i + 1].is_title: + doc[i + 1].is_sent_start = True + else: + # Explicitly set sentence start to False otherwise, to tell + # the parser to leave those tokens alone + doc[i + 1].is_sent_start = False + return doc + +nlp = spacy.load("en_core_web_sm") +nlp.add_pipe("custom_sentencizer", before="parser") # Insert before the parser +doc = nlp("This is. A sentence. | This is. Another sentence.") +for sent in doc.sents: + print(sent.text) +``` + +### Component factories and stateful components {id="custom-components-factories"} + +Component factories are callables that take settings and return a **pipeline +component function**. This is useful if your component is stateful and if you +need to customize their creation, or if you need access to the current `nlp` +object or the shared vocab. Component factories can be registered using the +[`@Language.factory`](/api/language#factory) decorator and they need at least +**two named arguments** that are filled in automatically when the component is +added to the pipeline: + +> #### Example +> +> ```python +> from spacy.language import Language +> +> @Language.factory("my_component") +> def my_component(nlp, name): +> return MyComponent() +> ``` + +| Argument | Description | +| -------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The current `nlp` object. Can be used to access the shared vocab. ~~Language~~ | +| `name` | The **instance name** of the component in the pipeline. This lets you identify different instances of the same component. ~~str~~ | + +All other settings can be passed in by the user via the `config` argument on +[`nlp.add_pipe`](/api/language). The +[`@Language.factory`](/api/language#factory) decorator also lets you define a +`default_config` that's used as a fallback. + +```python {title="With config",highlight="4,9"} +import spacy +from spacy.language import Language + +@Language.factory("my_component", default_config={"some_setting": True}) +def my_component(nlp, name, some_setting: bool): + return MyComponent(some_setting=some_setting) + +nlp = spacy.blank("en") +nlp.add_pipe("my_component", config={"some_setting": False}) +``` + + + +The [`@Language.component`](/api/language#component) decorator is essentially a +**shortcut** for stateless pipeline components that don't need any settings. +This means you don't have to always write a function that returns your function +if there's no state to be passed through – spaCy can just take care of this for +you. The following two code examples are equivalent: + +```python +# Stateless component with @Language.factory +@Language.factory("my_component") +def create_my_component(): + def my_component(doc): + # Do something to the doc + return doc + + return my_component + +# Stateless component with @Language.component +@Language.component("my_component") +def my_component(doc): + # Do something to the doc + return doc +``` + + + + + +Yes, the [`@Language.factory`](/api/language#factory) decorator can be added to +a function or a class. If it's added to a class, it expects the `__init__` +method to take the arguments `nlp` and `name`, and will populate all other +arguments from the config. That said, it's often cleaner and more intuitive to +make your factory a separate function. That's also how spaCy does it internally. + + + +### Language-specific factories {id="factories-language",version="3"} + +There are many use cases where you might want your pipeline components to be +language-specific. Sometimes this requires entirely different implementation per +language, sometimes the only difference is in the settings or data. spaCy allows +you to register factories of the **same name** on both the `Language` base +class, as well as its **subclasses** like `English` or `German`. Factories are +resolved starting with the specific subclass. If the subclass doesn't define a +component of that name, spaCy will check the `Language` base class. + +Here's an example of a pipeline component that overwrites the normalized form of +a token, the `Token.norm_` with an entry from a language-specific lookup table. +It's registered twice under the name `"token_normalizer"` – once using +`@English.factory` and once using `@German.factory`: + +```python {executable="true"} +from spacy.lang.en import English +from spacy.lang.de import German + +class TokenNormalizer: + def __init__(self, norm_table): + self.norm_table = norm_table + + def __call__(self, doc): + for token in doc: + # Overwrite the token.norm_ if there's an entry in the data + token.norm_ = self.norm_table.get(token.text, token.norm_) + return doc + +@English.factory("token_normalizer") +def create_en_normalizer(nlp, name): + return TokenNormalizer({"realise": "realize", "colour": "color"}) + +@German.factory("token_normalizer") +def create_de_normalizer(nlp, name): + return TokenNormalizer({"daß": "dass", "wußte": "wusste"}) + +nlp_en = English() +nlp_en.add_pipe("token_normalizer") # uses the English factory +print([token.norm_ for token in nlp_en("realise colour daß wußte")]) + +nlp_de = German() +nlp_de.add_pipe("token_normalizer") # uses the German factory +print([token.norm_ for token in nlp_de("realise colour daß wußte")]) +``` + + + +Under the hood, language-specific factories are added to the +[`factories` registry](/api/top-level#registry) prefixed with the language code, +e.g. `"en.token_normalizer"`. When resolving the factory in +[`nlp.add_pipe`](/api/language#add_pipe), spaCy first checks for a +language-specific version of the factory using `nlp.lang` and if none is +available, falls back to looking up the regular factory name. + + + +### Example: Stateful component with settings {id="example-stateful-components"} + +This example shows a **stateful** pipeline component for handling acronyms: +based on a dictionary, it will detect acronyms and their expanded forms in both +directions and add them to a list as the custom `doc._.acronyms` +[extension attribute](#custom-components-attributes). Under the hood, it uses +the [`PhraseMatcher`](/api/phrasematcher) to find instances of the phrases. + +The factory function takes three arguments: the shared `nlp` object and +component instance `name`, which are passed in automatically by spaCy, and a +`case_sensitive` config setting that makes the matching and acronym detection +case-sensitive. + +> #### ✏️ Things to try +> +> 1. Change the `config` passed to `nlp.add_pipe` and set `"case_sensitive"` to +> `True`. You should see that the expanded acronym for "LOL" isn't detected +> anymore. +> 2. Add some more terms to the `DICTIONARY` and update the processed text so +> they're detected. +> 3. Add a `name` argument to `nlp.add_pipe` to change the component name. Print +> `nlp.pipe_names` to see the change reflected in the pipeline. +> 4. Print the config of the current `nlp` object with +> `print(nlp.config.to_str())` and inspect the `[components]` block. You +> should see an entry for the acronyms component, referencing the factory +> `acronyms` and the config settings. + +```python {executable="true"} +from spacy.language import Language +from spacy.tokens import Doc +from spacy.matcher import PhraseMatcher +import spacy + +DICTIONARY = {"lol": "laughing out loud", "brb": "be right back"} +DICTIONARY.update({value: key for key, value in DICTIONARY.items()}) + +@Language.factory("acronyms", default_config={"case_sensitive": False}) +def create_acronym_component(nlp: Language, name: str, case_sensitive: bool): + return AcronymComponent(nlp, case_sensitive) + +class AcronymComponent: + def __init__(self, nlp: Language, case_sensitive: bool): + # Create the matcher and match on Token.lower if case-insensitive + matcher_attr = "TEXT" if case_sensitive else "LOWER" + self.matcher = PhraseMatcher(nlp.vocab, attr=matcher_attr) + self.matcher.add("ACRONYMS", [nlp.make_doc(term) for term in DICTIONARY]) + self.case_sensitive = case_sensitive + # Register custom extension on the Doc + if not Doc.has_extension("acronyms"): + Doc.set_extension("acronyms", default=[]) + + def __call__(self, doc: Doc) -> Doc: + # Add the matched spans when doc is processed + for _, start, end in self.matcher(doc): + span = doc[start:end] + acronym = DICTIONARY.get(span.text if self.case_sensitive else span.text.lower()) + doc._.acronyms.append((span, acronym)) + return doc + +# Add the component to the pipeline and configure it +nlp = spacy.blank("en") +nlp.add_pipe("acronyms", config={"case_sensitive": False}) + +# Process a doc and see the results +doc = nlp("LOL, be right back") +print(doc._.acronyms) +``` + +## Initializing and serializing component data {id="component-data"} + +Many stateful components depend on **data resources** like dictionaries and +lookup tables that should ideally be **configurable**. For example, it makes +sense to make the `DICTIONARY` in the above example an argument of the +registered function, so the `AcronymComponent` can be re-used with different +data. One logical solution would be to make it an argument of the component +factory, and allow it to be initialized with different dictionaries. + +> #### config.cfg +> +> ```ini +> [components.acronyms.data] +> # 🚨 Problem: you don't want the data in the config +> lol = "laugh out loud" +> brb = "be right back" +> ``` + +```python +@Language.factory("acronyms", default_config={"data": {}, "case_sensitive": False}) +def create_acronym_component(nlp: Language, name: str, data: Dict[str, str], case_sensitive: bool): + # 🚨 Problem: data ends up in the config file + return AcronymComponent(nlp, data, case_sensitive) +``` + +However, passing in the dictionary directly is problematic, because it means +that if a component saves out its config and settings, the +[`config.cfg`](/usage/training#config) will include a dump of the entire data, +since that's the config the component was created with. It will also fail if the +data is not JSON-serializable. + +### Option 1: Using a registered function {id="component-data-function"} + + + +- ✅ **Pros:** can load anything in Python, easy to add to and configure via + config +- ❌ **Cons:** requires the function and its dependencies to be available at + runtime + + + +If what you're passing in isn't JSON-serializable – e.g. a custom object like a +[model](#trainable-components) – saving out the component config becomes +impossible because there's no way for spaCy to know _how_ that object was +created, and what to do to create it again. This makes it much harder to save, +load and train custom pipelines with custom components. A simple solution is to +**register a function** that returns your resources. The +[registry](/api/top-level#registry) lets you **map string names to functions** +that create objects, so given a name and optional arguments, spaCy will know how +to recreate the object. To register a function that returns your custom +dictionary, you can use the `@spacy.registry.misc` decorator with a single +argument, the name: + +> #### What's the misc registry? +> +> The [`registry`](/api/top-level#registry) provides different categories for +> different types of functions – for example, model architectures, tokenizers or +> batchers. `misc` is intended for miscellaneous functions that don't fit +> anywhere else. + +```python {title="Registered function for assets",highlight="1"} +@spacy.registry.misc("acronyms.slang_dict.v1") +def create_acronyms_slang_dict(): + dictionary = {"lol": "laughing out loud", "brb": "be right back"} + dictionary.update({value: key for key, value in dictionary.items()}) + return dictionary +``` + +In your `default_config` (and later in your +[training config](/usage/training#config)), you can now refer to the function +registered under the name `"acronyms.slang_dict.v1"` using the `@misc` key. This +tells spaCy how to create the value, and when your component is created, the +result of the registered function is passed in as the key `"dictionary"`. + +> #### config.cfg +> +> ```ini +> [components.acronyms] +> factory = "acronyms" +> +> [components.acronyms.data] +> @misc = "acronyms.slang_dict.v1" +> ``` + +```diff +- default_config = {"dictionary:" DICTIONARY} ++ default_config = {"dictionary": {"@misc": "acronyms.slang_dict.v1"}} +``` + +Using a registered function also means that you can easily include your custom +components in pipelines that you [train](/usage/training). To make sure spaCy +knows where to find your custom `@misc` function, you can pass in a Python file +via the argument `--code`. If someone else is using your component, all they +have to do to customize the data is to register their own function and swap out +the name. Registered functions can also take **arguments**, by the way, that can +be defined in the config as well – you can read more about this in the docs on +[training with custom code](/usage/training#custom-code). + +### Option 2: Save data with the pipeline and load it in once on initialization {id="component-data-initialization"} + + + +- ✅ **Pros:** lets components save and load their own data and reflect user + changes, load in data assets before training without depending on them at + runtime +- ❌ **Cons:** requires more component methods, more complex config and data + flow + + + +Just like models save out their binary weights when you call +[`nlp.to_disk`](/api/language#to_disk), components can also **serialize** any +other data assets – for instance, an acronym dictionary. If a pipeline component +implements its own `to_disk` and `from_disk` methods, those will be called +automatically by `nlp.to_disk` and will receive the path to the directory to +save to or load from. The component can then perform any custom saving or +loading. If a user makes changes to the component data, they will be reflected +when the `nlp` object is saved. For more examples of this, see the usage guide +on [serialization methods](/usage/saving-loading/#serialization-methods). + +> #### About the data path +> +> The `path` argument spaCy passes to the serialization methods consists of the +> path provided by the user, plus a directory of the component name. This means +> that when you call `nlp.to_disk("/path")`, the `acronyms` component will +> receive the directory path `/path/acronyms` and can then create files in this +> directory. + +```python {title="Custom serialization methods",highlight="7-11,13-15"} +import srsly +from spacy.util import ensure_path + +class AcronymComponent: + # other methods here... + + def to_disk(self, path, exclude=tuple()): + path = ensure_path(path) + if not path.exists(): + path.mkdir() + srsly.write_json(path / "data.json", self.data) + + def from_disk(self, path, exclude=tuple()): + self.data = srsly.read_json(path / "data.json") + return self +``` + +Now the component can save to and load from a directory. The only remaining +question: How do you **load in the initial data**? In Python, you could just +call the pipe's `from_disk` method yourself. But if you're adding the component +to your [training config](/usage/training#config), spaCy will need to know how +to set it up, from start to finish, including the data to initialize it with. + +While you could use a registered function or a file loader like +[`srsly.read_json.v1`](/api/top-level#file_readers) as an argument of the +component factory, this approach is problematic: the component factory runs +**every time the component is created**. This means it will run when creating +the `nlp` object before training, but also every time a user loads your +pipeline. So your runtime pipeline would either depend on a local path on your +file system, or it's loaded twice: once when the component is created, and then +again when the data is by `from_disk`. + +> ```ini +> ### config.cfg +> [components.acronyms.data] +> # 🚨 Problem: Runtime pipeline depends on local path +> @readers = "srsly.read_json.v1" +> path = "/path/to/slang_dict.json" +> ``` +> +> ```ini +> ### config.cfg +> [components.acronyms.data] +> # 🚨 Problem: this always runs +> @misc = "acronyms.slang_dict.v1" +> ``` + +```python +@Language.factory("acronyms", default_config={"data": {}, "case_sensitive": False}) +def create_acronym_component(nlp: Language, name: str, data: Dict[str, str], case_sensitive: bool): + # 🚨 Problem: data will be loaded every time component is created + return AcronymComponent(nlp, data, case_sensitive) +``` + +To solve this, your component can implement a separate method, `initialize`, +which will be called by [`nlp.initialize`](/api/language#initialize) if +available. This typically happens before training, but not at runtime when the +pipeline is loaded. For more background on this, see the usage guides on the +[config lifecycle](/usage/training#config-lifecycle) and +[custom initialization](/usage/training#initialization). + +![Illustration of pipeline lifecycle](/images/lifecycle.svg) + +A component's `initialize` method needs to take at least **two named +arguments**: a `get_examples` callback that gives it access to the training +examples, and the current `nlp` object. This is mostly used by trainable +components so they can initialize their models and label schemes from the data, +so we can ignore those arguments here. All **other arguments** on the method can +be defined via the config – in this case a dictionary `data`. + +> #### config.cfg +> +> ```ini +> [initialize.components.my_component] +> +> [initialize.components.my_component.data] +> # ✅ This only runs on initialization +> @readers = "srsly.read_json.v1" +> path = "/path/to/slang_dict.json" +> ``` + +```python {title="Custom initialize method",highlight="5-6"} +class AcronymComponent: + def __init__(self): + self.data = {} + + def initialize(self, get_examples=None, nlp=None, data={}): + self.data = data +``` + +When [`nlp.initialize`](/api/language#initialize) runs before training (or when +you call it in your own code), the +[`[initialize]`](/api/data-formats#config-initialize) block of the config is +loaded and used to construct the `nlp` object. The custom acronym component will +then be passed the data loaded from the JSON file. After training, the `nlp` +object is saved to disk, which will run the component's `to_disk` method. When +the pipeline is loaded back into spaCy later to use it, the `from_disk` method +will load the data back in. + +## Python type hints and validation {id="type-hints",version="3"} + +spaCy's configs are powered by our machine learning library Thinc's +[configuration system](https://thinc.ai/docs/usage-config), which supports +[type hints](https://docs.python.org/3/library/typing.html) and even +[advanced type annotations](https://thinc.ai/docs/usage-config#advanced-types) +using [`pydantic`](https://github.com/samuelcolvin/pydantic). If your component +factory provides type hints, the values that are passed in will be **checked +against the expected types**. If the value can't be cast to an integer, spaCy +will raise an error. `pydantic` also provides strict types like `StrictFloat`, +which will force the value to be an integer and raise an error if it's not – for +instance, if your config defines a float. + + + +If you're not using +[strict types](https://pydantic-docs.helpmanual.io/usage/types/#strict-types), +values that can be **cast to** the given type will still be accepted. For +example, `1` can be cast to a `float` or a `bool` type, but not to a +`List[str]`. However, if the type is +[`StrictFloat`](https://pydantic-docs.helpmanual.io/usage/types/#strict-types), +only a float will be accepted. + + + +The following example shows a custom pipeline component for debugging. It can be +added anywhere in the pipeline and logs information about the `nlp` object and +the `Doc` that passes through. The `log_level` config setting lets the user +customize what log statements are shown – for instance, `"INFO"` will show info +logs and more critical logging statements, whereas `"DEBUG"` will show +everything. The value is annotated as a `StrictStr`, so it will only accept a +string value. + +> #### ✏️ Things to try +> +> 1. Change the `config` passed to `nlp.add_pipe` to use the log level `"INFO"`. +> You should see that only the statement logged with `logger.info` is shown. +> 2. Change the `config` passed to `nlp.add_pipe` so that it contains unexpected +> values – for example, a boolean instead of a string: `"log_level": False`. +> You should see a validation error. +> 3. Check out the docs on `pydantic`'s +> [constrained types](https://pydantic-docs.helpmanual.io/usage/types/#constrained-types) +> and write a type hint for `log_level` that only accepts the exact string +> values `"DEBUG"`, `"INFO"` or `"CRITICAL"`. + +```python {executable="true"} +import spacy +from spacy.language import Language +from spacy.tokens import Doc +from pydantic import StrictStr +import logging + +@Language.factory("debug", default_config={"log_level": "DEBUG"}) +class DebugComponent: + def __init__(self, nlp: Language, name: str, log_level: StrictStr): + self.logger = logging.getLogger(f"spacy.{name}") + self.logger.setLevel(log_level) + self.logger.info(f"Pipeline: {nlp.pipe_names}") + + def __call__(self, doc: Doc) -> Doc: + is_tagged = doc.has_annotation("TAG") + self.logger.debug(f"Doc: {len(doc)} tokens, is tagged: {is_tagged}") + return doc + +nlp = spacy.load("en_core_web_sm") +nlp.add_pipe("debug", config={"log_level": "DEBUG"}) +doc = nlp("This is a text...") +``` + +## Trainable components {id="trainable-components",version="3"} + +spaCy's [`TrainablePipe`](/api/pipe) class helps you implement your own +trainable components that have their own model instance, make predictions over +`Doc` objects and can be updated using [`spacy train`](/api/cli#train). This +lets you plug fully custom machine learning components into your pipeline. + +![Illustration of Pipe methods](/images/trainable_component.svg) + +You'll need the following: + +1. **Model:** A Thinc [`Model`](https://thinc.ai/docs/api-model) instance. This + can be a model implemented in [Thinc](/usage/layers-architectures#thinc), or + a [wrapped model](/usage/layers-architectures#frameworks) implemented in + PyTorch, TensorFlow, MXNet or a fully custom solution. The model must take a + list of [`Doc`](/api/doc) objects as input and can have any type of output. +2. **TrainablePipe subclass:** A subclass of [`TrainablePipe`](/api/pipe) that + implements at least two methods: [`TrainablePipe.predict`](/api/pipe#predict) + and [`TrainablePipe.set_annotations`](/api/pipe#set_annotations). +3. **Component factory:** A component factory registered with + [`@Language.factory`](/api/language#factory) that takes the `nlp` object and + component `name` and optional settings provided by the config and returns an + instance of your trainable component. + +> #### Example +> +> ```python +> from spacy.pipeline import TrainablePipe +> from spacy.language import Language +> +> class TrainableComponent(TrainablePipe): +> def predict(self, docs): +> ... +> +> def set_annotations(self, docs, scores): +> ... +> +> @Language.factory("my_trainable_component") +> def make_component(nlp, name, model): +> return TrainableComponent(nlp.vocab, model, name=name) +> ``` + +| Name | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| [`predict`](/api/pipe#predict) | Apply the component's model to a batch of [`Doc`](/api/doc) objects (without modifying them) and return the scores. | +| [`set_annotations`](/api/pipe#set_annotations) | Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores generated by `predict`. | + +By default, [`TrainablePipe.__init__`](/api/pipe#init) takes the shared vocab, +the [`Model`](https://thinc.ai/docs/api-model) and the name of the component +instance in the pipeline, which you can use as a key in the losses. All other +keyword arguments will become available as [`TrainablePipe.cfg`](/api/pipe#cfg) +and will also be serialized with the component. + + + +spaCy's [config system](/usage/training#config) resolves the config describing +the pipeline components and models **bottom-up**. This means that it will +_first_ create a `Model` from a [registered architecture](/api/architectures), +validate its arguments and _then_ pass the object forward to the component. This +means that the config can express very complex, nested trees of objects – but +the objects don't have to pass the model settings all the way down to the +components. It also makes the components more **modular** and lets you +[swap](/usage/layers-architectures#swap-architectures) different architectures +in your config, and re-use model definitions. + +```ini {title="config.cfg (excerpt)"} +[components] + +[components.textcat] +factory = "textcat" +labels = [] + +# This function is created and then passed to the "textcat" component as +# the argument "model" +[components.textcat.model] +@architectures = "spacy.TextCatBOW.v3" +exclusive_classes = true +length = 262144 +ngram_size = 1 +no_output_layer = false + +[components.other_textcat] +factory = "textcat" +# This references the [components.textcat.model] block above +model = ${components.textcat.model} +labels = [] +``` + +Your trainable pipeline component factories should therefore always take a +`model` argument instead of instantiating the +[`Model`](https://thinc.ai/docs/api-model) inside the component. To register +custom architectures, you can use the +[`@spacy.registry.architectures`](/api/top-level#registry) decorator. Also see +the [training guide](/usage/training#config) for details. + + + +For some use cases, it makes sense to also overwrite additional methods to +customize how the model is updated from examples, how it's initialized, how the +loss is calculated and to add evaluation scores to the training output. + +| Name | Description | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`update`](/api/pipe#update) | Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. | +| [`initialize`](/api/pipe#initialize) | Initialize the model. Typically calls into [`Model.initialize`](https://thinc.ai/docs/api-model#initialize) and can be passed custom arguments via the [`[initialize]`](/api/data-formats#config-initialize) config block that are only loaded during training or when you call [`nlp.initialize`](/api/language#initialize), not at runtime. | +| [`get_loss`](/api/pipe#get_loss) | Return a tuple of the loss and the gradient for a batch of [`Example`](/api/example) objects. | +| [`score`](/api/pipe#score) | Score a batch of [`Example`](/api/example) objects and return a dictionary of scores. The [`@Language.factory`](/api/language#factory) decorator can define the `default_score_weights` of the component to decide which keys of the scores to display during training and how they count towards the final score. | + + + +For more details on how to implement your own trainable components and model +architectures, and plug existing models implemented in PyTorch or TensorFlow +into your spaCy pipeline, see the usage guide on +[layers and model architectures](/usage/layers-architectures#components). + + + +## Extension attributes {id="custom-components-attributes",version="2"} + +spaCy allows you to set any custom attributes and methods on the `Doc`, `Span` +and `Token`, which become available as `Doc._`, `Span._` and `Token._` – for +example, `Token._.my_attr`. This lets you store additional information relevant +to your application, add new features and functionality to spaCy, and implement +your own models trained with other machine learning libraries. It also lets you +take advantage of spaCy's data structures and the `Doc` object as the "single +source of truth". + + + +Writing to a `._` attribute instead of to the `Doc` directly keeps a clearer +separation and makes it easier to ensure backwards compatibility. For example, +if you've implemented your own `.coref` property and spaCy claims it one day, +it'll break your code. Similarly, just by looking at the code, you'll +immediately know what's built-in and what's custom – for example, +`doc.sentiment` is spaCy, while `doc._.sent_score` isn't. + + + + + +Extension definitions – the defaults, methods, getters and setters you pass in +to `set_extension` – are stored in class attributes on the `Underscore` class. +If you write to an extension attribute, e.g. `doc._.hello = True`, the data is +stored within the [`Doc.user_data`](/api/doc#attributes) dictionary. To keep the +underscore data separate from your other dictionary entries, the string `"._."` +is placed before the name, in a tuple. + + + +--- + +There are three main types of extensions, which can be defined using the +[`Doc.set_extension`](/api/doc#set_extension), +[`Span.set_extension`](/api/span#set_extension) and +[`Token.set_extension`](/api/token#set_extension) methods. + +## Description + +1. **Attribute extensions.** Set a default value for an attribute, which can be + overwritten manually at any time. Attribute extensions work like "normal" + variables and are the quickest way to store arbitrary information on a `Doc`, + `Span` or `Token`. + + ```python + Doc.set_extension("hello", default=True) + assert doc._.hello + doc._.hello = False + ``` + +2. **Property extensions.** Define a getter and an optional setter function. If + no setter is provided, the extension is immutable. Since the getter and + setter functions are only called when you _retrieve_ the attribute, you can + also access values of previously added attribute extensions. For example, a + `Doc` getter can average over `Token` attributes. For `Span` extensions, + you'll almost always want to use a property – otherwise, you'd have to write + to _every possible_ `Span` in the `Doc` to set up the values correctly. + + ```python + Doc.set_extension("hello", getter=get_hello_value, setter=set_hello_value) + assert doc._.hello + doc._.hello = "Hi!" + ``` + +3. **Method extensions.** Assign a function that becomes available as an object + method. Method extensions are always immutable. For more details and + implementation ideas, see + [these examples](/usage/examples#custom-components-attr-methods). + + ```python + Doc.set_extension("hello", method=lambda doc, name: f"Hi {name}!") + assert doc._.hello("Bob") == "Hi Bob!" + ``` + +Before you can access a custom extension, you need to register it using the +`set_extension` method on the object you want to add it to, e.g. the `Doc`. Keep +in mind that extensions are always **added globally** and not just on a +particular instance. If an attribute of the same name already exists, or if +you're trying to access an attribute that hasn't been registered, spaCy will +raise an `AttributeError`. + +```python {title="Example"} +from spacy.tokens import Doc, Span, Token + +fruits = ["apple", "pear", "banana", "orange", "strawberry"] +is_fruit_getter = lambda token: token.text in fruits +has_fruit_getter = lambda obj: any([t.text in fruits for t in obj]) + +Token.set_extension("is_fruit", getter=is_fruit_getter) +Doc.set_extension("has_fruit", getter=has_fruit_getter) +Span.set_extension("has_fruit", getter=has_fruit_getter) +``` + +> #### Usage example +> +> ```python +> doc = nlp("I have an apple and a melon") +> assert doc[3]._.is_fruit # get Token attributes +> assert not doc[0]._.is_fruit +> assert doc._.has_fruit # get Doc attributes +> assert doc[1:4]._.has_fruit # get Span attributes +> ``` + +Once you've registered your custom attribute, you can also use the built-in +`set`, `get` and `has` methods to modify and retrieve the attributes. This is +especially useful it you want to pass in a string instead of calling +`doc._.my_attr`. + +### Example: Pipeline component for GPE entities and country meta data via a REST API {id="component-example3"} + +This example shows the implementation of a pipeline component that fetches +country meta data via the [REST Countries API](https://restcountries.com), sets +entity annotations for countries and sets custom attributes on the `Doc` and +`Span` – for example, the capital, latitude/longitude coordinates and even the +country flag. + +```python {executable="true"} +import requests +from spacy.lang.en import English +from spacy.language import Language +from spacy.matcher import PhraseMatcher +from spacy.tokens import Doc, Span, Token + +@Language.factory("rest_countries") +class RESTCountriesComponent: + def __init__(self, nlp, name, label="GPE"): + r = requests.get("https://restcountries.com/v2/all") + r.raise_for_status() # make sure requests raises an error if it fails + countries = r.json() + # Convert API response to dict keyed by country name for easy lookup + self.countries = {c["name"]: c for c in countries} + self.label = label + # Set up the PhraseMatcher with Doc patterns for each country name + self.matcher = PhraseMatcher(nlp.vocab) + self.matcher.add("COUNTRIES", [nlp.make_doc(c) for c in self.countries.keys()]) + # Register attributes on the Span. We'll be overwriting this based on + # the matches, so we're only setting a default value, not a getter. + Span.set_extension("is_country", default=None) + Span.set_extension("country_capital", default=None) + Span.set_extension("country_latlng", default=None) + Span.set_extension("country_flag", default=None) + # Register attribute on Doc via a getter that checks if the Doc + # contains a country entity + Doc.set_extension("has_country", getter=self.has_country) + + def __call__(self, doc): + spans = [] # keep the spans for later so we can merge them afterwards + for _, start, end in self.matcher(doc): + # Generate Span representing the entity & set label + entity = Span(doc, start, end, label=self.label) + # Set custom attributes on entity. Can be extended with other data + # returned by the API, like currencies, country code, calling code etc. + entity._.set("is_country", True) + entity._.set("country_capital", self.countries[entity.text]["capital"]) + entity._.set("country_latlng", self.countries[entity.text]["latlng"]) + entity._.set("country_flag", self.countries[entity.text]["flag"]) + spans.append(entity) + # Overwrite doc.ents and add entity – be careful not to replace! + doc.ents = list(doc.ents) + spans + return doc # don't forget to return the Doc! + + def has_country(self, doc): + """Getter for Doc attributes. Since the getter is only called + when we access the attribute, we can refer to the Span's 'is_country' + attribute here, which is already set in the processing step.""" + return any([entity._.get("is_country") for entity in doc.ents]) + +nlp = English() +nlp.add_pipe("rest_countries", config={"label": "GPE"}) +doc = nlp("Some text about Colombia and the Czech Republic") +print("Pipeline", nlp.pipe_names) # pipeline contains component name +print("Doc has countries", doc._.has_country) # Doc contains countries +for ent in doc.ents: + if ent._.is_country: + print(ent.text, ent.label_, ent._.country_capital, ent._.country_latlng, ent._.country_flag) +``` + +In this case, all data can be fetched on initialization in one request. However, +if you're working with text that contains incomplete country names, spelling +mistakes or foreign-language versions, you could also implement a +`like_country`-style getter function that makes a request to the search API +endpoint and returns the best-matching result. + +### User hooks {id="custom-components-user-hooks"} + +While it's generally recommended to use the `Doc._`, `Span._` and `Token._` +proxies to add your own custom attributes, spaCy offers a few exceptions to +allow **customizing the built-in methods** like +[`Doc.similarity`](/api/doc#similarity) or [`Doc.vector`](/api/doc#vector) with +your own hooks, which can rely on components you train yourself. For instance, +you can provide your own on-the-fly sentence segmentation algorithm or document +similarity method. + +Hooks let you customize some of the behaviors of the `Doc`, `Span` or `Token` +objects by adding a component to the pipeline. For instance, to customize the +[`Doc.similarity`](/api/doc#similarity) method, you can add a component that +sets a custom function to `doc.user_hooks["similarity"]`. The built-in +`Doc.similarity` method will check the `user_hooks` dict, and delegate to your +function if you've set one. Similar results can be achieved by setting functions +to `Doc.user_span_hooks` and `Doc.user_token_hooks`. + +> #### Implementation note +> +> The hooks live on the `Doc` object because the `Span` and `Token` objects are +> created lazily, and don't own any data. They just proxy to their parent `Doc`. +> This turns out to be convenient here – we only have to worry about installing +> hooks in one place. + +| Name | Customizes | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `user_hooks` | [`Doc.similarity`](/api/doc#similarity), [`Doc.vector`](/api/doc#vector), [`Doc.has_vector`](/api/doc#has_vector), [`Doc.vector_norm`](/api/doc#vector_norm), [`Doc.sents`](/api/doc#sents) | +| `user_token_hooks` | [`Token.similarity`](/api/token#similarity), [`Token.vector`](/api/token#vector), [`Token.has_vector`](/api/token#has_vector), [`Token.vector_norm`](/api/token#vector_norm), [`Token.conjuncts`](/api/token#conjuncts) | +| `user_span_hooks` | [`Span.similarity`](/api/span#similarity), [`Span.vector`](/api/span#vector), [`Span.has_vector`](/api/span#has_vector), [`Span.vector_norm`](/api/span#vector_norm), [`Span.root`](/api/span#root) | + +```python {title="Add custom similarity hooks"} +from spacy.language import Language + + +class SimilarityModel: + def __init__(self, name: str, index: int): + self.name = name + self.index = index + + def __call__(self, doc): + doc.user_hooks["similarity"] = self.similarity + doc.user_span_hooks["similarity"] = self.similarity + doc.user_token_hooks["similarity"] = self.similarity + return doc + + def similarity(self, obj1, obj2): + return obj1.vector[self.index] + obj2.vector[self.index] + + +@Language.factory("similarity_component", default_config={"index": 0}) +def create_similarity_component(nlp, name, index: int): + return SimilarityModel(name, index) +``` + +## Developing plugins and wrappers {id="plugins"} + +We're very excited about all the new possibilities for community extensions and +plugins in spaCy, and we can't wait to see what you build with it! To get you +started, here are a few tips, tricks and best +practices. [See here](/universe/?category=pipeline) for examples of other spaCy +extensions. + +### Usage ideas {id="custom-components-usage-ideas"} + +- **Adding new features and hooking in models.** For example, a sentiment + analysis model, or your preferred solution for lemmatization or sentiment + analysis. spaCy's built-in tagger, parser and entity recognizer respect + annotations that were already set on the `Doc` in a previous step of the + pipeline. +- **Integrating other libraries and APIs.** For example, your pipeline component + can write additional information and data directly to the `Doc` or `Token` as + custom attributes, while making sure no information is lost in the process. + This can be output generated by other libraries and models, or an external + service with a REST API. +- **Debugging and logging.** For example, a component which stores and/or + exports relevant information about the current state of the processed + document, and insert it at any point of your pipeline. + +### Best practices {id="custom-components-best-practices"} + +Extensions can claim their own `._` namespace and exist as standalone packages. +If you're developing a tool or library and want to make it easy for others to +use it with spaCy and add it to their pipeline, all you have to do is expose a +function that takes a `Doc`, modifies it and returns it. + +- Make sure to choose a **descriptive and specific name** for your pipeline + component class, and set it as its `name` attribute. Avoid names that are too + common or likely to clash with built-in or a user's other custom components. + While it's fine to call your package `"spacy_my_extension"`, avoid component + names including `"spacy"`, since this can easily lead to confusion. + + ```diff + + name = "myapp_lemmatizer" + - name = "lemmatizer" + ``` + +- When writing to `Doc`, `Token` or `Span` objects, **use getter functions** + wherever possible, and avoid setting values explicitly. Tokens and spans don't + own any data themselves, and they're implemented as C extension classes – so + you can't usually add new attributes to them like you could with most pure + Python objects. + + ```diff + + is_fruit = lambda token: token.text in ("apple", "orange") + + Token.set_extension("is_fruit", getter=is_fruit) + + - token._.set_extension("is_fruit", default=False) + - if token.text in ('"apple", "orange"): + - token._.set("is_fruit", True) + ``` + +- Always add your custom attributes to the **global** `Doc`, `Token` or `Span` + objects, not a particular instance of them. Add the attributes **as early as + possible**, e.g. in your extension's `__init__` method or in the global scope + of your module. This means that in the case of namespace collisions, the user + will see an error immediately, not just when they run their pipeline. + + ```diff + + from spacy.tokens import Doc + + def __init__(attr="my_attr"): + + Doc.set_extension(attr, getter=self.get_doc_attr) + + - def __call__(doc): + - doc.set_extension("my_attr", getter=self.get_doc_attr) + ``` + +- If your extension is setting properties on the `Doc`, `Token` or `Span`, + include an option to **let the user to change those attribute names**. This + makes it easier to avoid namespace collisions and accommodate users with + different naming preferences. We recommend adding an `attrs` argument to the + `__init__` method of your class so you can write the names to class attributes + and reuse them across your component. + + ```diff + + Doc.set_extension(self.doc_attr, default="some value") + - Doc.set_extension("my_doc_attr", default="some value") + ``` + +- Ideally, extensions should be **standalone packages** with spaCy and + optionally, other packages specified as a dependency. They can freely assign + to their own `._` namespace, but should stick to that. If your extension's + only job is to provide a better `.similarity` implementation, and your docs + state this explicitly, there's no problem with writing to the + [`user_hooks`](#custom-components-user-hooks) and overwriting spaCy's built-in + method. However, a third-party extension should **never silently overwrite + built-ins**, or attributes set by other extensions. + +- If you're looking to publish a pipeline package that depends on a custom + pipeline component, you can either **require it** in the package's + dependencies, or – if the component is specific and lightweight – choose to + **ship it with your pipeline package**. Just make sure the + [`@Language.component`](/api/language#component) or + [`@Language.factory`](/api/language#factory) decorator that registers the + custom component runs in your package's `__init__.py` or is exposed via an + [entry point](/usage/saving-loading#entry-points). + +- Once you're ready to share your extension with others, make sure to **add docs + and installation instructions** (you can always link to this page for more + info). Make it easy for others to install and use your extension, for example + by uploading it to [PyPi](https://pypi.python.org). If you're sharing your + code on GitHub, don't forget to tag it with + [`spacy`](https://github.com/topics/spacy?o=desc&s=stars) and + [`spacy-extension`](https://github.com/topics/spacy-extension?o=desc&s=stars) + to help people find it. + +### Wrapping other models and libraries {id="wrapping-models-libraries"} + +Let's say you have a custom entity recognizer that takes a list of strings and +returns their [BILUO tags](/usage/linguistic-features#accessing-ner). Given an +input like `["A", "text", "about", "Facebook"]`, it will predict and return +`["O", "O", "O", "U-ORG"]`. To integrate it into your spaCy pipeline and make it +add those entities to the `doc.ents`, you can wrap it in a custom pipeline +component function and pass it the token texts from the `Doc` object received by +the component. + +The [`training.biluo_tags_to_spans`](/api/top-level#biluo_tags_to_spans) is very +helpful here, because it takes a `Doc` object and token-based BILUO tags and +returns a sequence of `Span` objects in the `Doc` with added labels. So all your +wrapper has to do is compute the entity spans and overwrite the `doc.ents`. + +> #### How the doc.ents work +> +> When you add spans to the `doc.ents`, spaCy will automatically resolve them +> back to the underlying tokens and set the `Token.ent_type` and `Token.ent_iob` +> attributes. By definition, each token can only be part of one entity, so +> overlapping entity spans are not allowed. + +```python {highlight="1,8-9"} +import your_custom_entity_recognizer +from spacy.training import biluo_tags_to_spans +from spacy.language import Language + +@Language.component("custom_ner_wrapper") +def custom_ner_wrapper(doc): + words = [token.text for token in doc] + custom_entities = your_custom_entity_recognizer(words) + doc.ents = biluo_tags_to_spans(doc, custom_entities) + return doc +``` + +The `custom_ner_wrapper` can then be added to a blank pipeline using +[`nlp.add_pipe`](/api/language#add_pipe). You can also replace the existing +entity recognizer of a trained pipeline with +[`nlp.replace_pipe`](/api/language#replace_pipe). + +Here's another example of a custom model, `your_custom_model`, that takes a list +of tokens and returns lists of fine-grained part-of-speech tags, coarse-grained +part-of-speech tags, dependency labels and head token indices. Here, we can use +the [`Doc.from_array`](/api/doc#from_array) to create a new `Doc` object using +those values. To create a numpy array we need integers, so we can look up the +string labels in the [`StringStore`](/api/stringstore). The +[`doc.vocab.strings.add`](/api/stringstore#add) method comes in handy here, +because it returns the integer ID of the string _and_ makes sure it's added to +the vocab. This is especially important if the custom model uses a different +label scheme than spaCy's default models. + +> #### Example: spacy-stanza +> +> For an example of an end-to-end wrapper for statistical tokenization, tagging +> and parsing, check out +> [`spacy-stanza`](https://github.com/explosion/spacy-stanza). It uses a very +> similar approach to the example in this section – the only difference is that +> it fully replaces the `nlp` object instead of providing a pipeline component, +> since it also needs to handle tokenization. + +```python {highlight="1,11,17-19"} +import your_custom_model +from spacy.language import Language +from spacy.symbols import POS, TAG, DEP, HEAD +from spacy.tokens import Doc +import numpy + +@Language.component("custom_model_wrapper") +def custom_model_wrapper(doc): + words = [token.text for token in doc] + spaces = [token.whitespace for token in doc] + pos, tags, deps, heads = your_custom_model(words) + # Convert the strings to integers and add them to the string store + pos = [doc.vocab.strings.add(label) for label in pos] + tags = [doc.vocab.strings.add(label) for label in tags] + deps = [doc.vocab.strings.add(label) for label in deps] + # Create a new Doc from a numpy array + attrs = [POS, TAG, DEP, HEAD] + arr = numpy.array(list(zip(pos, tags, deps, heads)), dtype="uint64") + new_doc = Doc(doc.vocab, words=words, spaces=spaces).from_array(attrs, arr) + return new_doc +``` + + + +If you create a `Doc` object with dependencies and heads, spaCy is able to +resolve the sentence boundaries automatically. However, note that the `HEAD` +value used to construct a `Doc` is the token index **relative** to the current +token – e.g. `-1` for the previous token. The CoNLL format typically annotates +heads as `1`-indexed absolute indices with `0` indicating the root. If that's +the case in your annotations, you need to convert them first: + +```python +heads = [2, 0, 4, 2, 2] +new_heads = [head - i - 1 if head != 0 else 0 for i, head in enumerate(heads)] +``` + + + + + +For more details on how to write and package custom components, make them +available to spaCy via entry points and implement your own serialization +methods, check out the usage guide on +[saving and loading](/usage/saving-loading). + + diff --git a/website/docs/usage/projects.mdx b/website/docs/usage/projects.mdx new file mode 100644 index 0000000..e10ba4c --- /dev/null +++ b/website/docs/usage/projects.mdx @@ -0,0 +1,1141 @@ +--- +title: Projects +version: 3 +menu: + - ['Intro & Workflow', 'intro'] + - ['Directory & Assets', 'directory'] + - ['Custom Projects', 'custom'] + - ['Remote Storage', 'remote'] + - ['Integrations', 'integrations'] +--- + +## Introduction and workflow {id="intro",hidden="true"} + +> #### 🪐 Project templates +> +> Our [`projects`](https://github.com/explosion/projects) repo includes various +> project templates for different NLP tasks, models, workflows and integrations +> that you can clone and run. The easiest way to get started is to pick a +> template, clone it and start modifying it! + +spaCy projects let you manage and share **end-to-end spaCy workflows** for +different **use cases and domains**, and orchestrate training, packaging and +serving your custom pipelines. You can start off by cloning a pre-defined +project template, adjust it to fit your needs, load in your data, train a +pipeline, export it as a Python package, upload your outputs to a remote storage +and share your results with your team. spaCy projects can be used via the new +[`spacy project`](/api/cli#project) command and we provide templates in our +[`projects`](https://github.com/explosion/projects) repo. + +![Illustration of project workflow and commands](/images/projects.svg) + + + +The easiest way to get started is to clone a project template and run it – for +example, this end-to-end template that lets you train a **part-of-speech +tagger** and **dependency parser** on a Universal Dependencies treebank. + + + +spaCy projects make it easy to integrate with many other **awesome tools** in +the data science and machine learning ecosystem to track and manage your data +and experiments, iterate on demos and prototypes and ship your models into +production. + + + + Manage and version your data + + + Create labelled training data + + + Visualize and demo your pipelines + + + Serve your models and host APIs + + + Distributed and parallel training + + + Track your experiments and results + + + Upload your pipelines to the Hugging Face Hub + + + +### 1. Clone a project template {id="clone"} + +> #### Cloning under the hood +> +> To clone a project, spaCy calls into `git` and uses the "sparse checkout" +> feature to only clone the relevant directory or directories. + +The [`spacy project clone`](/api/cli#project-clone) command clones an existing +project template and copies the files to a local directory. You can then run the +project, e.g. to train a pipeline and edit the commands and scripts to build +fully custom workflows. + +```bash +python -m spacy project clone pipelines/tagger_parser_ud +``` + +By default, the project will be cloned into the current working directory. You +can specify an optional second argument to define the output directory. The +`--repo` option lets you define a custom repo to clone from if you don't want to +use the spaCy [`projects`](https://github.com/explosion/projects) repo. You can +also use any private repo you have access to with Git. + +### 2. Fetch the project assets {id="assets"} + +> #### project.yml +> +> ```yaml +> assets: +> - dest: 'assets/training.spacy' +> url: 'https://example.com/data.spacy' +> checksum: '63373dd656daa1fd3043ce166a59474c' +> - dest: 'assets/development.spacy' +> git: +> repo: 'https://github.com/example/repo' +> branch: 'master' +> path: 'path/development.spacy' +> checksum: '5113dc04e03f079525edd8df3f4f39e3' +> ``` + +Assets are data files your project needs – for example, the training and +evaluation data or pretrained vectors and embeddings to initialize your model +with. Each project template comes with a `project.yml` that defines the assets +to download and where to put them. The [`spacy project assets`](/api/cli#run) +will fetch the project assets for you: + +```bash +$ cd some_example_project +$ python -m spacy project assets +``` + +Asset URLs can be a number of different protocols: HTTP, HTTPS, FTP, SSH, and +even cloud storage such as GCS and S3. You can also fetch assets using git, by +replacing the `url` string with a `git` block. spaCy will use Git's "sparse +checkout" feature to avoid downloading the whole repository. + +Sometimes your project configuration may include large assets that you don't +necessarily want to download when you run `spacy project assets`. That's why +assets can be marked as [`extra`](#data-assets-url) - by default, these assets +are not downloaded. If they should be, run `spacy project assets --extra`. + +### 3. Run a command {id="run"} + +> #### project.yml +> +> ```yaml +> commands: +> - name: preprocess +> help: "Convert the input data to spaCy's format" +> script: +> - 'python -m spacy convert assets/train.conllu corpus/' +> - 'python -m spacy convert assets/eval.conllu corpus/' +> deps: +> - 'assets/train.conllu' +> - 'assets/eval.conllu' +> outputs: +> - 'corpus/train.spacy' +> - 'corpus/eval.spacy' +> ``` + +Commands consist of one or more steps and can be run with +[`spacy project run`](/api/cli#project-run). The following will run the command +`preprocess` defined in the `project.yml`: + +```bash +$ python -m spacy project run preprocess +``` + +Commands can define their expected [dependencies and outputs](#deps-outputs) +using the `deps` (files the commands require) and `outputs` (files the commands +create) keys. This allows your project to track changes and determine whether a +command needs to be re-run. For instance, if your input data changes, you want +to re-run the `preprocess` command. But if nothing changed, this step can be +skipped. You can also set `--force` to force re-running a command, or `--dry` to +perform a "dry run" and see what would happen (without actually running the +script). + +Since spaCy v3.4.2, `spacy projects run` checks your installed dependencies to +verify that your environment is properly set up and aligns with the project's +`requirements.txt`, if there is one. If missing or conflicting dependencies are +detected, a corresponding warning is displayed. If you'd like to disable the +dependency check, set `check_requirements: false` in your project's +`project.yml`. + +### 4. Run a workflow {id="run-workflow"} + +> #### project.yml +> +> ```yaml +> workflows: +> all: +> - preprocess +> - train +> - package +> ``` + +Workflows are series of commands that are run in order and often depend on each +other. For instance, to generate a pipeline package, you might start by +converting your data, then run [`spacy train`](/api/cli#train) to train your +pipeline on the converted data and if that's successful, run +[`spacy package`](/api/cli#package) to turn the best trained artifact into an +installable Python package. The following command runs the workflow named `all` +defined in the `project.yml`, and executes the commands it specifies, in order: + +```bash +$ python -m spacy project run all +``` + +Using the expected [dependencies and outputs](#deps-outputs) defined in the +commands, spaCy can determine whether to re-run a command (if its inputs or +outputs have changed) or whether to skip it. If you're looking to implement more +advanced data pipelines and track your changes in Git, check out the +[Data Version Control (DVC) integration](#dvc). The +[`spacy project dvc`](/api/cli#project-dvc) command generates a DVC config file +from a workflow defined in your `project.yml` so you can manage your spaCy +project as a DVC repo. + +### 5. Optional: Push to remote storage {id="push"} + +> ```yaml +> ### project.yml +> remotes: +> default: 's3://my-spacy-bucket' +> local: '/mnt/scratch/cache' +> ``` + +After training a pipeline, you can optionally use the +[`spacy project push`](/api/cli#project-push) command to upload your outputs to +a remote storage, using protocols like [S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage) or SSH. This can help +you **export** your pipeline packages, **share** work with your team, or **cache +results** to avoid repeating work. + +```bash +$ python -m spacy project push +``` + +The `remotes` section in your `project.yml` lets you assign names to the +different storages. To download state from a remote storage, you can use the +[`spacy project pull`](/api/cli#project-pull) command. For more details, see the +docs on [remote storage](#remote). + +## Project directory and assets {id="directory"} + +### project.yml {id="project-yml"} + +The `project.yml` defines the assets a project depends on, like datasets and +pretrained weights, as well as a series of commands that can be run separately +or as a workflow – for instance, to preprocess the data, convert it to spaCy's +format, train a pipeline, evaluate it and export metrics, package it and spin up +a quick web demo. It looks pretty similar to a config file used to define CI +pipelines. + +> #### Tip: Multi-line YAML syntax for long values +> +> YAML has [multi-line syntax](https://yaml-multiline.info/) that can be helpful +> for readability with longer values such as project descriptions or commands +> that take several arguments. + +```yaml +%%GITHUB_PROJECTS/pipelines/tagger_parser_ud/project.yml +``` + +> #### Tip: Overriding variables on the CLI +> +> If you want to override one or more variables on the CLI and are not already +> specifying a project directory, you need to add `.` as a placeholder: +> +> ``` +> python -m spacy project run test . --vars.foo bar +> ``` + +> #### Tip: Environment Variables +> +> Commands in a project file are not executed in a shell, so they don't have +> direct access to environment variables. But you can insert environment +> variables using the `env` dictionary to make values available for +> interpolation, just like values in `vars`. Here's an example `env` dict that +> makes `$PATH` available as `ENV_PATH`: +> +> ```yaml +> env: +> ENV_PATH: PATH +> ``` +> +> This can be used in a project command like so: +> +> ```yaml +> - name: 'echo-path' +> script: +> - 'echo ${env.ENV_PATH}' +> ``` + +| Section | Description | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | +| `description` | An optional project description used in [auto-generated docs](#custom-docs). | +| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overridden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. | +| `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. | +| `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. | +| `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. | +| `workflows` | A dictionary of workflow names, mapped to a list of command names, to execute in order. Workflows can be run with the [`project run`](/api/cli#project-run) command. | +| `commands` | A list of named commands. A command can define an optional help message (shown in the CLI when the user adds `--help`) and the `script`, a list of commands to run. The `deps` and `outputs` let you define the created file the command depends on and produces, respectively. This lets spaCy determine whether a command needs to be re-run because its dependencies or outputs changed. Commands can be run as part of a workflow, or separately with the [`project run`](/api/cli#project-run) command. | +| `spacy_version` | Optional spaCy version range like `>=3.0.0,<3.1.0` that the project is compatible with. If it's loaded with an incompatible version, an error is raised when the project is loaded. | +| `check_requirements` 3.4.2 | A flag determining whether to verify that the installed dependencies align with the project's `requirements.txt`. Defaults to `true`. | + +### Data assets {id="data-assets"} + +Assets are any files that your project might need, like training and development +corpora or pretrained weights for initializing your model. Assets are defined in +the `assets` block of your `project.yml` and can be downloaded using the +[`project assets`](/api/cli#project-assets) command. Defining checksums lets you +verify that someone else running your project will use the same files you used. +Asset URLs can be a number of different **protocols**: HTTP, HTTPS, FTP, SSH, +and even **cloud storage** such as GCS and S3. You can also download assets from +a **Git repo** instead. + +#### Downloading from a URL or cloud storage {id="data-assets-url"} + +Under the hood, spaCy uses the +[`smart-open`](https://github.com/RaRe-Technologies/smart_open) library so you +can use any protocol it supports. Note that you may need to install extra +dependencies to use certain protocols. + +> #### project.yml +> +> ```yaml +> assets: +> # Download from public HTTPS URL +> - dest: 'assets/training.spacy' +> url: 'https://example.com/data.spacy' +> checksum: '63373dd656daa1fd3043ce166a59474c' +> # Optional download from Google Cloud Storage bucket +> - dest: 'assets/development.spacy' +> extra: True +> url: 'gs://your-bucket/corpora' +> checksum: '5113dc04e03f079525edd8df3f4f39e3' +> ``` + +| Name | Description | +| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dest` | The destination path to save the downloaded asset to (relative to the project directory), including the file name. | +| `extra` | Optional flag determining whether this asset is downloaded only if `spacy project assets` is run with `--extra`. `False` by default. | +| `url` | The URL to download from, using the respective protocol. | +| `checksum` | Optional checksum of the file. If provided, it will be used to verify that the file matches and downloads will be skipped if a local file with the same checksum already exists. | +| `description` | Optional asset description, used in [auto-generated docs](#custom-docs). | + +#### Downloading from a Git repo {id="data-assets-git"} + +If a `git` block is provided, the asset is downloaded from the given Git +repository. You can download from any repo that you have access to. Under the +hood, this uses Git's "sparse checkout" feature, so you're only downloading the +files you need and not the whole repo. + +> #### project.yml +> +> ```yaml +> assets: +> - dest: 'assets/training.spacy' +> git: +> repo: 'https://github.com/example/repo' +> branch: 'master' +> path: 'path/training.spacy' +> checksum: '63373dd656daa1fd3043ce166a59474c' +> description: 'The training data (5000 examples)' +> ``` + +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dest` | The destination path to save the downloaded asset to (relative to the project directory), including the file name. | +| `git` | `repo`: The URL of the repo to download from.
`path`: Path of the file or directory to download, relative to the repo root. "" specifies the root directory.
`branch`: The branch to download from. Defaults to `"master"`. | +| `checksum` | Optional checksum of the file. If provided, it will be used to verify that the file matches and downloads will be skipped if a local file with the same checksum already exists. | +| `description` | Optional asset description, used in [auto-generated docs](#custom-docs). | + +#### Working with private assets {id="data-asets-private"} + +> #### project.yml +> +> ```yaml +> assets: +> - dest: 'assets/private_training_data.json' +> checksum: '63373dd656daa1fd3043ce166a59474c' +> - dest: 'assets/private_vectors.bin' +> checksum: '5113dc04e03f079525edd8df3f4f39e3' +> ``` + +For many projects, the datasets and weights you're working with might be +company-internal and not available over the internet. In that case, you can +specify the destination paths and a checksum, and leave out the URL. When your +teammates clone and run your project, they can place the files in the respective +directory themselves. The [`project assets`](/api/cli#project-assets) command +will alert you about missing files and mismatched checksums, so you can ensure +that others are running your project with the same data. + +### Dependencies and outputs {id="deps-outputs"} + +Each command defined in the `project.yml` can optionally define a list of +dependencies and outputs. These are the files the command requires and creates. +For example, a command for training a pipeline may depend on a +[`config.cfg`](/usage/training#config) and the training and evaluation data, and +it will export a directory `model-best`, which you can then re-use in other +commands. + +{/* prettier-ignore */} +```yaml {title="project.yml"} +commands: + - name: train + help: 'Train a spaCy pipeline using the specified corpus and config' + script: + - 'python -m spacy train ./configs/config.cfg -o training/ --paths.train ./corpus/training.spacy --paths.dev ./corpus/evaluation.spacy' + deps: + - 'configs/config.cfg' + - 'corpus/training.spacy' + - 'corpus/evaluation.spacy' + outputs: + - 'training/model-best' +``` + +> #### Re-running vs. skipping +> +> Under the hood, spaCy uses a `project.lock` lockfile that stores the details +> for each command, as well as its dependencies and outputs and their checksums. +> It's updated on each run. If any of this information changes, the command will +> be re-run. Otherwise, it will be skipped. + +If you're running a command and it depends on files that are missing, spaCy will +show you an error. If a command defines dependencies and outputs that haven't +changed since the last run, the command will be skipped. This means that you're +only re-running commands if they need to be re-run. Commands can also set +`no_skip: true` if they should never be skipped – for example commands that run +tests. Commands without outputs are also never skipped. To force re-running a +command or workflow, even if nothing changed, you can set the `--force` flag. + +Note that [`spacy project`](/api/cli#project) doesn't compile any dependency +graphs based on the dependencies and outputs, and won't re-run previous steps +automatically. For instance, if you only run the command `train` that depends on +data created by `preprocess` and those files are missing, spaCy will show an +error – it won't just re-run `preprocess`. If you're looking for more advanced +data management, check out the [Data Version Control (DVC) integration](#dvc). +If you're planning on integrating your spaCy project with DVC, you can also use +`outputs_no_cache` instead of `outputs` to define outputs that won't be cached +or tracked. + +### Files and directory structure {id="project-files"} + +The `project.yml` can define a list of `directories` that should be created +within a project – for instance, `assets`, `training`, `corpus` and so on. spaCy +will make sure that these directories are always available, so your commands can +write to and read from them. Project directories will also include all files and +directories copied from the project template with +[`spacy project clone`](/api/cli#project-clone). Here's an example of a project +directory: + +> #### project.yml +> +> {/* prettier-ignore */} +> ```yaml +> directories: ['assets', 'configs', 'corpus', 'metas', 'metrics', 'notebooks', 'packages', 'scripts', 'training'] +> ``` + +```yaml {title="Example project directory"} +├── project.yml # the project settings +├── project.lock # lockfile that tracks inputs/outputs +├── assets/ # downloaded data assets +├── configs/ # pipeline config.cfg files used for training +├── corpus/ # output directory for training corpus +├── metas/ # pipeline meta.json templates used for packaging +├── metrics/ # output directory for evaluation metrics +├── notebooks/ # directory for Jupyter notebooks +├── packages/ # output directory for pipeline Python packages +├── scripts/ # directory for scripts, e.g. referenced in commands +├── training/ # output directory for trained pipelines +└── ... # any other files, like a requirements.txt etc. +``` + +If you don't want a project to create a directory, you can delete it and remove +its entry from the `project.yml` – just make sure it's not required by any of +the commands. [Custom templates](#custom) can use any directories they need – +the only file that's required for a project is the `project.yml`. + +--- + +## Custom scripts and projects {id="custom"} + +The `project.yml` lets you define any custom commands and run them as part of +your training, evaluation or deployment workflows. The `script` section defines +a list of commands that are called in a subprocess, in order. This lets you +execute other Python scripts or command-line tools. Let's say you've written a +few integration tests that load the best model produced by the training command +and check that it works correctly. You can now define a `test` command that +calls into [`pytest`](https://docs.pytest.org/en/latest/), runs your tests and +uses [`pytest-html`](https://github.com/pytest-dev/pytest-html) to export a test +report: + +```yaml {title="project.yml"} +commands: + - name: test + help: 'Test the trained pipeline' + script: + - 'pip install pytest pytest-html' + - 'python -m pytest ./scripts/tests --html=metrics/test-report.html' + deps: + - 'training/model-best' + outputs: + - 'metrics/test-report.html' + no_skip: true +``` + +Adding `training/model-best` to the command's `deps` lets you ensure that the +file is available. If not, spaCy will show an error and the command won't run. +Setting `no_skip: true` means that the command will always run, even if the +dependencies (the trained pipeline) haven't changed. This makes sense here, +because you typically don't want to skip your tests. + +### Writing custom scripts {id="custom-scripts"} + +Your project commands can include any custom scripts – essentially, anything you +can run from the command line. Here's an example of a custom script that uses +[`typer`](https://typer.tiangolo.com/) for quick and easy command-line arguments +that you can define via your `project.yml`: + +> #### About Typer +> +> [`typer`](https://typer.tiangolo.com/) is a modern library for building Python +> CLIs using type hints. It's a dependency of spaCy, so it will already be +> pre-installed in your environment. Function arguments automatically become +> positional CLI arguments and using Python type hints, you can define the value +> types. For instance, `batch_size: int` means that the value provided via the +> command line is converted to an integer. + +```python {title="scripts/custom_evaluation.py"} +import typer + +def custom_evaluation(batch_size: int = 128, model_path: str, data_path: str): + # The arguments are now available as positional CLI arguments + print(batch_size, model_path, data_path) + +if __name__ == "__main__": + typer.run(custom_evaluation) +``` + +In your `project.yml`, you can then run the script by calling +`python scripts/custom_evaluation.py` with the function arguments. You can also +use the `vars` section to define reusable variables that will be substituted in +commands, paths and URLs. In this example, the batch size is defined as a +variable will be added in place of `${vars.batch_size}` in the script. Just like +in the [training config](/usage/training##config-overrides), you can also +override settings on the command line – for example using `--vars.batch_size`. + +> #### Calling into Python +> +> If any of your command scripts call into `python`, spaCy will take care of +> replacing that with your `sys.executable`, to make sure you're executing +> everything with the same Python (not some other Python installed on your +> system). It also normalizes references to `python3`, `pip3` and `pip`. + +{/* prettier-ignore */} +```yaml {title="project.yml"} +vars: + batch_size: 128 + +commands: + - name: evaluate + script: + - 'python scripts/custom_evaluation.py ${vars.batch_size} ./training/model-best ./corpus/eval.json' + deps: + - 'training/model-best' + - 'corpus/eval.json' +``` + +You can also use the `env` section to reference **environment variables** and +make their values available to the commands. This can be useful for overriding +settings on the command line and passing through system-level settings. + +> #### Usage example +> +> ```bash +> export GPU_ID=1 +> BATCH_SIZE=128 python -m spacy project run evaluate +> ``` + +```yaml {title="project.yml"} +env: + batch_size: BATCH_SIZE + gpu_id: GPU_ID + +commands: + - name: evaluate + script: + - 'python scripts/custom_evaluation.py ${env.batch_size}' +``` + +### Documenting your project {id="custom-docs"} + +> #### Readme Example +> +> For more examples, see the [`projects`](https://github.com/explosion/projects) +> repo. +> +> ![Screenshot of auto-generated Markdown Readme](/images/project_document.jpg) + +When your custom project is ready and you want to share it with others, you can +use the [`spacy project document`](/api/cli#project-document) command to +**auto-generate** a pretty, Markdown-formatted `README` file based on your +project's `project.yml`. It will list all commands, workflows and assets defined +in the project and include details on how to run the project, as well as links +to the relevant spaCy documentation to make it easy for others to get started +using your project. + +```bash +$ python -m spacy project document --output README.md +``` + +Under the hood, hidden markers are added to identify where the auto-generated +content starts and ends. This means that you can add your own custom content +before or after it and re-running the `project document` command will **only +update the auto-generated part**. This makes it easy to keep your documentation +up to date. + + + +Note that the contents of an existing file will be **replaced** if no existing +auto-generated docs are found. If you want spaCy to ignore a file and not update +it, you can add the comment marker `{/* SPACY PROJECT: IGNORE */}` anywhere in +your markup. + + + +### Cloning from your own repo {id="custom-repo"} + +The [`spacy project clone`](/api/cli#project-clone) command lets you customize +the repo to clone from using the `--repo` option. It calls into `git`, so you'll +be able to clone from any repo that you have access to, including private repos. + +```bash +python -m spacy project clone your_project --repo https://github.com/you/repo +``` + +At a minimum, a valid project template needs to contain a +[`project.yml`](#project-yml). It can also include +[other files](/usage/projects#project-files), like custom scripts, a +`requirements.txt` listing additional dependencies, +[training configs](/usage/training#config) and model meta templates, or Jupyter +notebooks with usage examples. + + + +It's typically not a good idea to check large data assets, trained pipelines or +other artifacts into a Git repo and you should exclude them from your project +template by adding a `.gitignore`. If you want to version your data and models, +check out [Data Version Control](#dvc) (DVC), which integrates with spaCy +projects. + + + +## Remote Storage {id="remote"} + +You can persist your project outputs to a remote storage using the +[`project push`](/api/cli#project-push) command. This can help you **export** +your pipeline packages, **share** work with your team, or **cache results** to +avoid repeating work. The [`project pull`](/api/cli#project-pull) command will +download any outputs that are in the remote storage and aren't available +locally. + +You can list one or more remotes in the `remotes` section of your +[`project.yml`](#project-yml) by mapping a string name to the URL of the +storage. Under the hood, spaCy uses +[`cloudpathlib`](https://cloudpathlib.drivendata.org) to communicate with the +remote storages, so you can use any protocol that `cloudpathlib` supports, +including [S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. + +> #### Example +> +> ```bash +> $ python -m spacy project pull local +> ``` + +```yaml {title="project.yml"} +remotes: + default: 's3://my-spacy-bucket' + local: '/mnt/scratch/cache' +``` + + + +Inside the remote storage, spaCy uses a clever **directory structure** to avoid +overwriting files. The top level of the directory structure is a URL-encoded +version of the output's path. Within this directory are subdirectories named +according to a hash of the command string and the command's dependencies. +Finally, within those directories are files, named according to an MD5 hash of +their contents. + +{/* TODO: update with actual real example? */} + +{/* prettier-ignore */} +```yaml +└── urlencoded_file_path # Path of original file + ├── some_command_hash # Hash of command you ran + │ ├── some_content_hash # Hash of file content + │ └── another_content_hash + └── another_command_hash + └── third_content_hash +``` + + + +For instance, let's say you had the following command in your `project.yml`: + +```yaml {title="project.yml"} +- name: train + help: 'Train a spaCy pipeline using the specified corpus and config' + script: + - 'spacy train ./config.cfg --output training/' + deps: + - 'corpus/train' + - 'corpus/dev' + - 'config.cfg' + outputs: + - 'training/model-best' +``` + +> #### Example +> +> ``` +> └── s3://my-spacy-bucket/training%2Fmodel-best +> └── 1d8cb33a06cc345ad3761c6050934a1b +> └── d8e20c3537a084c5c10d95899fe0b1ff +> ``` + +After you finish training, you run [`project push`](/api/cli#project-push) to +make sure the `training/model-best` output is saved to remote storage. spaCy +will then construct a hash from your command script and the listed dependencies, +`corpus/train`, `corpus/dev` and `config.cfg`, in order to identify the +execution context of your output. It would then compute an MD5 hash of the +`training/model-best` directory, and use those three pieces of information to +construct the storage URL. + +```bash +$ python -m spacy project run train +$ python -m spacy project push +``` + +If you change the command or one of its dependencies (for instance, by editing +the [`config.cfg`](/usage/training#config) file to tune the hyperparameters, a +different creation hash will be calculated, so when you use +[`project push`](/api/cli#project-push) you won't be overwriting your previous +file. The system even supports multiple outputs for the same file and the same +context, which can happen if your training process is not deterministic, or if +you have dependencies that aren't represented in the command. + +In summary, the [`spacy project`](/api/cli#project) remote storages are designed +to make a particular set of trade-offs. Priority is placed on **convenience**, +**correctness** and **avoiding data loss**. You can use +[`project push`](/api/cli#project-push) freely, as you'll never overwrite remote +state, and you don't have to come up with names or version numbers. However, +it's up to you to manage the size of your remote storage, and to remove files +that are no longer relevant to you. + +## Integrations {id="integrations"} + +{

Data Version Control (DVC) + + +

} + +Data assets like training corpora or pretrained weights are at the core of any +NLP project, but they're often difficult to manage: you can't just check them +into your Git repo to version and keep track of them. And if you have multiple +steps that depend on each other, like a preprocessing step that generates your +training data, you need to make sure the data is always up-to-date, and re-run +all steps of your process every time, just to be safe. + +[Data Version Control](https://dvc.org) (DVC) is a standalone open-source tool +that integrates into your workflow like Git, builds a dependency graph for your +data pipelines and tracks and caches your data files. If you're downloading data +from an external source, like a storage bucket, DVC can tell whether the +resource has changed. It can also determine whether to re-run a step, depending +on whether its input have changed or not. All metadata can be checked into a Git +repo, so you'll always be able to reproduce your experiments. + +To set up DVC, install the package and initialize your spaCy project as a Git +and DVC repo. You can also +[customize your DVC installation](https://dvc.org/doc/install/macos#install-with-pip) +to include support for remote storage like Google Cloud Storage, S3, Azure, SSH +and more. + +```bash +$ pip install dvc # Install DVC +$ git init # Initialize a Git repo +$ dvc init # Initialize a DVC project +``` + + + +DVC enables usage analytics by default, so if you're working in a +privacy-sensitive environment, make sure to +[**opt-out manually**](https://dvc.org/doc/user-guide/analytics#opting-out). + + + +The [`spacy project dvc`](/api/cli#project-dvc) command creates a `dvc.yaml` +config file based on a workflow defined in your `project.yml`. Whenever you +update your project, you can re-run the command to update your DVC config. You +can then manage your spaCy project like any other DVC project, run +[`dvc add`](https://dvc.org/doc/command-reference/add) to add and track assets +and [`dvc repro`](https://dvc.org/doc/command-reference/repro) to reproduce the +workflow or individual commands. + +```bash +$ python -m spacy project dvc [project_dir] [workflow_name] +``` + + + +DVC currently expects a single workflow per project, so when creating the config +with [`spacy project dvc`](/api/cli#project-dvc), you need to specify the name +of a workflow defined in your `project.yml`. You can still use multiple +workflows, but only one can be tracked by DVC. + + + +{/* { TODO: } */} + +--- + +{

Prodigy + + +

} + +[Prodigy](https://prodi.gy) is a modern annotation tool for creating training +data for machine learning models, developed by us. It integrates with spaCy +out-of-the-box and provides many different +[annotation recipes](https://prodi.gy/docs/recipes) for a variety of NLP tasks, +with and without a model in the loop. If Prodigy is installed in your project, +you can start the annotation server from your `project.yml` for a tight feedback +loop between data development and training. + + + +This integration requires [Prodigy v1.11](https://prodi.gy/docs/changelog#v1.11) +or higher. If you're using an older version of Prodigy, you can still use your +annotations in spaCy v3 by exporting your data with +[`data-to-spacy`](https://prodi.gy/docs/recipes#data-to-spacy) and running +[`spacy convert`](/api/cli#convert) to convert it to the binary format. + + + +The following example shows a workflow for merging and exporting NER annotations +collected with Prodigy and training a spaCy pipeline: + +> #### Example usage +> +> ```bash +> $ python -m spacy project run all +> ``` + +{/* prettier-ignore */} +```yaml {title="project.yml"} +vars: + prodigy: + train_dataset: "fashion_brands_training" + eval_dataset: "fashion_brands_eval" + +workflows: + all: + - data-to-spacy + - train_spacy + +commands: + - name: "data-to-spacy" + help: "Merge your annotations and create data in spaCy's binary format" + script: + - "python -m prodigy data-to-spacy corpus/ --ner ${vars.prodigy.train_dataset},eval:${vars.prodigy.eval_dataset}" + outputs: + - "corpus/train.spacy" + - "corpus/dev.spacy" + - name: "train_spacy" + help: "Train a named entity recognition model with spaCy" + script: + - "python -m spacy train configs/config.cfg --output training/ --paths.train corpus/train.spacy --paths.dev corpus/dev.spacy" + deps: + - "corpus/train.spacy" + - "corpus/dev.spacy" + outputs: + - "training/model-best" +``` + +> #### Example train curve output +> +> src="/images/prodigy_train_curve.jpg" +> href="https://prodi.gy/docs/recipes#train-curve" +> alt="Screenshot of train curve terminal output" +> /> + +The [`train-curve`](https://prodi.gy/docs/recipes#train-curve) recipe is another +cool workflow you can include in your project. It will run the training with +different portions of the data, e.g. 25%, 50%, 75% and 100%. As a rule of thumb, +if accuracy increases in the last segment, this could indicate that collecting +more annotations of the same type might improve the model further. + +{/* prettier-ignore */} +```yaml {title="project.yml (excerpt)"} +- name: "train_curve" + help: "Train the model with Prodigy by using different portions of training examples to evaluate if more annotations can potentially improve the performance" + script: + - "python -m prodigy train-curve --ner ${vars.prodigy.train_dataset},eval:${vars.prodigy.eval_dataset} --config configs/${vars.config} --show-plot" +``` + +You can use the same approach for various types of projects and annotation +workflows, including +[named entity recognition](https://prodi.gy/docs/named-entity-recognition), +[span categorization](https://prodi.gy/docs/span-categorization), +[text classification](https://prodi.gy/docs/text-classification), +[dependency parsing](https://prodi.gy/docs/dependencies-relations), +[part-of-speech tagging](https://prodi.gy/docs/recipes#pos) or fully +[custom recipes](https://prodi.gy/docs/custom-recipes). You can also use spaCy +project templates to quickly start the annotation server to collect more +annotations and add them to your Prodigy dataset. + + + +Get started with spaCy and Prodigy using our project template. It includes +commands to create a merged training corpus from your Prodigy annotations, +training and packaging a spaCy pipeline and analyzing if more annotations may +improve performance. + + + +--- + +{

Streamlit + + +

} + +[Streamlit](https://streamlit.io) is a Python framework for building interactive +data apps. The [`spacy-streamlit`](https://github.com/explosion/spacy-streamlit) +package helps you integrate spaCy visualizations into your Streamlit apps and +quickly spin up demos to explore your pipelines interactively. It includes a +full embedded visualizer, as well as individual components. + +{/* TODO: update once version is stable */} + +> #### Installation +> +> ```bash +> $ pip install spacy-streamlit --pre +> ``` + +![Screenshot of the spacy-streamlit package in Streamlit](/images/spacy-streamlit.png) + +Using [`spacy-streamlit`](https://github.com/explosion/spacy-streamlit), your +projects can easily define their own scripts that spin up an interactive +visualizer, using the latest pipeline you trained, or a selection of pipelines +so you can compare their results. + + + +Get started with spaCy and Streamlit using our project template. It includes a +script to spin up a custom visualizer and commands you can adjust to showcase +and explore your own custom trained pipelines. + + + +> #### Example usage +> +> ```bash +> $ python -m spacy project run visualize +> ``` + +{/* prettier-ignore */} +```yaml {title="project.yml"} +commands: + - name: visualize + help: "Visualize the pipeline's output interactively using Streamlit" + script: + - 'streamlit run ./scripts/visualize.py ./training/model-best "I like Adidas shoes."' + deps: + - "training/model-best" +``` + +The following script is called from the `project.yml` and takes two positional +command-line argument: a comma-separated list of paths or packages to load the +pipelines from and an example text to use as the default text. + +```python +https://github.com/explosion/projects/blob/v3/integrations/streamlit/scripts/visualize.py +``` + +--- + +{

FastAPI + + +

} + +[FastAPI](https://fastapi.tiangolo.com/) is a modern high-performance framework +for building REST APIs with Python, based on Python +[type hints](https://fastapi.tiangolo.com/python-types/). It's become a popular +library for serving machine learning models and you can use it in your spaCy +projects to quickly serve up a trained pipeline and make it available behind a +REST API. + + + +Get started with spaCy and FastAPI using our project template. It includes a +simple REST API for processing batches of text, and usage examples for how to +query your API from Python and JavaScript (Vanilla JS and React). + + + +> #### Example usage +> +> ```bash +> $ python -m spacy project run serve +> ``` + +{/* prettier-ignore */} +```yaml {title="project.yml"} + - name: "serve" + help: "Serve the models via a FastAPI REST API using the given host and port" + script: + - "uvicorn scripts.main:app --reload --host 127.0.0.1 --port 5000" + deps: + - "scripts/main.py" + no_skip: true +``` + +The script included in the template shows a simple REST API with a `POST` +endpoint that accepts batches of texts and returns batches of predictions, e.g. +named entities found in the documents. Type hints and +[`pydantic`](https://github.com/samuelcolvin/pydantic) are used to define the +expected data types. + +```python +https://github.com/explosion/projects/blob/v3/integrations/fastapi/scripts/main.py +``` + +--- + +{

Weights & Biases + + +

} + +[Weights & Biases](https://www.wandb.com/) is a popular platform for experiment +tracking. spaCy integrates with it out-of-the-box via the +[`WandbLogger`](https://github.com/explosion/spacy-loggers#wandblogger), which +you can add as the `[training.logger]` block of your training +[config](/usage/training#config). The results of each step are then logged in +your project, together with the full **training config**. This means that +_every_ hyperparameter, registered function name and argument will be tracked +and you'll be able to see the impact it has on your results. + +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.WandbLogger.v3" +> project_name = "monitor_spacy_training" +> remove_config_values = ["paths.train", "paths.dev", "corpora.train.path", "corpora.dev.path"] +> log_dataset_dir = "corpus" +> model_log_interval = 1000 +> ``` + +![Screenshot: Visualized training results](/images/wandb1.jpg) + +![Screenshot: Parameter importance using config values](/images/wandb2.jpg 'Parameter importance using config values') + + + +Get started with tracking your spaCy training runs in Weights & Biases using our +project template. It trains on the IMDB Movie Review Dataset and includes a +simple config with the built-in `WandbLogger`, as well as a custom example of +creating variants of the config for a simple hyperparameter grid search and +logging the results. + + + +--- + +{

Hugging Face Hub + + +

} + +The [Hugging Face Hub](https://huggingface.co/) lets you upload models and share +them with others. It hosts models as Git-based repositories which are storage +spaces that can contain all your files. It support versioning, branches and +custom metadata out-of-the-box, and provides browser-based visualizers for +exploring your models interactively, as well as an API for production use. The +[`spacy-huggingface-hub`](https://github.com/explosion/spacy-huggingface-hub) +package automatically adds the `huggingface-hub` command to your `spacy` CLI if +it's installed. + +> #### Installation +> +> ```bash +> $ pip install spacy-huggingface-hub +> # Check that the CLI is registered +> $ python -m spacy huggingface-hub --help +> ``` + +You can then upload any pipeline packaged with +[`spacy package`](/api/cli#package). Make sure to set `--build wheel` to output +a binary `.whl` file. The uploader will read all metadata from the pipeline +package, including the auto-generated pretty `README.md` and the model details +available in the `meta.json`. For examples, check out the +[spaCy pipelines](https://huggingface.co/spacy) we've uploaded. + +```bash +$ huggingface-cli login +$ python -m spacy package ./en_ner_fashion ./output --build wheel +$ cd ./output/en_ner_fashion-0.0.0/dist +$ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl +``` + +After uploading, you will see the live URL of your pipeline packages, as well as +the direct URL to the model wheel you can install via `pip install`. You'll also +be able to test your pipeline interactively from your browser: + +![Screenshot: interactive NER visualizer](/images/huggingface_hub.jpg) + +In your `project.yml`, you can add a command that uploads your trained and +packaged pipeline to the hub. You can either run this as a manual step, or +automatically as part of a workflow. Make sure to set `--build wheel` when +running `spacy package` to build a wheel file for your pipeline package. + +{/* prettier-ignore */} +```yaml {title="project.yml"} +- name: "push_to_hub" + help: "Upload the trained model to the Hugging Face Hub" + script: + - "python -m spacy huggingface-hub push packages/en_${vars.name}-${vars.version}/dist/en_${vars.name}-${vars.version}-py3-none-any.whl" + deps: + - "packages/en_${vars.name}-${vars.version}/dist/en_${vars.name}-${vars.version}-py3-none-any.whl" +``` + + + +Get started with uploading your models to the Hugging Face hub using our project +template. It trains a simple pipeline, packages it and uploads it if the +packaged model has changed. This makes it easy to deploy your models end-to-end. + + diff --git a/website/docs/usage/rule-based-matching.mdx b/website/docs/usage/rule-based-matching.mdx new file mode 100644 index 0000000..8632f45 --- /dev/null +++ b/website/docs/usage/rule-based-matching.mdx @@ -0,0 +1,1880 @@ +--- +title: Rule-based matching +teaser: Find phrases and tokens, and match entities +menu: + - ['Token Matcher', 'matcher'] + - ['Phrase Matcher', 'phrasematcher'] + - ['Dependency Matcher', 'dependencymatcher'] + - ['Entity Ruler', 'entityruler'] + - ['Span Ruler', 'spanruler'] + - ['Models & Rules', 'models-rules'] +--- + +Compared to using regular expressions on raw text, spaCy's rule-based matcher +engines and components not only let you find the words and phrases you're +looking for – they also give you access to the tokens within the document and +their relationships. This means you can easily access and analyze the +surrounding tokens, merge spans into single tokens or add entries to the named +entities in `doc.ents`. + + + +For complex tasks, it's usually better to train a statistical entity recognition +model. However, statistical models require training data, so for many +situations, rule-based approaches are more practical. This is especially true at +the start of a project: you can use a rule-based approach as part of a data +collection process, to help you "bootstrap" a statistical model. + +Training a model is useful if you have some examples and you want your system to +be able to **generalize** based on those examples. It works especially well if +there are clues in the _local context_. For instance, if you're trying to detect +person or company names, your application may benefit from a statistical named +entity recognition model. + +Rule-based systems are a good choice if there's a more or less **finite number** +of examples that you want to find in the data, or if there's a very **clear, +structured pattern** you can express with token rules or regular expressions. +For instance, country names, IP addresses or URLs are things you might be able +to handle well with a purely rule-based approach. + +You can also combine both approaches and improve a statistical model with rules +to handle very specific cases and boost accuracy. For details, see the section +on [rule-based entity recognition](#entityruler). + + + + + +The `PhraseMatcher` is useful if you already have a large terminology list or +gazetteer consisting of single or multi-token phrases that you want to find +exact instances of in your data. As of spaCy v2.1.0, you can also match on the +`LOWER` attribute for fast and case-insensitive matching. + +The `Matcher` isn't as blazing fast as the `PhraseMatcher`, since it compares +across individual token attributes. However, it allows you to write very +abstract representations of the tokens you're looking for, using lexical +attributes, linguistic features predicted by the model, operators, set +membership and rich comparison. For example, you can find a noun, followed by a +verb with the lemma "love" or "like", followed by an optional determiner and +another token that's at least 10 characters long. + + + +## Token-based matching {id="matcher"} + +spaCy features a rule-matching engine, the [`Matcher`](/api/matcher), that +operates over tokens, similar to regular expressions. The rules can refer to +token annotations (e.g. the token `text` or `tag_`, and flags like `IS_PUNCT`). +The rule matcher also lets you pass in a custom callback to act on matches – for +example, to merge entities and apply custom labels. You can also associate +patterns with entity IDs, to allow some basic entity linking or disambiguation. +To match large terminology lists, you can use the +[`PhraseMatcher`](/api/phrasematcher), which accepts `Doc` objects as match +patterns. + +### Adding patterns {id="adding-patterns"} + +Let's say we want to enable spaCy to find a combination of three tokens: + +1. A token whose **lowercase form matches "hello"**, e.g. "Hello" or "HELLO". +2. A token whose **`is_punct` flag is set to `True`**, i.e. any punctuation. +3. A token whose **lowercase form matches "world"**, e.g. "World" or "WORLD". + +```python +[{"LOWER": "hello"}, {"IS_PUNCT": True}, {"LOWER": "world"}] +``` + + + +When writing patterns, keep in mind that **each dictionary** represents **one +token**. If spaCy's tokenization doesn't match the tokens defined in a pattern, +the pattern is not going to produce any results. When developing complex +patterns, make sure to check examples against spaCy's tokenization: + +```python +doc = nlp("A complex-example,!") +print([token.text for token in doc]) +``` + + + +First, we initialize the `Matcher` with a vocab. The matcher must always share +the same vocab with the documents it will operate on. We can now call +[`matcher.add()`](/api/matcher#add) with an ID and a list of patterns. + +```python {executable="true"} +import spacy +from spacy.matcher import Matcher + +nlp = spacy.load("en_core_web_sm") +matcher = Matcher(nlp.vocab) +# Add match ID "HelloWorld" with no callback and one pattern +pattern = [{"LOWER": "hello"}, {"IS_PUNCT": True}, {"LOWER": "world"}] +matcher.add("HelloWorld", [pattern]) + +doc = nlp("Hello, world! Hello world!") +matches = matcher(doc) +for match_id, start, end in matches: + string_id = nlp.vocab.strings[match_id] # Get string representation + span = doc[start:end] # The matched span + print(match_id, string_id, start, end, span.text) +``` + +The matcher returns a list of `(match_id, start, end)` tuples – in this case, +`[('15578876784678163569', 0, 3)]`, which maps to the span `doc[0:3]` of our +original document. The `match_id` is the [hash value](/usage/spacy-101#vocab) of +the string ID "HelloWorld". To get the string value, you can look up the ID in +the [`StringStore`](/api/stringstore). + +```python +for match_id, start, end in matches: + string_id = nlp.vocab.strings[match_id] # 'HelloWorld' + span = doc[start:end] # The matched span +``` + +Optionally, we could also choose to add more than one pattern, for example to +also match sequences without punctuation between "hello" and "world": + +```python +patterns = [ + [{"LOWER": "hello"}, {"IS_PUNCT": True}, {"LOWER": "world"}], + [{"LOWER": "hello"}, {"LOWER": "world"}] +] +matcher.add("HelloWorld", patterns) +``` + +By default, the matcher will only return the matches and **not do anything +else**, like merge entities or assign labels. This is all up to you and can be +defined individually for each pattern, by passing in a callback function as the +`on_match` argument on `add()`. This is useful, because it lets you write +entirely custom and **pattern-specific logic**. For example, you might want to +merge _some_ patterns into one token, while adding entity labels for other +pattern types. You shouldn't have to create different matchers for each of those +processes. + +#### Available token attributes {id="adding-patterns-attributes"} + +The available token pattern keys correspond to a number of +[`Token` attributes](/api/token#attributes). The supported attributes for +rule-based matching are: + +| Attribute | Description | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ORTH` | The exact verbatim text of a token. ~~str~~ | +| `TEXT` | The exact verbatim text of a token. ~~str~~ | +| `NORM` | The normalized form of the token text. ~~str~~ | +| `LOWER` | The lowercase form of the token text. ~~str~~ | +| `LENGTH` | The length of the token text. ~~int~~ | +| `IS_ALPHA`, `IS_ASCII`, `IS_DIGIT` | Token text consists of alphabetic characters, ASCII characters, digits. ~~bool~~ | +| `IS_LOWER`, `IS_UPPER`, `IS_TITLE` | Token text is in lowercase, uppercase, titlecase. ~~bool~~ | +| `IS_PUNCT`, `IS_SPACE`, `IS_STOP` | Token is punctuation, whitespace, stop word. ~~bool~~ | +| `IS_SENT_START` | Token is start of sentence. ~~bool~~ | +| `LIKE_NUM`, `LIKE_URL`, `LIKE_EMAIL` | Token text resembles a number, URL, email. ~~bool~~ | +| `SPACY` | Token has a trailing space. ~~bool~~ | +| `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ | +| `ENT_TYPE` | The token's entity label. ~~str~~ | +| `_` | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ | + + + +No, it shouldn't. spaCy will normalize the names internally and +`{"LOWER": "text"}` and `{"lower": "text"}` will both produce the same result. +Using the uppercase version is mostly a convention to make it clear that the +attributes are "special" and don't exactly map to the token attributes like +`Token.lower` and `Token.lower_`. + + + + + +spaCy can't provide access to all of the attributes because the `Matcher` loops +over the Cython data, not the Python objects. Inside the matcher, we're dealing +with a [`TokenC` struct](/api/cython-structs#tokenc) – we don't have an instance +of [`Token`](/api/token). This means that all of the attributes that refer to +computed properties can't be accessed. + +The uppercase attribute names like `LOWER` or `IS_PUNCT` refer to symbols from +the [`spacy.attrs`](%%GITHUB_SPACY/spacy/attrs.pyx) enum table. They're passed +into a function that essentially is a big case/switch statement, to figure out +which struct field to return. The same attribute identifiers are used in +[`Doc.to_array`](/api/doc#to_array), and a few other places in the code where +you need to describe fields like this. + + + +--- + + + +Matcher demo + +The [Matcher Explorer](https://explosion.ai/demos/matcher) lets you test the +rule-based `Matcher` by creating token patterns interactively and running them +over your text. Each token can set multiple attributes like text value, +part-of-speech tag or boolean flags. The token-based view lets you explore how +spaCy processes your text – and why your pattern matches, or why it doesn't. + + + +#### Extended pattern syntax and attributes {id="adding-patterns-attributes-extended",version="2.1"} + +Instead of mapping to a single value, token patterns can also map to a +**dictionary of properties**. For example, to specify that the value of a lemma +should be part of a list of values, or to set a minimum character length. The +following rich comparison attributes are available: + +> #### Example +> +> ```python +> # Matches "love cats" or "likes flowers" +> pattern1 = [{"LEMMA": {"IN": ["like", "love"]}}, +> {"POS": "NOUN"}] +> +> # Matches tokens of length >= 10 +> pattern2 = [{"LENGTH": {">=": 10}}] +> +> # Match based on morph attributes +> pattern3 = [{"MORPH": {"IS_SUBSET": ["Number=Sing", "Gender=Neut"]}}] +> # "", "Number=Sing" and "Number=Sing|Gender=Neut" will match as subsets +> # "Number=Plur|Gender=Neut" will not match +> # "Number=Sing|Gender=Neut|Polite=Infm" will not match because it's a superset +> ``` + +| Attribute | Description | +| -------------------------- | --------------------------------------------------------------------------------------------------------- | +| `IN` | Attribute value is member of a list. ~~Any~~ | +| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | +| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | +| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | +| `INTERSECTS` | Attribute value (for `MORPH` or custom list attributes) has a non-empty intersection with a list. ~~Any~~ | +| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | + +#### Regular expressions {id="regex",version="2.1"} + +In some cases, only matching tokens and token attributes isn't enough – for +example, you might want to match different spellings of a word, without having +to add a new pattern for each spelling. + +```python +pattern = [{"TEXT": {"REGEX": "^[Uu](\\.?|nited)$"}}, + {"TEXT": {"REGEX": "^[Ss](\\.?|tates)$"}}, + {"LOWER": "president"}] +``` + +The `REGEX` operator allows defining rules for any attribute string value, +including custom attributes. It always needs to be applied to an attribute like +`TEXT`, `LOWER` or `TAG`: + +```python +# Match different spellings of token texts +pattern = [{"TEXT": {"REGEX": "deff?in[ia]tely"}}] + +# Match tokens with fine-grained POS tags starting with 'V' +pattern = [{"TAG": {"REGEX": "^V"}}] + +# Match custom attribute values with regular expressions +pattern = [{"_": {"country": {"REGEX": "^[Uu](nited|\\.?) ?[Ss](tates|\\.?)$"}}}] +``` + + + +When using the `REGEX` operator, keep in mind that it operates on **single +tokens**, not the whole text. Each expression you provide will be matched on a +token. If you need to match on the whole text instead, see the details on +[regex matching on the whole text](#regex-text). + + + +##### Matching regular expressions on the full text {id="regex-text"} + +If your expressions apply to multiple tokens, a simple solution is to match on +the `doc.text` with `re.finditer` and use the +[`Doc.char_span`](/api/doc#char_span) method to create a `Span` from the +character indices of the match. If the matched characters don't map to one or +more valid tokens, `Doc.char_span` returns `None`. + +> #### What's a valid token sequence? +> +> In the example, the expression will also match `"US"` in `"USA"`. However, +> `"USA"` is a single token and `Span` objects are **sequences of tokens**. So +> `"US"` cannot be its own span, because it does not end on a token boundary. + +```python {executable="true"} +import spacy +import re + +nlp = spacy.load("en_core_web_sm") +doc = nlp("The United States of America (USA) are commonly known as the United States (U.S. or US) or America.") + +expression = r"[Uu](nited|\.?) ?[Ss](tates|\.?)" +for match in re.finditer(expression, doc.text): + start, end = match.span() + span = doc.char_span(start, end) + # This is a Span object or None if match doesn't map to valid token sequence + if span is not None: + print("Found match:", span.text) +``` + + + +In some cases, you might want to expand the match to the closest token +boundaries, so you can create a `Span` for `"USA"`, even though only the +substring `"US"` is matched. You can calculate this using the character offsets +of the tokens in the document, available as +[`Token.idx`](/api/token#attributes). This lets you create a list of valid token +start and end boundaries and leaves you with a rather basic algorithmic problem: +Given a number, find the next lowest (start token) or the next highest (end +token) number that's part of a given list of numbers. This will be the closest +valid token boundary. + +There are many ways to do this and the most straightforward one is to create a +dict keyed by characters in the `Doc`, mapped to the token they're part of. It's +easy to write and less error-prone, and gives you a constant lookup time: you +only ever need to create the dict once per `Doc`. + +```python +chars_to_tokens = {} +for token in doc: + for i in range(token.idx, token.idx + len(token.text)): + chars_to_tokens[i] = token.i +``` + +You can then look up character at a given position, and get the index of the +corresponding token that the character is part of. Your span would then be +`doc[token_start:token_end]`. If a character isn't in the dict, it means it's +the (white)space tokens are split on. That hopefully shouldn't happen, though, +because it'd mean your regex is producing matches with leading or trailing +whitespace. + +```python {highlight="5-8"} +span = doc.char_span(start, end) +if span is not None: + print("Found match:", span.text) +else: + start_token = chars_to_tokens.get(start) + end_token = chars_to_tokens.get(end) + if start_token is not None and end_token is not None: + span = doc[start_token:end_token + 1] + print("Found closest match:", span.text) +``` + + + +#### Fuzzy matching {id="fuzzy", version="3.5"} + +Fuzzy matching allows you to match tokens with alternate spellings, typos, etc. +without specifying every possible variant. + +```python +# Matches "favourite", "favorites", "gavorite", "theatre", "theatr", ... +pattern = [{"TEXT": {"FUZZY": "favorite"}}, + {"TEXT": {"FUZZY": "theater"}}] +``` + +The `FUZZY` attribute allows fuzzy matches for any attribute string value, +including custom attributes. Just like `REGEX`, it always needs to be applied to +an attribute like `TEXT` or `LOWER`. By default `FUZZY` allows a Levenshtein +edit distance of at least 2 and up to 30% of the pattern string length. Using +the more specific attributes `FUZZY1`..`FUZZY9` you can specify the maximum +allowed edit distance directly. + +```python +# Match lowercase with fuzzy matching (allows 3 edits) +pattern = [{"LOWER": {"FUZZY": "definitely"}}] + +# Match custom attribute values with fuzzy matching (allows 3 edits) +pattern = [{"_": {"country": {"FUZZY": "Kyrgyzstan"}}}] + +# Match with exact Levenshtein edit distance limits (allows 4 edits) +pattern = [{"_": {"country": {"FUZZY4": "Kyrgyzstan"}}}] +``` + +#### Regex and fuzzy matching with lists {id="regex-fuzzy-lists", version="3.5"} + +Starting in spaCy v3.5, both `REGEX` and `FUZZY` can be combined with the +attributes `IN` and `NOT_IN`: + +```python +pattern = [{"TEXT": {"FUZZY": {"IN": ["awesome", "cool", "wonderful"]}}}] + +pattern = [{"TEXT": {"REGEX": {"NOT_IN": ["^awe(some)?$", "^wonder(ful)?"]}}}] +``` + +--- + +#### Operators and quantifiers {id="quantifiers"} + +The matcher also lets you use quantifiers, specified as the `'OP'` key. +Quantifiers let you define sequences of tokens to be matched, e.g. one or more +punctuation marks, or specify optional tokens. Note that there are no nested or +scoped quantifiers – instead, you can build those behaviors with `on_match` +callbacks. + +| OP | Description | +| ------- | ---------------------------------------------------------------------- | +| `!` | Negate the pattern, by requiring it to match exactly 0 times. | +| `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | +| `+` | Require the pattern to match 1 or more times. | +| `*` | Allow the pattern to match zero or more times. | +| `{n}` | Require the pattern to match exactly _n_ times. | +| `{n,m}` | Require the pattern to match at least _n_ but not more than _m_ times. | +| `{n,}` | Require the pattern to match at least _n_ times. | +| `{,m}` | Require the pattern to match at most _m_ times. | + +> #### Example +> +> ```python +> pattern = [{"LOWER": "hello"}, +> {"IS_PUNCT": True, "OP": "?"}] +> ``` + + + +In versions before v2.1.0, the semantics of the `+` and `*` operators behave +inconsistently. They were usually interpreted "greedily", i.e. longer matches +are returned where possible. However, if you specify two `+` and `*` patterns in +a row and their matches overlap, the first operator will behave non-greedily. +This quirk in the semantics is corrected in spaCy v2.1.0. + + + +#### Using wildcard token patterns {id="adding-patterns-wildcard",version="2"} + +While the token attributes offer many options to write highly specific patterns, +you can also use an empty dictionary, `{}` as a wildcard representing **any +token**. This is useful if you know the context of what you're trying to match, +but very little about the specific token and its characters. For example, let's +say you're trying to extract people's user names from your data. All you know is +that they are listed as "User name: \{username\}". The name itself may contain +any character, but no whitespace – so you'll know it will be handled as one +token. + +```python +[{"ORTH": "User"}, {"ORTH": "name"}, {"ORTH": ":"}, {}] +``` + +#### Validating and debugging patterns {id="pattern-validation",version="2.1"} + +The `Matcher` can validate patterns against a JSON schema with the option +`validate=True`. This is useful for debugging patterns during development, in +particular for catching unsupported attributes. + +```python {executable="true"} +import spacy +from spacy.matcher import Matcher + +nlp = spacy.load("en_core_web_sm") +matcher = Matcher(nlp.vocab, validate=True) +# Add match ID "HelloWorld" with unsupported attribute CASEINSENSITIVE +pattern = [{"LOWER": "hello"}, {"IS_PUNCT": True}, {"CASEINSENSITIVE": "world"}] +matcher.add("HelloWorld", [pattern]) +# 🚨 Raises an error: +# MatchPatternError: Invalid token patterns for matcher rule 'HelloWorld' +# Pattern 0: +# - [pattern -> 2 -> CASEINSENSITIVE] extra fields not permitted + +``` + +### Adding on_match rules {id="on_match"} + +To move on to a more realistic example, let's say you're working with a large +corpus of blog articles, and you want to match all mentions of "Google I/O" +(which spaCy tokenizes as `['Google', 'I', '/', 'O'`]). To be safe, you only +match on the uppercase versions, avoiding matches with phrases such as "Google +i/o". + +```python {executable="true"} +from spacy.lang.en import English +from spacy.matcher import Matcher +from spacy.tokens import Span + +nlp = English() +matcher = Matcher(nlp.vocab) + +def add_event_ent(matcher, doc, i, matches): + # Get the current match and create tuple of entity label, start and end. + # Append entity to the doc's entity. (Don't overwrite doc.ents!) + match_id, start, end = matches[i] + entity = Span(doc, start, end, label="EVENT") + doc.ents += (entity,) + print(entity.text) + +pattern = [{"ORTH": "Google"}, {"ORTH": "I"}, {"ORTH": "/"}, {"ORTH": "O"}] +matcher.add("GoogleIO", [pattern], on_match=add_event_ent) +doc = nlp("This is a text about Google I/O") +matches = matcher(doc) +``` + +A very similar logic has been implemented in the built-in +[`EntityRuler`](/api/entityruler) by the way. It also takes care of handling +overlapping matches, which you would otherwise have to take care of yourself. + +> #### Tip: Visualizing matches +> +> When working with entities, you can use [displaCy](/api/top-level#displacy) to +> quickly generate a NER visualization from your updated `Doc`, which can be +> exported as an HTML file: +> +> ```python +> from spacy import displacy +> html = displacy.render(doc, style="ent", page=True, +> options={"ents": ["EVENT"]}) +> ``` +> +> For more info and examples, see the usage guide on +> [visualizing spaCy](/usage/visualizers). + +We can now call the matcher on our documents. The patterns will be matched in +the order they occur in the text. The matcher will then iterate over the +matches, look up the callback for the match ID that was matched, and invoke it. + +```python +doc = nlp(YOUR_TEXT_HERE) +matcher(doc) +``` + +When the callback is invoked, it is passed four arguments: the matcher itself, +the document, the position of the current match, and the total list of matches. +This allows you to write callbacks that consider the entire set of matched +phrases, so that you can resolve overlaps and other conflicts in whatever way +you prefer. + +| Argument | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `matcher` | The matcher instance. ~~Matcher~~ | +| `doc` | The document the matcher was used on. ~~Doc~~ | +| `i` | Index of the current match (`matches[i`]). ~~int~~ | +| `matches` | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. ~~List[Tuple[int, int int]]~~ | + +### Creating spans from matches {id="matcher-spans"} + +Creating [`Span`](/api/span) objects from the returned matches is a very common +use case. spaCy makes this easy by giving you access to the `start` and `end` +token of each match, which you can use to construct a new span with an optional +label. As of spaCy v3.0, you can also set `as_spans=True` when calling the +matcher on a `Doc`, which will return a list of [`Span`](/api/span) objects +using the `match_id` as the span label. + +```python {executable="true"} +import spacy +from spacy.matcher import Matcher +from spacy.tokens import Span + +nlp = spacy.blank("en") +matcher = Matcher(nlp.vocab) +matcher.add("PERSON", [[{"lower": "barack"}, {"lower": "obama"}]]) +doc = nlp("Barack Obama was the 44th president of the United States") + +# 1. Return (match_id, start, end) tuples +matches = matcher(doc) +for match_id, start, end in matches: + # Create the matched span and assign the match_id as a label + span = Span(doc, start, end, label=match_id) + print(span.text, span.label_) + +# 2. Return Span objects directly +matches = matcher(doc, as_spans=True) +for span in matches: + print(span.text, span.label_) +``` + +### Using custom pipeline components {id="matcher-pipeline"} + +Let's say your data also contains some annoying pre-processing artifacts, like +leftover HTML line breaks (e.g. `
` or `
`). To make your text easier to +analyze, you want to merge those into one token and flag them, to make sure you +can ignore them later. Ideally, this should all be done automatically as you +process the text. You can achieve this by adding a +[custom pipeline component](/usage/processing-pipelines#custom-components) +that's called on each `Doc` object, merges the leftover HTML spans and sets an +attribute `bad_html` on the token. + +```python {executable="true"} +import spacy +from spacy.language import Language +from spacy.matcher import Matcher +from spacy.tokens import Token + +# We're using a component factory because the component needs to be +# initialized with the shared vocab via the nlp object +@Language.factory("html_merger") +def create_bad_html_merger(nlp, name): + return BadHTMLMerger(nlp.vocab) + +class BadHTMLMerger: + def __init__(self, vocab): + patterns = [ + [{"ORTH": "<"}, {"LOWER": "br"}, {"ORTH": ">"}], + [{"ORTH": "<"}, {"LOWER": "br/"}, {"ORTH": ">"}], + ] + # Register a new token extension to flag bad HTML + Token.set_extension("bad_html", default=False) + self.matcher = Matcher(vocab) + self.matcher.add("BAD_HTML", patterns) + + def __call__(self, doc): + # This method is invoked when the component is called on a Doc + matches = self.matcher(doc) + spans = [] # Collect the matched spans here + for match_id, start, end in matches: + spans.append(doc[start:end]) + with doc.retokenize() as retokenizer: + for span in spans: + retokenizer.merge(span) + for token in span: + token._.bad_html = True # Mark token as bad HTML + return doc + +nlp = spacy.load("en_core_web_sm") +nlp.add_pipe("html_merger", last=True) # Add component to the pipeline +doc = nlp("Hello
world!
This is a test.") +for token in doc: + print(token.text, token._.bad_html) + +``` + +Instead of hard-coding the patterns into the component, you could also make it +take a path to a JSON file containing the patterns. This lets you reuse the +component with different patterns, depending on your application. When adding +the component to the pipeline with [`nlp.add_pipe`](/api/language#add_pipe), you +can pass in the argument via the `config`: + +```python +@Language.factory("html_merger", default_config={"path": None}) +def create_bad_html_merger(nlp, name, path): + return BadHTMLMerger(nlp, path=path) + +nlp.add_pipe("html_merger", config={"path": "/path/to/patterns.json"}) +``` + + + +For more details and examples of how to **create custom pipeline components** +and **extension attributes**, see the +[usage guide](/usage/processing-pipelines). + + + +### Example: Using linguistic annotations {id="example1"} + +Let's say you're analyzing user comments and you want to find out what people +are saying about Facebook. You want to start off by finding adjectives following +"Facebook is" or "Facebook was". This is obviously a very rudimentary solution, +but it'll be fast, and a great way to get an idea for what's in your data. Your +pattern could look like this: + +```python +[{"LOWER": "facebook"}, {"LEMMA": "be"}, {"POS": "ADV", "OP": "*"}, {"POS": "ADJ"}] +``` + +This translates to a token whose lowercase form matches "facebook" (like +Facebook, facebook or FACEBOOK), followed by a token with the lemma "be" (for +example, is, was, or 's), followed by an **optional** adverb, followed by an +adjective. Using the linguistic annotations here is especially useful, because +you can tell spaCy to match "Facebook's annoying", but **not** "Facebook's +annoying ads". The optional adverb makes sure you won't miss adjectives with +intensifiers, like "pretty awful" or "very nice". + +To get a quick overview of the results, you could collect all sentences +containing a match and render them with the +[displaCy visualizer](/usage/visualizers). In the callback function, you'll have +access to the `start` and `end` of each match, as well as the parent `Doc`. This +lets you determine the sentence containing the match, `doc[start:end].sent`, and +calculate the start and end of the matched span within the sentence. Using +displaCy in ["manual" mode](/usage/visualizers#manual-usage) lets you pass in a +list of dictionaries containing the text and entities to render. + +```python {executable="true"} +import spacy +from spacy import displacy +from spacy.matcher import Matcher + +nlp = spacy.load("en_core_web_sm") +matcher = Matcher(nlp.vocab) +matched_sents = [] # Collect data of matched sentences to be visualized + +def collect_sents(matcher, doc, i, matches): + match_id, start, end = matches[i] + span = doc[start:end] # Matched span + sent = span.sent # Sentence containing matched span + # Append mock entity for match in displaCy style to matched_sents + # get the match span by ofsetting the start and end of the span with the + # start and end of the sentence in the doc + match_ents = [{ + "start": span.start_char - sent.start_char, + "end": span.end_char - sent.start_char, + "label": "MATCH", + }] + matched_sents.append({"text": sent.text, "ents": match_ents}) + +pattern = [{"LOWER": "facebook"}, {"LEMMA": "be"}, {"POS": "ADV", "OP": "*"}, + {"POS": "ADJ"}] +matcher.add("FacebookIs", [pattern], on_match=collect_sents) # add pattern +doc = nlp("I'd say that Facebook is evil. – Facebook is pretty cool, right?") +matches = matcher(doc) + +# Serve visualization of sentences containing match with displaCy +# set manual=True to make displaCy render straight from a dictionary +# (if you're not running the code within a Jupyter environment, you can +# use displacy.serve instead) +displacy.render(matched_sents, style="ent", manual=True) +``` + +### Example: Phone numbers {id="example2"} + +Phone numbers can have many different formats and matching them is often tricky. +During tokenization, spaCy will leave sequences of numbers intact and only split +on whitespace and punctuation. This means that your match pattern will have to +look out for number sequences of a certain length, surrounded by specific +punctuation – depending on the +[national conventions](https://en.wikipedia.org/wiki/National_conventions_for_writing_telephone_numbers). + +The `IS_DIGIT` flag is not very helpful here, because it doesn't tell us +anything about the length. However, you can use the `SHAPE` flag, with each `d` +representing a digit (up to 4 digits / characters): + +```python +[{"ORTH": "("}, {"SHAPE": "ddd"}, {"ORTH": ")"}, {"SHAPE": "dddd"}, + {"ORTH": "-", "OP": "?"}, {"SHAPE": "dddd"}] +``` + +This will match phone numbers of the format **(123) 4567 8901** or **(123) +4567-8901**. To also match formats like **(123) 456 789**, you can add a second +pattern using `'ddd'` in place of `'dddd'`. By hard-coding some values, you can +match only certain, country-specific numbers. For example, here's a pattern to +match the most common formats of +[international German numbers](https://en.wikipedia.org/wiki/National_conventions_for_writing_telephone_numbers#Germany): + +```python +[{"ORTH": "+"}, {"ORTH": "49"}, {"ORTH": "(", "OP": "?"}, {"SHAPE": "dddd"}, + {"ORTH": ")", "OP": "?"}, {"SHAPE": "dddd", "LENGTH": 6}] +``` + +Depending on the formats your application needs to match, creating an extensive +set of rules like this is often better than training a model. It'll produce more +predictable results, is much easier to modify and extend, and doesn't require +any training data – only a set of test cases. + +```python {executable="true"} +import spacy +from spacy.matcher import Matcher + +nlp = spacy.load("en_core_web_sm") +matcher = Matcher(nlp.vocab) +pattern = [{"ORTH": "("}, {"SHAPE": "ddd"}, {"ORTH": ")"}, {"SHAPE": "ddd"}, + {"ORTH": "-", "OP": "?"}, {"SHAPE": "ddd"}] +matcher.add("PHONE_NUMBER", [pattern]) + +doc = nlp("Call me at (123) 456 789 or (123) 456 789!") +print([t.text for t in doc]) +matches = matcher(doc) +for match_id, start, end in matches: + span = doc[start:end] + print(span.text) +``` + +### Example: Hashtags and emoji on social media {id="example3"} + +Social media posts, especially tweets, can be difficult to work with. They're +very short and often contain various emoji and hashtags. By only looking at the +plain text, you'll lose a lot of valuable semantic information. + +Let's say you've extracted a large sample of social media posts on a specific +topic, for example posts mentioning a brand name or product. As the first step +of your data exploration, you want to filter out posts containing certain emoji +and use them to assign a general sentiment score, based on whether the expressed +emotion is positive or negative, e.g. 😀 or 😞. You also want to find, merge and +label hashtags like `#MondayMotivation`, to be able to ignore or analyze them +later. + +> #### Note on sentiment analysis +> +> Ultimately, sentiment analysis is not always _that_ easy. In addition to the +> emoji, you'll also want to take specific words into account and check the +> `subtree` for intensifiers like "very", to increase the sentiment score. At +> some point, you might also want to train a sentiment model. However, the +> approach described in this example is very useful for **bootstrapping rules to +> collect training data**. It's also an incredibly fast way to gather first +> insights into your data – with about 1 million tweets, you'd be looking at a +> processing time of **under 1 minute**. + +By default, spaCy's tokenizer will split emoji into separate tokens. This means +that you can create a pattern for one or more emoji tokens. Valid hashtags +usually consist of a `#`, plus a sequence of ASCII characters with no +whitespace, making them easy to match as well. + +```python {executable="true"} +from spacy.lang.en import English +from spacy.matcher import Matcher + +nlp = English() # We only want the tokenizer, so no need to load a pipeline +matcher = Matcher(nlp.vocab) + +pos_emoji = ["😀", "😃", "😂", "🤣", "😊", "😍"] # Positive emoji +neg_emoji = ["😞", "😠", "😩", "😢", "😭", "😒"] # Negative emoji + +# Add patterns to match one or more emoji tokens +pos_patterns = [[{"ORTH": emoji}] for emoji in pos_emoji] +neg_patterns = [[{"ORTH": emoji}] for emoji in neg_emoji] + +# Function to label the sentiment +def label_sentiment(matcher, doc, i, matches): + match_id, start, end = matches[i] + if doc.vocab.strings[match_id] == "HAPPY": # Don't forget to get string! + doc.sentiment += 0.1 # Add 0.1 for positive sentiment + elif doc.vocab.strings[match_id] == "SAD": + doc.sentiment -= 0.1 # Subtract 0.1 for negative sentiment + +matcher.add("HAPPY", pos_patterns, on_match=label_sentiment) # Add positive pattern +matcher.add("SAD", neg_patterns, on_match=label_sentiment) # Add negative pattern + +# Add pattern for valid hashtag, i.e. '#' plus any ASCII token +matcher.add("HASHTAG", [[{"ORTH": "#"}, {"IS_ASCII": True}]]) + +doc = nlp("Hello world 😀 #MondayMotivation") +matches = matcher(doc) +for match_id, start, end in matches: + string_id = doc.vocab.strings[match_id] # Look up string ID + span = doc[start:end] + print(string_id, span.text) +``` + +Because the `on_match` callback receives the ID of each match, you can use the +same function to handle the sentiment assignment for both the positive and +negative pattern. To keep it simple, we'll either add or subtract `0.1` points – +this way, the score will also reflect combinations of emoji, even positive _and_ +negative ones. + +With a library like [emoji](https://github.com/carpedm20/emoji), we can also +retrieve a short description for each emoji – for example, 😍's official title +is "Smiling Face With Heart-Eyes". Assigning it to a +[custom attribute](/usage/processing-pipelines#custom-components-attributes) on +the emoji span will make it available as `span._.emoji_desc`. + +```python +import emoji # Installation: pip install emoji +from spacy.tokens import Span # Get the global Span object + +Span.set_extension("emoji_desc", default=None) # Register the custom attribute + +def label_sentiment(matcher, doc, i, matches): + match_id, start, end = matches[i] + if doc.vocab.strings[match_id] == "HAPPY": # Don't forget to get string! + doc.sentiment += 0.1 # Add 0.1 for positive sentiment + elif doc.vocab.strings[match_id] == "SAD": + doc.sentiment -= 0.1 # Subtract 0.1 for negative sentiment + span = doc[start:end] + # Verify if it is an emoji and set the extension attribute correctly. + if emoji.is_emoji(span[0].text): + span._.emoji_desc = emoji.demojize(span[0].text, delimiters=("", ""), language=doc.lang_).replace("_", " ") +``` + +To label the hashtags, we can use a +[custom attribute](/usage/processing-pipelines#custom-components-attributes) set +on the respective token: + +```python {executable="true"} +import spacy +from spacy.matcher import Matcher +from spacy.tokens import Token + +nlp = spacy.load("en_core_web_sm") +matcher = Matcher(nlp.vocab) + +# Add pattern for valid hashtag, i.e. '#' plus any ASCII token +matcher.add("HASHTAG", [[{"ORTH": "#"}, {"IS_ASCII": True}]]) + +# Register token extension +Token.set_extension("is_hashtag", default=False) + +doc = nlp("Hello world 😀 #MondayMotivation") +matches = matcher(doc) +hashtags = [] +for match_id, start, end in matches: + if doc.vocab.strings[match_id] == "HASHTAG": + hashtags.append(doc[start:end]) +with doc.retokenize() as retokenizer: + for span in hashtags: + retokenizer.merge(span) + for token in span: + token._.is_hashtag = True + +for token in doc: + print(token.text, token._.is_hashtag) +``` + +## Efficient phrase matching {id="phrasematcher"} + +If you need to match large terminology lists, you can also use the +[`PhraseMatcher`](/api/phrasematcher) and create [`Doc`](/api/doc) objects +instead of token patterns, which is much more efficient overall. The `Doc` +patterns can contain single or multiple tokens. + +### Adding phrase patterns {id="adding-phrase-patterns"} + +```python {executable="true"} +import spacy +from spacy.matcher import PhraseMatcher + +nlp = spacy.load("en_core_web_sm") +matcher = PhraseMatcher(nlp.vocab) +terms = ["Barack Obama", "Angela Merkel", "Washington, D.C."] +# Only run nlp.make_doc to speed things up +patterns = [nlp.make_doc(text) for text in terms] +matcher.add("TerminologyList", patterns) + +doc = nlp("German Chancellor Angela Merkel and US President Barack Obama " + "converse in the Oval Office inside the White House in Washington, D.C.") +matches = matcher(doc) +for match_id, start, end in matches: + span = doc[start:end] + print(span.text) +``` + +Since spaCy is used for processing both the patterns and the text to be matched, +you won't have to worry about specific tokenization – for example, you can +simply pass in `nlp("Washington, D.C.")` and won't have to write a complex token +pattern covering the exact tokenization of the term. + + + +To create the patterns, each phrase has to be processed with the `nlp` object. +If you have a trained pipeline loaded, doing this in a loop or list +comprehension can easily become inefficient and slow. If you **only need the +tokenization and lexical attributes**, you can run +[`nlp.make_doc`](/api/language#make_doc) instead, which will only run the +tokenizer. For an additional speed boost, you can also use the +[`nlp.tokenizer.pipe`](/api/tokenizer#pipe) method, which will process the texts +as a stream. + +```diff +- patterns = [nlp(term) for term in LOTS_OF_TERMS] ++ patterns = [nlp.make_doc(term) for term in LOTS_OF_TERMS] ++ patterns = list(nlp.tokenizer.pipe(LOTS_OF_TERMS)) +``` + + + +### Matching on other token attributes {id="phrasematcher-attrs",version="2.1"} + +By default, the `PhraseMatcher` will match on the verbatim token text, e.g. +`Token.text`. By setting the `attr` argument on initialization, you can change +**which token attribute the matcher should use** when comparing the phrase +pattern to the matched `Doc`. For example, using the attribute `LOWER` lets you +match on `Token.lower` and create case-insensitive match patterns: + +```python {executable="true"} +from spacy.lang.en import English +from spacy.matcher import PhraseMatcher + +nlp = English() +matcher = PhraseMatcher(nlp.vocab, attr="LOWER") +patterns = [nlp.make_doc(name) for name in ["Angela Merkel", "Barack Obama"]] +matcher.add("Names", patterns) + +doc = nlp("angela merkel and us president barack Obama") +for match_id, start, end in matcher(doc): + print("Matched based on lowercase token text:", doc[start:end]) +``` + + + +The examples here use [`nlp.make_doc`](/api/language#make_doc) to create `Doc` +object patterns as efficiently as possible and without running any of the other +pipeline components. If the token attribute you want to match on is set by a +pipeline component, **make sure that the pipeline component runs** when you +create the pattern. For example, to match on `POS` or `LEMMA`, the pattern `Doc` +objects need to have part-of-speech tags set by the `tagger` or `morphologizer`. +You can either call the `nlp` object on your pattern texts instead of +`nlp.make_doc`, or use [`nlp.select_pipes`](/api/language#select_pipes) to +disable components selectively. + + + +Another possible use case is matching number tokens like IP addresses based on +their shape. This means that you won't have to worry about how those strings +will be tokenized and you'll be able to find tokens and combinations of tokens +based on a few examples. Here, we're matching on the shapes `ddd.d.d.d` and +`ddd.ddd.d.d`: + +```python {executable="true"} +from spacy.lang.en import English +from spacy.matcher import PhraseMatcher + +nlp = English() +matcher = PhraseMatcher(nlp.vocab, attr="SHAPE") +matcher.add("IP", [nlp("127.0.0.1"), nlp("127.127.0.0")]) + +doc = nlp("Often the router will have an IP address such as 192.168.1.1 or 192.168.2.1.") +for match_id, start, end in matcher(doc): + print("Matched based on token shape:", doc[start:end]) +``` + +In theory, the same also works for attributes like `POS`. For example, a pattern +`nlp("I like cats")` matched based on its part-of-speech tag would return a +match for "I love dogs". You could also match on boolean flags like `IS_PUNCT` +to match phrases with the same sequence of punctuation and non-punctuation +tokens as the pattern. But this can easily get confusing and doesn't have much +of an advantage over writing one or two token patterns. + +## Dependency Matcher {id="dependencymatcher",version="3",model="parser"} + +The [`DependencyMatcher`](/api/dependencymatcher) lets you match patterns within +the dependency parse using +[Semgrex](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html) +operators. It requires a model containing a parser such as the +[`DependencyParser`](/api/dependencyparser). Instead of defining a list of +adjacent tokens as in `Matcher` patterns, the `DependencyMatcher` patterns match +tokens in the dependency parse and specify the relations between them. + +> ```python +> ### Example +> from spacy.matcher import DependencyMatcher +> +> # "[subject] ... initially founded" +> pattern = [ +> # anchor token: founded +> { +> "RIGHT_ID": "founded", +> "RIGHT_ATTRS": {"ORTH": "founded"} +> }, +> # founded -> subject +> { +> "LEFT_ID": "founded", +> "REL_OP": ">", +> "RIGHT_ID": "subject", +> "RIGHT_ATTRS": {"DEP": "nsubj"} +> }, +> # "founded" follows "initially" +> { +> "LEFT_ID": "founded", +> "REL_OP": ";", +> "RIGHT_ID": "initially", +> "RIGHT_ATTRS": {"ORTH": "initially"} +> } +> ] +> +> matcher = DependencyMatcher(nlp.vocab) +> matcher.add("FOUNDED", [pattern]) +> matches = matcher(doc) +> ``` + +A pattern added to the dependency matcher consists of a **list of +dictionaries**, with each dictionary describing a **token to match** and its +**relation to an existing token** in the pattern. Except for the first +dictionary, which defines an anchor token using only `RIGHT_ID` and +`RIGHT_ATTRS`, each pattern should have the following keys: + +| Name | Description | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `LEFT_ID` | The name of the left-hand node in the relation, which has been defined in an earlier node. ~~str~~ | +| `REL_OP` | An operator that describes how the two nodes are related. ~~str~~ | +| `RIGHT_ID` | A unique name for the right-hand node in the relation. ~~str~~ | +| `RIGHT_ATTRS` | The token attributes to match for the right-hand node in the same format as patterns provided to the regular token-based [`Matcher`](/api/matcher). ~~Dict[str, Any]~~ | + +Each additional token added to the pattern is linked to an existing token +`LEFT_ID` by the relation `REL_OP`. The new token is given the name `RIGHT_ID` +and described by the attributes `RIGHT_ATTRS`. + + + +Because the unique token **names** in `LEFT_ID` and `RIGHT_ID` are used to +identify tokens, the order of the dicts in the patterns is important: a token +name needs to be defined as `RIGHT_ID` in one dict in the pattern **before** it +can be used as `LEFT_ID` in another dict. + + + +### Dependency matcher operators {id="dependencymatcher-operators"} + +The following operators are supported by the `DependencyMatcher`, most of which +come directly from +[Semgrex](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html): + +| Symbol | Description | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `A < B` | `A` is the immediate dependent of `B`. | +| `A > B` | `A` is the immediate head of `B`. | +| `A << B` | `A` is the dependent in a chain to `B` following dep → head paths. | +| `A >> B` | `A` is the head in a chain to `B` following head → dep paths. | +| `A . B` | `A` immediately precedes `B`, i.e. `A.i == B.i - 1`, and both are within the same dependency tree. | +| `A .* B` | `A` precedes `B`, i.e. `A.i < B.i`, and both are within the same dependency tree _(Semgrex counterpart: `..`)_. | +| `A ; B` | `A` immediately follows `B`, i.e. `A.i == B.i + 1`, and both are within the same dependency tree _(Semgrex counterpart: `-`)_. | +| `A ;* B` | `A` follows `B`, i.e. `A.i > B.i`, and both are within the same dependency tree _(Semgrex counterpart: `--`)_. | +| `A $+ B` | `B` is a right immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i - 1`. | +| `A $- B` | `B` is a left immediate sibling of `A`, i.e. `A` and `B` have the same parent and `A.i == B.i + 1`. | +| `A $++ B` | `B` is a right sibling of `A`, i.e. `A` and `B` have the same parent and `A.i < B.i`. | +| `A $-- B` | `B` is a left sibling of `A`, i.e. `A` and `B` have the same parent and `A.i > B.i`. | +| `A >+ B` 3.5.1 | `B` is a right immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A >- B` 3.5.1 | `B` is a left immediate child of `A`, i.e. `A` is a parent of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A >++ B` | `B` is a right child of `A`, i.e. `A` is a parent of `B` and `A.i < B.i`. | +| `A >-- B` | `B` is a left child of `A`, i.e. `A` is a parent of `B` and `A.i > B.i`. | +| `A <+ B` 3.5.1 | `B` is a right immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i - 1` _(not in Semgrex)_. | +| `A <- B` 3.5.1 | `B` is a left immediate parent of `A`, i.e. `A` is a child of `B` and `A.i == B.i + 1` _(not in Semgrex)_. | +| `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i`. | +| `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i`. | + +### Designing dependency matcher patterns {id="dependencymatcher-patterns"} + +Let's say we want to find sentences describing who founded what kind of company: + +- _Smith founded a healthcare company in 2005._ +- _Williams initially founded an insurance company in 1987._ +- _Lee, an experienced CEO, has founded two AI startups._ + +The dependency parse for "Smith founded a healthcare company" shows types of +relations and tokens we want to match: + +> #### Visualizing the parse +> +> The [`displacy` visualizer](/usage/visualizers) lets you render `Doc` objects +> and their dependency parse and part-of-speech tags: +> +> ```python +> import spacy +> from spacy import displacy +> +> nlp = spacy.load("en_core_web_sm") +> doc = nlp("Smith founded a healthcare company") +> displacy.serve(doc) +> ``` + + + +The relations we're interested in are: + +- the founder is the **subject** (`nsubj`) of the token with the text `founded` +- the company is the **object** (`dobj`) of `founded` +- the kind of company may be an **adjective** (`amod`, not shown above) or a + **compound** (`compound`) + +The first step is to pick an **anchor token** for the pattern. Since it's the +root of the dependency parse, `founded` is a good choice here. It is often +easier to construct patterns when all dependency relation operators point from +the head to the children. In this example, we'll only use `>`, which connects a +head to an immediate dependent as `head > child`. + +The simplest dependency matcher pattern will identify and name a single token in +the tree: + +```python {executable="true"} +import spacy +from spacy.matcher import DependencyMatcher + +nlp = spacy.load("en_core_web_sm") +matcher = DependencyMatcher(nlp.vocab) +pattern = [ + { + "RIGHT_ID": "anchor_founded", # unique name + "RIGHT_ATTRS": {"ORTH": "founded"} # token pattern for "founded" + } +] +matcher.add("FOUNDED", [pattern]) +doc = nlp("Smith founded two companies.") +matches = matcher(doc) +print(matches) # [(4851363122962674176, [1])] +``` + +Now that we have a named anchor token (`anchor_founded`), we can add the founder +as the immediate dependent (`>`) of `founded` with the dependency label `nsubj`: + +```python {title="Step 1",highlight="8,10"} +pattern = [ + { + "RIGHT_ID": "anchor_founded", + "RIGHT_ATTRS": {"ORTH": "founded"} + }, + { + "LEFT_ID": "anchor_founded", + "REL_OP": ">", + "RIGHT_ID": "founded_subject", + "RIGHT_ATTRS": {"DEP": "nsubj"}, + } + # ... +] +``` + +The direct object (`dobj`) is added in the same way: + +```python {title="Step 2"} +pattern = [ + #... + { + "LEFT_ID": "anchor_founded", + "REL_OP": ">", + "RIGHT_ID": "founded_object", + "RIGHT_ATTRS": {"DEP": "dobj"}, + } + # ... +] +``` + +When the subject and object tokens are added, they are required to have names +under the key `RIGHT_ID`, which are allowed to be any unique string, e.g. +`founded_subject`. These names can then be used as `LEFT_ID` to **link new +tokens into the pattern**. For the final part of our pattern, we'll specify that +the token `founded_object` should have a modifier with the dependency relation +`amod` or `compound`: + +```python {title="Step 3",highlight="7"} +pattern = [ + # ... + { + "LEFT_ID": "founded_object", + "REL_OP": ">", + "RIGHT_ID": "founded_object_modifier", + "RIGHT_ATTRS": {"DEP": {"IN": ["amod", "compound"]}}, + } +] +``` + +You can picture the process of creating a dependency matcher pattern as defining +an anchor token on the left and building up the pattern by linking tokens +one-by-one on the right using relation operators. To create a valid pattern, +each new token needs to be linked to an existing token on its left. As for +`founded` in this example, a token may be linked to more than one token on its +right: + +![Dependency matcher pattern](/images/dep-match-diagram.svg) + +The full pattern comes together as shown in the example below: + +```python {executable="true"} +import spacy +from spacy.matcher import DependencyMatcher + +nlp = spacy.load("en_core_web_sm") +matcher = DependencyMatcher(nlp.vocab) + +pattern = [ + { + "RIGHT_ID": "anchor_founded", + "RIGHT_ATTRS": {"ORTH": "founded"} + }, + { + "LEFT_ID": "anchor_founded", + "REL_OP": ">", + "RIGHT_ID": "founded_subject", + "RIGHT_ATTRS": {"DEP": "nsubj"}, + }, + { + "LEFT_ID": "anchor_founded", + "REL_OP": ">", + "RIGHT_ID": "founded_object", + "RIGHT_ATTRS": {"DEP": "dobj"}, + }, + { + "LEFT_ID": "founded_object", + "REL_OP": ">", + "RIGHT_ID": "founded_object_modifier", + "RIGHT_ATTRS": {"DEP": {"IN": ["amod", "compound"]}}, + } +] + +matcher.add("FOUNDED", [pattern]) +doc = nlp("Lee, an experienced CEO, has founded two AI startups.") +matches = matcher(doc) + +print(matches) # [(4851363122962674176, [6, 0, 10, 9])] +# Each token_id corresponds to one pattern dict +match_id, token_ids = matches[0] +for i in range(len(token_ids)): + print(pattern[i]["RIGHT_ID"] + ":", doc[token_ids[i]].text) +``` + + + +The dependency matcher may be slow when token patterns can potentially match +many tokens in the sentence or when relation operators allow longer paths in the +dependency parse, e.g. `<<`, `>>`, `.*` and `;*`. + +To improve the matcher speed, try to make your token patterns and operators as +specific as possible. For example, use `>` instead of `>>` if possible and use +token patterns that include dependency labels and other token attributes instead +of patterns such as `{}` that match any token in the sentence. + + + +## Rule-based entity recognition {id="entityruler",version="2.1"} + +The [`EntityRuler`](/api/entityruler) is a component that lets you add named +entities based on pattern dictionaries, which makes it easy to combine +rule-based and statistical named entity recognition for even more powerful +pipelines. + +### Entity Patterns {id="entityruler-patterns"} + +Entity patterns are dictionaries with two keys: `"label"`, specifying the label +to assign to the entity if the pattern is matched, and `"pattern"`, the match +pattern. The entity ruler accepts two types of patterns: + +1. **Phrase patterns** for exact string matches (string). + + ```python + {"label": "ORG", "pattern": "Apple"} + ``` + +2. **Token patterns** with one dictionary describing one token (list). + + ```python + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} + ``` + +### Using the entity ruler {id="entityruler-usage"} + +The [`EntityRuler`](/api/entityruler) is a pipeline component that's typically +added via [`nlp.add_pipe`](/api/language#add_pipe). When the `nlp` object is +called on a text, it will find matches in the `doc` and add them as entities to +the `doc.ents`, using the specified pattern label as the entity label. If any +matches were to overlap, the pattern matching most tokens takes priority. If +they also happen to be equally long, then the match occurring first in the `Doc` +is chosen. + +```python {executable="true"} +from spacy.lang.en import English + +nlp = English() +ruler = nlp.add_pipe("entity_ruler") +patterns = [{"label": "ORG", "pattern": "Apple"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}] +ruler.add_patterns(patterns) + +doc = nlp("Apple is opening its first big office in San Francisco.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +The entity ruler is designed to integrate with spaCy's existing pipeline +components and enhance the named entity recognizer. If it's added **before the +`"ner"` component**, the entity recognizer will respect the existing entity +spans and adjust its predictions around it. This can significantly improve +accuracy in some cases. If it's added **after the `"ner"` component**, the +entity ruler will only add spans to the `doc.ents` if they don't overlap with +existing entities predicted by the model. To overwrite overlapping entities, you +can set `overwrite_ents=True` on initialization. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +ruler = nlp.add_pipe("entity_ruler") +patterns = [{"label": "ORG", "pattern": "MyCorp Inc."}] +ruler.add_patterns(patterns) + +doc = nlp("MyCorp Inc. is a company in the U.S.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +#### Validating and debugging EntityRuler patterns {id="entityruler-pattern-validation",version="2.1.8"} + +The entity ruler can validate patterns against a JSON schema with the config +setting `"validate"`. See details under +[Validating and debugging patterns](#pattern-validation). + +```python +ruler = nlp.add_pipe("entity_ruler", config={"validate": True}) +``` + +### Adding IDs to patterns {id="entityruler-ent-ids",version="2.2.2"} + +The [`EntityRuler`](/api/entityruler) can also accept an `id` attribute for each +pattern. Using the `id` attribute allows multiple patterns to be associated with +the same entity. + +```python {executable="true"} +from spacy.lang.en import English + +nlp = English() +ruler = nlp.add_pipe("entity_ruler") +patterns = [{"label": "ORG", "pattern": "Apple", "id": "apple"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}], "id": "san-francisco"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "fran"}], "id": "san-francisco"}] +ruler.add_patterns(patterns) + +doc1 = nlp("Apple is opening its first big office in San Francisco.") +print([(ent.text, ent.label_, ent.ent_id_) for ent in doc1.ents]) + +doc2 = nlp("Apple is opening its first big office in San Fran.") +print([(ent.text, ent.label_, ent.ent_id_) for ent in doc2.ents]) +``` + +If the `id` attribute is included in the [`EntityRuler`](/api/entityruler) +patterns, the `ent_id_` property of the matched entity is set to the `id` given +in the patterns. So in the example above it's easy to identify that "San +Francisco" and "San Fran" are both the same entity. + +### Using pattern files {id="entityruler-files"} + +The [`to_disk`](/api/entityruler#to_disk) and +[`from_disk`](/api/entityruler#from_disk) let you save and load patterns to and +from JSONL (newline-delimited JSON) files, containing one pattern object per +line. + +```json {title="patterns.jsonl"} +{"label": "ORG", "pattern": "Apple"} +{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} +``` + +```python +ruler.to_disk("./patterns.jsonl") +new_ruler = nlp.add_pipe("entity_ruler").from_disk("./patterns.jsonl") +``` + + + +If you're using the [Prodigy](https://prodi.gy) annotation tool, you might +recognize these pattern files from bootstrapping your named entity and text +classification labelling. The patterns for the `EntityRuler` follow the same +syntax, so you can use your existing Prodigy pattern files in spaCy, and vice +versa. + + + +When you save out an `nlp` object that has an `EntityRuler` added to its +pipeline, its patterns are automatically exported to the pipeline directory: + +```python +nlp = spacy.load("en_core_web_sm") +ruler = nlp.add_pipe("entity_ruler") +ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +nlp.to_disk("/path/to/pipeline") +``` + +The saved pipeline now includes the `"entity_ruler"` in its +[`config.cfg`](/api/data-formats#config) and the pipeline directory contains a +file `patterns.jsonl` with the patterns. When you load the pipeline back in, all +pipeline components will be restored and deserialized – including the entity +ruler. This lets you ship powerful pipeline packages with binary weights _and_ +rules included! + +### Using a large number of phrase patterns {id="entityruler-large-phrase-patterns",version="2.2.4"} + +{/* TODO: double-check that this still works if the ruler is added to the pipeline on creation, and include suggestion if needed */} + +When using a large amount of **phrase patterns** (roughly > 10000) it's useful +to understand how the `add_patterns` function of the entity ruler works. For +each **phrase pattern**, the EntityRuler calls the nlp object to construct a doc +object. This happens in case you try to add the EntityRuler at the end of an +existing pipeline with, for example, a POS tagger and want to extract matches +based on the pattern's POS signature. In this case you would pass a config value +of `"phrase_matcher_attr": "POS"` for the entity ruler. + +Running the full language pipeline across every pattern in a large list scales +linearly and can therefore take a long time on large amounts of phrase patterns. +As of spaCy v2.2.4 the `add_patterns` function has been refactored to use +`nlp.pipe` on all phrase patterns resulting in about a 10x-20x speed up with +5,000-100,000 phrase patterns respectively. Even with this speedup (but +especially if you're using an older version) the `add_patterns` function can +still take a long time. An easy workaround to make this function run faster is +disabling the other language pipes while adding the phrase patterns. + +```python +ruler = nlp.add_pipe("entity_ruler") +patterns = [{"label": "TEST", "pattern": str(i)} for i in range(100000)] +with nlp.select_pipes(enable="tagger"): + ruler.add_patterns(patterns) +``` + +## Rule-based span matching {id="spanruler",version="3.3.1"} + +The [`SpanRuler`](/api/spanruler) is a generalized version of the entity ruler +that lets you add spans to `doc.spans` or `doc.ents` based on pattern +dictionaries, which makes it easy to combine rule-based and statistical pipeline +components. + +### Span patterns {id="spanruler-patterns"} + +The [pattern format](#entityruler-patterns) is the same as for the entity ruler: + +1. **Phrase patterns** for exact string matches (string). + + ```python + {"label": "ORG", "pattern": "Apple"} + ``` + +2. **Token patterns** with one dictionary describing one token (list). + + ```python + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} + ``` + +### Using the span ruler {id="spanruler-usage"} + +The [`SpanRuler`](/api/spanruler) is a pipeline component that's typically added +via [`nlp.add_pipe`](/api/language#add_pipe). When the `nlp` object is called on +a text, it will find matches in the `doc` and add them as spans to +`doc.spans["ruler"]`, using the specified pattern label as the entity label. +Unlike in `doc.ents`, overlapping matches are allowed in `doc.spans`, so no +filtering is required, but optional filtering and sorting can be applied to the +spans before they're saved. + +```python {executable="true"} +import spacy + +nlp = spacy.blank("en") +ruler = nlp.add_pipe("span_ruler") +patterns = [{"label": "ORG", "pattern": "Apple"}, + {"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]}] +ruler.add_patterns(patterns) + +doc = nlp("Apple is opening its first big office in San Francisco.") +print([(span.text, span.label_) for span in doc.spans["ruler"]]) +``` + +The span ruler is designed to integrate with spaCy's existing pipeline +components and enhance the [SpanCategorizer](/api/spancat) and +[EntityRecognizer](/api/entityrecognizer). The `overwrite` setting determines +whether the existing annotation in `doc.spans` or `doc.ents` is preserved. +Because overlapping entities are not allowed for `doc.ents`, the entities are +always filtered, using [`util.filter_spans`](/api/top-level#util.filter_spans) +by default. See the [`SpanRuler` API docs](/api/spanruler) for more information +about how to customize the sorting and filtering of matched spans. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +# only annotate doc.ents, not doc.spans +config = {"spans_key": None, "annotate_ents": True, "overwrite": False} +ruler = nlp.add_pipe("span_ruler", config=config) +patterns = [{"label": "ORG", "pattern": "MyCorp Inc."}] +ruler.add_patterns(patterns) + +doc = nlp("MyCorp Inc. is a company in the U.S.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +### Using pattern files {id="spanruler-files"} + +You can save patterns in a JSONL file (newline-delimited JSON) to load with +[`SpanRuler.initialize`](/api/spanruler#initialize) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns). + +```json {title="patterns.jsonl"} +{"label": "ORG", "pattern": "Apple"} +{"label": "GPE", "pattern": [{"LOWER": "san"}, {"LOWER": "francisco"}]} +``` + +```python +import srsly + +patterns = srsly.read_jsonl("patterns.jsonl") +ruler = nlp.add_pipe("span_ruler") +ruler.add_patterns(patterns) +``` + + + +Unlike the entity ruler, the span ruler cannot load patterns on initialization +with `SpanRuler(patterns=patterns)` or directly from a JSONL file path with +`SpanRuler.from_disk(jsonl_path)`. Patterns should be loaded from the JSONL file +separately and then added through +[`SpanRuler.initialize`](/api/spanruler#initialize]) or +[`SpanRuler.add_patterns`](/api/spanruler#add_patterns) as shown above. + + + +## Combining models and rules {id="models-rules"} + +You can combine statistical and rule-based components in a variety of ways. +Rule-based components can be used to improve the accuracy of statistical models, +by presetting tags, entities or sentence boundaries for specific tokens. The +statistical models will usually respect these preset annotations, which +sometimes improves the accuracy of other decisions. You can also use rule-based +components after a statistical model to correct common errors. Finally, +rule-based components can reference the attributes set by statistical models, in +order to implement more abstract logic. + +### Example: Expanding named entities {id="models-rules-ner"} + +When using a trained +[named entity recognition](/usage/linguistic-features/#named-entities) model to +extract information from your texts, you may find that the predicted span only +includes parts of the entity you're looking for. Sometimes, this happens if +statistical model predicts entities incorrectly. Other times, it happens if the +way the entity type was defined in the original training corpus doesn't match +what you need for your application. + +> #### Where corpora come from +> +> Corpora used to train pipelines from scratch are often produced in academia. +> They contain text from various sources with linguistic features labeled +> manually by human annotators (following a set of specific guidelines). The +> corpora are then distributed with evaluation data, so other researchers can +> benchmark their algorithms and everyone can report numbers on the same data. +> However, most applications need to learn information that isn't contained in +> any available corpus. + +For example, the corpus spaCy's [English pipelines](/models/en) were trained on +defines a `PERSON` entity as just the **person name**, without titles like "Mr." +or "Dr.". This makes sense, because it makes it easier to resolve the entity +type back to a knowledge base. But what if your application needs the full +names, _including_ the titles? + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Dr. Alex Smith chaired first board meeting of Acme Corp Inc.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +While you could try and teach the model a new definition of the `PERSON` entity +by [updating it](/usage/training/#example-train-ner) with more examples of spans +that include the title, this might not be the most efficient approach. The +existing model was trained on over 2 million words, so in order to completely +change the definition of an entity type, you might need a lot of training +examples. However, if you already have the predicted `PERSON` entities, you can +use a rule-based approach that checks whether they come with a title and if so, +expands the entity span by one token. After all, what all titles in this example +have in common is that _if_ they occur, they occur in the **previous token** +right before the person entity. + +```python {highlight="9-13"} +from spacy.language import Language +from spacy.tokens import Span + +@Language.component("expand_person_entities") +def expand_person_entities(doc): + new_ents = [] + for ent in doc.ents: + # Only check for title if it's a person and not the first token + if ent.label_ == "PERSON" and ent.start != 0: + prev_token = doc[ent.start - 1] + if prev_token.text in ("Dr", "Dr.", "Mr", "Mr.", "Ms", "Ms."): + new_ent = Span(doc, ent.start - 1, ent.end, label=ent.label) + new_ents.append(new_ent) + else: + new_ents.append(ent) + else: + new_ents.append(ent) + doc.ents = new_ents + return doc +``` + +The above function takes a `Doc` object, modifies its `doc.ents` and returns it. +Using the [`@Language.component`](/api/language#component) decorator, we can +register it as a [pipeline component](/usage/processing-pipelines) so it can run +automatically when processing a text. We can use +[`nlp.add_pipe`](/api/language#add_pipe) to add it to the current pipeline. + +```python {executable="true"} +import spacy +from spacy.language import Language +from spacy.tokens import Span + +nlp = spacy.load("en_core_web_sm") + +@Language.component("expand_person_entities") +def expand_person_entities(doc): + new_ents = [] + for ent in doc.ents: + if ent.label_ == "PERSON" and ent.start != 0: + prev_token = doc[ent.start - 1] + if prev_token.text in ("Dr", "Dr.", "Mr", "Mr.", "Ms", "Ms."): + new_ent = Span(doc, ent.start - 1, ent.end, label=ent.label) + new_ents.append(new_ent) + else: + new_ents.append(ent) + else: + new_ents.append(ent) + doc.ents = new_ents + return doc + +# Add the component after the named entity recognizer +nlp.add_pipe("expand_person_entities", after="ner") + +doc = nlp("Dr. Alex Smith chaired first board meeting of Acme Corp Inc.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +An alternative approach would be to use an +[extension attribute](/usage/processing-pipelines/#custom-components-attributes) +like `._.person_title` and add it to `Span` objects (which includes entity spans +in `doc.ents`). The advantage here is that the entity text stays intact and can +still be used to look up the name in a knowledge base. The following function +takes a `Span` object, checks the previous token if it's a `PERSON` entity and +returns the title if one is found. The `Span.doc` attribute gives us easy access +to the span's parent document. + +```python +def get_person_title(span): + if span.label_ == "PERSON" and span.start != 0: + prev_token = span.doc[span.start - 1] + if prev_token.text in ("Dr", "Dr.", "Mr", "Mr.", "Ms", "Ms."): + return prev_token.text +``` + +We can now use the [`Span.set_extension`](/api/span#set_extension) method to add +the custom extension attribute `"person_title"`, using `get_person_title` as the +getter function. + +```python {executable="true"} +import spacy +from spacy.tokens import Span + +nlp = spacy.load("en_core_web_sm") + +def get_person_title(span): + if span.label_ == "PERSON" and span.start != 0: + prev_token = span.doc[span.start - 1] + if prev_token.text in ("Dr", "Dr.", "Mr", "Mr.", "Ms", "Ms."): + return prev_token.text + +# Register the Span extension as 'person_title' +Span.set_extension("person_title", getter=get_person_title) + +doc = nlp("Dr Alex Smith chaired first board meeting of Acme Corp Inc.") +print([(ent.text, ent.label_, ent._.person_title) for ent in doc.ents]) +``` + +### Example: Using entities, part-of-speech tags and the dependency parse {id="models-rules-pos-dep"} + +> #### Linguistic features +> +> This example makes extensive use of part-of-speech tag and dependency +> attributes and related `Doc`, `Token` and `Span` methods. For an introduction +> on this, see the guide on [linguistic features](/usage/linguistic-features/). +> Also see the label schemes in the [models directory](/models) for details on +> the labels. + +Let's say you want to parse professional biographies and extract the person +names and company names, and whether it's a company they're _currently_ working +at, or a _previous_ company. One approach could be to try and train a named +entity recognizer to predict `CURRENT_ORG` and `PREVIOUS_ORG` – but this +distinction is very subtle and something the entity recognizer may struggle to +learn. Nothing about "Acme Corp Inc." is inherently "current" or "previous". + +However, the syntax of the sentence holds some very important clues: we can +check for trigger words like "work", whether they're **past tense** or **present +tense**, whether company names are attached to it and whether the person is the +subject. All of this information is available in the part-of-speech tags and the +dependency parse. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Alex Smith worked at Acme Corp Inc.") +print([(ent.text, ent.label_) for ent in doc.ents]) +``` + +> - `nsubj`: Nominal subject. +> - `prep`: Preposition. +> - `pobj`: Object of preposition. +> - `NNP`: Proper noun, singular. +> - `VBD`: Verb, past tense. +> - `IN`: Conjunction, subordinating or preposition. + +![Visualization of dependency parse](/images/displacy-model-rules.svg "[`spacy.displacy`](/api/top-level#displacy) visualization with `options={'fine_grained': True}` to output the fine-grained part-of-speech tags, i.e. `Token.tag_`") + +In this example, "worked" is the root of the sentence and is a past tense verb. +Its subject is "Alex Smith", the person who worked. "at Acme Corp Inc." is a +prepositional phrase attached to the verb "worked". To extract this +relationship, we can start by looking at the predicted `PERSON` entities, find +their heads and check whether they're attached to a trigger word like "work". +Next, we can check for prepositional phrases attached to the head and whether +they contain an `ORG` entity. Finally, to determine whether the company +affiliation is current, we can check the head's part-of-speech tag. + +```python +person_entities = [ent for ent in doc.ents if ent.label_ == "PERSON"] +for ent in person_entities: + # Because the entity is a span, we need to use its root token. The head + # is the syntactic governor of the person, e.g. the verb + head = ent.root.head + if head.lemma_ == "work": + # Check if the children contain a preposition + preps = [token for token in head.children if token.dep_ == "prep"] + for prep in preps: + # Check if tokens part of ORG entities are in the preposition's + # children, e.g. at -> Acme Corp Inc. + orgs = [token for token in prep.children if token.ent_type_ == "ORG"] + # If the verb is in past tense, the company was a previous company + print({"person": ent, "orgs": orgs, "past": head.tag_ == "VBD"}) +``` + +To apply this logic automatically when we process a text, we can add it to the +`nlp` object as a +[custom pipeline component](/usage/processing-pipelines/#custom-components). The +above logic also expects that entities are merged into single tokens. spaCy +ships with a handy built-in `merge_entities` that takes care of that. Instead of +just printing the result, you could also write it to +[custom attributes](/usage/processing-pipelines#custom-components-attributes) on +the entity `Span` – for example `._.orgs` or `._.prev_orgs` and +`._.current_orgs`. + +> #### Merging entities +> +> Under the hood, entities are merged using the +> [`Doc.retokenize`](/api/doc#retokenize) context manager: +> +> ```python +> with doc.retokenize() as retokenizer: +> for ent in doc.ents: +> retokenizer.merge(ent) +> ``` + +```python {executable="true"} +import spacy +from spacy.language import Language +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") + +@Language.component("extract_person_orgs") +def extract_person_orgs(doc): + person_entities = [ent for ent in doc.ents if ent.label_ == "PERSON"] + for ent in person_entities: + head = ent.root.head + if head.lemma_ == "work": + preps = [token for token in head.children if token.dep_ == "prep"] + for prep in preps: + orgs = [token for token in prep.children if token.ent_type_ == "ORG"] + print({'person': ent, 'orgs': orgs, 'past': head.tag_ == "VBD"}) + return doc + +# To make the entities easier to work with, we'll merge them into single tokens +nlp.add_pipe("merge_entities") +nlp.add_pipe("extract_person_orgs") + +doc = nlp("Alex Smith worked at Acme Corp Inc.") +# If you're not in a Jupyter / IPython environment, use displacy.serve +displacy.render(doc, options={"fine_grained": True}) +``` + +If you change the sentence structure above, for example to "was working", you'll +notice that our current logic fails and doesn't correctly detect the company as +a past organization. That's because the root is a participle and the tense +information is in the attached auxiliary "was": + +![Visualization of dependency parse](/images/displacy-model-rules2.svg) + +To solve this, we can adjust the rules to also check for the above construction: + +```python {highlight="10-12"} +@Language.component("extract_person_orgs") +def extract_person_orgs(doc): + person_entities = [ent for ent in doc.ents if ent.label_ == "PERSON"] + for ent in person_entities: + head = ent.root.head + if head.lemma_ == "work": + preps = [token for token in head.children if token.dep_ == "prep"] + for prep in preps: + orgs = [t for t in prep.children if t.ent_type_ == "ORG"] + aux = [token for token in head.children if token.dep_ == "aux"] + past_aux = any(t.tag_ == "VBD" for t in aux) + past = head.tag_ == "VBD" or head.tag_ == "VBG" and past_aux + print({'person': ent, 'orgs': orgs, 'past': past}) + return doc +``` + +In your final rule-based system, you may end up with **several different code +paths** to cover the types of constructions that occur in your data. diff --git a/website/docs/usage/saving-loading.mdx b/website/docs/usage/saving-loading.mdx new file mode 100644 index 0000000..0b0b759 --- /dev/null +++ b/website/docs/usage/saving-loading.mdx @@ -0,0 +1,777 @@ +--- +title: Saving and Loading +menu: + - ['Basics', 'basics'] + - ['Serializing Docs', 'docs'] + - ['Serialization Methods', 'serialization-methods'] + - ['Entry Points', 'entry-points'] + - ['Trained Pipelines', 'models'] +--- + +## Basics {id="basics",hidden="true"} + + + +### Serializing the pipeline {id="pipeline"} + +When serializing the pipeline, keep in mind that this will only save out the +**binary data for the individual components** to allow spaCy to restore them – +not the entire objects. This is a good thing, because it makes serialization +safe. But it also means that you have to take care of storing the config, which +contains the pipeline configuration and all the relevant settings. + +> #### Saving the meta and config +> +> The [`nlp.meta`](/api/language#meta) attribute is a JSON-serializable +> dictionary and contains all pipeline meta information like the author and +> license information. The [`nlp.config`](/api/language#config) attribute is a +> dictionary containing the training configuration, pipeline component factories +> and other settings. It is saved out with a pipeline as the `config.cfg`. + +```python {title="Serialize"} +config = nlp.config +bytes_data = nlp.to_bytes() +``` + +```python {title="Deserialize"} +lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"]) +nlp = lang_cls.from_config(config) +nlp.from_bytes(bytes_data) +``` + +This is also how spaCy does it under the hood when loading a pipeline: it loads +the `config.cfg` containing the language and pipeline information, initializes +the language class, creates and adds the pipeline components based on the config +and _then_ loads in the binary data. You can read more about this process +[here](/usage/processing-pipelines#pipelines). + +## Serializing Doc objects efficiently {id="docs",version="2.2"} + +If you're working with lots of data, you'll probably need to pass analyses +between machines, either to use something like [Dask](https://dask.org) or +[Spark](https://spark.apache.org), or even just to save out work to disk. Often +it's sufficient to use the [`Doc.to_array`](/api/doc#to_array) functionality for +this, and just serialize the numpy arrays – but other times you want a more +general way to save and restore `Doc` objects. + +The [`DocBin`](/api/docbin) class makes it easy to serialize and deserialize a +collection of `Doc` objects together, and is much more efficient than calling +[`Doc.to_bytes`](/api/doc#to_bytes) on each individual `Doc` object. You can +also control what data gets saved, and you can merge pallets together for easy +map/reduce-style processing. + +```python {highlight="4,8,9,13,14"} +import spacy +from spacy.tokens import DocBin + +doc_bin = DocBin(attrs=["LEMMA", "ENT_IOB", "ENT_TYPE"], store_user_data=True) +texts = ["Some text", "Lots of texts...", "..."] +nlp = spacy.load("en_core_web_sm") +for doc in nlp.pipe(texts): + doc_bin.add(doc) +bytes_data = doc_bin.to_bytes() + +# Deserialize later, e.g. in a new process +nlp = spacy.blank("en") +doc_bin = DocBin().from_bytes(bytes_data) +docs = list(doc_bin.get_docs(nlp.vocab)) +``` + +If `store_user_data` is set to `True`, the `Doc.user_data` will be serialized as +well, which includes the values of +[extension attributes](/usage/processing-pipelines#custom-components-attributes) +(if they're serializable with msgpack). + + + +Including the `Doc.user_data` and extension attributes will only serialize the +**values** of the attributes. To restore the values and access them via the +`doc._.` property, you need to register the global attribute on the `Doc` again. + +```python +docs = list(doc_bin.get_docs(nlp.vocab)) +Doc.set_extension("my_custom_attr", default=None) +print([doc._.my_custom_attr for doc in docs]) +``` + + + +### Using Pickle {id="pickle"} + +> #### Example +> +> ```python +> doc = nlp("This is a text.") +> data = pickle.dumps(doc) +> ``` + +When pickling spaCy's objects like the [`Doc`](/api/doc) or the +[`EntityRecognizer`](/api/entityrecognizer), keep in mind that they all require +the shared [`Vocab`](/api/vocab) (which includes the string to hash mappings, +label schemes and optional vectors). This means that their pickled +representations can become very large, especially if you have word vectors +loaded, because it won't only include the object itself, but also the entire +shared vocab it depends on. + +If you need to pickle multiple objects, try to pickle them **together** instead +of separately. For instance, instead of pickling all pipeline components, pickle +the entire pipeline once. And instead of pickling several `Doc` objects +separately, pickle a list of `Doc` objects. Since they all share a reference to +the _same_ `Vocab` object, it will only be included once. + +```python {title="Pickling objects with shared data",highlight="8-9"} +doc1 = nlp("Hello world") +doc2 = nlp("This is a test") + +doc1_data = pickle.dumps(doc1) +doc2_data = pickle.dumps(doc2) +print(len(doc1_data) + len(doc2_data)) # 6636116 😞 + +doc_data = pickle.dumps([doc1, doc2]) +print(len(doc_data)) # 3319761 😃 +``` + + + +Pickling `Token` and `Span` objects isn't supported. They're only views of the +`Doc` and can't exist on their own. Pickling them would always mean pulling in +the parent document and its vocabulary, which has practically no advantage over +pickling the parent `Doc`. + +```diff +- data = pickle.dumps(doc[10:20]) ++ data = pickle.dumps(doc) +``` + +If you really only need a span – for example, a particular sentence – you can +use [`Span.as_doc`](/api/span#as_doc) to make a copy of it and convert it to a +`Doc` object. However, note that this will not let you recover contextual +information from _outside_ the span. + +```diff ++ span_doc = doc[10:20].as_doc() +data = pickle.dumps(span_doc) +``` + + + +## Implementing serialization methods {id="serialization-methods"} + +When you call [`nlp.to_disk`](/api/language#to_disk), +[`nlp.from_disk`](/api/language#from_disk) or load a pipeline package, spaCy +will iterate over the components in the pipeline, check if they expose a +`to_disk` or `from_disk` method and if so, call it with the path to the pipeline +directory plus the string name of the component. For example, if you're calling +`nlp.to_disk("/path")`, the data for the named entity recognizer will be saved +in `/path/ner`. + +If you're using custom pipeline components that depend on external data – for +example, model weights or terminology lists – you can take advantage of spaCy's +built-in component serialization by making your custom component expose its own +`to_disk` and `from_disk` or `to_bytes` and `from_bytes` methods. When an `nlp` +object with the component in its pipeline is saved or loaded, the component will +then be able to serialize and deserialize itself. + + + +For more details on how to work with pipeline components that depend on data +resources and manage data loading and initialization at training and runtime, +see the usage guide on initializing and serializing +[component data](/usage/processing-pipelines#component-data). + + + +The following example shows a custom component that keeps arbitrary +JSON-serializable data, allows the user to add to that data and saves and loads +the data to and from a JSON file. + +> #### Real-world example +> +> To see custom serialization methods in action, check out the new +> [`EntityRuler`](/api/entityruler) component and its +> [source](%%GITHUB_SPACY/spacy/pipeline/entityruler.py). Patterns added to the +> component will be saved to a `.jsonl` file if the pipeline is serialized to +> disk, and to a bytestring if the pipeline is serialized to bytes. This allows +> saving out a pipeline with a rule-based entity recognizer and including all +> rules _with_ the component data. + +```python {highlight="16-23,25-30"} +import json +from spacy import Language +from spacy.util import ensure_path + +@Language.factory("my_component") +class CustomComponent: + def __init__(self, nlp: Language, name: str = "my_component"): + self.name = name + self.data = [] + + def __call__(self, doc): + # Do something to the doc here + return doc + + def add(self, data): + # Add something to the component's data + self.data.append(data) + + def to_disk(self, path, exclude=tuple()): + # This will receive the directory path + /my_component + path = ensure_path(path) + if not path.exists(): + path.mkdir() + data_path = path / "data.json" + with data_path.open("w", encoding="utf8") as f: + f.write(json.dumps(self.data)) + + def from_disk(self, path, exclude=tuple()): + # This will receive the directory path + /my_component + data_path = path / "data.json" + with data_path.open("r", encoding="utf8") as f: + self.data = json.load(f) + return self +``` + +After adding the component to the pipeline and adding some data to it, we can +serialize the `nlp` object to a directory, which will call the custom +component's `to_disk` method. + +```python {highlight="2-4"} +nlp = spacy.load("en_core_web_sm") +my_component = nlp.add_pipe("my_component") +my_component.add({"hello": "world"}) +nlp.to_disk("/path/to/pipeline") +``` + +The contents of the directory would then look like this. +`CustomComponent.to_disk` converted the data to a JSON string and saved it to a +file `data.json` in its subdirectory: + +```yaml {title="Directory structure",highlight="2-3"} +└── /path/to/pipeline + ├── my_component # data serialized by "my_component" + │ └── data.json + ├── ner # data for "ner" component + ├── parser # data for "parser" component + ├── tagger # data for "tagger" component + ├── vocab # pipeline vocabulary + ├── meta.json # pipeline meta.json + ├── config.cfg # pipeline config + └── tokenizer # tokenization rules +``` + +When you load the data back in, spaCy will call the custom component's +`from_disk` method with the given file path, and the component can then load the +contents of `data.json`, convert them to a Python object and restore the +component state. The same works for other types of data, of course – for +instance, you could add a +[wrapper for a model](/usage/layers-architectures#frameworks) trained with a +different library like TensorFlow or PyTorch and make spaCy load its weights +automatically when you load the pipeline package. + + + +When you load back a pipeline with custom components, make sure that the +components are **available** and that the +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorators are executed _before_ +your pipeline is loaded back. Otherwise, spaCy won't know how to resolve the +string name of a component factory like `"my_component"` back to a function. For +more details, see the documentation on +[adding factories](/usage/processing-pipelines#custom-components-factories) or +use [entry points](#entry-points) to make your extension package expose your +custom components to spaCy automatically. + + + +{/* ## Initializing components with data {id="initialization",version="3"} */} + +## Using entry points {id="entry-points",version="2.1"} + +Entry points let you expose parts of a Python package you write to other Python +packages. This lets one application easily customize the behavior of another, by +exposing an entry point in its `setup.py`. For a quick and fun intro to entry +points in Python, check out +[this excellent blog post](https://amir.rachum.com/blog/2017/07/28/python-entry-points/). +spaCy can load custom functions from several different entry points to add +pipeline component factories, language classes and other settings. To make spaCy +use your entry points, your package needs to expose them and it needs to be +installed in the same environment – that's it. + +| Entry point | Description | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`spacy_factories`](#entry-points-components) | Group of entry points for pipeline component factories, keyed by component name. Can be used to expose custom components defined by another package. | +| [`spacy_languages`](#entry-points-languages) | Group of entry points for custom [`Language` subclasses](/usage/linguistic-features#language-data), keyed by language shortcut. | +| `spacy_lookups` | Group of entry points for custom [`Lookups`](/api/lookups), including lemmatizer data. Used by spaCy's [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) package. | +| [`spacy_displacy_colors`](#entry-points-displacy) | Group of entry points of custom label colors for the [displaCy visualizer](/usage/visualizers#ent). The key name doesn't matter, but it should point to a dict of labels and color values. Useful for custom models that predict different entity types. | + +### Loading probability tables into existing models + +You can load a probability table from +[spacy-lookups-data](https://github.com/explosion/spacy-lookups-data) into an +existing spaCy model like `en_core_web_sm`. + +```python +# Requirements: pip install spacy-lookups-data +import spacy +from spacy.lookups import load_lookups +nlp = spacy.load("en_core_web_sm") +lookups = load_lookups("en", ["lexeme_prob"]) +nlp.vocab.lookups.add_table("lexeme_prob", lookups.get_table("lexeme_prob")) +``` + +When training a model from scratch you can also specify probability tables in +the `config.cfg`. + +```ini {title="config.cfg (excerpt)"} +[initialize.lookups] +@misc = "spacy.LookupsDataLoader.v1" +lang = ${nlp.lang} +tables = ["lexeme_prob"] +``` + +### Custom components via entry points {id="entry-points-components"} + +When you load a pipeline, spaCy will generally use its `config.cfg` to set up +the language class and construct the pipeline. The pipeline is specified as a +list of strings, e.g. `pipeline = ["tagger", "parser", "ner"]`. For each of +those strings, spaCy will call `nlp.add_pipe` and look up the name in all +factories defined by the decorators +[`@Language.component`](/api/language#component) and +[`@Language.factory`](/api/language#factory). This means that you have to import +your custom components _before_ loading the pipeline. + +Using entry points, pipeline packages and extension packages can define their +own `"spacy_factories"`, which will be loaded automatically in the background +when the `Language` class is initialized. So if a user has your package +installed, they'll be able to use your components – even if they **don't import +them**! + +To stick with the theme of +[this entry points blog post](https://amir.rachum.com/blog/2017/07/28/python-entry-points/), +consider the following custom spaCy +[pipeline component](/usage/processing-pipelines#custom-components) that prints +a snake when it's called: + +> #### Package directory structure +> +> ```yaml +> ├── snek.py # the extension code +> └── setup.py # setup file for pip installation +> ``` + +```python {title="snek.py"} +from spacy.language import Language + +snek = """ + --..,_ _,.--. + `'.'. .'`__ o `;__. {text} + '.'. .'.'` '---'` ` + '.`'--....--'`.' + `'--....--'` +""" + +@Language.component("snek") +def snek_component(doc): + print(snek.format(text=doc.text)) + return doc +``` + +Since it's a very complex and sophisticated module, you want to split it off +into its own package so you can version it and upload it to PyPi. You also want +your custom package to be able to define `pipeline = ["snek"]` in its +`config.cfg`. For that, you need to be able to tell spaCy where to find the +component `"snek"`. If you don't do this, spaCy will raise an error when you try +to load the pipeline because there's no built-in `"snek"` component. To add an +entry to the factories, you can now expose it in your `setup.py` via the +`entry_points` dictionary: + +> #### Entry point syntax +> +> Python entry points for a group are formatted as a **list of strings**, with +> each string following the syntax of `name = module:object`. In this example, +> the created entry point is named `snek` and points to the function +> `snek_component` in the module `snek`, i.e. `snek.py`. + +```python {title="setup.py",highlight="5-7"} +from setuptools import setup + +setup( + name="snek", + entry_points={ + "spacy_factories": ["snek = snek:snek_component"] + } +) +``` + +The same package can expose multiple entry points, by the way. To make them +available to spaCy, all you need to do is install the package in your +environment: + +```bash +$ python -m pip install . +``` + +spaCy is now able to create the pipeline component `"snek"` – even though you +never imported `snek_component`. When you save the +[`nlp.config`](/api/language#config) to disk, it includes an entry for your +`"snek"` component and any pipeline you train with this config will include the +component and know how to load it – if your `snek` package is installed. + +> #### config.cfg (excerpt) +> +> ```diff +> [nlp] +> lang = "en" +> + pipeline = ["snek"] +> +> [components] +> +> + [components.snek] +> + factory = "snek" +> ``` + +``` +>>> from spacy.lang.en import English +>>> nlp = English() +>>> nlp.add_pipe("snek") # this now works! 🐍🎉 +>>> doc = nlp("I am snek") + --..,_ _,.--. + `'.'. .'`__ o `;__. I am snek + '.'. .'.'` '---'` ` + '.`'--....--'`.' + `'--....--'` +``` + +Instead of making your snek component a simple +[stateless component](/usage/processing-pipelines#custom-components-simple), you +could also make it a +[factory](/usage/processing-pipelines#custom-components-factories) that takes +settings. Your users can then pass in an optional `config` when they add your +component to the pipeline and customize its appearance – for example, the +`snek_style`. + +> #### config.cfg (excerpt) +> +> ```diff +> [components.snek] +> factory = "snek" +> + snek_style = "basic" +> ``` + +```python +SNEKS = {"basic": snek, "cute": cute_snek} # collection of sneks + +@Language.factory("snek", default_config={"snek_style": "basic"}) +class SnekFactory: + def __init__(self, nlp: Language, name: str, snek_style: str): + self.nlp = nlp + self.snek_style = snek_style + self.snek = SNEKS[self.snek_style] + + def __call__(self, doc): + print(self.snek) + return doc +``` + +```diff {title="setup.py"} +entry_points={ +- "spacy_factories": ["snek = snek:snek_component"] ++ "spacy_factories": ["snek = snek:SnekFactory"] +} +``` + +The factory can also implement other pipeline component methods like `to_disk` +and `from_disk` for serialization, or even `update` to make the component +trainable. If a component exposes a `from_disk` method and is included in a +pipeline, spaCy will call it on load. This lets you ship custom data with your +pipeline package. When you save out a pipeline using `nlp.to_disk` and the +component exposes a `to_disk` method, it will be called with the disk path. + +```python +from spacy.util import ensure_path + +def to_disk(self, path, exclude=tuple()): + path = ensure_path(path) + if not path.exists(): + path.mkdir() + snek_path = path / "snek.txt" + with snek_path.open("w", encoding="utf8") as snek_file: + snek_file.write(self.snek) + +def from_disk(self, path, exclude=tuple()): + snek_path = path / "snek.txt" + with snek_path.open("r", encoding="utf8") as snek_file: + self.snek = snek_file.read() + return self +``` + +The above example will serialize the current snake in a `snek.txt` in the data +directory. When a pipeline using the `snek` component is loaded, it will open +the `snek.txt` and make it available to the component. + +### Custom language classes via entry points {id="entry-points-languages"} + +To stay with the theme of the previous example and +[this blog post on entry points](https://amir.rachum.com/blog/2017/07/28/python-entry-points/), +let's imagine you wanted to implement your own `SnekLanguage` class for your +custom pipeline – but you don't necessarily want to modify spaCy's code to add a +language. In your package, you could then implement the following +[custom language subclass](/usage/linguistic-features#language-subclass): + +```python {title="snek.py"} +from spacy.language import Language + +class SnekDefaults(Language.Defaults): + stop_words = set(["sss", "hiss"]) + +class SnekLanguage(Language): + lang = "snk" + Defaults = SnekDefaults +``` + +Alongside the `spacy_factories`, there's also an entry point option for +`spacy_languages`, which maps language codes to language-specific `Language` +subclasses: + +```diff {title="setup.py"} +from setuptools import setup + +setup( + name="snek", + entry_points={ + "spacy_factories": ["snek = snek:SnekFactory"], ++ "spacy_languages": ["snk = snek:SnekLanguage"] + } +) +``` + +In spaCy, you can then load the custom `snk` language and it will be resolved to +`SnekLanguage` via the custom entry point. This is especially relevant for +pipeline packages you [train](/usage/training), which could then specify +`lang = snk` in their `config.cfg` without spaCy raising an error because the +language is not available in the core library. + +### Custom displaCy colors via entry points {id="entry-points-displacy",version="2.2"} + +If you're training a named entity recognition model for a custom domain, you may +end up training different labels that don't have pre-defined colors in the +[`displacy` visualizer](/usage/visualizers#ent). The `spacy_displacy_colors` +entry point lets you define a dictionary of entity labels mapped to their color +values. It's added to the pre-defined colors and can also overwrite existing +values. + +> #### Domain-specific NER labels +> +> Good examples of pipelines with domain-specific label schemes are +> [scispaCy](/universe/project/scispacy) and +> [Blackstone](/universe/project/blackstone). + +```python {title="snek.py"} +displacy_colors = {"SNEK": "#3dff74", "HUMAN": "#cfc5ff"} +``` + +Given the above colors, the entry point can be defined as follows. Entry points +need to have a name, so we use the key `colors`. However, the name doesn't +matter and whatever is defined in the entry point group will be used. + +```diff {title="setup.py"} +from setuptools import setup + +setup( + name="snek", + entry_points={ ++ "spacy_displacy_colors": ["colors = snek:displacy_colors"] + } +) +``` + +After installing the package, the custom colors will be used when visualizing +text with `displacy`. Whenever the label `SNEK` is assigned, it will be +displayed in `#3dff74`. + + +
🌱🌿 🐍 SNEK ____ 🌳🌲 ____ 👨‍🌾 HUMAN 🏘️
+ + +## Saving, loading and distributing trained pipelines {id="models"} + +After training your pipeline, you'll usually want to save its state, and load it +back later. You can do this with the [`Language.to_disk`](/api/language#to_disk) +method: + +```python +nlp.to_disk("./en_example_pipeline") +``` + +The directory will be created if it doesn't exist, and the whole pipeline data, +meta and configuration will be written out. To make the pipeline more convenient +to deploy, we recommend wrapping it as a [Python package](/api/cli#package). + + + +When you save a pipeline in spaCy v3.0+, two files will be exported: a +[`config.cfg`](/api/data-formats#config) based on +[`nlp.config`](/api/language#config) and a [`meta.json`](/api/data-formats#meta) +based on [`nlp.meta`](/api/language#meta). + +- **config**: Configuration used to create the current `nlp` object, its + pipeline components and models, as well as training settings and + hyperparameters. Can include references to registered functions like + [pipeline components](/usage/processing-pipelines#custom-components) or + [model architectures](/api/architectures). Given a config, spaCy is able + reconstruct the whole tree of objects and the `nlp` object. An exported config + can also be used to [train a pipeline](/usage/training#config) with the same + settings. +- **meta**: Meta information about the pipeline and the Python package, such as + the author information, license, version, data sources and label scheme. This + is mostly used for documentation purposes and for packaging pipelines. It has + no impact on the functionality of the `nlp` object. + + + + + +The easiest way to get started with an end-to-end workflow is to clone a +[project template](/usage/projects) and run it – for example, this template that +lets you train a **part-of-speech tagger** and **dependency parser** on a +Universal Dependencies treebank and generates an installable Python package. + + + +### Generating a pipeline package {id="models-generating"} + + + +Pipeline packages are typically **not suitable** for the public +[pypi.python.org](https://pypi.python.org) directory, which is not designed for +binary data and files over 50 MB. However, if your company is running an +**internal installation** of PyPi, publishing your pipeline packages on there +can be a convenient way to share them with your team. + + + +spaCy comes with a handy CLI command that will create all required files, and +walk you through generating the meta data. You can also create the +[`meta.json`](/api/data-formats#meta) manually and place it in the data +directory, or supply a path to it using the `--meta` flag. For more info on +this, see the [`package`](/api/cli#package) docs. + +> #### meta.json (example) +> +> ```json +> { +> "name": "example_pipeline", +> "lang": "en", +> "version": "1.0.0", +> "spacy_version": ">=2.0.0,<3.0.0", +> "description": "Example pipeline for spaCy", +> "author": "You", +> "email": "you@example.com", +> "license": "CC BY-SA 3.0" +> } +> ``` + +```bash +$ python -m spacy package ./en_example_pipeline ./packages +``` + +This command will create a pipeline package directory and will run +`python -m build` in that directory to create a binary `.whl` file or +`.tar.gz` archive of your package that can be installed using `pip install`. +Installing the binary wheel is usually more efficient. + +```yaml {title="Directory structure"} +└── / + ├── MANIFEST.in # to include meta.json + ├── meta.json # pipeline meta data + ├── setup.py # setup file for pip installation + ├── en_example_pipeline # pipeline directory + │ ├── __init__.py # init for pip installation + │ └── en_example_pipeline-1.0.0 # pipeline data + │ ├── config.cfg # pipeline config + │ ├── meta.json # pipeline meta + │ └── ... # directories with component data + └── dist + └── en_example_pipeline-1.0.0.tar.gz # installable package +``` + +You can also find templates for all files in the +[`cli/package.py` source](https://github.com/explosion/spacy/tree/master/spacy/cli/package.py). +If you're creating the package manually, keep in mind that the directories need +to be named according to the naming conventions of `lang_name` and +`lang_name-version`. + +### Including custom functions and components {id="models-custom"} + +If your pipeline includes +[custom components](/usage/processing-pipelines#custom-components), model +architectures or other [code](/usage/training#custom-code), those functions need +to be registered **before** your pipeline is loaded. Otherwise, spaCy won't know +how to create the objects referenced in the config. If you're loading your own +pipeline in Python, you can make custom components available just by importing +the code that defines them before calling +[`spacy.load`](/api/top-level#spacy.load). This is also how the `--code` +argument to CLI commands works. + +With the [`spacy package`](/api/cli#package) command, you can provide one or +more paths to Python files containing custom registered functions using the +`--code` argument. + +> #### \_\_init\_\_.py (excerpt) +> +> ```python +> from . import functions +> +> def load(**overrides): +> ... +> ``` + +```bash +$ python -m spacy package ./en_example_pipeline ./packages --code functions.py +``` + +The Python files will be copied over into the root of the package, and the +package's `__init__.py` will import them as modules. This ensures that functions +are registered when the pipeline is imported, e.g. when you call `spacy.load`. A +simple import is all that's needed to make registered functions available. + +Make sure to include **all Python files** that are referenced in your custom +code, including modules imported by others. If your custom code depends on +**external packages**, make sure they're listed in the list of `"requirements"` +in your [`meta.json`](/api/data-formats#meta). For the majority of use cases, +registered functions should provide you with all customizations you need, from +custom components to custom model architectures and lifecycle hooks. However, if +you do want to customize the setup in more detail, you can edit the package's +`__init__.py` and the package's `load` function that's called by +[`spacy.load`](/api/top-level#spacy.load). + + + +While it's no problem to edit the package code or meta information, avoid making +edits to the `config.cfg` **after** training, as this can easily lead to data +incompatibility. For instance, changing an architecture or hyperparameter can +mean that the trained weights are now incompatible. If you want to make +adjustments, you can do so before training. Otherwise, you should always trust +spaCy to export the current state of its `nlp` objects via +[`nlp.config`](/api/language#config). + + + +### Loading a custom pipeline package {id="loading"} + +To load a pipeline from a data directory, you can use +[`spacy.load()`](/api/top-level#spacy.load) with the local path. This will look +for a `config.cfg` in the directory and use the `lang` and `pipeline` settings +to initialize a `Language` class with a processing pipeline and load in the +model data. + +```python +nlp = spacy.load("/path/to/pipeline") +``` + +If you want to **load only the binary data**, you'll have to create a `Language` +class and call [`from_disk`](/api/language#from_disk) instead. + +```python +nlp = spacy.blank("en").from_disk("/path/to/data") +``` diff --git a/website/docs/usage/spacy-101.mdx b/website/docs/usage/spacy-101.mdx new file mode 100644 index 0000000..a3cd7b4 --- /dev/null +++ b/website/docs/usage/spacy-101.mdx @@ -0,0 +1,584 @@ +--- +title: 'spaCy 101: Everything you need to know' +teaser: The most important concepts, explained in simple terms +menu: + - ["What's spaCy?", 'whats-spacy'] + - ['Features', 'features'] + - ['Linguistic Annotations', 'annotations'] + - ['Pipelines', 'pipelines'] + - ['Architecture', 'architecture'] + - ['Vocab', 'vocab'] + - ['Serialization', 'serialization'] + - ['Training', 'training'] + - ['Language Data', 'language-data'] + - ['Community & FAQ', 'community-faq'] +--- + +Whether you're new to spaCy, or just want to brush up on some NLP basics and +implementation details – this page should have you covered. Each section will +explain one of spaCy's features in simple terms and with examples or +illustrations. Some sections will also reappear across the usage guides as a +quick introduction. + +> #### Help us improve the docs +> +> Did you spot a mistake or come across explanations that are unclear? We always +> appreciate improvement +> [suggestions](https://github.com/explosion/spaCy/issues) or +> [pull requests](https://github.com/explosion/spaCy/pulls). You can find a +> "Suggest edits" link at the bottom of each page that points you to the source. + + + +Advanced NLP with spaCy + +In this course you'll learn how to use spaCy to build advanced natural language +understanding systems, using both rule-based and machine learning approaches. It +includes 55 exercises featuring interactive coding practice, multiple-choice +questions and slide decks. + + + + + +## What's spaCy? {id="whats-spacy"} + + + +
+ +spaCy is a **free, open-source library** for advanced **Natural Language +Processing** (NLP) in Python. + +If you're working with a lot of text, you'll eventually want to know more about +it. For example, what's it about? What do the words mean in context? Who is +doing what to whom? What companies and products are mentioned? Which texts are +similar to each other? + +spaCy is designed specifically for **production use** and helps you build +applications that process and "understand" large volumes of text. It can be used +to build **information extraction** or **natural language understanding** +systems, or to pre-process text for **deep learning**. + +
+ + + +- [Features](#features) +- [Linguistic annotations](#annotations) +- [Tokenization](#annotations-token) +- [POS tags and dependencies](#annotations-pos-deps) +- [Named entities](#annotations-ner) +- [Word vectors and similarity](#vectors-similarity) +- [Pipelines](#pipelines) +- [Library architecture](#architecture) +- [Vocab, hashes and lexemes](#vocab) +- [Serialization](#serialization) +- [Training](#training) +- [Language data](#language-data) +- [Community & FAQ](#community) + + + + + +### What spaCy isn't {id="what-spacy-isnt"} + +- ❌ **spaCy is not a platform or "an API"**. Unlike a platform, spaCy does not + provide a software as a service, or a web application. It's an open-source + library designed to help you build NLP applications, not a consumable service. +- ❌ **spaCy is not an out-of-the-box chat bot engine**. While spaCy can be used + to power conversational applications, it's not designed specifically for chat + bots, and only provides the underlying text processing capabilities. +- ❌**spaCy is not research software**. It's built on the latest research, but + it's designed to get things done. This leads to fairly different design + decisions than [NLTK](https://github.com/nltk/nltk) or + [CoreNLP](https://stanfordnlp.github.io/CoreNLP/), which were created as + platforms for teaching and research. The main difference is that spaCy is + integrated and opinionated. spaCy tries to avoid asking the user to choose + between multiple algorithms that deliver equivalent functionality. Keeping the + menu small lets spaCy deliver generally better performance and developer + experience. +- ❌ **spaCy is not a company**. It's an open-source library. Our company + publishing spaCy and other software is called + [Explosion](https://explosion.ai). + +## Features {id="features"} + +In the documentation, you'll come across mentions of spaCy's features and +capabilities. Some of them refer to linguistic concepts, while others are +related to more general machine learning functionality. + +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| **Tokenization** | Segmenting text into words, punctuations marks etc. | +| **Part-of-speech** (POS) **Tagging** | Assigning word types to tokens, like verb or noun. | +| **Dependency Parsing** | Assigning syntactic dependency labels, describing the relations between individual tokens, like subject or object. | +| **Lemmatization** | Assigning the base forms of words. For example, the lemma of "was" is "be", and the lemma of "rats" is "rat". | +| **Sentence Boundary Detection** (SBD) | Finding and segmenting individual sentences. | +| **Named Entity Recognition** (NER) | Labelling named "real-world" objects, like persons, companies or locations. | +| **Entity Linking** (EL) | Disambiguating textual entities to unique identifiers in a knowledge base. | +| **Similarity** | Comparing words, text spans and documents and how similar they are to each other. | +| **Text Classification** | Assigning categories or labels to a whole document, or parts of a document. | +| **Rule-based Matching** | Finding sequences of tokens based on their texts and linguistic annotations, similar to regular expressions. | +| **Training** | Updating and improving a statistical model's predictions. | +| **Serialization** | Saving objects to files or byte strings. | + +### Statistical models {id="statistical-models"} + +While some of spaCy's features work independently, others require +[trained pipelines](/models) to be loaded, which enable spaCy to **predict** +linguistic annotations – for example, whether a word is a verb or a noun. A +trained pipeline can consist of multiple components that use a statistical model +trained on labeled data. spaCy currently offers trained pipelines for a variety +of languages, which can be installed as individual Python modules. Pipeline +packages can differ in size, speed, memory usage, accuracy and the data they +include. The package you choose always depends on your use case and the texts +you're working with. For a general-purpose use case, the small, default packages +are always a good start. They typically include the following components: + +- **Binary weights** for the part-of-speech tagger, dependency parser and named + entity recognizer to predict those annotations in context. +- **Lexical entries** in the vocabulary, i.e. words and their + context-independent attributes like the shape or spelling. +- **Data files** like lemmatization rules and lookup tables. +- **Word vectors**, i.e. multi-dimensional meaning representations of words that + let you determine how similar they are to each other. +- **Configuration** options, like the language and processing pipeline settings + and model implementations to use, to put spaCy in the correct state when you + load the pipeline. + +## Linguistic annotations {id="annotations"} + +spaCy provides a variety of linguistic annotations to give you **insights into a +text's grammatical structure**. This includes the word types, like the parts of +speech, and how the words are related to each other. For example, if you're +analyzing text, it makes a huge difference whether a noun is the subject of a +sentence, or the object – or whether "google" is used as a verb, or refers to +the website or company in a specific context. + +> #### Loading pipelines +> +> ```bash +> $ python -m spacy download en_core_web_sm +> +> >>> import spacy +> >>> nlp = spacy.load("en_core_web_sm") +> ``` + +Once you've [downloaded and installed](/usage/models) a trained pipeline, you +can load it via [`spacy.load`](/api/top-level#spacy.load). This will return a +`Language` object containing all components and data needed to process text. We +usually call it `nlp`. Calling the `nlp` object on a string of text will return +a processed `Doc`: + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("Apple is looking at buying U.K. startup for $1 billion") +for token in doc: + print(token.text, token.pos_, token.dep_) +``` + +Even though a `Doc` is processed – e.g. split into individual words and +annotated – it still holds **all information of the original text**, like +whitespace characters. You can always get the offset of a token into the +original string, or reconstruct the original by joining the tokens and their +trailing whitespace. This way, you'll never lose any information when processing +text with spaCy. + +### Tokenization {id="annotations-token"} + + + + + +To learn more about how spaCy's tokenization rules work in detail, how to +**customize and replace** the default tokenizer and how to **add +language-specific data**, see the usage guides on +[language data](/usage/linguistic-features#language-data) and +[customizing the tokenizer](/usage/linguistic-features#tokenization). + + + +### Part-of-speech tags and dependencies {id="annotations-pos-deps",model="parser"} + + + + + +To learn more about **part-of-speech tagging** and rule-based morphology, and +how to **navigate and use the parse tree** effectively, see the usage guides on +[part-of-speech tagging](/usage/linguistic-features#pos-tagging) and +[using the dependency parse](/usage/linguistic-features#dependency-parse). + + + +### Named Entities {id="annotations-ner",model="ner"} + + + + + +To learn more about entity recognition in spaCy, how to **add your own +entities** to a document and how to **train and update** the entity predictions +of a model, see the usage guides on +[named entity recognition](/usage/linguistic-features#named-entities) and +[training pipelines](/usage/training). + + + +### Word vectors and similarity {id="vectors-similarity",model="vectors"} + + + + + +To learn more about word vectors, how to **customize them** and how to load +**your own vectors** into spaCy, see the usage guide on +[using word vectors and semantic similarities](/usage/linguistic-features#vectors-similarity). + + + +## Pipelines {id="pipelines"} + + + + + +To learn more about **how processing pipelines work** in detail, how to enable +and disable their components, and how to **create your own**, see the usage +guide on [language processing pipelines](/usage/processing-pipelines). + + + +## Architecture {id="architecture"} + + + +## Vocab, hashes and lexemes {id="vocab"} + +Whenever possible, spaCy tries to store data in a vocabulary, the +[`Vocab`](/api/vocab), that will be **shared by multiple documents**. To save +memory, spaCy also encodes all strings to **hash values** – in this case for +example, "coffee" has the hash `3197928453018144401`. Entity labels like "ORG" +and part-of-speech tags like "VERB" are also encoded. Internally, spaCy only +"speaks" in hash values. + +> - **Token**: A word, punctuation mark etc. _in context_, including its +> attributes, tags and dependencies. +> - **Lexeme**: A "word type" with no context. Includes the word shape and +> flags, e.g. if it's lowercase, a digit or punctuation. +> - **Doc**: A processed container of tokens in context. +> - **Vocab**: The collection of lexemes. +> - **StringStore**: The dictionary mapping hash values to strings, for example +> `3197928453018144401` → "coffee". + +![Doc, Vocab, Lexeme and StringStore](/images/vocab_stringstore.svg) + +If you process lots of documents containing the word "coffee" in all kinds of +different contexts, storing the exact string "coffee" every time would take up +way too much space. So instead, spaCy hashes the string and stores it in the +[`StringStore`](/api/stringstore). You can think of the `StringStore` as a +**lookup table that works in both directions** – you can look up a string to get +its hash, or a hash to get its string: + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I love coffee") +print(doc.vocab.strings["coffee"]) # 3197928453018144401 +print(doc.vocab.strings[3197928453018144401]) # 'coffee' +``` + +Now that all strings are encoded, the entries in the vocabulary **don't need to +include the word text** themselves. Instead, they can look it up in the +`StringStore` via its hash value. Each entry in the vocabulary, also called +[`Lexeme`](/api/lexeme), contains the **context-independent** information about +a word. For example, no matter if "love" is used as a verb or a noun in some +context, its spelling and whether it consists of alphabetic characters won't +ever change. Its hash value will also always be the same. + +```python {executable="true"} +import spacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I love coffee") +for word in doc: + lexeme = doc.vocab[word.text] + print(lexeme.text, lexeme.orth, lexeme.shape_, lexeme.prefix_, lexeme.suffix_, + lexeme.is_alpha, lexeme.is_digit, lexeme.is_title, lexeme.lang_) +``` + +> - **Text**: The original text of the lexeme. +> - **Orth**: The hash value of the lexeme. +> - **Shape**: The abstract word shape of the lexeme. +> - **Prefix**: By default, the first letter of the word string. +> - **Suffix**: By default, the last three letters of the word string. +> - **is alpha**: Does the lexeme consist of alphabetic characters? +> - **is digit**: Does the lexeme consist of digits? + +| Text | Orth | Shape | Prefix | Suffix | is_alpha | is_digit | +| ------ | --------------------- | ------ | ------ | ------ | -------- | -------- | +| I | `4690420944186131903` | `X` | I | I | `True` | `False` | +| love | `3702023516439754181` | `xxxx` | l | ove | `True` | `False` | +| coffee | `3197928453018144401` | `xxxx` | c | fee | `True` | `False` | + +The mapping of words to hashes doesn't depend on any state. To make sure each +value is unique, spaCy uses a +[hash function](https://en.wikipedia.org/wiki/Hash_function) to calculate the +hash **based on the word string**. This also means that the hash for "coffee" +will always be the same, no matter which pipeline you're using or how you've +configured spaCy. + +However, hashes **cannot be reversed** and there's no way to resolve +`3197928453018144401` back to "coffee". All spaCy can do is look it up in the +vocabulary. That's why you always need to make sure all objects you create have +access to the same vocabulary. If they don't, spaCy might not be able to find +the strings it needs. + +```python {executable="true"} +import spacy +from spacy.tokens import Doc +from spacy.vocab import Vocab + +nlp = spacy.load("en_core_web_sm") +doc = nlp("I love coffee") # Original Doc +print(doc.vocab.strings["coffee"]) # 3197928453018144401 +print(doc.vocab.strings[3197928453018144401]) # 'coffee' 👍 + +empty_doc = Doc(Vocab()) # New Doc with empty Vocab +# empty_doc.vocab.strings[3197928453018144401] will raise an error :( + +empty_doc.vocab.strings.add("coffee") # Add "coffee" and generate hash +print(empty_doc.vocab.strings[3197928453018144401]) # 'coffee' 👍 + +new_doc = Doc(doc.vocab) # Create new doc with first doc's vocab +print(new_doc.vocab.strings[3197928453018144401]) # 'coffee' 👍 +``` + +If the vocabulary doesn't contain a string for `3197928453018144401`, spaCy will +raise an error. You can re-add "coffee" manually, but this only works if you +actually _know_ that the document contains that word. To prevent this problem, +spaCy will also export the `Vocab` when you save a `Doc` or `nlp` object. This +will give you the object and its encoded annotations, plus the "key" to decode +it. + +## Serialization {id="serialization"} + + + + + +To learn more about how to **save and load your own pipelines**, see the usage +guide on [saving and loading](/usage/saving-loading#models). + + + +## Training {id="training"} + + + + + +To learn more about **training and updating** pipelines, how to create training +data and how to improve spaCy's named models, see the usage guides on +[training](/usage/training). + + + +### Training config and lifecycle {id="training-config"} + +Training config files include all **settings and hyperparameters** for training +your pipeline. Instead of providing lots of arguments on the command line, you +only need to pass your `config.cfg` file to [`spacy train`](/api/cli#train). +This also makes it easy to integrate custom models and architectures, written in +your framework of choice. A pipeline's `config.cfg` is considered the "single +source of truth", both at **training** and **runtime**. + +> ```ini +> ### config.cfg (excerpt) +> [training] +> accumulate_gradient = 3 +> +> [training.optimizer] +> @optimizers = "Adam.v1" +> +> [training.optimizer.learn_rate] +> @schedules = "warmup_linear.v1" +> warmup_steps = 250 +> total_steps = 20000 +> initial_rate = 0.01 +> ``` + +![Illustration of pipeline lifecycle](/images/lifecycle.svg) + + + +For more details on spaCy's **configuration system** and how to use it to +customize your pipeline components, component models, training settings and +hyperparameters, see the [training config](/usage/training#config) usage guide. + + + +### Trainable components {id="training-components"} + +spaCy's [`Pipe`](/api/pipe) class helps you implement your own trainable +components that have their own model instance, make predictions over `Doc` +objects and can be updated using [`spacy train`](/api/cli#train). This lets you +plug fully custom machine learning components into your pipeline that can be +configured via a single training config. + +> #### config.cfg (excerpt) +> +> ```ini +> [components.my_component] +> factory = "my_component" +> +> [components.my_component.model] +> @architectures = "my_model.v1" +> width = 128 +> ``` + +![Illustration of Pipe methods](/images/trainable_component.svg) + + + +To learn more about how to implement your own **model architectures** and use +them to power custom **trainable components**, see the usage guides on the +[trainable component API](/usage/processing-pipelines#trainable-components) and +implementing [layers and architectures](/usage/layers-architectures#components) +for trainable components. + + + +## Language data {id="language-data"} + + + +## Community & FAQ {id="community-faq"} + +We're very happy to see the spaCy community grow and include a mix of people +from all kinds of different backgrounds – computational linguistics, data +science, deep learning, research and more. If you'd like to get involved, below +are some answers to the most important questions and resources for further +reading. + +### Help, my code isn't working! {id="faq-help-code"} + +Bugs suck, and we're doing our best to continuously improve the tests and fix +bugs as soon as possible. Before you submit an issue, do a quick search and +check if the problem has already been reported. If you're having installation or +loading problems, make sure to also check out the +[troubleshooting guide](/usage/#troubleshooting). Help with spaCy is available +via the following platforms: + +> #### How do I know if something is a bug? +> +> Of course, it's always hard to know for sure, so don't worry – we're not going +> to be mad if a bug report turns out to be a typo in your code. As a simple +> rule, any C-level error without a Python traceback, like a **segmentation +> fault** or **memory error**, is **always** a spaCy bug. +> +> Because models are statistical, their performance will never be _perfect_. +> However, if you come across **patterns that might indicate an underlying +> issue**, please do file a report. Similarly, we also care about behaviors that +> **contradict our docs**. + +- [Stack Overflow](https://stackoverflow.com/questions/tagged/spacy): **Usage + questions** and everything related to problems with your specific code. The + Stack Overflow community is much larger than ours, so if your problem can be + solved by others, you'll receive help much quicker. +- [GitHub discussions](https://github.com/explosion/spaCy/discussions): + **General discussion**, **project ideas** and **usage questions**. Meet other + community members to get help with a specific code implementation, discuss + ideas for new projects/plugins, support more languages, and share best + practices. +- [GitHub issue tracker](https://github.com/explosion/spaCy/issues): **Bug + reports** and **improvement suggestions**, i.e. everything that's likely + spaCy's fault. This also includes problems with the trained pipelines beyond + statistical imprecisions, like patterns that point to a bug. + + + +Please understand that we won't be able to provide individual support via email. +We also believe that help is much more valuable if it's shared publicly, so that +**more people can benefit from it**. If you come across an issue and you think +you might be able to help, consider posting a quick update with your solution. +No matter how simple, it can easily save someone a lot of time and headache – +and the next time you need help, they might repay the favor. + + + +### How can I contribute to spaCy? {id="faq-contributing"} + +You don't have to be an NLP expert or Python pro to contribute, and we're happy +to help you get started. If you're new to spaCy, a good place to start is the +[`help wanted (easy)` label](https://github.com/explosion/spaCy/issues?q=is%3Aissue+is%3Aopen+label%3A"help+wanted+%28easy%29") +on GitHub, which we use to tag bugs and feature requests that are easy and +self-contained. We also appreciate contributions to the docs – whether it's +fixing a typo, improving an example or adding additional explanations. You'll +find a "Suggest edits" link at the bottom of each page that points you to the +source. + +Another way of getting involved is to help us improve the +[language data](/usage/linguistic-features#language-data) – especially if you +happen to speak one of the languages currently in +[alpha support](/usage/models#languages). Even adding simple tokenizer +exceptions, stop words or lemmatizer data can make a big difference. It will +also make it easier for us to provide a trained pipeline for the language in the +future. Submitting a test that documents a bug or performance issue, or covers +functionality that's especially important for your application is also very +helpful. This way, you'll also make sure we never accidentally introduce +regressions to the parts of the library that you care about the most. + +**For more details on the types of contributions we're looking for, the code +conventions and other useful tips, make sure to check out the +[contributing guidelines](%%GITHUB_SPACY/CONTRIBUTING.md).** + + + +spaCy adheres to the +[Contributor Covenant Code of Conduct](http://contributor-covenant.org/version/1/4/). +By participating, you are expected to uphold this code. + + + +### I've built something cool with spaCy – how can I get the word out? {id="faq-project-with-spacy"} + +First, congrats – we'd love to check it out! If you think your project would be a good fit for the +[spaCy Universe](/universe), **feel free to submit it!** Tutorials are also +incredibly valuable to other users and a great way to get exposure. So we +strongly encourage **writing up your experiences**, or sharing your code and +some tips and tricks on your blog. Since our website is open-source, you can add +your project or tutorial by making a pull request on GitHub. + +If you would like to use the spaCy logo on your site, please get in touch and +ask us first. However, if you want to show support and tell others that your +project is using spaCy, you can grab one of our **spaCy badges** here: + +Built with spaCy + +```markdown +[![Built with spaCy](https://img.shields.io/badge/built%20with-spaCy-09a3d5.svg)](https://spacy.io) +``` + +Made with love and spaCy + +```markdown +[![Made with love and spaCy](https://img.shields.io/badge/made%20with%20❤%20and-spaCy-09a3d5.svg)](https://spacy.io) +``` diff --git a/website/docs/usage/training.mdx b/website/docs/usage/training.mdx new file mode 100644 index 0000000..abb1b9c --- /dev/null +++ b/website/docs/usage/training.mdx @@ -0,0 +1,1723 @@ +--- +title: Training Pipelines & Models +teaser: Train and update components on your own data and integrate custom models +next: /usage/layers-architectures +menu: + - ['Introduction', 'basics'] + - ['Quickstart', 'quickstart'] + - ['Config System', 'config'] + - ['Training Data', 'training-data'] + - ['Custom Training', 'config-custom'] + - ['Custom Functions', 'custom-functions'] + - ['Initialization', 'initialization'] + - ['Data Utilities', 'data'] + - ['Internal API', 'api'] +--- + +## Introduction to training {id="basics",hidden="true"} + + + + + +Prodigy: Radically efficient machine teaching + +If you need to label a lot of data, check out [Prodigy](https://prodi.gy), a +new, active learning-powered annotation tool we've developed. Prodigy is fast +and extensible, and comes with a modern **web application** that helps you +collect training data faster. It integrates seamlessly with spaCy, pre-selects +the **most relevant examples** for annotation, and lets you train and evaluate +ready-to-use spaCy pipelines. + + + +## Quickstart {id="quickstart",tag="new"} + +The recommended way to train your spaCy pipelines is via the +[`spacy train`](/api/cli#train) command on the command line. It only needs a +single [`config.cfg`](#config) **configuration file** that includes all settings +and hyperparameters. You can optionally [overwrite](#config-overrides) settings +on the command line, and load in a Python file to register +[custom functions](#custom-code) and architectures. This quickstart widget helps +you generate a starter config with the **recommended settings** for your +specific use case. It's also available in spaCy as the +[`init config`](/api/cli#init-config) command. + + + +Upgrade to the [latest version of spaCy](/usage) to use the quickstart widget. +For earlier releases, follow the CLI instructions to generate a compatible +config. + + + +> #### Instructions: widget +> +> 1. Select your requirements and settings. +> 2. Use the buttons at the bottom to save the result to your clipboard or a +> file `base_config.cfg`. +> 3. Run [`init fill-config`](/api/cli#init-fill-config) to create a full +> config. +> 4. Run [`train`](/api/cli#train) with your config and data. +> +> #### Instructions: CLI +> +> 1. Run the [`init config`](/api/cli#init-config) command and specify your +> requirements and settings as CLI arguments. +> 2. Run [`train`](/api/cli#train) with the exported config and data. + + + +After you've saved the starter config to a file `base_config.cfg`, you can use +the [`init fill-config`](/api/cli#init-fill-config) command to fill in the +remaining defaults. Training configs should always be **complete and without +hidden defaults**, to keep your experiments reproducible. + +```bash +$ python -m spacy init fill-config base_config.cfg config.cfg +``` + +> #### Tip: Debug your data +> +> The [`debug data` command](/api/cli#debug-data) lets you analyze and validate +> your training and development data, get useful stats, and find problems like +> invalid entity annotations, cyclic dependencies, low data labels and more. +> +> ```bash +> $ python -m spacy debug data config.cfg +> ``` + +Instead of exporting your starter config from the quickstart widget and +auto-filling it, you can also use the [`init config`](/api/cli#init-config) +command and specify your requirement and settings as CLI arguments. You can now +add your data and run [`train`](/api/cli#train) with your config. See the +[`convert`](/api/cli#convert) command for details on how to convert your data to +spaCy's binary `.spacy` format. You can either include the data paths in the +`[paths]` section of your config, or pass them in via the command line. + +```bash +$ python -m spacy train config.cfg --output ./output --paths.train ./train.spacy --paths.dev ./dev.spacy +``` + +> #### Tip: Enable your GPU +> +> Use the `--gpu-id` option to select the GPU: +> +> ```bash +> $ python -m spacy train config.cfg --gpu-id 0 +> ``` + + + +The recommended config settings generated by the quickstart widget and the +[`init config`](/api/cli#init-config) command are based on some general **best +practices** and things we've found to work well in our experiments. The goal is +to provide you with the most **useful defaults**. + +Under the hood, the +[`quickstart_training.jinja`](%%GITHUB_SPACY/spacy/cli/templates/quickstart_training.jinja) +template defines the different combinations – for example, which parameters to +change if the pipeline should optimize for efficiency vs. accuracy. The file +[`quickstart_training_recommendations.yml`](%%GITHUB_SPACY/spacy/cli/templates/quickstart_training_recommendations.yml) +collects the recommended settings and available resources for each language +including the different transformer weights. For some languages, we include +different transformer recommendations, depending on whether you want the model +to be more efficient or more accurate. The recommendations will be **evolving** +as we run more experiments. + + + + + +The easiest way to get started is to clone a [project template](/usage/projects) +and run it – for example, this end-to-end template that lets you train a +**part-of-speech tagger** and **dependency parser** on a Universal Dependencies +treebank. + + + +## Training config system {id="config"} + +Training config files include all **settings and hyperparameters** for training +your pipeline. Instead of providing lots of arguments on the command line, you +only need to pass your `config.cfg` file to [`spacy train`](/api/cli#train). +Under the hood, the training config uses the +[configuration system](https://thinc.ai/docs/usage-config) provided by our +machine learning library [Thinc](https://thinc.ai). This also makes it easy to +integrate custom models and architectures, written in your framework of choice. +Some of the main advantages and features of spaCy's training config are: + +- **Structured sections.** The config is grouped into sections, and nested + sections are defined using the `.` notation. For example, `[components.ner]` + defines the settings for the pipeline's named entity recognizer. The config + can be loaded as a Python dict. +- **References to registered functions.** Sections can refer to registered + functions like [model architectures](/api/architectures), + [optimizers](https://thinc.ai/docs/api-optimizers) or + [schedules](https://thinc.ai/docs/api-schedules) and define arguments that are + passed into them. You can also + [register your own functions](#custom-functions) to define custom + architectures or methods, reference them in your config and tweak their + parameters. +- **Interpolation.** If you have hyperparameters or other settings used by + multiple components, define them once and reference them as + [variables](#config-interpolation). +- **Reproducibility with no hidden defaults.** The config file is the "single + source of truth" and includes all settings. +- **Automated checks and validation.** When you load a config, spaCy checks if + the settings are complete and if all values have the correct types. This lets + you catch potential mistakes early. In your custom architectures, you can use + Python [type hints](https://docs.python.org/3/library/typing.html) to tell the + config which types of data to expect. + +```ini +%%GITHUB_SPACY/spacy/default_config.cfg +``` + +Under the hood, the config is parsed into a dictionary. It's divided into +sections and subsections, indicated by the square brackets and dot notation. For +example, `[training]` is a section and `[training.batcher]` a subsection. +Subsections can define values, just like a dictionary, or use the `@` syntax to +refer to [registered functions](#config-functions). This allows the config to +not just define static settings, but also construct objects like architectures, +schedules, optimizers or any other custom components. The main top-level +sections of a config file are: + +| Section | Description | +| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | Definition of the `nlp` object, its tokenizer and [processing pipeline](/usage/processing-pipelines) component names. | +| `components` | Definitions of the [pipeline components](/usage/processing-pipelines) and their models. | +| `paths` | Paths to data and other assets. Re-used across the config as variables, e.g. `${paths.train}`, and can be [overwritten](#config-overrides) on the CLI. | +| `system` | Settings related to system and hardware. Re-used across the config as variables, e.g. `${system.seed}`, and can be [overwritten](#config-overrides) on the CLI. | +| `training` | Settings and controls for the training and evaluation process. | +| `pretraining` | Optional settings and controls for the [language model pretraining](/usage/embeddings-transformers#pretraining). | +| `initialize` | Data resources and arguments passed to components when [`nlp.initialize`](/api/language#initialize) is called before training (but not at runtime). | + + + +For a full overview of spaCy's config format and settings, see the +[data format documentation](/api/data-formats#config) and +[Thinc's config system docs](https://thinc.ai/docs/usage-config). The settings +available for the different architectures are documented with the +[model architectures API](/api/architectures). See the Thinc documentation for +[optimizers](https://thinc.ai/docs/api-optimizers) and +[schedules](https://thinc.ai/docs/api-schedules). + + + + + +### Config lifecycle at runtime and training {id="config-lifecycle"} + +A pipeline's `config.cfg` is considered the "single source of truth", both at +**training** and **runtime**. Under the hood, +[`Language.from_config`](/api/language#from_config) takes care of constructing +the `nlp` object using the settings defined in the config. An `nlp` object's +config is available as [`nlp.config`](/api/language#config) and it includes all +information about the pipeline, as well as the settings used to train and +initialize it. + +![Illustration of pipeline lifecycle](/images/lifecycle.svg) + +At runtime spaCy will only use the `[nlp]` and `[components]` blocks of the +config and load all data, including tokenization rules, model weights and other +resources from the pipeline directory. The `[training]` block contains the +settings for training the model and is only used during training. Similarly, the +`[initialize]` block defines how the initial `nlp` object should be set up +before training and whether it should be initialized with vectors or pretrained +tok2vec weights, or any other data needed by the components. + +The initialization settings are only loaded and used when +[`nlp.initialize`](/api/language#initialize) is called (typically right before +training). This allows you to set up your pipeline using local data resources +and custom functions, and preserve the information in your config – but without +requiring it to be available at runtime. You can also use this mechanism to +provide data paths to custom pipeline components and custom tokenizers – see the +section on [custom initialization](#initialization) for details. + +### Overwriting config settings on the command line {id="config-overrides"} + +The config system means that you can define all settings **in one place** and in +a consistent format. There are no command-line arguments that need to be set, +and no hidden defaults. However, there can still be scenarios where you may want +to override config settings when you run [`spacy train`](/api/cli#train). This +includes **file paths** to vectors or other resources that shouldn't be +hard-coded in a config file, or **system-dependent settings**. + +For cases like this, you can set additional command-line options starting with +`--` that correspond to the config section and value to override. For example, +`--paths.train ./corpus/train.spacy` sets the `train` value in the `[paths]` +block. + +```bash +$ python -m spacy train config.cfg --paths.train ./corpus/train.spacy --paths.dev ./corpus/dev.spacy --training.max_epochs 3 +``` + +Only existing sections and values in the config can be overwritten. At the end +of the training, the final filled `config.cfg` is exported with your pipeline, +so you'll always have a record of the settings that were used, including your +overrides. Overrides are added before [variables](#config-interpolation) are +resolved, by the way – so if you need to use a value in multiple places, +reference it across your config and override it on the CLI once. + +> #### 💡 Tip: Verbose logging +> +> If you're using config overrides, you can set the `--verbose` flag on +> [`spacy train`](/api/cli#train) to make spaCy log more info, including which +> overrides were set via the CLI and environment variables. + +#### Adding overrides via environment variables {id="config-overrides-env"} + +Instead of defining the overrides as CLI arguments, you can also use the +`SPACY_CONFIG_OVERRIDES` environment variable using the same argument syntax. +This is especially useful if you're training models as part of an automated +process. Environment variables **take precedence** over CLI overrides and values +defined in the config file. + +```bash +$ SPACY_CONFIG_OVERRIDES="--system.gpu_allocator pytorch --training.max_epochs 3" ./your_script.sh +``` + +### Reading from standard input {id="config-stdin"} + +Setting the config path to `-` on the command line lets you read the config from +standard input and pipe it forward from a different process, like +[`init config`](/api/cli#init-config) or your own custom script. This is +especially useful for quick experiments, as it lets you generate a config on the +fly without having to save to and load from disk. + +> #### 💡 Tip: Writing to stdout +> +> When you run `init config`, you can set the output path to `-` to write to +> stdout. In a custom script, you can print the string config, e.g. +> `print(nlp.config.to_str())`. + +```bash +$ python -m spacy init config - --lang en --pipeline ner,textcat --optimize accuracy | python -m spacy train - --paths.train ./corpus/train.spacy --paths.dev ./corpus/dev.spacy +``` + +### Using variable interpolation {id="config-interpolation"} + +Another very useful feature of the config system is that it supports variable +interpolation for both **values and sections**. This means that you only need to +define a setting once and can reference it across your config using the +`${section.value}` syntax. In this example, the value of `seed` is reused within +the `[training]` block, and the whole block of `[training.optimizer]` is reused +in `[pretraining]` and will become `pretraining.optimizer`. + +```ini {title="config.cfg (excerpt)",highlight="5,18"} +[system] +seed = 0 + +[training] +seed = ${system.seed} + +[training.optimizer] +@optimizers = "Adam.v1" +beta1 = 0.9 +beta2 = 0.999 +L2_is_weight_decay = true +L2 = 0.01 +grad_clip = 1.0 +use_averages = false +eps = 1e-8 + +[pretraining] +optimizer = ${training.optimizer} +``` + +You can also use variables inside strings. In that case, it works just like +f-strings in Python. If the value of a variable is not a string, it's converted +to a string. + +```ini +[paths] +version = 5 +root = "/Users/you/data" +train = "${paths.root}/train_${paths.version}.spacy" +# Result: /Users/you/data/train_5.spacy +``` + + + +If you need to change certain values between training runs, you can define them +once, reference them as variables and then [override](#config-overrides) them on +the CLI. For example, `--paths.root /other/root` will change the value of `root` +in the block `[paths]` and the change will be reflected across all other values +that reference this variable. + + + +## Preparing Training Data {id="training-data"} + +Training data for NLP projects comes in many different formats. For some common +formats such as CoNLL, spaCy provides [converters](/api/cli#convert) you can use +from the command line. In other cases you'll have to prepare the training data +yourself. + +When converting training data for use in spaCy, the main thing is to create +[`Doc`](/api/doc) objects just like the results you want as output from the +pipeline. For example, if you're creating an NER pipeline, loading your +annotations and setting them as the `.ents` property on a `Doc` is all you need +to worry about. On disk the annotations will be saved as a +[`DocBin`](/api/docbin) in the +[`.spacy` format](/api/data-formats#binary-training), but the details of that +are handled automatically. + +Here's an example of creating a `.spacy` file from some NER annotations. + +```python {title="preprocess.py"} +import spacy +from spacy.tokens import DocBin + +nlp = spacy.blank("en") +training_data = [ + ("Tokyo Tower is 333m tall.", [(0, 11, "BUILDING")]), +] +# the DocBin will store the example documents +db = DocBin() +for text, annotations in training_data: + doc = nlp(text) + ents = [] + for start, end, label in annotations: + span = doc.char_span(start, end, label=label) + ents.append(span) + doc.ents = ents + db.add(doc) +db.to_disk("./train.spacy") +``` + +For more examples of how to convert training data from a wide variety of formats +for use with spaCy, look at the preprocessing steps in the +[tutorial projects](https://github.com/explosion/projects/tree/v3/tutorials). + + + +In spaCy v2, the recommended way to store training data was in +[a particular JSON format](/api/data-formats#json-input), but in v3 this format +is deprecated. It's fine as a readable storage format, but there's no need to +convert your data to JSON before creating a `.spacy` file. + + + +## Customizing the pipeline and training {id="config-custom"} + +### Defining pipeline components {id="config-components"} + +You typically train a [pipeline](/usage/processing-pipelines) of **one or more +components**. The `[components]` block in the config defines the available +pipeline components and how they should be created – either by a built-in or +custom [factory](/usage/processing-pipelines#built-in), or +[sourced](/usage/processing-pipelines#sourced-components) from an existing +trained pipeline. For example, `[components.parser]` defines the component named +`"parser"` in the pipeline. There are different ways you might want to treat +your components during training, and the most common scenarios are: + +1. Train a **new component** from scratch on your data. +2. Update an existing **trained component** with more examples. +3. Include an existing trained component without updating it. +4. Include a non-trainable component, like a rule-based + [`EntityRuler`](/api/entityruler) or [`Sentencizer`](/api/sentencizer), or a + fully [custom component](/usage/processing-pipelines#custom-components). + +If a component block defines a `factory`, spaCy will look it up in the +[built-in](/usage/processing-pipelines#built-in) or +[custom](/usage/processing-pipelines#custom-components) components and create a +new component from scratch. All settings defined in the config block will be +passed to the component factory as arguments. This lets you configure the model +settings and hyperparameters. If a component block defines a `source`, the +component will be copied over from an existing trained pipeline, with its +existing weights. This lets you include an already trained component in your +pipeline, or update a trained component with more data specific to your use +case. + +```ini {title="config.cfg (excerpt)"} +[components] + +# "parser" and "ner" are sourced from a trained pipeline +[components.parser] +source = "en_core_web_sm" + +[components.ner] +source = "en_core_web_sm" + +# "textcat" and "custom" are created blank from a built-in / custom factory +[components.textcat] +factory = "textcat" + +[components.custom] +factory = "your_custom_factory" +your_custom_setting = true +``` + +The `pipeline` setting in the `[nlp]` block defines the pipeline components +added to the pipeline, in order. For example, `"parser"` here references +`[components.parser]`. By default, spaCy will **update all components that can +be updated**. Trainable components that are created from scratch are initialized +with random weights. For sourced components, spaCy will keep the existing +weights and [resume training](/api/language#resume_training). + +If you don't want a component to be updated, you can **freeze** it by adding it +to the `frozen_components` list in the `[training]` block. Frozen components are +**not updated** during training and are included in the final trained pipeline +as-is. They are also excluded when calling +[`nlp.initialize`](/api/language#initialize). + +> #### Note on frozen components +> +> Even though frozen components are not **updated** during training, they will +> still **run** during evaluation. This is very important, because they may +> still impact your model's performance – for instance, a sentence boundary +> detector can impact what the parser or entity recognizer considers a valid +> parse. So the evaluation results should always reflect what your pipeline will +> produce at runtime. If you want a frozen component to run (without updating) +> during training as well, so that downstream components can use its +> **predictions**, you should add it to the list of +> [`annotating_components`](/usage/training#annotating-components). + +```ini +[nlp] +lang = "en" +pipeline = ["parser", "ner", "textcat", "custom"] + +[training] +frozen_components = ["parser", "custom"] +``` + + + +When the components in your pipeline +[share an embedding layer](/usage/embeddings-transformers#embedding-layers), the +**performance** of your frozen component will be **degraded** if you continue +training other layers with the same underlying `Tok2Vec` instance. As a rule of +thumb, ensure that your frozen components are truly **independent** in the +pipeline. + +To automatically replace a shared token-to-vector listener with an independent +copy of the token-to-vector layer, you can use the `replace_listeners` setting +of a sourced component, pointing to the listener layer(s) in the config. For +more details on how this works under the hood, see +[`Language.replace_listeners`](/api/language#replace_listeners). + +```ini +[training] +frozen_components = ["tagger"] + +[components.tagger] +source = "en_core_web_sm" +replace_listeners = ["model.tok2vec"] +``` + + + +### Using predictions from preceding components {id="annotating-components",version="3.1"} + +By default, components are updated in isolation during training, which means +that they don't see the predictions of any earlier components in the pipeline. A +component receives [`Example.predicted`](/api/example) as input and compares its +predictions to [`Example.reference`](/api/example) without saving its +annotations in the `predicted` doc. + +Instead, if certain components should **set their annotations** during training, +use the setting `annotating_components` in the `[training]` block to specify a +list of components. For example, the feature `DEP` from the parser could be used +as a tagger feature by including `DEP` in the tok2vec `attrs` and including +`parser` in `annotating_components`: + +```ini {title="config.cfg (excerpt)",highlight="7,12"} +[nlp] +pipeline = ["parser", "tagger"] + +[components.tagger.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = ${components.tagger.model.tok2vec.encode.width} +attrs = ["NORM","DEP"] +rows = [5000,2500] +include_static_vectors = false + +[training] +annotating_components = ["parser"] +``` + +Any component in the pipeline can be included as an annotating component, +including frozen components. Frozen components can set annotations during +training just as they would set annotations during evaluation or when the final +pipeline is run. The config excerpt below shows how a frozen `ner` component and +a `sentencizer` can provide the required `doc.sents` and `doc.ents` for the +entity linker during training: + +```ini {title="config.cfg (excerpt)"} +[nlp] +pipeline = ["sentencizer", "ner", "entity_linker"] + +[components.ner] +source = "en_core_web_sm" + +[training] +frozen_components = ["ner"] +annotating_components = ["sentencizer", "ner"] +``` + +Similarly, a pretrained `tok2vec` layer can be frozen and specified in the list +of `annotating_components` to ensure that a downstream component can use the +embedding layer without updating it. + + + +Be aware that non-frozen annotating components with statistical models will +**run twice** on each batch, once to update the model and once to apply the +now-updated model to the predicted docs. + + + +### Using registered functions {id="config-functions"} + +The training configuration defined in the config file doesn't have to only +consist of static values. Some settings can also be **functions**. For instance, +the batch size can be a number that doesn't change, or a schedule, like a +sequence of compounding values, which has shown to be an effective trick (see +[Smith et al., 2017](https://arxiv.org/abs/1711.00489)). + +```ini {title="With static value"} +[training.batcher] +@batchers = "spacy.batch_by_words.v1" +size = 3000 +``` + +To refer to a function instead, you can make `[training.batcher.size]` its own +section and use the `@` syntax to specify the function and its arguments – in +this case [`compounding.v1`](https://thinc.ai/docs/api-schedules#compounding) +defined in the [function registry](/api/top-level#registry). All other values +defined in the block are passed to the function as keyword arguments when it's +initialized. You can also use this mechanism to register +[custom implementations and architectures](#custom-functions) and reference them +from your configs. + +> #### How the config is resolved +> +> The config file is parsed into a regular dictionary and is resolved and +> validated **bottom-up**. Arguments provided for registered functions are +> checked against the function's signature and type annotations. The return +> value of a registered function can also be passed into another function – for +> instance, a learning rate schedule can be provided as the an argument of an +> optimizer. + +```ini {title="With registered function"} +[training.batcher.size] +@schedules = "compounding.v1" +start = 100 +stop = 1000 +compound = 1.001 +``` + +### Model architectures {id="model-architectures"} + +> #### 💡 Model type annotations +> +> In the documentation and code base, you may come across type annotations and +> descriptions of [Thinc](https://thinc.ai) model types, like ~~Model[List[Doc], +> List[Floats2d]]~~. This so-called generic type describes the layer and its +> input and output type – in this case, it takes a list of `Doc` objects as the +> input and list of 2-dimensional arrays of floats as the output. You can read +> more about defining Thinc models [here](https://thinc.ai/docs/usage-models). +> Also see the [type checking](https://thinc.ai/docs/usage-type-checking) for +> how to enable linting in your editor to see live feedback if your inputs and +> outputs don't match. + +A **model architecture** is a function that wires up a Thinc +[`Model`](https://thinc.ai/docs/api-model) instance, which you can then use in a +component or as a layer of a larger network. You can use Thinc as a thin +[wrapper around frameworks](https://thinc.ai/docs/usage-frameworks) such as +PyTorch, TensorFlow or MXNet, or you can implement your logic in Thinc +[directly](https://thinc.ai/docs/usage-models). For more details and examples, +see the usage guide on [layers and architectures](/usage/layers-architectures). + +spaCy's built-in components will never construct their `Model` instances +themselves, so you won't have to subclass the component to change its model +architecture. You can just **update the config** so that it refers to a +different registered function. Once the component has been created, its `Model` +instance has already been assigned, so you cannot change its model architecture. +The architecture is like a recipe for the network, and you can't change the +recipe once the dish has already been prepared. You have to make a new one. +spaCy includes a variety of built-in [architectures](/api/architectures) for +different tasks. For example: + +| Architecture | Description | +| ----------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [HashEmbedCNN](/api/architectures#HashEmbedCNN) | Build spaCy’s "standard" embedding layer, which uses hash embedding with subword features and a CNN with layer-normalized maxout. ~~Model[List[Doc], List[Floats2d]]~~ | +| [TransitionBasedParser](/api/architectures#TransitionBasedParser) | Build a [transition-based parser](https://explosion.ai/blog/parsing-english-in-python) model used in the default [`EntityRecognizer`](/api/entityrecognizer) and [`DependencyParser`](/api/dependencyparser). ~~Model[List[Docs], List[List[Floats2d]]]~~ | +| [TextCatEnsemble](/api/architectures#TextCatEnsemble) | Stacked ensemble of a bag-of-words model and a neural network model with an internal CNN embedding layer. Used in the default [`TextCategorizer`](/api/textcategorizer). ~~Model[List[Doc], Floats2d]~~ | + +### Metrics, training output and weighted scores {id="metrics"} + +When you train a pipeline using the [`spacy train`](/api/cli#train) command, +you'll see a table showing the metrics after each pass over the data. The +available metrics **depend on the pipeline components**. Pipeline components +also define which scores are shown and how they should be **weighted in the +final score** that decides about the best model. + +The `training.score_weights` setting in your `config.cfg` lets you customize the +scores shown in the table and how they should be weighted. In this example, the +labeled dependency accuracy and NER F-score count towards the final score with +40% each and the tagging accuracy makes up the remaining 20%. The tokenization +accuracy and speed are both shown in the table, but not counted towards the +score. + +> #### Why do I need score weights? +> +> At the end of your training process, you typically want to select the **best +> model** – but what "best" means depends on the available components and your +> specific use case. For instance, you may prefer a pipeline with higher NER and +> lower POS tagging accuracy over a pipeline with lower NER and higher POS +> accuracy. You can express this preference in the score weights, e.g. by +> assigning `ents_f` (NER F-score) a higher weight. + +```ini +[training.score_weights] +dep_las = 0.4 +dep_uas = null +ents_f = 0.4 +tag_acc = 0.2 +token_acc = 0.0 +speed = 0.0 +``` + +The `score_weights` don't _have to_ sum to `1.0` – but it's recommended. When +you generate a config for a given pipeline, the score weights are generated by +combining and normalizing the default score weights of the pipeline components. +The default score weights are defined by each pipeline component via the +`default_score_weights` setting on the +[`@Language.factory`](/api/language#factory) decorator. By default, all pipeline +components are weighted equally. If a score weight is set to `null`, it will be +excluded from the logs and the score won't be weighted. + + + +| Name | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------- | +| **Loss** | The training loss representing the amount of work left for the optimizer. Should decrease, but usually not to `0`. | +| **Precision** (P) | Percentage of predicted annotations that were correct. Should increase. | +| **Recall** (R) | Percentage of reference annotations recovered. Should increase. | +| **F-Score** (F) | Harmonic mean of precision and recall. Should increase. | +| **UAS** / **LAS** | Unlabeled and labeled attachment score for the dependency parser, i.e. the percentage of correct arcs. Should increase. | +| **Speed** | Prediction speed in words per second (WPS). Should stay stable. | + +Note that if the development data has raw text, some of the gold-standard +entities might not align to the predicted tokenization. These tokenization +errors are **excluded from the NER evaluation**. If your tokenization makes it +impossible for the model to predict 50% of your entities, your NER F-score might +still look good. + + + +## Custom functions {id="custom-functions"} + +Registered functions in the training config files can refer to built-in +implementations, but you can also plug in fully **custom implementations**. All +you need to do is register your function using the `@spacy.registry` decorator +with the name of the respective [registry](/api/top-level#registry), e.g. +`@spacy.registry.architectures`, and a string name to assign to your function. +Registering custom functions allows you to **plug in models** defined in PyTorch +or TensorFlow, make **custom modifications** to the `nlp` object, create custom +optimizers or schedules, or **stream in data** and preprocess it on the fly +while training. + +Each custom function can have any number of arguments that are passed in via the +[config](#config), just the built-in functions. If your function defines +**default argument values**, spaCy is able to auto-fill your config when you run +[`init fill-config`](/api/cli#init-fill-config). If you want to make sure that a +given parameter is always explicitly set in the config, avoid setting a default +value for it. + +### Training with custom code {id="custom-code"} + +> ```bash +> ### Training +> $ python -m spacy train config.cfg --code functions.py +> ``` +> +> ```bash +> ### Packaging +> $ python -m spacy package ./model-best ./packages --code functions.py +> ``` + +The [`spacy train`](/api/cli#train) recipe lets you specify an optional argument +`--code` that points to a Python file. The file is imported before training and +allows you to add custom functions and architectures to the function registry +that can then be referenced from your `config.cfg`. This lets you train spaCy +pipelines with custom components, without having to re-implement the whole +training workflow. When you package your trained pipeline later using +[`spacy package`](/api/cli#package), you can provide one or more Python files to +be included in the package and imported in its `__init__.py`. This means that +any custom architectures, functions or +[components](/usage/processing-pipelines#custom-components) will be shipped with +your pipeline and registered when it's loaded. See the documentation on +[saving and loading pipelines](/usage/saving-loading#models-custom) for details. + + + +Note that the unpackaged models produced by `spacy train` are data directories +that **do not include custom code**. You need to import the code in your script +before loading in unpackaged models. For more details, see +[`spacy.load`](/api/top-level#spacy.load). + + + +#### Example: Modifying the nlp object {id="custom-code-nlp-callbacks"} + +For many use cases, you don't necessarily want to implement the whole `Language` +subclass and language data from scratch – it's often enough to make a few small +modifications, like adjusting the +[tokenization rules](/usage/linguistic-features#native-tokenizer-additions) or +[language defaults](/api/language#defaults) like stop words. The config lets you +provide five optional **callback functions** that give you access to the +language class and `nlp` object at different points of the lifecycle: + +| Callback | Description | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp.before_creation` | Called before the `nlp` object is created and receives the language subclass like `English` (not the instance). Useful for writing to the [`Language.Defaults`](/api/language#defaults) aside from the tokenizer settings. | +| `nlp.after_creation` | Called right after the `nlp` object is created, but before the pipeline components are added to the pipeline and receives the `nlp` object. | +| `nlp.after_pipeline_creation` | Called right after the pipeline components are created and added and receives the `nlp` object. Useful for modifying pipeline components. | +| `initialize.before_init` | Called before the pipeline components are initialized and receives the `nlp` object for in-place modification. Useful for modifying the tokenizer settings, similar to the v2 base model option. | +| `initialize.after_init` | Called after the pipeline components are initialized and receives the `nlp` object for in-place modification. | + +The `@spacy.registry.callbacks` decorator lets you register your custom function +in the `callbacks` [registry](/api/top-level#registry) under a given name. You +can then reference the function in a config block using the `@callbacks` key. If +a block contains a key starting with an `@`, it's interpreted as a reference to +a function. Because you've registered the function, spaCy knows how to create it +when you reference `"customize_language_data"` in your config. Here's an example +of a callback that runs before the `nlp` object is created and adds a custom +stop word to the defaults: + +> #### config.cfg +> +> ```ini +> [nlp.before_creation] +> @callbacks = "customize_language_data" +> ``` + +```python {title="functions.py",highlight="3,6"} +import spacy + +@spacy.registry.callbacks("customize_language_data") +def create_callback(): + def customize_language_data(lang_cls): + lang_cls.Defaults.stop_words.add("good") + return lang_cls + + return customize_language_data +``` + + + +Remember that a registered function should always be a function that spaCy +**calls to create something**. In this case, it **creates a callback** – it's +not the callback itself. + + + +Any registered function – in this case `create_callback` – can also take +**arguments** that can be **set by the config**. This lets you implement and +keep track of different configurations, without having to hack at your code. You +can choose any arguments that make sense for your use case. In this example, +we're adding the arguments `extra_stop_words` (a list of strings) and `debug` +(boolean) for printing additional info when the function runs. + +> #### config.cfg +> +> ```ini +> [nlp.before_creation] +> @callbacks = "customize_language_data" +> extra_stop_words = ["ooh", "aah"] +> debug = true +> ``` + +```python {title="functions.py",highlight="5,7-9"} +from typing import List +import spacy + +@spacy.registry.callbacks("customize_language_data") +def create_callback(extra_stop_words: List[str] = [], debug: bool = False): + def customize_language_data(lang_cls): + lang_cls.Defaults.stop_words.update(extra_stop_words) + if debug: + print("Updated stop words") + return lang_cls + + return customize_language_data +``` + + + +spaCy's configs are powered by our machine learning library Thinc's +[configuration system](https://thinc.ai/docs/usage-config), which supports +[type hints](https://docs.python.org/3/library/typing.html) and even +[advanced type annotations](https://thinc.ai/docs/usage-config#advanced-types) +using [`pydantic`](https://github.com/samuelcolvin/pydantic). If your registered +function provides type hints, the values that are passed in will be checked +against the expected types. For example, `debug: bool` in the example above will +ensure that the value received as the argument `debug` is a boolean. If the +value can't be coerced into a boolean, spaCy will raise an error. +`debug: pydantic.StrictBool` will force the value to be a boolean and raise an +error if it's not – for instance, if your config defines `1` instead of `true`. + + + +With your `functions.py` defining additional code and the updated `config.cfg`, +you can now run [`spacy train`](/api/cli#train) and point the argument `--code` +to your Python file. Before loading the config, spaCy will import the +`functions.py` module and your custom functions will be registered. + +```bash +$ python -m spacy train config.cfg --output ./output --code ./functions.py +``` + +#### Example: Modifying tokenizer settings {id="custom-tokenizer"} + +Use the `initialize.before_init` callback to modify the tokenizer settings when +training a new pipeline. Write a registered callback that modifies the tokenizer +settings and specify this callback in your config: + +> #### config.cfg +> +> ```ini +> [initialize] +> +> [initialize.before_init] +> @callbacks = "customize_tokenizer" +> ``` + +```python {title="functions.py"} +from spacy.util import registry, compile_suffix_regex + +@registry.callbacks("customize_tokenizer") +def make_customize_tokenizer(): + def customize_tokenizer(nlp): + # remove a suffix + suffixes = list(nlp.Defaults.suffixes) + suffixes.remove("\\[") + suffix_regex = compile_suffix_regex(suffixes) + nlp.tokenizer.suffix_search = suffix_regex.search + + # add a special case + nlp.tokenizer.add_special_case("_SPECIAL_", [{"ORTH": "_SPECIAL_"}]) + return customize_tokenizer +``` + +When training, provide the function above with the `--code` option: + +```bash +$ python -m spacy train config.cfg --code ./functions.py +``` + +Because this callback is only called in the one-time initialization step before +training, the callback code does not need to be packaged with the final pipeline +package. However, to make it easier for others to replicate your training setup, +you can choose to package the initialization callbacks with the pipeline package +or to publish them separately. + + + +- `nlp.before_creation` is the best place to modify language defaults other than + the tokenizer settings. +- `initialize.before_init` is the best place to modify tokenizer settings when + training a new pipeline. + +Unlike the other language defaults, the tokenizer settings are saved with the +pipeline with `nlp.to_disk()`, so modifications made in `nlp.before_creation` +will be clobbered by the saved settings when the trained pipeline is loaded from +disk. + + + +#### Example: Custom logging function {id="custom-logging"} + +During training, the results of each step are passed to a logger function. By +default, these results are written to the console with the +[`ConsoleLogger`](/api/top-level#ConsoleLogger). There is also built-in support +for writing the log files to [Weights & Biases](https://www.wandb.com/) with the +[`WandbLogger`](https://github.com/explosion/spacy-loggers#wandblogger). On each +step, the logger function receives a **dictionary** with the following keys: + +| Key | Value | +| -------------- | ----------------------------------------------------------------------------------------------------- | +| `epoch` | How many passes over the data have been completed. ~~int~~ | +| `step` | How many steps have been completed. ~~int~~ | +| `score` | The main score from the last evaluation, measured on the dev set. ~~float~~ | +| `other_scores` | The other scores from the last evaluation, measured on the dev set. ~~Dict[str, Any]~~ | +| `losses` | The accumulated training losses, keyed by component name. ~~Dict[str, float]~~ | +| `checkpoints` | A list of previous results, where each result is a `(score, step)` tuple. ~~List[Tuple[float, int]]~~ | + +You can easily implement and plug in your own logger that records the training +results in a custom way, or sends them to an experiment management tracker of +your choice. In this example, the function `my_custom_logger.v1` writes the +tabular results to a file: + +> ```ini +> ### config.cfg (excerpt) +> [training.logger] +> @loggers = "my_custom_logger.v1" +> log_path = "my_file.tab" +> ``` + +```python {title="functions.py"} +import sys +from typing import IO, Tuple, Callable, Dict, Any, Optional +import spacy +from spacy import Language +from pathlib import Path + +@spacy.registry.loggers("my_custom_logger.v1") +def custom_logger(log_path): + def setup_logger( + nlp: Language, + stdout: IO=sys.stdout, + stderr: IO=sys.stderr + ) -> Tuple[Callable, Callable]: + stdout.write(f"Logging to {log_path}\\n") + log_file = Path(log_path).open("w", encoding="utf8") + log_file.write("step\\t") + log_file.write("score\\t") + for pipe in nlp.pipe_names: + log_file.write(f"loss_{pipe}\\t") + log_file.write("\\n") + + def log_step(info: Optional[Dict[str, Any]]): + if info: + log_file.write(f"{info['step']}\\t") + log_file.write(f"{info['score']}\\t") + for pipe in nlp.pipe_names: + log_file.write(f"{info['losses'][pipe]}\\t") + log_file.write("\\n") + + def finalize(): + log_file.close() + + return log_step, finalize + + return setup_logger +``` + +#### Example: Custom batch size schedule {id="custom-code-schedule"} + +You can also implement your own batch size schedule to use during training. The +`@spacy.registry.schedules` decorator lets you register that function in the +`schedules` [registry](/api/top-level#registry) and assign it a string name: + +> #### Why the version in the name? +> +> A big benefit of the config system is that it makes your experiments +> reproducible. We recommend versioning the functions you register, especially +> if you expect them to change (like a new model architecture). This way, you +> know that a config referencing `v1` means a different function than a config +> referencing `v2`. + +```python {title="functions.py"} +import spacy + +@spacy.registry.schedules("my_custom_schedule.v1") +def my_custom_schedule(start: int = 1, factor: float = 1.001): + while True: + yield start + start = start * factor +``` + +In your config, you can now reference the schedule in the +`[training.batcher.size]` block via `@schedules`. If a block contains a key +starting with an `@`, it's interpreted as a reference to a function. All other +settings in the block will be passed to the function as keyword arguments. Keep +in mind that the config shouldn't have any hidden defaults and all arguments on +the functions need to be represented in the config. + +```ini {title="config.cfg (excerpt)"} +[training.batcher.size] +@schedules = "my_custom_schedule.v1" +start = 2 +factor = 1.005 +``` + +### Defining custom architectures {id="custom-architectures"} + +Built-in pipeline components such as the tagger or named entity recognizer are +constructed with default neural network [models](/api/architectures). You can +change the model architecture entirely by implementing your own custom models +and providing those in the config when creating the pipeline component. See the +documentation on [layers and model architectures](/usage/layers-architectures) +for more details. + +> ```ini +> ### config.cfg +> [components.tagger] +> factory = "tagger" +> +> [components.tagger.model] +> @architectures = "custom_neural_network.v1" +> output_width = 512 +> ``` + +```python {title="functions.py"} +from typing import List +from thinc.types import Floats2d +from thinc.api import Model +import spacy +from spacy.tokens import Doc + +@spacy.registry.architectures("custom_neural_network.v1") +def custom_neural_network(output_width: int) -> Model[List[Doc], List[Floats2d]]: + return create_model(output_width) +``` + +## Customizing the initialization {id="initialization"} + +When you start training a new model from scratch, +[`spacy train`](/api/cli#train) will call +[`nlp.initialize`](/api/language#initialize) to initialize the pipeline and load +the required data. All settings for this are defined in the +[`[initialize]`](/api/data-formats#config-initialize) block of the config, so +you can keep track of how the initial `nlp` object was created. The +initialization process typically includes the following: + +> #### config.cfg (excerpt) +> +> ```ini +> [initialize] +> vectors = ${paths.vectors} +> init_tok2vec = ${paths.init_tok2vec} +> +> [initialize.components] +> # Settings for components +> ``` + +1. Load in **data resources** defined in the `[initialize]` config, including + **word vectors** and + [pretrained](/usage/embeddings-transformers/#pretraining) **tok2vec + weights**. +2. Call the `initialize` methods of the tokenizer (if implemented, e.g. for + [Chinese](/usage/models#chinese)) and pipeline components with a callback to + access the training data, the current `nlp` object and any **custom + arguments** defined in the `[initialize]` config. +3. In **pipeline components**: if needed, use the data to + [infer missing shapes](/usage/layers-architectures#thinc-shape-inference) and + set up the label scheme if no labels are provided. Components may also load + other data like lookup tables or dictionaries. + +The initialization step allows the config to define **all settings** required +for the pipeline, while keeping a separation between settings and functions that +should only be used **before training** to set up the initial pipeline, and +logic and configuration that needs to be available **at runtime**. Without that +separation, it would be very difficult to use the same, reproducible config file +because the component settings required for training (load data from an external +file) wouldn't match the component settings required at runtime (load what's +included with the saved `nlp` object and don't depend on external file). + +![Illustration of pipeline lifecycle](/images/lifecycle.svg) + + + +For details and examples of how pipeline components can **save and load data +assets** like model weights or lookup tables, and how the component +initialization is implemented under the hood, see the usage guide on +[serializing and initializing component data](/usage/processing-pipelines#component-data-initialization). + + + +#### Initializing labels {id="initialization-labels"} + +Built-in pipeline components like the +[`EntityRecognizer`](/api/entityrecognizer) or +[`DependencyParser`](/api/dependencyparser) need to know their available labels +and associated internal meta information to initialize their model weights. +Using the `get_examples` callback provided on initialization, they're able to +**read the labels off the training data** automatically, which is very +convenient – but it can also slow down the training process to compute this +information on every run. + +The [`init labels`](/api/cli#init-labels) command lets you auto-generate JSON +files containing the label data for all supported components. You can then pass +in the labels in the `[initialize]` settings for the respective components to +allow them to initialize faster. + +> #### config.cfg +> +> ```ini +> [initialize.components.ner] +> +> [initialize.components.ner.labels] +> @readers = "spacy.read_labels.v1" +> path = "corpus/labels/ner.json +> ``` + +```bash +$ python -m spacy init labels config.cfg ./corpus --paths.train ./corpus/train.spacy +``` + +Under the hood, the command delegates to the `label_data` property of the +pipeline components, for instance +[`EntityRecognizer.label_data`](/api/entityrecognizer#label_data). + + + +The JSON format differs for each component and some components need additional +meta information about their labels. The format exported by +[`init labels`](/api/cli#init-labels) matches what the components need, so you +should always let spaCy **auto-generate the labels** for you. + + + +## Data utilities {id="data"} + +spaCy includes various features and utilities to make it easy to train models +using your own data, manage training and evaluation corpora, convert existing +annotations and configure data augmentation strategies for more robust models. + +### Converting existing corpora and annotations {id="data-convert"} + +If you have training data in a standard format like `.conll` or `.conllu`, the +easiest way to convert it for use with spaCy is to run +[`spacy convert`](/api/cli#convert) and pass it a file and an output directory. +By default, the command will pick the converter based on the file extension. + +```bash +$ python -m spacy convert ./train.gold.conll ./corpus +``` + +> #### 💡 Tip: Converting from Prodigy +> +> If you're using the [Prodigy](https://prodi.gy) annotation tool to create +> training data, you can run the +> [`data-to-spacy` command](https://prodi.gy/docs/recipes#data-to-spacy) to +> merge and export multiple datasets for use with +> [`spacy train`](/api/cli#train). Different types of annotations on the same +> text will be combined, giving you one corpus to train multiple components. + + + +Training workflows often consist of multiple steps, from preprocessing the data +all the way to packaging and deploying the trained model. +[spaCy projects](/usage/projects) let you define all steps in one file, manage +data assets, track changes and share your end-to-end processes with your team. + + + +The binary `.spacy` format is a serialized [`DocBin`](/api/docbin) containing +one or more [`Doc`](/api/doc) objects. It's extremely **efficient in storage**, +especially when packing multiple documents together. You can also create `Doc` +objects manually, so you can write your own custom logic to convert and store +existing annotations for use in spaCy. + +```python {title="Training data from Doc objects",highlight="6-9"} +import spacy +from spacy.tokens import Doc, DocBin + +nlp = spacy.blank("en") +docbin = DocBin() +words = ["Apple", "is", "looking", "at", "buying", "U.K.", "startup", "."] +spaces = [True, True, True, True, True, True, True, False] +ents = ["B-ORG", "O", "O", "O", "O", "B-GPE", "O", "O"] +doc = Doc(nlp.vocab, words=words, spaces=spaces, ents=ents) +docbin.add(doc) +docbin.to_disk("./train.spacy") +``` + +### Working with corpora {id="data-corpora"} + +> #### Example +> +> ```ini +> [corpora] +> +> [corpora.train] +> @readers = "spacy.Corpus.v1" +> path = ${paths.train} +> gold_preproc = false +> max_length = 0 +> limit = 0 +> augmenter = null +> +> [training] +> train_corpus = "corpora.train" +> ``` + +The [`[corpora]`](/api/data-formats#config-corpora) block in your config lets +you define **data resources** to use for training, evaluation, pretraining or +any other custom workflows. `corpora.train` and `corpora.dev` are used as +conventions within spaCy's default configs, but you can also define any other +custom blocks. Each section in the corpora config should resolve to a +[`Corpus`](/api/corpus) – for example, using spaCy's built-in +[corpus reader](/api/top-level#corpus-readers) that takes a path to a binary +`.spacy` file. The `train_corpus` and `dev_corpus` fields in the +[`[training]`](/api/data-formats#config-training) block specify where to find +the corpus in your config. This makes it easy to **swap out** different corpora +by only changing a single config setting. + +Instead of making `[corpora]` a block with multiple subsections for each portion +of the data, you can also use a single function that returns a dictionary of +corpora, keyed by corpus name, e.g. `"train"` and `"dev"`. This can be +especially useful if you need to split a single file into corpora for training +and evaluation, without loading the same file twice. + +By default, the training data is loaded into memory and shuffled before each +epoch. If the corpus is **too large to fit into memory** during training, stream +the corpus using a custom reader as described in the next section. + +### Custom data reading and batching {id="custom-code-readers-batchers"} + +Some use-cases require **streaming in data** or manipulating datasets on the +fly, rather than generating all data beforehand and storing it to disk. Instead +of using the built-in [`Corpus`](/api/corpus) reader, which uses static file +paths, you can create and register a custom function that generates +[`Example`](/api/example) objects. + +In the following example we assume a custom function `read_custom_data` which +loads or generates texts with relevant text classification annotations. Then, +small lexical variations of the input text are created before generating the +final [`Example`](/api/example) objects. The `@spacy.registry.readers` decorator +lets you register the function creating the custom reader in the `readers` +[registry](/api/top-level#registry) and assign it a string name, so it can be +used in your config. All arguments on the registered function become available +as **config settings** – in this case, `source`. + +> #### config.cfg +> +> ```ini +> [corpora.train] +> @readers = "corpus_variants.v1" +> source = "s3://your_bucket/path/data.csv" +> ``` + +```python {title="functions.py",highlight="7-8"} +from typing import Callable, Iterator, List +import spacy +from spacy.training import Example +from spacy.language import Language +import random + +@spacy.registry.readers("corpus_variants.v1") +def stream_data(source: str) -> Callable[[Language], Iterator[Example]]: + def generate_stream(nlp): + for text, cats in read_custom_data(source): + # Create a random variant of the example text + i = random.randint(0, len(text) - 1) + variant = text[:i] + text[i].upper() + text[i + 1:] + doc = nlp.make_doc(variant) + example = Example.from_dict(doc, {"cats": cats}) + yield example + + return generate_stream +``` + + + +Remember that a registered function should always be a function that spaCy +**calls to create something**. In this case, it **creates the reader function** +– it's not the reader itself. + + + +If the corpus is **too large to load into memory** or the corpus reader is an +**infinite generator**, use the setting `max_epochs = -1` to indicate that the +train corpus should be streamed. With this setting the train corpus is merely +streamed and batched, not shuffled, so any shuffling needs to be implemented in +the corpus reader itself. In the example below, a corpus reader that generates +sentences containing even or odd numbers is used with an unlimited number of +examples for the train corpus and a limited number of examples for the dev +corpus. The dev corpus should always be finite and fit in memory during the +evaluation step. `max_steps` and/or `patience` are used to determine when the +training should stop. + +> #### config.cfg +> +> ```ini +> [corpora.dev] +> @readers = "even_odd.v1" +> limit = 100 +> +> [corpora.train] +> @readers = "even_odd.v1" +> limit = -1 +> +> [training] +> max_epochs = -1 +> patience = 500 +> max_steps = 2000 +> ``` + +```python {title="functions.py"} +from typing import Callable, Iterable, Iterator +from spacy import util +import random +from spacy.training import Example +from spacy import Language + + +@util.registry.readers("even_odd.v1") +def create_even_odd_corpus(limit: int = -1) -> Callable[[Language], Iterable[Example]]: + return EvenOddCorpus(limit) + + +class EvenOddCorpus: + def __init__(self, limit): + self.limit = limit + + def __call__(self, nlp: Language) -> Iterator[Example]: + i = 0 + while i < self.limit or self.limit < 0: + r = random.randint(0, 1000) + cat = r % 2 == 0 + text = "This is sentence " + str(r) + yield Example.from_dict( + nlp.make_doc(text), {"cats": {"EVEN": cat, "ODD": not cat}} + ) + i += 1 +``` + +> #### config.cfg +> +> ```ini +> [initialize.components.textcat.labels] +> @readers = "spacy.read_labels.v1" +> path = "labels/textcat.json" +> require = true +> ``` + +If the train corpus is streamed, the initialize step peeks at the first 100 +examples in the corpus to find the labels for each component. If this isn't +sufficient, you'll need to [provide the labels](#initialization-labels) for each +component in the `[initialize]` block. [`init labels`](/api/cli#init-labels) can +be used to generate JSON files in the correct format, which you can extend with +the full label set. + +We can also customize the **batching strategy** by registering a new batcher +function in the `batchers` [registry](/api/top-level#registry). A batcher turns +a stream of items into a stream of batches. spaCy has several useful built-in +[batching strategies](/api/top-level#batchers) with customizable sizes, but it's +also easy to implement your own. For instance, the following function takes the +stream of generated [`Example`](/api/example) objects, and removes those which +have the same underlying raw text, to avoid duplicates within each batch. Note +that in a more realistic implementation, you'd also want to check whether the +annotations are the same. + +> #### config.cfg +> +> ```ini +> [training.batcher] +> @batchers = "filtering_batch.v1" +> size = 150 +> ``` + +```python {title="functions.py"} +from typing import Callable, Iterable, Iterator, List +import spacy +from spacy.training import Example + +@spacy.registry.batchers("filtering_batch.v1") +def filter_batch(size: int) -> Callable[[Iterable[Example]], Iterator[List[Example]]]: + def create_filtered_batches(examples): + batch = [] + for eg in examples: + # Remove duplicate examples with the same text from batch + if eg.text not in [x.text for x in batch]: + batch.append(eg) + if len(batch) == size: + yield batch + batch = [] + + return create_filtered_batches +``` + +{/* TODO: Custom corpus class, Minibatching */} + +### Data augmentation {id="data-augmentation"} + +Data augmentation is the process of applying small **modifications** to the +training data. It can be especially useful for punctuation and case replacement +– for example, if your corpus only uses smart quotes and you want to include +variations using regular quotes, or to make the model less sensitive to +capitalization by including a mix of capitalized and lowercase examples. + +The easiest way to use data augmentation during training is to provide an +`augmenter` to the training corpus, e.g. in the `[corpora.train]` section of +your config. The built-in [`orth_variants`](/api/top-level#orth_variants) +augmenter creates a data augmentation callback that uses orth-variant +replacement. + +```ini {title="config.cfg (excerpt)",highlight="8,14"} +[corpora.train] +@readers = "spacy.Corpus.v1" +path = ${paths.train} +gold_preproc = false +max_length = 0 +limit = 0 + +[corpora.train.augmenter] +@augmenters = "spacy.orth_variants.v1" +# Percentage of texts that will be augmented / lowercased +level = 0.1 +lower = 0.5 + +[corpora.train.augmenter.orth_variants] +@readers = "srsly.read_json.v1" +path = "corpus/orth_variants.json" +``` + +The `orth_variants` argument lets you pass in a dictionary of replacement rules, +typically loaded from a JSON file. There are two types of orth variant rules: +`"single"` for single tokens that should be replaced (e.g. hyphens) and +`"paired"` for pairs of tokens (e.g. quotes). + +```json {title="orth_variants.json"} +{ + "single": [{ "tags": ["NFP"], "variants": ["…", "..."] }], + "paired": [ + { + "tags": ["``", "''"], + "variants": [ + ["'", "'"], + ["‘", "’"] + ] + } + ] +} +``` + + + +```json +https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/en_orth_variants.json +``` + +```json +https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/de_orth_variants.json +``` + + + + + +When adding data augmentation, keep in mind that it typically only makes sense +to apply it to the **training corpus**, not the development data. + + + +#### Writing custom data augmenters {id="data-augmentation-custom"} + +Using the [`@spacy.augmenters`](/api/top-level#registry) registry, you can also +register your own data augmentation callbacks. The callback should be a function +that takes the current `nlp` object and a training [`Example`](/api/example) and +yields `Example` objects. Keep in mind that the augmenter should yield **all +examples** you want to use in your corpus, not only the augmented examples +(unless you want to augment all examples). + +Here'a an example of a custom augmentation callback that produces text variants +in ["SpOnGeBoB cAsE"](https://knowyourmeme.com/memes/mocking-spongebob). The +registered function takes one argument `randomize` that can be set via the +config and decides whether the uppercase/lowercase transformation is applied +randomly or not. The augmenter yields two `Example` objects: the original +example and the augmented example. + +> #### config.cfg +> +> ```ini +> [corpora.train.augmenter] +> @augmenters = "spongebob_augmenter.v1" +> randomize = false +> ``` + +```python +import spacy +import random + +@spacy.registry.augmenters("spongebob_augmenter.v1") +def create_augmenter(randomize: bool = False): + def augment(nlp, example): + text = example.text + if randomize: + # Randomly uppercase/lowercase characters + chars = [c.lower() if random.random() < 0.5 else c.upper() for c in text] + else: + # Uppercase followed by lowercase + chars = [c.lower() if i % 2 else c.upper() for i, c in enumerate(text)] + # Create augmented training example + example_dict = example.to_dict() + doc = nlp.make_doc("".join(chars)) + example_dict["token_annotation"]["ORTH"] = [t.text for t in doc] + # Original example followed by augmented example + yield example + yield example.from_dict(doc, example_dict) + + return augment +``` + +An easy way to create modified `Example` objects is to use the +[`Example.from_dict`](/api/example#from_dict) method with a new reference +[`Doc`](/api/doc) created from the modified text. In this case, only the +capitalization changes, so only the `ORTH` values of the tokens will be +different between the original and augmented examples. + +Note that if your data augmentation strategy involves changing the tokenization +(for instance, removing or adding tokens) and your training examples include +token-based annotations like the dependency parse or entity labels, you'll need +to take care to adjust the `Example` object so its annotations match and remain +valid. + +## Internal training API {id="api"} + + + +spaCy gives you full control over the training loop. However, for most use +cases, it's recommended to train your pipelines via the +[`spacy train`](/api/cli#train) command with a [`config.cfg`](#config) to keep +track of your settings and hyperparameters, instead of writing your own training +scripts from scratch. [Custom registered functions](#custom-code) should +typically give you everything you need to train fully custom pipelines with +[`spacy train`](/api/cli#train). + + + +### Training from a Python script {id="api-train",version="3.2"} + +If you want to run the training from a Python script instead of using the +[`spacy train`](/api/cli#train) CLI command, you can call into the +[`train`](/api/cli#train-function) helper function directly. It takes the path +to the config file, an optional output directory and an optional dictionary of +[config overrides](#config-overrides). + +```python +from spacy.cli.train import train + +train("./config.cfg", overrides={"paths.train": "./train.spacy", "paths.dev": "./dev.spacy"}) +``` + +### Internal training loop API {id="api-loop"} + + + +This section documents how the training loop and updates to the `nlp` object +work internally. You typically shouldn't have to implement this in Python unless +you're writing your own trainable components. To train a pipeline, use +[`spacy train`](/api/cli#train) or the [`train`](/api/cli#train-function) helper +function instead. + + + +The [`Example`](/api/example) object contains annotated training data, also +called the **gold standard**. It's initialized with a [`Doc`](/api/doc) object +that will hold the predictions, and another `Doc` object that holds the +gold-standard annotations. It also includes the **alignment** between those two +documents if they differ in tokenization. The `Example` class ensures that spaCy +can rely on one **standardized format** that's passed through the pipeline. For +instance, let's say we want to define gold-standard part-of-speech tags: + +```python +words = ["I", "like", "stuff"] +predicted = Doc(vocab, words=words) +# create the reference Doc with gold-standard TAG annotations +tags = ["NOUN", "VERB", "NOUN"] +tag_ids = [vocab.strings.add(tag) for tag in tags] +reference = Doc(vocab, words=words).from_array("TAG", numpy.array(tag_ids, dtype="uint64")) +example = Example(predicted, reference) +``` + +As this is quite verbose, there's an alternative way to create the reference +`Doc` with the gold-standard annotations. The function `Example.from_dict` takes +a dictionary with keyword arguments specifying the annotations, like `tags` or +`entities`. Using the resulting `Example` object and its gold-standard +annotations, the model can be updated to learn a sentence of three words with +their assigned part-of-speech tags. + +```python +words = ["I", "like", "stuff"] +tags = ["NOUN", "VERB", "NOUN"] +predicted = Doc(nlp.vocab, words=words) +example = Example.from_dict(predicted, {"tags": tags}) +``` + +Here's another example that shows how to define gold-standard named entities. +The letters added before the labels refer to the tags of the +[BILUO scheme](/usage/linguistic-features#updating-biluo) – `O` is a token +outside an entity, `U` a single entity unit, `B` the beginning of an entity, `I` +a token inside an entity and `L` the last token of an entity. + +```python +doc = Doc(nlp.vocab, words=["Facebook", "released", "React", "in", "2014"]) +example = Example.from_dict(doc, {"entities": ["U-ORG", "O", "U-TECHNOLOGY", "O", "U-DATE"]}) +``` + + + +As of v3.0, the [`Example`](/api/example) object replaces the `GoldParse` class. +It can be constructed in a very similar way – from a `Doc` and a dictionary of +annotations. For more details, see the +[migration guide](/usage/v3#migrating-training). + +```diff +- gold = GoldParse(doc, entities=entities) ++ example = Example.from_dict(doc, {"entities": entities}) +``` + + + +Of course, it's not enough to only show a model a single example once. +Especially if you only have few examples, you'll want to train for a **number of +iterations**. At each iteration, the training data is **shuffled** to ensure the +model doesn't make any generalizations based on the order of examples. Another +technique to improve the learning results is to set a **dropout rate**, a rate +at which to randomly "drop" individual features and representations. This makes +it harder for the model to memorize the training data. For example, a `0.25` +dropout means that each feature or internal representation has a 1/4 likelihood +of being dropped. + +> - [`nlp`](/api/language): The `nlp` object with the pipeline components and +> their models. +> - [`nlp.initialize`](/api/language#initialize): Initialize the pipeline and +> return an optimizer to update the component model weights. +> - [`Optimizer`](https://thinc.ai/docs/api-optimizers): Function that holds +> state between updates. +> - [`nlp.update`](/api/language#update): Update component models with examples. +> - [`Example`](/api/example): object holding predictions and gold-standard +> annotations. +> - [`nlp.to_disk`](/api/language#to_disk): Save the updated pipeline to a +> directory. + +```python {title="Example training loop"} +optimizer = nlp.initialize() +for itn in range(100): + random.shuffle(train_data) + for raw_text, entity_offsets in train_data: + doc = nlp.make_doc(raw_text) + example = Example.from_dict(doc, {"entities": entity_offsets}) + nlp.update([example], sgd=optimizer) +nlp.to_disk("/output") +``` + +The [`nlp.update`](/api/language#update) method takes the following arguments: + +| Name | Description | +| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | [`Example`](/api/example) objects. The `update` method takes a sequence of them, so you can batch up your training examples. | +| `drop` | Dropout rate. Makes it harder for the model to just memorize the data. | +| `sgd` | An [`Optimizer`](https://thinc.ai/docs/api-optimizers) object, which updates the model's weights. If not set, spaCy will create a new one and save it for further use. | + + + +As of v3.0, the [`Example`](/api/example) object replaces the `GoldParse` class +and the "simple training style" of calling `nlp.update` with a text and a +dictionary of annotations. Updating your code to use the `Example` object should +be very straightforward: you can call +[`Example.from_dict`](/api/example#from_dict) with a [`Doc`](/api/doc) and the +dictionary of annotations: + +```diff +text = "Facebook released React in 2014" +annotations = {"entities": ["U-ORG", "O", "U-TECHNOLOGY", "O", "U-DATE"]} ++ example = Example.from_dict(nlp.make_doc(text), annotations) +- nlp.update([text], [annotations]) ++ nlp.update([example]) +``` + + diff --git a/website/docs/usage/v2-1.mdx b/website/docs/usage/v2-1.mdx new file mode 100644 index 0000000..261525d --- /dev/null +++ b/website/docs/usage/v2-1.mdx @@ -0,0 +1,309 @@ +--- +title: What's New in v2.1 +teaser: New features, backwards incompatibilities and migration guide +menu: + - ['New Features', 'features'] + - ['Backwards Incompatibilities', 'incompat'] +--- + +## New Features {id="features",hidden="true"} + +spaCy v2.1 has focussed primarily on stability and performance, solidifying the +design changes introduced in [v2.0](/usage/v2). As well as smaller models, +faster runtime, and many bug fixes, v2.1 also introduces experimental support +for some exciting new NLP innovations. For the full changelog, see the +[release notes on GitHub](https://github.com/explosion/spaCy/releases/tag/v2.1.0). +For more details and a behind-the-scenes look at the new release, +[see our blog post](https://explosion.ai/blog/spacy-v2-1). + +### BERT/ULMFit/Elmo-style pre-training {id="pretraining",tag="experimental"} + +> #### Example +> +> ```bash +> $ python -m spacy pretrain ./raw_text.jsonl +> en_core_web_lg ./pretrained-model +> ``` + +spaCy v2.1 introduces a new CLI command, `spacy pretrain`, that can make your +models much more accurate. It's especially useful when you have **limited +training data**. The `spacy pretrain` command lets you use transfer learning to +initialize your models with information from raw text, using a language model +objective similar to the one used in Google's BERT system. We've taken +particular care to ensure that pretraining works well even with spaCy's small +default architecture sizes, so you don't have to compromise on efficiency to use +it. + + + +**API:** [`spacy pretrain`](/api/cli#pretrain) **Usage: ** +[Improving accuracy with transfer learning](/usage/training#transfer-learning) + + + +### Extended match pattern API {id="matcher-api"} + +> #### Example +> +> ```python +> # Matches "love cats" or "likes flowers" +> pattern1 = [{"LEMMA": {"IN": ["like", "love"]}}, {"POS": "NOUN"}] +> # Matches tokens of length >= 10 +> pattern2 = [{"LENGTH": {">=": 10}}] +> # Matches custom attribute with regex +> pattern3 = [{"_": {"country": {"REGEX": "^([Uu](\\.?|nited) ?[Ss](\\.?|tates)"}}}] +> ``` + +Instead of mapping to a single value, token patterns can now also map to a +**dictionary of properties**. For example, to specify that the value of a lemma +should be part of a list of values, or to set a minimum character length. It now +also supports a `REGEX` property, as well as set membership via `IN` and +`NOT_IN`, custom extension attributes via `_` and rich comparison for numeric +values. + + + +**API:** [`Matcher`](/api/matcher) **Usage: ** +[Extended pattern syntax and attributes](/usage/rule-based-matching#adding-patterns-attributes-extended), +[Regular expressions](/usage/rule-based-matching#regex) + + + +### Easy rule-based entity recognition {id="entity-ruler"} + +> #### Example +> +> ```python +> from spacy.pipeline import EntityRuler +> ruler = EntityRuler(nlp) +> ruler.add_patterns([{"label": "ORG", "pattern": "Apple"}]) +> nlp.add_pipe(ruler, before="ner") +> ``` + +The `EntityRuler` is an exciting new component that lets you add named entities +based on pattern dictionaries, and makes it easy to combine rule-based and +statistical named entity recognition for even more powerful models. Entity rules +can be phrase patterns for exact string matches, or token patterns for full +flexibility. + + + +**API:** [`EntityRuler`](/api/entityruler) **Usage: ** +[Rule-based entity recognition](/usage/rule-based-matching#entityruler) + + + +### Phrase matching with other attributes {id="phrasematcher"} + +> #### Example +> +> ```python +> matcher = PhraseMatcher(nlp.vocab, attr="POS") +> matcher.add("PATTERN", None, nlp("I love cats")) +> doc = nlp("You like dogs") +> matches = matcher(doc) +> ``` + +By default, the `PhraseMatcher` will match on the verbatim token text, e.g. +`Token.text`. By setting the `attr` argument on initialization, you can change +**which token attribute the matcher should use** when comparing the phrase +pattern to the matched `Doc`. For example, `LOWER` for case-insensitive matches +or `POS` for finding sequences of the same part-of-speech tags. + + + +**API:** [`PhraseMatcher`](/api/phrasematcher) **Usage: ** +[Matching on other token attributes](/usage/rule-based-matching#phrasematcher-attrs) + + + +### Retokenizer for merging and splitting {id="retokenizer"} + +> #### Example +> +> ```python +> doc = nlp("I like David Bowie") +> with doc.retokenize() as retokenizer: +> attrs = {"LEMMA": "David Bowie"} +> retokenizer.merge(doc[2:4], attrs=attrs) +> ``` + +The new `Doc.retokenize` context manager allows merging spans of multiple tokens +into one single token, and splitting single tokens into multiple tokens. +Modifications to the `Doc`'s tokenization are stored, and then made all at once +when the context manager exits. This is much more efficient, and less +error-prone. `Doc.merge` and `Span.merge` still work, but they're considered +deprecated. + + + +**API:** [`Doc.retokenize`](/api/doc#retokenize), +[`Retokenizer.merge`](/api/doc#retokenizer.merge), +[`Retokenizer.split`](/api/doc#retokenizer.split)
**Usage: +**[Merging and splitting](/usage/linguistic-features#retokenization) + + + +### Components and languages via entry points {id="entry-points"} + +> #### Example +> +> ```python +> from setuptools import setup +> setup( +> name="custom_extension_package", +> entry_points={ +> "spacy_factories": ["your_component = component:ComponentFactory"] +> "spacy_languages": ["xyz = language:XYZLanguage"] +> } +> ) +> ``` + +Using entry points, model packages and extension packages can now define their +own `"spacy_factories"` and `"spacy_languages"`, which will be added to the +built-in factories and languages. If a package in the same environment exposes +spaCy entry points, all of this happens automatically and no further user action +is required. + + + +**Usage:** [Using entry points](/usage/saving-loading#entry-points) + + + +### Improved documentation {id="docs"} + +Although it looks pretty much the same, we've rebuilt the entire documentation +using [Gatsby](https://www.gatsbyjs.org/) and [MDX](https://mdxjs.com/). It's +now an even faster progressive web app and allows us to write all content +entirely **in Markdown**, without having to compromise on easy-to-use custom UI +components. We're hoping that the Markdown source will make it even easier to +contribute to the documentation. For more details, check out the +[styleguide](/styleguide) and +[source](https://github.com/explosion/spacy/tree/v2.x/website). While converting +the pages to Markdown, we've also fixed a bunch of typos, improved the existing +pages and added some new content: + +- **Usage Guide:** [Rule-based Matching](/usage/rule-based-matching)
How to + use the `Matcher`, `PhraseMatcher` and the new `EntityRuler`, and write + powerful components to combine statistical models and rules. +- **Usage Guide:** [Saving and Loading](/usage/saving-loading)
Everything + you need to know about serialization, and how to save and load pipeline + components, package your spaCy models as Python modules and use entry points. +- **Usage Guide: ** + [Merging and Splitting](/usage/linguistic-features#retokenization)
How to + retokenize a `Doc` using the new `retokenize` context manager and merge spans + into single tokens and split single tokens into multiple. +- **Universe:** [Videos](/universe/category/videos) and + [Podcasts](/universe/category/podcasts) +- **API:** [`EntityRuler`](/api/entityruler) +- **API:** [`Sentencizer`](/api/sentencizer) +- **API:** [Pipeline functions](/api/pipeline-functions) + +## Backwards incompatibilities {id="incompat"} + + + +If you've been training **your own models**, you'll need to **retrain** them +with the new version. Also don't forget to upgrade all models to the latest +versions. Models for v2.0.x aren't compatible with models for v2.1.x. To check +if all of your models are up to date, you can run the +[`spacy validate`](/api/cli#validate) command. + + + +- Due to difficulties linking our new + [`blis`](https://github.com/explosion/cython-blis) for faster + platform-independent matrix multiplication, this release currently **doesn't + work on Python 2.7 on Windows**. We expect this to be corrected in the future. + +- While the [`Matcher`](/api/matcher) API is fully backwards compatible, its + algorithm has changed to fix a number of bugs and performance issues. This + means that the `Matcher` in v2.1.x may produce different results compared to + the `Matcher` in v2.0.x. + +- The deprecated [`Doc.merge`](/api/doc#merge) and + [`Span.merge`](/api/span#merge) methods still work, but you may notice that + they now run slower when merging many objects in a row. That's because the + merging engine was rewritten to be more reliable and to support more efficient + merging **in bulk**. To take advantage of this, you should rewrite your logic + to use the [`Doc.retokenize`](/api/doc#retokenize) context manager and perform + as many merges as possible together in the `with` block. + + ```diff + - doc[1:5].merge() + - doc[6:8].merge() + + with doc.retokenize() as retokenizer: + + retokenizer.merge(doc[1:5]) + + retokenizer.merge(doc[6:8]) + ``` + +- The serialization methods `to_disk`, `from_disk`, `to_bytes` and `from_bytes` + now support a single `exclude` argument to provide a list of string names to + exclude. The docs have been updated to list the available serialization fields + for each class. The `disable` argument on the [`Language`](/api/language) + serialization methods has been renamed to `exclude` for consistency. + + ```diff + - nlp.to_disk("/path", disable=["parser", "ner"]) + + nlp.to_disk("/path", exclude=["parser", "ner"]) + - data = nlp.tokenizer.to_bytes(vocab=False) + + data = nlp.tokenizer.to_bytes(exclude=["vocab"]) + ``` + +- The .pos value for several common English words has changed, due to + corrections to long-standing mistakes in the English tag map (see + [issue #593](https://github.com/explosion/spaCy/issues/593) and + [issue #3311](https://github.com/explosion/spaCy/issues/3311) for details). + +- For better compatibility with the Universal Dependencies data, the lemmatizer + now preserves capitalization, e.g. for proper nouns. See + [issue #3256](https://github.com/explosion/spaCy/issues/3256) for details. + +- The built-in rule-based sentence boundary detector is now only called + `"sentencizer"` – the name `"sbd"` is deprecated. + + ```diff + - sentence_splitter = nlp.create_pipe("sbd") + + sentence_splitter = nlp.create_pipe("sentencizer") + ``` + +- The `is_sent_start` attribute of the first token in a `Doc` now correctly + defaults to `True`. It previously defaulted to `None`. + +- The keyword argument `n_threads` on the `.pipe` methods is now deprecated, as + the v2.x models cannot release the global interpreter lock. (Future versions + may introduce a `n_process` argument for parallel inference via + multiprocessing.) + +- The `Doc.print_tree` method is now deprecated. If you need a custom nested + JSON representation of a `Doc` object, you might want to write your own helper + function. For a simple and consistent JSON representation of the `Doc` object + and its annotations, you can now use the [`Doc.to_json`](/api/doc#to_json) + method. Going forward, this method will output the same format as the JSON + training data expected by [`spacy train`](/api/cli#train). + +- The [`spacy train`](/api/cli#train) command now lets you specify a + comma-separated list of pipeline component names, instead of separate flags + like `--no-parser` to disable components. This is more flexible and also + handles custom components out-of-the-box. + + ```diff + - $ spacy train en /output train_data.json dev_data.json --no-parser + + $ spacy train en /output train_data.json dev_data.json --pipeline tagger,ner + ``` + +- The [`spacy init-model`](/api/cli#init-model) command now uses a `--jsonl-loc` + argument to pass in a a newline-delimited JSON (JSONL) file containing one + lexical entry per line instead of a separate `--freqs-loc` and + `--clusters-loc`. + + ```diff + - $ spacy init-model en ./model --freqs-loc ./freqs.txt --clusters-loc ./clusters.txt + + $ spacy init-model en ./model --jsonl-loc ./vocab.jsonl + ``` + +- Also note that some of the model licenses have changed: + [`it_core_news_sm`](/models/it#it_core_news_sm) is now correctly licensed + under CC BY-NC-SA 3.0, and all [English](/models/en) and [German](/models/de) + models are now published under the MIT license. diff --git a/website/docs/usage/v2-2.mdx b/website/docs/usage/v2-2.mdx new file mode 100644 index 0000000..cf4f7c5 --- /dev/null +++ b/website/docs/usage/v2-2.mdx @@ -0,0 +1,455 @@ +--- +title: What's New in v2.2 +teaser: New features, backwards incompatibilities and migration guide +menu: + - ['New Features', 'features'] + - ['Backwards Incompatibilities', 'incompat'] + - ['Migrating from v2.1', 'migrating'] +--- + +## New Features {id="features",hidden="true"} + +spaCy v2.2 features improved statistical models, new pretrained models for +Norwegian and Lithuanian, better Dutch NER, as well as a new mechanism for +storing language data that makes the installation about **5-10× smaller** +on disk. We've also added a new class to efficiently **serialize annotations**, +an improved and **10× faster** phrase matching engine, built-in scoring +and **CLI training for text classification**, a new command to analyze and +**debug training data**, data augmentation during training and more. For the +full changelog, see the +[release notes on GitHub](https://github.com/explosion/spaCy/releases/tag/v2.2.0). + +For more details and a behind-the-scenes look at the new release, +[see our blog post](https://explosion.ai/blog/spacy-v2-2). + +### Better pretrained models and more languages {id="models"} + +> #### Example +> +> ```bash +> python -m spacy download nl_core_news_sm +> python -m spacy download nb_core_news_sm +> python -m spacy download lt_core_news_sm +> ``` + +The new version also features new and re-trained models for all languages and +resolves a number of data bugs. The [Dutch model](/models/nl) has been retrained +with a new and custom-labelled NER corpus using the same extended label scheme +as the English models. It should now produce significantly better NER results +overall. We've also added new core models for [Norwegian](/models/nb) (MIT) and +[Lithuanian](/models/lt) (CC BY-SA). + + + +**Usage:** [Models directory](/models) **Benchmarks: ** +[Release notes](https://github.com/explosion/spaCy/releases/tag/v2.2.0) + + + +### Text classification scores and CLI training {id="train-textcat-cli"} + +> #### Example +> +> ```bash +> $ python -m spacy train en /output /train /dev \\ +> --pipeline textcat --textcat-arch simple_cnn \\ +> --textcat-multilabel +> ``` + +When training your models using the `spacy train` command, you can now also +include text categories in the JSON-formatted training data. The `Scorer` and +`nlp.evaluate` now report the text classification scores, calculated as the +F-score on positive label for binary exclusive tasks, the macro-averaged F-score +for 3+ exclusive labels or the macro-averaged AUC ROC score for multilabel +classification. + + + +**API:** [`spacy train`](/api/cli#train), [`Scorer`](/api/scorer), +[`Language.evaluate`](/api/language#evaluate) + + + +### New DocBin class to efficiently serialize Doc collections + +> #### Example +> +> ```python +> from spacy.tokens import DocBin +> doc_bin = DocBin(attrs=["LEMMA", "ENT_IOB", "ENT_TYPE"], store_user_data=True) +> for doc in nlp.pipe(texts): +> doc_bin.add(doc) +> bytes_data = doc_bin.to_bytes() +> # Deserialize later, e.g. in a new process +> nlp = spacy.blank("en") +> doc_bin = DocBin().from_bytes(bytes_data) +> docs = list(doc_bin.get_docs(nlp.vocab)) +> ``` + +If you're working with lots of data, you'll probably need to pass analyses +between machines, either to use something like [Dask](https://dask.org) or +[Spark](https://spark.apache.org), or even just to save out work to disk. Often +it's sufficient to use the `Doc.to_array` functionality for this, and just +serialize the numpy arrays – but other times you want a more general way to save +and restore `Doc` objects. + +The new `DocBin` class makes it easy to serialize and deserialize a collection +of `Doc` objects together, and is much more efficient than calling +`Doc.to_bytes` on each individual `Doc` object. You can also control what data +gets saved, and you can merge pallets together for easy map/reduce-style +processing. + + + +**API:** [`DocBin`](/api/docbin) **Usage: ** +[Serializing Doc objects](/usage/saving-loading#docs) + + + +### Serializable lookup tables and smaller installation {id="lookups"} + +> #### Example +> +> ```python +> data = {"foo": "bar"} +> nlp.vocab.lookups.add_table("my_dict", data) +> +> def custom_component(doc): +> table = doc.vocab.lookups.get_table("my_dict") +> print(table.get("foo")) # look something up +> return doc +> ``` + +The new `Lookups` API lets you add large dictionaries and lookup tables to the +`Vocab` and access them from the tokenizer or custom components and extension +attributes. Internally, the tables use Bloom filters for efficient lookup +checks. They're also fully serializable out-of-the-box. All large data resources +like lemmatization tables have been moved to a separate package, +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) that can +be installed alongside the core library. This allowed us to make the spaCy +installation **5-10× smaller on disk** (depending on your platform). +[Pretrained models](/models) now include their data files, so you only need to +install the lookups if you want to build blank models or use lemmatization with +languages that don't yet ship with pretrained models. + + + +**API:** [`Lookups`](/api/lookups), +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) **Usage: +** [Adding languages: Lemmatizer](/usage/adding-languages#lemmatizer) + + + +### CLI command to debug and validate training data {id="debug-data"} + +> #### Example +> +> ```bash +> $ python -m spacy debug-data en train.json dev.json +> ``` + +The new `debug-data` command lets you analyze and validate your training and +development data, get useful stats, and find problems like invalid entity +annotations, cyclic dependencies, low data labels and more. If you're training a +model with `spacy train` and the results seem surprising or confusing, +`debug-data` may help you track down the problems and improve your training +data. + + + +``` +=========================== Data format validation =========================== +✔ Corpus is loadable + +=============================== Training stats =============================== +Training pipeline: tagger, parser, ner +Starting with blank model 'en' +18127 training docs +2939 evaluation docs +⚠ 34 training examples also in evaluation data + +============================== Vocab & Vectors ============================== +ℹ 2083156 total words in the data (56962 unique) +⚠ 13020 misaligned tokens in the training data +⚠ 2423 misaligned tokens in the dev data +10 most common words: 'the' (98429), ',' (91756), '.' (87073), 'to' (50058), +'of' (49559), 'and' (44416), 'a' (34010), 'in' (31424), 'that' (22792), 'is' +(18952) +ℹ No word vectors present in the model + +========================== Named Entity Recognition ========================== +ℹ 18 new labels, 0 existing labels +528978 missing values (tokens with '-' label) +New: 'ORG' (23860), 'PERSON' (21395), 'GPE' (21193), 'DATE' (18080), 'CARDINAL' +(10490), 'NORP' (9033), 'MONEY' (5164), 'PERCENT' (3761), 'ORDINAL' (2122), +'LOC' (2113), 'TIME' (1616), 'WORK_OF_ART' (1229), 'QUANTITY' (1150), 'FAC' +(1134), 'EVENT' (974), 'PRODUCT' (935), 'LAW' (444), 'LANGUAGE' (338) +✔ Good amount of examples for all labels +✔ Examples without occurrences available for all labels +✔ No entities consisting of or starting/ending with whitespace + +=========================== Part-of-speech Tagging =========================== +ℹ 49 labels in data (57 labels in tag map) +'NN' (266331), 'IN' (227365), 'DT' (185600), 'NNP' (164404), 'JJ' (119830), +'NNS' (110957), '.' (101482), ',' (92476), 'RB' (90090), 'PRP' (90081), 'VB' +(74538), 'VBD' (68199), 'CC' (62862), 'VBZ' (50712), 'VBP' (43420), 'VBN' +(42193), 'CD' (40326), 'VBG' (34764), 'TO' (31085), 'MD' (25863), 'PRP$' +(23335), 'HYPH' (13833), 'POS' (13427), 'UH' (13322), 'WP' (10423), 'WDT' +(9850), 'RP' (8230), 'WRB' (8201), ':' (8168), '''' (7392), '``' (6984), 'NNPS' +(5817), 'JJR' (5689), '$' (3710), 'EX' (3465), 'JJS' (3118), 'RBR' (2872), +'-RRB-' (2825), '-LRB-' (2788), 'PDT' (2078), 'XX' (1316), 'RBS' (1142), 'FW' +(794), 'NFP' (557), 'SYM' (440), 'WP$' (294), 'LS' (293), 'ADD' (191), 'AFX' +(24) +✔ All labels present in tag map for language 'en' + +============================= Dependency Parsing ============================= +ℹ Found 111703 sentences with an average length of 18.6 words. +ℹ Found 2251 nonprojective train sentences +ℹ Found 303 nonprojective dev sentences +ℹ 47 labels in train data +ℹ 211 labels in projectivized train data +'punct' (236796), 'prep' (188853), 'pobj' (182533), 'det' (172674), 'nsubj' +(169481), 'compound' (116142), 'ROOT' (111697), 'amod' (107945), 'dobj' (93540), +'aux' (86802), 'advmod' (86197), 'cc' (62679), 'conj' (59575), 'poss' (36449), +'ccomp' (36343), 'advcl' (29017), 'mark' (27990), 'nummod' (24582), 'relcl' +(21359), 'xcomp' (21081), 'attr' (18347), 'npadvmod' (17740), 'acomp' (17204), +'auxpass' (15639), 'appos' (15368), 'neg' (15266), 'nsubjpass' (13922), 'case' +(13408), 'acl' (12574), 'pcomp' (10340), 'nmod' (9736), 'intj' (9285), 'prt' +(8196), 'quantmod' (7403), 'dep' (4300), 'dative' (4091), 'agent' (3908), 'expl' +(3456), 'parataxis' (3099), 'oprd' (2326), 'predet' (1946), 'csubj' (1494), +'subtok' (1147), 'preconj' (692), 'meta' (469), 'csubjpass' (64), 'iobj' (1) +⚠ Low number of examples for label 'iobj' (1) +⚠ Low number of examples for 130 labels in the projectivized dependency +trees used for training. You may want to projectivize labels such as punct +before training in order to improve parser performance. +⚠ Projectivized labels with low numbers of examples: appos||attr: 12 +advmod||dobj: 13 prep||ccomp: 12 nsubjpass||ccomp: 15 pcomp||prep: 14 +amod||dobj: 9 attr||xcomp: 14 nmod||nsubj: 17 prep||advcl: 2 prep||prep: 5 +nsubj||conj: 12 advcl||advmod: 18 ccomp||advmod: 11 ccomp||pcomp: 5 acl||pobj: +10 npadvmod||acomp: 7 dobj||pcomp: 14 nsubjpass||pcomp: 1 nmod||pobj: 8 +amod||attr: 6 nmod||dobj: 12 aux||conj: 1 neg||conj: 1 dative||xcomp: 11 +pobj||dative: 3 xcomp||acomp: 19 advcl||pobj: 2 nsubj||advcl: 2 csubj||ccomp: 1 +advcl||acl: 1 relcl||nmod: 2 dobj||advcl: 10 advmod||advcl: 3 nmod||nsubjpass: 6 +amod||pobj: 5 cc||neg: 1 attr||ccomp: 16 advcl||xcomp: 3 nmod||attr: 4 +advcl||nsubjpass: 5 advcl||ccomp: 4 ccomp||conj: 1 punct||acl: 1 meta||acl: 1 +parataxis||acl: 1 prep||acl: 1 amod||nsubj: 7 ccomp||ccomp: 3 acomp||xcomp: 5 +dobj||acl: 5 prep||oprd: 6 advmod||acl: 2 dative||advcl: 1 pobj||agent: 5 +xcomp||amod: 1 dep||advcl: 1 prep||amod: 8 relcl||compound: 1 advcl||csubj: 3 +npadvmod||conj: 2 npadvmod||xcomp: 4 advmod||nsubj: 3 ccomp||amod: 7 +advcl||conj: 1 nmod||conj: 2 advmod||nsubjpass: 2 dep||xcomp: 2 appos||ccomp: 1 +advmod||dep: 1 advmod||advmod: 5 aux||xcomp: 8 dep||advmod: 1 dative||ccomp: 2 +prep||dep: 1 conj||conj: 1 dep||ccomp: 4 cc||ROOT: 1 prep||ROOT: 1 nsubj||pcomp: +3 advmod||prep: 2 relcl||dative: 1 acl||conj: 1 advcl||attr: 4 prep||npadvmod: 1 +nsubjpass||xcomp: 1 neg||advmod: 1 xcomp||oprd: 1 advcl||advcl: 1 dobj||dep: 3 +nsubjpass||parataxis: 1 attr||pcomp: 1 ccomp||parataxis: 1 advmod||attr: 1 +nmod||oprd: 1 appos||nmod: 2 advmod||relcl: 1 appos||npadvmod: 1 appos||conj: 1 +prep||expl: 1 nsubjpass||conj: 1 punct||pobj: 1 cc||pobj: 1 conj||pobj: 1 +punct||conj: 1 ccomp||dep: 1 oprd||xcomp: 3 ccomp||xcomp: 1 ccomp||nsubj: 1 +nmod||dep: 1 xcomp||ccomp: 1 acomp||advcl: 1 intj||advmod: 1 advmod||acomp: 2 +relcl||oprd: 1 advmod||prt: 1 advmod||pobj: 1 appos||nummod: 1 relcl||npadvmod: +3 mark||advcl: 1 aux||ccomp: 1 amod||nsubjpass: 1 npadvmod||advmod: 1 conj||dep: +1 nummod||pobj: 1 amod||npadvmod: 1 intj||pobj: 1 nummod||npadvmod: 1 +xcomp||xcomp: 1 aux||dep: 1 advcl||relcl: 1 +⚠ The following labels were found only in the train data: xcomp||amod, +advcl||relcl, prep||nsubjpass, acl||nsubj, nsubjpass||conj, xcomp||oprd, +advmod||conj, advmod||advmod, iobj, advmod||nsubjpass, dobj||conj, ccomp||amod, +meta||acl, xcomp||xcomp, prep||attr, prep||ccomp, advcl||acomp, acl||dobj, +advcl||advcl, pobj||agent, prep||advcl, nsubjpass||xcomp, prep||dep, +acomp||xcomp, aux||ccomp, ccomp||dep, conj||dep, relcl||compound, +nsubjpass||ccomp, nmod||dobj, advmod||advcl, advmod||acl, dobj||advcl, +dative||xcomp, prep||nsubj, ccomp||ccomp, nsubj||ccomp, xcomp||acomp, +prep||acomp, dep||advmod, acl||pobj, appos||dobj, npadvmod||acomp, cc||ROOT, +relcl||nsubj, nmod||pobj, acl||nsubjpass, ccomp||advmod, pcomp||prep, +amod||dobj, advmod||attr, advcl||csubj, appos||attr, dobj||pcomp, prep||ROOT, +relcl||pobj, advmod||pobj, amod||nsubj, ccomp||xcomp, prep||oprd, +npadvmod||advmod, appos||nummod, advcl||pobj, neg||advmod, acl||attr, +appos||nsubjpass, csubj||ccomp, amod||nsubjpass, intj||pobj, dep||advcl, +cc||neg, xcomp||ccomp, dative||ccomp, nmod||oprd, pobj||dative, prep||dobj, +dep||ccomp, relcl||attr, ccomp||nsubj, advcl||xcomp, nmod||dep, advcl||advmod, +ccomp||conj, pobj||prep, advmod||acomp, advmod||relcl, attr||pcomp, +ccomp||parataxis, oprd||xcomp, intj||advmod, nmod||nsubjpass, prep||npadvmod, +parataxis||acl, prep||pobj, advcl||dobj, amod||pobj, prep||acl, conj||pobj, +advmod||dep, punct||pobj, ccomp||acomp, acomp||advcl, nummod||npadvmod, +dobj||dep, npadvmod||xcomp, advcl||conj, relcl||npadvmod, punct||acl, +relcl||dobj, dobj||xcomp, nsubjpass||parataxis, dative||advcl, relcl||nmod, +advcl||ccomp, appos||npadvmod, ccomp||pcomp, prep||amod, mark||advcl, +prep||advmod, prep||xcomp, appos||nsubj, attr||ccomp, advmod||prt, dobj||ccomp, +aux||conj, advcl||nsubj, conj||conj, advmod||ccomp, advcl||nsubjpass, +attr||xcomp, nmod||conj, npadvmod||conj, relcl||dative, prep||expl, +nsubjpass||pcomp, advmod||xcomp, advmod||dobj, appos||pobj, nsubj||conj, +relcl||nsubjpass, advcl||attr, appos||ccomp, advmod||prep, prep||conj, +nmod||attr, punct||conj, neg||conj, dep||xcomp, aux||xcomp, dobj||acl, +nummod||pobj, amod||npadvmod, nsubj||pcomp, advcl||acl, appos||nmod, +relcl||oprd, prep||prep, cc||pobj, nmod||nsubj, amod||attr, aux||dep, +appos||conj, advmod||nsubj, nsubj||advcl, acl||conj +To train a parser, your data should include at least 20 instances of each label. +⚠ Multiple root labels (ROOT, nsubj, aux, npadvmod, prep) found in +training data. spaCy's parser uses a single root label ROOT so this distinction +will not be available. + +================================== Summary ================================== +✔ 5 checks passed +⚠ 8 warnings +``` + + + + + +**API:** [`spacy debug-data`](/api/cli#debug-data) + + + +## Backwards incompatibilities {id="incompat"} + + + +If you've been training **your own models**, you'll need to **retrain** them +with the new version. Also don't forget to upgrade all models to the latest +versions. Models for v2.0 or v2.1 aren't compatible with models for v2.2. To +check if all of your models are up to date, you can run the +[`spacy validate`](/api/cli#validate) command. + + + +> #### Install with lookups data +> +> ```bash +> $ pip install spacy[lookups] +> ``` +> +> You can also install +> [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) +> directly. + +- The lemmatization tables have been moved to their own package, + [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data), which + is not installed by default. If you're using pretrained models, **nothing + changes**, because the tables are now included in the model packages. If you + want to use the lemmatizer for other languages that don't yet have pretrained + models (e.g. Turkish or Croatian) or start off with a blank model that + contains lookup data (e.g. `spacy.blank("en")`), you'll need to **explicitly + install spaCy plus data** via `pip install spacy[lookups]`. +- Lemmatization tables (rules, exceptions, index and lookups) are now part of + the `Vocab` and serialized with it. This means that serialized objects (`nlp`, + pipeline components, vocab) will now include additional data, and models + written to disk will include additional files. +- The [`Lemmatizer`](/api/lemmatizer) class is now initialized with an instance + of [`Lookups`](/api/lookups) containing the rules and tables, instead of dicts + as separate arguments. This makes it easier to share data tables and modify + them at runtime. This is mostly internals, but if you've been implementing a + custom `Lemmatizer`, you'll need to update your code. +- The [Dutch model](/models/nl) has been trained on a new NER corpus (custom + labelled UD instead of WikiNER), so their predictions may be very different + compared to the previous version. The results should be significantly better + and more generalizable, though. +- The [`spacy download`](/api/cli#download) command does **not** set the + `--no-deps` pip argument anymore by default, meaning that model package + dependencies (if available) will now be also downloaded and installed. If + spaCy (which is also a model dependency) is not installed in the current + environment, e.g. if a user has built from source, `--no-deps` is added back + automatically to prevent spaCy from being downloaded and installed again from + pip. +- The built-in + [`biluo_tags_from_offsets`](/api/top-level#biluo_tags_from_offsets) converter + is now stricter and will raise an error if entities are overlapping (instead + of silently skipping them). If your data contains invalid entity annotations, + make sure to clean it and resolve conflicts. You can now also use the new + `debug-data` command to find problems in your data. +- Pipeline components can now overwrite IOB tags of tokens that are not yet part + of an entity. Once a token has an `ent_iob` value set, it won't be reset to an + "unset" state and will always have at least `O` assigned. `list(doc.ents)` now + actually keeps the annotations on the token level consistent, instead of + resetting `O` to an empty string. +- The default punctuation in the [`Sentencizer`](/api/sentencizer) has been + extended and now includes more characters common in various languages. This + also means that the results it produces may change, depending on your text. If + you want the previous behavior with limited characters, set + `punct_chars=[".", "!", "?"]` on initialization. +- The [`PhraseMatcher`](/api/phrasematcher) algorithm was rewritten from scratch + and it's now 10× faster. The rewrite also resolved a few subtle bugs + with very large terminology lists. So if you were matching large lists, you + may see slightly different results – however, the results should now be fully + correct. See [this PR](https://github.com/explosion/spaCy/pull/4309) for more + details. +- The `Serbian` language class (introduced in v2.1.8) incorrectly used the + language code `rs` instead of `sr`. This has now been fixed, so `Serbian` is + now available via `spacy.lang.sr`. +- The `"sources"` in the `meta.json` have changed from a list of strings to a + list of dicts. This is mostly internals, but if your code used + `nlp.meta["sources"]`, you might have to update it. + +### Migrating from spaCy 2.1 {id="migrating"} + +#### Lemmatization data and lookup tables + +If you application needs lemmatization for [languages](/usage/models#languages) +with only tokenizers, you now need to install that data explicitly via +`pip install spacy[lookups]` or `pip install spacy-lookups-data`. No additional +setup is required – the package just needs to be installed in the same +environment as spaCy. + +```python {highlight="3-4"} +nlp = Turkish() +doc = nlp("Bu bir cümledir.") +# 🚨 This now requires the lookups data to be installed explicitly +print([token.lemma_ for token in doc]) +``` + +The same applies to blank models that you want to update and train – for +instance, you might use [`spacy.blank`](/api/top-level#spacy.blank) to create a +blank English model and then train your own part-of-speech tagger on top. If you +don't explicitly install the lookups data, that `nlp` object won't have any +lemmatization rules available. spaCy will now show you a warning when you train +a new part-of-speech tagger and the vocab has no lookups available. + +#### Lemmatizer initialization + +This is mainly internals and should hopefully not affect your code. But if +you've been creating custom [`Lemmatizers`](/api/lemmatizer), you'll need to +update how they're initialized and pass in an instance of +[`Lookups`](/api/lookups) with the (optional) tables `lemma_index`, `lemma_exc`, +`lemma_rules` and `lemma_lookup`. + +```diff +from spacy.lemmatizer import Lemmatizer ++ from spacy.lookups import Lookups + +lemma_index = {"verb": ("cope", "cop")} +lemma_exc = {"verb": {"coping": ("cope",)}} +lemma_rules = {"verb": [["ing", ""]]} +- lemmatizer = Lemmatizer(lemma_index, lemma_exc, lemma_rules) ++ lookups = Lookups() ++ lookups.add_table("lemma_index", lemma_index) ++ lookups.add_table("lemma_exc", lemma_exc) ++ lookups.add_table("lemma_rules", lemma_rules) ++ lemmatizer = Lemmatizer(lookups) +``` + +#### Converting entity offsets to BILUO tags + +If you've been using the +[`biluo_tags_from_offsets`](/api/top-level#biluo_tags_from_offsets) helper to +convert character offsets into token-based BILUO tags, you may now see an error +if the offsets contain overlapping tokens and make it impossible to create a +valid BILUO sequence. This is helpful, because it lets you spot potential +problems in your data that can lead to inconsistent results later on. But it +also means that you need to adjust and clean up the offsets before converting +them: + +```diff +doc = nlp("I live in Berlin Kreuzberg") +- entities = [(10, 26, "LOC"), (10, 16, "GPE"), (17, 26, "LOC")] ++ entities = [(10, 16, "GPE"), (17, 26, "LOC")] +tags = get_biluo_tags_from_offsets(doc, entities) +``` + +#### Serbian language data + +If you've been working with `Serbian` (introduced in v2.1.8), you'll need to +change the language code from `rs` to the correct `sr`: + +```diff +- from spacy.lang.rs import Serbian ++ from spacy.lang.sr import Serbian +``` diff --git a/website/docs/usage/v2-3.mdx b/website/docs/usage/v2-3.mdx new file mode 100644 index 0000000..3331525 --- /dev/null +++ b/website/docs/usage/v2-3.mdx @@ -0,0 +1,344 @@ +--- +title: What's New in v2.3 +teaser: New features, backwards incompatibilities and migration guide +menu: + - ['New Features', 'features'] + - ['Backwards Incompatibilities', 'incompat'] + - ['Migrating from v2.2', 'migrating'] +--- + +## New Features {id="features",hidden="true"} + +spaCy v2.3 features new pretrained models for five languages, word vectors for +all language models, and decreased model size and loading times for models with +vectors. We've added pretrained models for **Chinese, Danish, Japanese, Polish +and Romanian** and updated the training data and vectors for most languages. +Model packages with vectors are about **2×** smaller on disk and load +**2-4×** faster. For the full changelog, see the +[release notes on GitHub](https://github.com/explosion/spaCy/releases/tag/v2.3.0). +For more details and a behind-the-scenes look at the new release, +[see our blog post](https://explosion.ai/blog/spacy-v2-3). + +### Expanded model families with vectors {id="models"} + +> #### Example +> +> ```bash +> python -m spacy download da_core_news_sm +> python -m spacy download ja_core_news_sm +> python -m spacy download pl_core_news_sm +> python -m spacy download ro_core_news_sm +> python -m spacy download zh_core_web_sm +> ``` + +With new model families for Chinese, Danish, Polish, Romanian and Chinese plus +`md` and `lg` models with word vectors for all languages, this release provides +a total of 46 model packages. For models trained using +[Universal Dependencies](https://universaldependencies.org) corpora, the +training data has been updated to UD v2.5 (v2.6 for Japanese, v2.3 for Polish) +and Dutch has been extended to include both UD Dutch Alpino and LassySmall. + + + +**Models:** [Models directory](/models) **Benchmarks: ** +[Release notes](https://github.com/explosion/spaCy/releases/tag/v2.3.0) + + + +### Chinese {id="chinese"} + +> #### Example +> +> ```python +> from spacy.lang.zh import Chinese +> +> # Load with "default" model provided by pkuseg +> cfg = {"pkuseg_model": "default", "require_pkuseg": True} +> nlp = Chinese(meta={"tokenizer": {"config": cfg}}) +> +> # Append words to user dict +> nlp.tokenizer.pkuseg_update_user_dict(["中国", "ABC"]) +> ``` + +This release adds support for +[`pkuseg`](https://github.com/lancopku/pkuseg-python) for word segmentation and +the new Chinese models ship with a custom pkuseg model trained on OntoNotes. The +Chinese tokenizer can be initialized with both `pkuseg` and custom models and +the `pkuseg` user dictionary is easy to customize. Note that +[`pkuseg`](https://github.com/lancopku/pkuseg-python) doesn't yet ship with +pre-compiled wheels for Python 3.8. See the +[usage documentation](/usage/models#chinese) for details on how to install it on +Python 3.8. + + + +**Models:** [Chinese models](/models/zh) **Usage: ** +[Chinese tokenizer usage](/usage/models#chinese) + + + +### Japanese {id="japanese"} + +The updated Japanese language class switches to +[`SudachiPy`](https://github.com/WorksApplications/SudachiPy) for word +segmentation and part-of-speech tagging. Using `SudachiPy` greatly simplifies +installing spaCy for Japanese, which is now possible with a single command: +`pip install spacy[ja]`. + + + +**Models:** [Japanese models](/models/ja) **Usage:** +[Japanese tokenizer usage](/usage/models#japanese) + + + +### Small CLI updates + +- [`spacy debug-data`](/api/cli#debug-data) provides the coverage of the vectors + in a base model with `spacy debug-data lang train dev -b base_model` +- [`spacy evaluate`](/api/cli#evaluate) supports `blank:lg` (e.g. + `spacy evaluate blank:en dev.json`) to evaluate the tokenization accuracy + without loading a model +- [`spacy train`](/api/cli#train) on GPU restricts the CPU timing evaluation to + the first iteration + +## Backwards incompatibilities {id="incompat"} + + + +If you've been training **your own models**, you'll need to **retrain** them +with the new version. Also don't forget to upgrade all models to the latest +versions. Models for earlier v2 releases (v2.0, v2.1, v2.2) aren't compatible +with models for v2.3. To check if all of your models are up to date, you can run +the [`spacy validate`](/api/cli#validate) command. + + + +> #### Install with lookups data +> +> ```bash +> $ pip install spacy[lookups] +> ``` +> +> You can also install +> [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) +> directly. + +- If you're training new models, you'll want to install the package + [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data), which + now includes both the lemmatization tables (as in v2.2) and the normalization + tables (new in v2.3). If you're using pretrained models, **nothing changes**, + because the relevant tables are included in the model packages. +- Due to the updated Universal Dependencies training data, the fine-grained + part-of-speech tags will change for many provided language models. The + coarse-grained part-of-speech tagset remains the same, but the mapping from + particular fine-grained to coarse-grained tags may show minor differences. +- For French, Italian, Portuguese and Spanish, the fine-grained part-of-speech + tagsets contain new merged tags related to contracted forms, such as `ADP_DET` + for French `"au"`, which maps to UPOS `ADP` based on the head `"à"`. This + increases the accuracy of the models by improving the alignment between + spaCy's tokenization and Universal Dependencies multi-word tokens used for + contractions. + +### Migrating from spaCy 2.2 {id="migrating"} + +#### Tokenizer settings + +In spaCy v2.2.2-v2.2.4, there was a change to the precedence of `token_match` +that gave prefixes and suffixes priority over `token_match`, which caused +problems for many custom tokenizer configurations. This has been reverted in +v2.3 so that `token_match` has priority over prefixes and suffixes as in v2.2.1 +and earlier versions. + +A new tokenizer setting `url_match` has been introduced in v2.3.0 to handle +cases like URLs where the tokenizer should remove prefixes and suffixes (e.g., a +comma at the end of a URL) before applying the match. See the full +[tokenizer documentation](/usage/linguistic-features#tokenization) and try out +[`nlp.tokenizer.explain()`](/usage/linguistic-features#tokenizer-debug) when +debugging your tokenizer configuration. + +#### Warnings configuration + +spaCy's custom warnings have been replaced with native Python +[`warnings`](https://docs.python.org/3/library/warnings.html). Instead of +setting `SPACY_WARNING_IGNORE`, use the +[`warnings` filters](https://docs.python.org/3/library/warnings.html#the-warnings-filter) +to manage warnings. + +```diff +import spacy ++ import warnings + +- spacy.errors.SPACY_WARNING_IGNORE.append('W007') ++ warnings.filterwarnings("ignore", message=r"\\[W007\\]", category=UserWarning) +``` + +#### Normalization tables + +The normalization tables have moved from the language data in +[`spacy/lang`](https://github.com/explosion/spacy/tree/v2.x/spacy/lang) to the +package [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). +If you're adding data for a new language, the normalization table should be +added to `spacy-lookups-data`. See +[adding norm exceptions](/usage/adding-languages#norm-exceptions). + +#### No preloaded vocab for models with vectors + +To reduce the initial loading time, the lexemes in `nlp.vocab` are no longer +loaded on initialization for models with vectors. As you process texts, the +lexemes will be added to the vocab automatically, just as in small models +without vectors. + +To see the number of unique vectors and number of words with vectors, see +`nlp.meta['vectors']`, for example for `en_core_web_md` there are `20000` unique +vectors and `684830` words with vectors: + +```python +{ + 'width': 300, + 'vectors': 20000, + 'keys': 684830, + 'name': 'en_core_web_md.vectors' +} +``` + +If required, for instance if you are working directly with word vectors rather +than processing texts, you can load all lexemes for words with vectors at once: + +```python +for orth in nlp.vocab.vectors: + _ = nlp.vocab[orth] +``` + +If your workflow previously iterated over `nlp.vocab`, a similar alternative is +to iterate over words with vectors instead: + +```diff +- lexemes = [w for w in nlp.vocab] ++ lexemes = [nlp.vocab[orth] for orth in nlp.vocab.vectors] +``` + +Be aware that the set of preloaded lexemes in a v2.2 model is not equivalent to +the set of words with vectors. For English, v2.2 `md/lg` models have 1.3M +provided lexemes but only 685K words with vectors. The vectors have been updated +for most languages in v2.2, but the English models contain the same vectors for +both v2.2 and v2.3. + +#### Lexeme.is_oov and Token.is_oov + + + +Due to a bug, the values for `is_oov` are reversed in v2.3.0, but this will be +fixed in the next patch release v2.3.1. + + + +In v2.3, `Lexeme.is_oov` and `Token.is_oov` are `True` if the lexeme does not +have a word vector. This is equivalent to `token.orth not in nlp.vocab.vectors`. + +Previously in v2.2, `is_oov` corresponded to whether a lexeme had stored +probability and cluster features. The probability and cluster features are no +longer included in the provided medium and large models (see the next section). + +#### Probability and cluster features + +> #### Load and save extra prob lookups table +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("the") +> print(doc[0].prob) # lazily loads extra prob table +> nlp.to_disk("/path/to/model") # includes prob table +> ``` + +The `Token.prob` and `Token.cluster` features, which are no longer used by the +core pipeline components as of spaCy v2, are no longer provided in the +pretrained models to reduce the model size. To keep these features available for +users relying on them, the `prob` and `cluster` features for the most frequent +1M tokens have been moved to +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) as +`extra` features for the relevant languages (English, German, Greek and +Spanish). + +The extra tables are loaded lazily, so if you have `spacy-lookups-data` +installed and your code accesses `Token.prob`, the full table is loaded into the +model vocab, which will take a few seconds on initial loading. When you save +this model after loading the `prob` table, the full `prob` table will be saved +as part of the model vocab. + +To load the probability table into a provided model, first make sure you have +`spacy-lookups-data` installed. To load the table, remove the empty provided +`lexeme_prob` table and then access `Lexeme.prob` for any word to load the table +from `spacy-lookups-data`: + +```diff ++ # prerequisite: pip install spacy-lookups-data +import spacy + +nlp = spacy.load("en_core_web_md") + +# remove the empty placeholder prob table ++ if nlp.vocab.lookups_extra.has_table("lexeme_prob"): ++ nlp.vocab.lookups_extra.remove_table("lexeme_prob") + +# access any `.prob` to load the full table into the model +assert nlp.vocab["a"].prob == -3.9297883511 + +# if desired, save this model with the probability table included +nlp.to_disk("/path/to/model") +``` + +If you'd like to include custom `cluster`, `prob`, or `sentiment` tables as part +of a new model, add the data to +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) under +the entry point `lg_extra`, e.g. `en_extra` for English. Alternatively, you can +initialize your [`Vocab`](/api/vocab) with the `lookups_extra` argument with a +[`Lookups`](/api/lookups) object that includes the tables `lexeme_cluster`, +`lexeme_prob`, `lexeme_sentiment` or `lexeme_settings`. `lexeme_settings` is +currently only used to provide a custom `oov_prob`. See examples in the +[`data` directory](https://github.com/explosion/spacy-lookups-data/tree/master/spacy_lookups_data/data) +in `spacy-lookups-data`. + +#### Initializing new models without extra lookups tables + +When you initialize a new model with [`spacy init-model`](/api/cli#init-model), +the `prob` table from `spacy-lookups-data` may be loaded as part of the +initialization. If you'd like to omit this extra data as in spaCy's provided +v2.3 models, use the new flag `--omit-extra-lookups`. + +#### Tag maps in provided models vs. blank models + +The tag maps in the provided models may differ from the tag maps in the spaCy +library. You can access the tag map in a loaded model under +`nlp.vocab.morphology.tag_map`. + +The tag map from `spacy.lang.lg.tag_map` is still used when a blank model is +initialized. If you want to provide an alternate tag map, update +`nlp.vocab.morphology.tag_map` after initializing the model or if you're using +the [train CLI](/api/cli#train), you can use the new `--tag-map-path` option to +provide in the tag map as a JSON dict. + +If you want to export a tag map from a provided model for use with the train +CLI, you can save it as a JSON dict. To only use string keys as required by JSON +and to make it easier to read and edit, any internal integer IDs need to be +converted back to strings: + +```python +import spacy +import srsly + +nlp = spacy.load("en_core_web_sm") +tag_map = {} + +# convert any integer IDs to strings for JSON +for tag, morph in nlp.vocab.morphology.tag_map.items(): + tag_map[tag] = {} + for feat, val in morph.items(): + feat = nlp.vocab.strings.as_string(feat) + if not isinstance(val, bool): + val = nlp.vocab.strings.as_string(val) + tag_map[tag][feat] = val + +srsly.write_json("tag_map.json", tag_map) +``` diff --git a/website/docs/usage/v2.mdx b/website/docs/usage/v2.mdx new file mode 100644 index 0000000..d40126e --- /dev/null +++ b/website/docs/usage/v2.mdx @@ -0,0 +1,613 @@ +--- +title: What's New in v2.0 +teaser: New features, backwards incompatibilities and migration guide +menu: + - ['Summary', 'summary'] + - ['New Features', 'features'] + - ['Backwards Incompatibilities', 'incompat'] + - ['Migrating from v1.x', 'migrating'] +--- + +We're very excited to finally introduce spaCy v2.0! On this page, you'll find a +summary of the new features, information on the backwards incompatibilities, +including a handy overview of what's been renamed or deprecated. To help you +make the most of v2.0, we also **re-wrote almost all of the usage guides and API +docs**, and added more [real-world examples](/usage/examples). If you're new to +spaCy, or just want to brush up on some NLP basics and the details of the +library, check out the [spaCy 101 guide](/usage/spacy-101) that explains the +most important concepts with examples and illustrations. + +## Summary {id="summary"} + + + +
+ +This release features entirely new **deep learning-powered models** for spaCy's +tagger, parser and entity recognizer. The new models are **10× smaller**, **20% +more accurate** and **even cheaper to run** than the previous generation. + +We've also made several usability improvements that are particularly helpful for +**production deployments**. spaCy v2 now fully supports the Pickle protocol, +making it easy to use spaCy with [Apache Spark](https://spark.apache.org/). The +string-to-integer mapping is **no longer stateful**, making it easy to reconcile +annotations made in different processes. Models are smaller and use less memory, +and the APIs for serialization are now much more consistent. Custom pipeline +components let you modify the `Doc` at any stage in the pipeline. You can now +also add your own custom attributes, properties and methods to the `Doc`, +`Token` and `Span`. + +
+ + + +- [Summary](#summary) +- [New features](#features) +- [Neural network models](#features-models) +- [Improved processing pipelines](#features-pipelines) +- [Text classification](#features-text-classification) +- [Hash values as IDs](#features-hash-ids) +- [Improved word vectors support](#features-vectors) +- [Saving, loading and serialization](#features-serializer) +- [displaCy visualizer](#features-displacy) +- [Language data and lazy loading](#features-language) +- [Revised matcher API and phrase matcher](#features-matcher) +- [Backwards incompatibilities](#incompat) +- [Migrating from spaCy v1.x](#migrating) + + + + + +The main usability improvements you'll notice in spaCy v2.0 are around +**defining, training and loading your own models** and components. The new +neural network models make it much easier to train a model from scratch, or +update an existing model with a few examples. In v1.x, the statistical models +depended on the state of the `Vocab`. If you taught the model a new word, you +would have to save and load a lot of data — otherwise the model wouldn't +correctly recall the features of your new example. That's no longer the case. + +Due to some clever use of hashing, the statistical models **never change size**, +even as they learn new vocabulary items. The whole pipeline is also now fully +differentiable. Even if you don't have explicitly annotated data, you can update +spaCy using all the **latest deep learning tricks** like adversarial training, +noise contrastive estimation or reinforcement learning. + +## New features {id="features"} + +This section contains an overview of the most important **new features and +improvements**. The [API docs](/api) include additional deprecation notes. + +### Convolutional neural network models {id="features-models"} + +> #### Example +> +> ```bash +> python -m spacy download en_core_web_sm +> python -m spacy download de_core_news_sm +> python -m spacy download xx_ent_wiki_sm +> ``` + +spaCy v2.0 features new neural models for tagging, parsing and entity +recognition. The models have been designed and implemented from scratch +specifically for spaCy, to give you an unmatched balance of speed, size and +accuracy. The new models are **10× smaller**, **20% more accurate**, and **even +cheaper to run** than the previous generation. + +spaCy v2.0's new neural network models bring significant improvements in +accuracy, especially for English Named Entity Recognition. The new +[`en_core_web_lg`](/models/en#en_core_web_lg) model makes about **25% fewer +mistakes** than the corresponding v1.x model and is within **1% of the current +state-of-the-art** +([Strubell et al., 2017](https://arxiv.org/pdf/1702.02098.pdf)). The v2.0 models +are also cheaper to run at scale, as they require **under 1 GB of memory** per +process. + + + +**Usage:** [Models directory](/models) + + + +### Improved processing pipelines {id="features-pipelines"} + +> #### Example +> +> ```python +> # Set custom attributes +> Doc.set_extension("my_attr", default=False) +> Token.set_extension("my_attr", getter=my_token_getter) +> assert doc._.my_attr, token._.my_attr +> +> # Add components to the pipeline +> my_component = lambda doc: doc +> nlp.add_pipe(my_component) +> ``` + +It's now much easier to **customize the pipeline** with your own components: +functions that receive a `Doc` object, modify and return it. Extensions let you +write any **attributes, properties and methods** to the `Doc`, `Token` and +`Span`. You can add data, implement new features, integrate other libraries with +spaCy or plug in your own machine learning models. + +![The processing pipeline](/images/pipeline.svg) + + + +**API:** [`Language`](/api/language), +[`Doc.set_extension`](/api/doc#set_extension), +[`Span.set_extension`](/api/span#set_extension), +[`Token.set_extension`](/api/token#set_extension) **Usage:** +[Processing pipelines](/usage/processing-pipelines) **Code:** +[Pipeline examples](/usage/examples#section-pipeline) + + + +### Text classification {id="features-text-classification"} + +> #### Example +> +> ```python +> textcat = nlp.create_pipe("textcat") +> nlp.add_pipe(textcat, last=True) +> nlp.begin_training() +> for itn in range(100): +> for doc, gold in train_data: +> nlp.update([doc], [gold]) +> doc = nlp("This is a text.") +> print(doc.cats) +> ``` + +spaCy v2.0 lets you add text categorization models to spaCy pipelines. The model +supports classification with multiple, non-mutually exclusive labels – so +multiple labels can apply at once. You can change the model architecture rather +easily, but by default, the `TextCategorizer` class uses a convolutional neural +network to assign position-sensitive vectors to each word in the document. + + + +**API:** [`TextCategorizer`](/api/textcategorizer), +[`Doc.cats`](/api/doc#attributes), `GoldParse.cats` **Usage:** +[Training a text classification model](/usage/training#textcat) + + + +### Hash values instead of integer IDs {id="features-hash-ids"} + +> #### Example +> +> ```python +> doc = nlp("I love coffee") +> assert doc.vocab.strings["coffee"] == 3197928453018144401 +> assert doc.vocab.strings[3197928453018144401] == "coffee" +> +> beer_hash = doc.vocab.strings.add("beer") +> assert doc.vocab.strings["beer"] == beer_hash +> assert doc.vocab.strings[beer_hash] == "beer" +> ``` + +The [`StringStore`](/api/stringstore) now resolves all strings to hash values +instead of integer IDs. This means that the string-to-int mapping **no longer +depends on the vocabulary state**, making a lot of workflows much simpler, +especially during training. Unlike integer IDs in spaCy v1.x, hash values will +**always match** – even across models. Strings can now be added explicitly using +the new [`Stringstore.add`](/api/stringstore#add) method. A token's hash is +available via `token.orth`. + + + +**API:** [`StringStore`](/api/stringstore) **Usage:** +[Vocab, hashes and lexemes 101](/usage/spacy-101#vocab) + + + +### Improved word vectors support {id="features-vectors"} + +> #### Example +> +> ```python +> for word, vector in vector_data: +> nlp.vocab.set_vector(word, vector) +> nlp.vocab.vectors.from_glove("/path/to/vectors") +> # Keep 10000 unique vectors and remap the rest +> nlp.vocab.prune_vectors(10000) +> nlp.to_disk("/model") +> ``` + +The new [`Vectors`](/api/vectors) class helps the `Vocab` manage the vectors +assigned to strings, and lets you assign vectors individually, or +[load in GloVe vectors](/usage/linguistic-features#adding-vectors) from a +directory. To help you strike a good balance between coverage and memory usage, +the `Vectors` class lets you map **multiple keys** to the **same row** of the +table. If you're using the [`spacy init-model`](/api/cli#init-model) command to +create a vocabulary, pruning the vectors will be taken care of automatically if +you set the `--prune-vectors` flag. Otherwise, you can use the new +[`Vocab.prune_vectors`](/api/vocab#prune_vectors). + + + +**API:** [`Vectors`](/api/vectors), [`Vocab`](/api/vocab) **Usage:** +[Word vectors and semantic similarity](/usage/vectors-similarity) + + + +### Saving, loading and serialization {id="features-serializer"} + +> #### Example +> +> ```python +> nlp = spacy.load("en") # shortcut link +> nlp = spacy.load("en_core_web_sm") # package +> nlp = spacy.load("/path/to/en") # unicode path +> nlp = spacy.load(Path("/path/to/en")) # pathlib Path +> +> nlp.to_disk("/path/to/nlp") +> nlp = English().from_disk("/path/to/nlp") +> ``` + +spaCy's serialization API has been made consistent across classes and objects. +All container classes, i.e. `Language`, `Doc`, `Vocab` and `StringStore` now +have a `to_bytes()`, `from_bytes()`, `to_disk()` and `from_disk()` method that +supports the Pickle protocol. + +The improved `spacy.load` makes loading models easier and more transparent. You +can load a model by supplying its shortcut link, the name of an installed +[model package](/models) or a path. The `Language` class to initialize will be +determined based on the model's settings. For a blank language, you can import +the class directly, e.g. `from spacy.lang.en import English` or use +[`spacy.blank()`](/api/top-level#spacy.blank). + + + +**API:** [`spacy.load`](/api/top-level#spacy.load), +[`Language.to_disk`](/api/language#to_disk) **Usage:** +[Models](/usage/models#usage), +[Saving and loading](/usage/saving-loading#models) + + + +### displaCy visualizer with Jupyter support {id="features-displacy"} + +> #### Example +> +> ```python +> from spacy import displacy +> doc = nlp("This is a sentence about Facebook.") +> displacy.serve(doc, style="dep") # run the web server +> html = displacy.render(doc, style="ent") # generate HTML +> ``` + +Our popular dependency and named entity visualizers are now an official part of +the spaCy library. displaCy can run a simple web server, or generate raw HTML +markup or SVG files to be exported. You can pass in one or more docs, and +customize the style. displaCy also auto-detects whether you're running +[Jupyter](https://jupyter.org) and will render the visualizations in your +notebook. + + + +**API:** [`displacy`](/api/top-level#displacy) **Usage:** +[Visualizing spaCy](/usage/visualizers) + + + +### Improved language data and lazy loading {id="features-language"} + +Language-specific data now lives in its own submodule, `spacy.lang`. Languages +are lazy-loaded, i.e. only loaded when you import a `Language` class, or load a +model that initializes one. This allows languages to contain more custom data, +e.g. lemmatizer lookup tables, or complex regular expressions. The language data +has also been tidied up and simplified. spaCy now also supports simple +lookup-based lemmatization – and **many new languages**! + + + +**API:** [`Language`](/api/language) **Code:** +[`spacy/lang`](https://github.com/explosion/spacy/tree/v2.x/spacy/lang) +**Usage:** [Adding languages](/usage/adding-languages) + + + +### Revised matcher API and phrase matcher {id="features-matcher"} + +> #### Example +> +> ```python +> from spacy.matcher import Matcher, PhraseMatcher +> +> matcher = Matcher(nlp.vocab) +> matcher.add('HEARTS', None, [{"ORTH": "❤️", "OP": '+'}]) +> +> phrasematcher = PhraseMatcher(nlp.vocab) +> phrasematcher.add("OBAMA", None, nlp("Barack Obama")) +> ``` + +Patterns can now be added to the matcher by calling +[`matcher.add()`](/api/matcher#add) with a match ID, an optional callback +function to be invoked on each match, and one or more patterns. This allows you +to write powerful, pattern-specific logic using only one matcher. For example, +you might only want to merge some entity types, and set custom flags for other +matched patterns. The new [`PhraseMatcher`](/api/phrasematcher) lets you +efficiently match very large terminology lists using `Doc` objects as match +patterns. + + + +**API:** [`Matcher`](/api/matcher), [`PhraseMatcher`](/api/phrasematcher) +**Usage:** [Rule-based matching](/usage/rule-based-matching) + + + +## Backwards incompatibilities {id="incompat"} + +The following modules, classes and methods have changed between v1.x and v2.0. + +| Old | New | +| ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spacy.download.en`, `spacy.download.de` | [`cli.download`](/api/cli#download) | +| `spacy.en` etc. | `spacy.lang.en` etc. | +| `spacy.en.word_sets` | `spacy.lang.en.stop_words` | +| `spacy.orth` | `spacy.lang.xx.lex_attrs` | +| `spacy.syntax.iterators` | `spacy.lang.xx.syntax_iterators` | +| `spacy.tagger.Tagger` | `spacy.pipeline.Tagger` | +| `spacy.cli.model` | [`spacy.cli.vocab`](/api/cli#vocab) | +| `Language.save_to_directory` | [`Language.to_disk`](/api/language#to_disk) | +| `Language.end_training` | [`Language.begin_training`](/api/language#begin_training) | +| `Language.create_make_doc` | [`Language.tokenizer`](/api/language#attributes) | +| `Vocab.resize_vectors` | [`Vectors.resize`](/api/vectors#resize) | +| `Vocab.load` `Vocab.load_lexemes` | [`Vocab.from_disk`](/api/vocab#from_disk) [`Vocab.from_bytes`](/api/vocab#from_bytes) | +| `Vocab.dump` | [`Vocab.to_disk`](/api/vocab#to_disk) [`Vocab.to_bytes`](/api/vocab#to_bytes) | +| `Vocab.load_vectors` `Vocab.load_vectors_from_bin_loc` | [`Vectors.from_disk`](/api/vectors#from_disk) [`Vectors.from_bytes`](/api/vectors#from_bytes) [`Vectors.from_glove`](/api/vectors#from_glove) | +| `Vocab.dump_vectors` | [`Vectors.to_disk`](/api/vectors#to_disk) [`Vectors.to_bytes`](/api/vectors#to_bytes) | +| `StringStore.load` | [`StringStore.from_disk`](/api/stringstore#from_disk) [`StringStore.from_bytes`](/api/stringstore#from_bytes) | +| `StringStore.dump` | [`StringStore.to_disk`](/api/stringstore#to_disk) [`StringStore.to_bytes`](/api/stringstore#to_bytes) | +| `Tokenizer.load` | [`Tokenizer.from_disk`](/api/tokenizer#from_disk) [`Tokenizer.from_bytes`](/api/tokenizer#from_bytes) | +| `Tagger.load` | [`Tagger.from_disk`](/api/tagger#from_disk) [`Tagger.from_bytes`](/api/tagger#from_bytes) | +| `Tagger.tag_names` | `Tagger.labels` | +| `DependencyParser.load` | [`DependencyParser.from_disk`](/api/dependencyparser#from_disk) [`DependencyParser.from_bytes`](/api/dependencyparser#from_bytes) | +| `EntityRecognizer.load` | [`EntityRecognizer.from_disk`](/api/entityrecognizer#from_disk) [`EntityRecognizer.from_bytes`](/api/entityrecognizer#from_bytes) | +| `Matcher.load` | - | +| `Matcher.add_pattern` `Matcher.add_entity` | [`Matcher.add`](/api/matcher#add) [`PhraseMatcher.add`](/api/phrasematcher#add) | +| `Matcher.get_entity` | [`Matcher.get`](/api/matcher#get) | +| `Matcher.has_entity` | [`Matcher.has_key`](/api/matcher#has_key) | +| `Doc.read_bytes` | [`Doc.to_bytes`](/api/doc#to_bytes) [`Doc.from_bytes`](/api/doc#from_bytes) [`Doc.to_disk`](/api/doc#to_disk) [`Doc.from_disk`](/api/doc#from_disk) | +| `Token.is_ancestor_of` | [`Token.is_ancestor`](/api/token#is_ancestor) | + +### Deprecated {id="deprecated"} + +The following methods are deprecated. They can still be used, but should be +replaced. + +| Old | New | +| ---------------------------- | ----------------------------------------------- | +| `Tokenizer.tokens_from_list` | [`Doc`](/api/doc) | +| `Span.sent_start` | [`Span.is_sent_start`](/api/span#is_sent_start) | + +## Migrating from spaCy 1.x {id="migrating"} + +Because we'e made so many architectural changes to the library, we've tried to +**keep breaking changes to a minimum**. A lot of projects follow the philosophy +that if you're going to break anything, you may as well break everything. We +think migration is easier if there's a logic to what has changed. We've +therefore followed a policy of avoiding breaking changes to the `Doc`, `Span` +and `Token` objects. This way, you can focus on only migrating the code that +does training, loading and serialization — in other words, code that works with +the `nlp` object directly. Code that uses the annotations should continue to +work. + + + +If you've trained your own models, keep in mind that your train and runtime +inputs must match. This means you'll have to **retrain your models** with spaCy +v2.0. + + + +### Document processing {id="migrating-document-processing"} + +The [`Language.pipe`](/api/language#pipe) method allows spaCy to batch +documents, which brings a **significant performance advantage** in v2.0. The new +neural networks introduce some overhead per batch, so if you're processing a +number of documents in a row, you should use `nlp.pipe` and process the texts as +a stream. + +```diff +- docs = (nlp(text) for text in texts) + ++ docs = nlp.pipe(texts) +``` + +To make usage easier, there's now a boolean `as_tuples` keyword argument, that +lets you pass in an iterator of `(text, context)` pairs, so you can get back an +iterator of `(doc, context)` tuples. + +### Saving, loading and serialization {id="migrating-saving-loading"} + +Double-check all calls to `spacy.load()` and make sure they don't use the `path` +keyword argument. If you're only loading in binary data and not a model package +that can construct its own `Language` class and pipeline, you should now use the +[`Language.from_disk`](/api/language#from_disk) method. + +```diff +- nlp = spacy.load("en", path="/model") + ++ nlp = spacy.load("/model") ++ nlp = spacy.blank("en").from_disk("/model/data") +``` + +Review all other code that writes state to disk or bytes. All containers, now +share the same, consistent API for saving and loading. Replace saving with +`to_disk()` or `to_bytes()`, and loading with `from_disk()` and `from_bytes()`. + +```diff +- nlp.save_to_directory("/model") +- nlp.vocab.dump("/vocab") + ++ nlp.to_disk("/model") ++ nlp.vocab.to_disk("/vocab") +``` + +If you've trained models with input from v1.x, you'll need to **retrain them** +with spaCy v2.0. All previous models will not be compatible with the new +version. + +### Processing pipelines and language data {id="migrating-languages"} + +If you're importing language data or `Language` classes, make sure to change +your import statements to import from `spacy.lang`. If you've added your own +custom language, it needs to be moved to `spacy/lang/xx` and adjusted +accordingly. + +```diff +- from spacy.en import English + ++ from spacy.lang.en import English +``` + +If you've been using custom pipeline components, check out the new guide on +[processing pipelines](/usage/processing-pipelines). Pipeline components are now +`(name, func)` tuples. Appending them to the pipeline still works – but the +[`add_pipe`](/api/language#add_pipe) method now makes this much more convenient. +Methods for removing, renaming, replacing and retrieving components have been +added as well. Components can now be disabled by passing a list of their names +to the `disable` keyword argument on load, or by using +[`disable_pipes`](/api/language#disable_pipes) as a method or context manager: + +```diff +- nlp = spacy.load("en_core_web_sm", tagger=False, entity=False) +- doc = nlp("I don't want parsed", parse=False) + ++ nlp = spacy.load("en_core_web_sm", disable=["tagger", "ner"]) ++ with nlp.disable_pipes("parser"): ++ doc = nlp("I don't want parsed") +``` + +To add spaCy's built-in pipeline components to your pipeline, you can still +import and instantiate them directly – but it's more convenient to use the new +[`create_pipe`](/api/language#create_pipe) method with the component name, i.e. +`'tagger'`, `'parser'`, `'ner'` or `'textcat'`. + +```diff +- from spacy.pipeline import Tagger +- tagger = Tagger(nlp.vocab) +- nlp.pipeline.insert(0, tagger) + ++ tagger = nlp.create_pipe("tagger") ++ nlp.add_pipe(tagger, first=True) +``` + +### Training {id="migrating-training"} + +All built-in pipeline components are now subclasses of [`Pipe`](/api/pipe), +fully trainable and serializable, and follow the same API. Instead of updating +the model and telling spaCy when to _stop_, you can now explicitly call +[`begin_training`](/api/language#begin_training), which returns an optimizer you +can pass into the [`update`](/api/language#update) function. While `update` +still accepts sequences of `Doc` and `GoldParse` objects, you can now also pass +in a list of strings and dictionaries describing the annotations. We call this +the ["simple training style"](/usage/training#training-simple-style). This is +also the recommended usage, as it removes one layer of abstraction from the +training. + +```diff +- for itn in range(1000): +- for text, entities in train_data: +- doc = Doc(text) +- gold = GoldParse(doc, entities=entities) +- nlp.update(doc, gold) +- nlp.end_training() +- nlp.save_to_directory("/model") + ++ nlp.begin_training() ++ for itn in range(1000): ++ for texts, annotations in train_data: ++ nlp.update(texts, annotations) ++ nlp.to_disk("/model") +``` + +### Attaching custom data to the Doc {id="migrating-doc"} + +Previously, you had to create a new container in order to attach custom data to +a `Doc` object. This often required converting the `Doc` objects to and from +arrays. In spaCy v2.0, you can set your own attributes, properties and methods +on the `Doc`, `Token` and `Span` via +[custom extensions](/usage/processing-pipelines#custom-components-attributes). +This means that your application can – and should – only pass around `Doc` +objects and refer to them as the single source of truth. + +```diff +- doc = nlp("This is a regular doc") +- doc_array = doc.to_array(["ORTH", "POS"]) +- doc_with_meta = {"doc_array": doc_array, "meta": get_doc_meta(doc_array)} + ++ Doc.set_extension("meta", getter=get_doc_meta) ++ doc_with_meta = nlp(u'This is a doc with meta data') ++ meta = doc._.meta +``` + +If you wrap your extension attributes in a +[custom pipeline component](/usage/processing-pipelines#custom-components), they +will be assigned automatically when you call `nlp` on a text. If your +application assigns custom data to spaCy's container objects, or includes other +utilities that interact with the pipeline, consider moving this logic into its +own extension module. + +```diff +- doc = nlp("Doc with a standard pipeline") +- meta = get_meta(doc) + ++ nlp.add_pipe(meta_component) ++ doc = nlp("Doc with a custom pipeline that assigns meta") ++ meta = doc._.meta +``` + +### Strings and hash values {id="migrating-strings"} + +The change from integer IDs to hash values may not actually affect your code +very much. However, if you're adding strings to the vocab manually, you now need +to call [`StringStore.add`](/api/stringstore#add) explicitly. You can also now +be sure that the string-to-hash mapping will always match across vocabularies. + +```diff +- nlp.vocab.strings["coffee"] # 3672 +- other_nlp.vocab.strings["coffee"] # 40259 + ++ nlp.vocab.strings.add("coffee") ++ nlp.vocab.strings["coffee"] # 3197928453018144401 ++ other_nlp.vocab.strings["coffee"] # 3197928453018144401 +``` + +### Adding patterns and callbacks to the matcher {id="migrating-matcher"} + +If you're using the matcher, you can now add patterns in one step. This should +be easy to update – simply merge the ID, callback and patterns into one call to +[`Matcher.add()`](/api/matcher#add). The matcher now also supports string keys, +which saves you an extra import. If you've been using **acceptor functions**, +you'll need to move this logic into the +[`on_match` callbacks](/usage/linguistic-features#on_match). The callback +function is invoked on every match and will give you access to the doc, the +index of the current match and all total matches. This lets you both accept or +reject the match, and define the actions to be triggered. + +```diff +- matcher.add_entity("GoogleNow", on_match=merge_phrases) +- matcher.add_pattern("GoogleNow", [{ORTH: "Google"}, {ORTH: "Now"}]) + ++ matcher.add("GoogleNow", merge_phrases, [{"ORTH": "Google"}, {"ORTH": "Now"}]) +``` + +If you need to match large terminology lists, you can now also use the +[`PhraseMatcher`](/api/phrasematcher), which accepts `Doc` objects as match +patterns and is more efficient than the regular, rule-based matcher. + +```diff +- matcher = Matcher(nlp.vocab) +- matcher.add_entity("PRODUCT") +- for text in large_terminology_list +- matcher.add_pattern("PRODUCT", [{ORTH: text}]) + ++ from spacy.matcher import PhraseMatcher ++ matcher = PhraseMatcher(nlp.vocab) ++ patterns = [nlp.make_doc(text) for text in large_terminology_list] ++ matcher.add("PRODUCT", None, *patterns) +``` diff --git a/website/docs/usage/v3-1.mdx b/website/docs/usage/v3-1.mdx new file mode 100644 index 0000000..702932a --- /dev/null +++ b/website/docs/usage/v3-1.mdx @@ -0,0 +1,322 @@ +--- +title: What's New in v3.1 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New Features {id="features",hidden="true"} + +It's been great to see the adoption of the new spaCy v3, which introduced +[transformer-based](/usage/embeddings-transformers) pipelines, a new +[config and training system](/usage/training) for reproducible experiments, +[projects](/usage/projects) for end-to-end workflows, and many +[other features](/usage/v3). Version 3.1 adds more on top of it, including the +ability to use predicted annotations during training, a new `SpanCategorizer` +component for predicting arbitrary and potentially overlapping spans, support +for partial incorrect annotations in the entity recognizer, new trained +pipelines for Catalan and Danish, as well as many bug fixes and improvements. + +### Using predicted annotations during training {id="predicted-annotations-training"} + +By default, components are updated in isolation during training, which means +that they don't see the predictions of any earlier components in the pipeline. +The new +[`[training.annotating_components]`](/usage/training#annotating-components) +config setting lets you specify pipeline components that should set annotations +on the predicted docs during training. This makes it easy to use the predictions +of a previous component in the pipeline as features for a subsequent component, +e.g. the dependency labels in the tagger: + +```ini {title="config.cfg (excerpt)",highlight="7,12"} +[nlp] +pipeline = ["parser", "tagger"] + +[components.tagger.model.tok2vec.embed] +@architectures = "spacy.MultiHashEmbed.v1" +width = ${components.tagger.model.tok2vec.encode.width} +attrs = ["NORM","DEP"] +rows = [5000,2500] +include_static_vectors = false + +[training] +annotating_components = ["parser"] +``` + + + +This project shows how to use the `token.dep` attribute predicted by the parser +as a feature for a subsequent tagger component in the pipeline. + + + +### SpanCategorizer for predicting arbitrary and overlapping spans {id="spancategorizer",tag="experimental"} + +A common task in applied NLP is extracting spans of texts from documents, +including longer phrases or nested expressions. Named entity recognition isn't +the right tool for this problem, since an entity recognizer typically predicts +single token-based tags that are very sensitive to boundaries. This is effective +for proper nouns and self-contained expressions, but less useful for other types +of phrases or overlapping spans. The new +[`SpanCategorizer`](/api/spancategorizer) component and +[SpanCategorizer](/api/architectures#spancategorizer) architecture let you label +arbitrary and potentially overlapping spans of texts. A span categorizer +consists of two parts: a [suggester function](/api/spancategorizer#suggesters) +that proposes candidate spans, which may or may not overlap, and a labeler model +that predicts zero or more labels for each candidate. The predicted spans are +available via the [`Doc.spans`](/api/doc#spans) container. + + + +This project trains a span categorizer for Indonesian NER. + + + + + +Prodigy: example of the new manual spans UI + +The upcoming version of our annotation tool [Prodigy](https://prodi.gy) +(currently available as a [pre-release](https://support.prodi.gy/t/3861) for all +users) features a [new workflow and UI](https://support.prodi.gy/t/3861) for +annotating overlapping and nested spans. You can use it to create training data +for spaCy's `SpanCategorizer` component. + + + +### Update the entity recognizer with partial incorrect annotations {id="negative-samples"} + +> #### config.cfg (excerpt) +> +> ```ini +> [components.ner] +> factory = "ner" +> incorrect_spans_key = "incorrect_spans" +> moves = null +> update_with_oracle_cut_size = 100 +> ``` + +The [`EntityRecognizer`](/api/entityrecognizer) can now be updated with known +incorrect annotations, which lets you take advantage of partial and sparse data. +For example, you'll be able to use the information that certain spans of text +are definitely **not** `PERSON` entities, without having to provide the complete +gold-standard annotations for the given example. The incorrect span annotations +can be added via the [`Doc.spans`](/api/doc#spans) in the training data under +the key defined as [`incorrect_spans_key`](/api/entityrecognizer#init) in the +component config. + +```python +train_doc = nlp.make_doc("Barack Obama was born in Hawaii.") +# The doc.spans key can be defined in the config +train_doc.spans["incorrect_spans"] = [ + Span(doc, 0, 2, label="ORG"), + Span(doc, 5, 6, label="PRODUCT") +] +``` + +{/* TODO: more details and/or example project? */} + +### New pipeline packages for Catalan and Danish {id="pipeline-packages"} + +spaCy v3.1 adds 5 new pipeline packages, including a new core family for Catalan +and a new transformer-based pipeline for Danish using the +[`danish-bert-botxo`](http://huggingface.co/Maltehb/danish-bert-botxo) weights. +See the [models directory](/models) for an overview of all available trained +pipelines and the [training guide](/usage/training) for details on how to train +your own. + +> Thanks to Carlos Rodríguez Penagos and the +> [Barcelona Supercomputing Center](https://temu.bsc.es/) for their +> contributions for Catalan and to Kenneth Enevoldsen for Danish. For additional +> Danish pipelines, check out [DaCy](https://github.com/KennethEnevoldsen/DaCy). + +| Package | Language | UPOS | Parser LAS | NER F | +| ------------------------------------------------- | -------- | ---: | ---------: | ----: | +| [`ca_core_news_sm`](/models/ca#ca_core_news_sm) | Catalan | 98.2 | 87.4 | 79.8 | +| [`ca_core_news_md`](/models/ca#ca_core_news_md) | Catalan | 98.3 | 88.2 | 84.0 | +| [`ca_core_news_lg`](/models/ca#ca_core_news_lg) | Catalan | 98.5 | 88.4 | 84.2 | +| [`ca_core_news_trf`](/models/ca#ca_core_news_trf) | Catalan | 98.9 | 93.0 | 91.2 | +| [`da_core_news_trf`](/models/da#da_core_news_trf) | Danish | 98.0 | 85.0 | 82.9 | + +### Resizable text classification architectures {id="resizable-textcat"} + +Previously, the [`TextCategorizer`](/api/textcategorizer) architectures could +not be resized, meaning that you couldn't add new labels to an already trained +model. In spaCy v3.1, the [TextCatCNN](/api/architectures#TextCatCNN) and +[TextCatBOW](/api/architectures#TextCatBOW) architectures are now resizable, +while ensuring that the predictions for the old labels remain the same. + +### CLI command to assemble pipeline from config {id="assemble"} + +The [`spacy assemble`](/api/cli#assemble) command lets you assemble a pipeline +from a config file without additional training. It can be especially useful for +creating a blank pipeline with a custom tokenizer, rule-based components or word +vectors. + +```bash +$ python -m spacy assemble config.cfg ./output +``` + +### Pretty pipeline package READMEs {id="package-readme"} + +The [`spacy package`](/api/cli#package) command now auto-generates a pretty +`README.md` based on the pipeline information defined in the `meta.json`. This +includes a table with a general overview, as well as the label scheme and +accuracy figures, if available. For an example, see the +[model releases](https://github.com/explosion/spacy-models/releases). + +### Support for streaming large or infinite corpora {id="streaming-corpora"} + +> #### config.cfg (excerpt) +> +> ```ini +> [training] +> max_epochs = -1 +> ``` + +The training process now supports streaming large or infinite corpora +out-of-the-box, which can be controlled via the +[`[training.max_epochs]`](/api/data-formats#training) config setting. Setting it +to `-1` means that the train corpus should be streamed rather than loaded into +memory with no shuffling within the training loop. For details on how to +implement a custom corpus loader, e.g. to stream in data from a remote storage, +see the usage guide on +[custom data reading](/usage/training#custom-code-readers-batchers). + +When streaming a corpus, only the first 100 examples will be used for +[initialization](/usage/training#config-lifecycle). This is no problem if you're +training a component like the text classifier with data that specifies all +available labels in every example. If necessary, you can use the +[`init labels`](/api/cli#init-labels) command to pre-generate the labels for +your components using a representative sample so the model can be initialized +correctly before training. + +### New lemmatizers for Catalan and Italian {id="pos-lemmatizers"} + +The trained pipelines for [Catalan](/models/ca) and [Italian](/models/it) now +include lemmatizers that use the predicted part-of-speech tags as part of the +lookup lemmatization for higher lemmatization accuracy. If you're training your +own pipelines for these languages and you want to include a lemmatizer, make +sure you have the +[`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) package +installed, which provides the relevant tables. + +### Upload your pipelines to the Hugging Face Hub {id="huggingface-hub"} + +The [Hugging Face Hub](https://huggingface.co/) lets you upload models and share +them with others, and it now supports spaCy pipelines out-of-the-box. The new +[`spacy-huggingface-hub`](https://github.com/explosion/spacy-huggingface-hub) +package automatically adds the `huggingface-hub` command to your `spacy` CLI. It +lets you upload any pipelines packaged with [`spacy package`](/api/cli#package) +and `--build wheel` and takes care of auto-generating all required meta +information. + +After uploading, you'll get a live URL for your model page that includes all +details, files and interactive visualizers, as well as a direct URL to the wheel +file that you can install via `pip install`. For examples, check out the +[spaCy pipelines](https://huggingface.co/spacy) we've uploaded. + +```bash +$ pip install spacy-huggingface-hub +$ huggingface-cli login +$ python -m spacy package ./en_ner_fashion ./output --build wheel +$ cd ./output/en_ner_fashion-0.0.0/dist +$ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl +``` + +You can also integrate the upload command into your +[project template](/usage/projects#huggingface_hub) to automatically upload your +packaged pipelines after training. + + + +Get started with uploading your models to the Hugging Face hub using our project +template. It trains a simple pipeline, packages it and uploads it if the +packaged model has changed. This makes it easy to deploy your models end-to-end. + + + +## Notes about upgrading from v3.0 {id="upgrading"} + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with spaCy v3.0, you will see a +warning telling you that the pipeline may be incompatible. This doesn't +necessarily have to be true, but we recommend running your pipelines against +your test suite or evaluation data to make sure there are no unexpected results. +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.0.0,<3.1.0", ++ "spacy_version": ">=3.0.0,<3.2.0", +``` + +### Updating v3.0 configs + +To update a config from spaCy v3.0 with the new v3.1 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```bash +python -m spacy init fill-config config-v3.0.cfg config-v3.1.cfg +``` + +In many cases (`spacy train`, `spacy.load()`), the new defaults will be filled +in automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). + +### Sourcing pipeline components with vectors {id="source-vectors"} + +If you're sourcing a pipeline component that requires static vectors (for +example, a tagger or parser from an `md` or `lg` pretrained pipeline), be sure +to include the source model's vectors in the setting `[initialize.vectors]`. In +spaCy v3.0, a bug allowed vectors to be loaded implicitly through `source`, +however in v3.1 this setting must be provided explicitly as +`[initialize.vectors]`: + +```ini {title="config.cfg (excerpt)"} +[components.ner] +source = "en_core_web_md" + +[initialize] +vectors = "en_core_web_md" +``` + + + +Each pipeline can only store one set of static vectors, so it's not possible to +assemble a pipeline with components that were trained on different static +vectors. + + + +[`spacy train`](/api/cli#train) and [`spacy assemble`](/api/cli#assemble) will +provide warnings if the source and target pipelines don't contain the same +vectors. If you are sourcing a rule-based component like an entity ruler or +lemmatizer that does not use the vectors as a model feature, then this warning +can be safely ignored. + +### Warnings {id="warnings"} + +Logger warnings have been converted to Python warnings. Use +[`warnings.filterwarnings`](https://docs.python.org/3/library/warnings.html#warnings.filterwarnings) +or the new helper method `spacy.errors.filter_warning(action, error_msg='')` to +manage warnings. diff --git a/website/docs/usage/v3-2.mdx b/website/docs/usage/v3-2.mdx new file mode 100644 index 0000000..b3ffd5d --- /dev/null +++ b/website/docs/usage/v3-2.mdx @@ -0,0 +1,242 @@ +--- +title: What's New in v3.2 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New Features {id="features",hidden="true"} + +spaCy v3.2 adds support for [`floret`](https://github.com/explosion/floret) +vectors, makes custom `Doc` creation and scoring easier, and includes many bug +fixes and improvements. For the trained pipelines, there's a new transformer +pipeline for Japanese and the Universal Dependencies training data has been +updated across the board to the most recent release. + + + +spaCy is now up to **8 × faster on M1 Macs** by calling into Apple's +native Accelerate library for matrix multiplication. For more details, see +[`thinc-apple-ops`](https://github.com/explosion/thinc-apple-ops). + +```bash +$ pip install spacy[apple] +``` + + + +### Registered scoring functions {id="registered-scoring-functions"} + +To customize the scoring, you can specify a scoring function for each component +in your config from the new [`scorers` registry](/api/top-level#registry): + +```ini {title="config.cfg (excerpt)",highlight="3"} +[components.tagger] +factory = "tagger" +scorer = {"@scorers":"spacy.tagger_scorer.v1"} +``` + +### Overwrite settings {id="overwrite"} + +Most pipeline components now include an `overwrite` setting in the config that +determines whether existing annotation in the `Doc` is preserved or overwritten: + +```ini {title="config.cfg (excerpt)",highlight="3"} +[components.tagger] +factory = "tagger" +overwrite = false +``` + +### Doc input for pipelines {id="doc-input"} + +[`nlp`](/api/language#call) and [`nlp.pipe`](/api/language#pipe) accept +[`Doc`](/api/doc) input, skipping the tokenizer if a `Doc` is provided instead +of a string. This makes it easier to create a `Doc` with custom tokenization or +to set custom extensions before processing: + +```python +doc = nlp.make_doc("This is text 500.") +doc._.text_id = 500 +doc = nlp(doc) +``` + +### Support for floret vectors {id="vectors"} + +We recently published [`floret`](https://github.com/explosion/floret), an +extended version of [fastText](https://fasttext.cc) that combines fastText's +subwords with Bloom embeddings for compact, full-coverage vectors. The use of +subwords means that there are no OOV words and due to Bloom embeddings, the +vector table can be kept very small at \<100K entries. Bloom embeddings are +already used by [HashEmbed](https://thinc.ai/docs/api-layers#hashembed) in +[tok2vec](/api/architectures#tok2vec-arch) for compact spaCy models. + +For easy integration, floret includes a +[Python wrapper](https://github.com/explosion/floret/blob/main/python/README.md): + +```bash +$ pip install floret +``` + +A demo project shows how to train and import floret vectors: + + + +Train toy English floret vectors and import them into a spaCy pipeline. + + + +Two additional demo projects compare standard fastText vectors with floret +vectors for full spaCy pipelines. For agglutinative languages like Finnish or +Korean, there are large improvements in performance due to the use of subwords +(no OOV words!), with a vector table containing merely 50K entries. + + + +Finnish UD+NER vector and pipeline training, comparing standard fasttext vs. +floret vectors. + +For the default project settings with 1M (2.6G) tokenized training texts and 50K +300-dim vectors, ~300K keys for the standard vectors: + +| Vectors | TAG | POS | DEP UAS | DEP LAS | NER F | +| -------------------------------------------- | -------: | -------: | -------: | -------: | -------: | +| none | 93.3 | 92.3 | 79.7 | 72.8 | 61.0 | +| standard (pruned: 50K vectors for 300K keys) | 95.9 | 94.7 | 83.3 | 77.9 | 68.5 | +| standard (unpruned: 300K vectors/keys) | 96.0 | 95.0 | **83.8** | 78.4 | 69.1 | +| floret (minn 4, maxn 5; 50K vectors, no OOV) | **96.6** | **95.5** | 83.5 | **78.5** | **70.9** | + + + + + +Korean UD vector and pipeline training, comparing standard fasttext vs. floret +vectors. + +For the default project settings with 1M (3.3G) tokenized training texts and 50K +300-dim vectors, ~800K keys for the standard vectors: + +| Vectors | TAG | POS | DEP UAS | DEP LAS | +| -------------------------------------------- | -------: | -------: | -------: | -------: | +| none | 72.5 | 85.0 | 73.2 | 64.3 | +| standard (pruned: 50K vectors for 800K keys) | 77.9 | 89.4 | 78.8 | 72.8 | +| standard (unpruned: 800K vectors/keys) | 79.0 | 90.2 | 79.2 | 73.9 | +| floret (minn 2, maxn 3; 50K vectors, no OOV) | **82.5** | **93.8** | **83.0** | **80.1** | + + + +### Updates for spacy-transformers v1.1 {id="spacy-transformers"} + +[`spacy-transformers`](https://github.com/explosion/spacy-transformers) v1.1 has +been refactored to improve serialization and support of inline transformer +components and replacing listeners. In addition, the transformer model output is +provided as +[`ModelOutput`](https://huggingface.co/transformers/main_classes/output.html?highlight=modeloutput#transformers.file_utils.ModelOutput) +instead of tuples in +`TransformerData.model_output and FullTransformerBatch.model_output.` For +backwards compatibility, the tuple format remains available under +`TransformerData.tensors` and `FullTransformerBatch.tensors`. See more details +in the [transformer API docs](/api/architectures#TransformerModel). + +`spacy-transformers` v1.1 also adds support for `transformer_config` settings +such as `output_attentions`. Additional output is stored under +`TransformerData.model_output`. More details are in the +[TransformerModel docs](/api/architectures#TransformerModel). The training speed +has been improved by streamlining allocations for tokenizer output and there is +new support for [mixed-precision training](/api/architectures#TransformerModel). + +### New transformer package for Japanese {id="pipeline-packages"} + +spaCy v3.2 adds a new transformer pipeline package for Japanese +[`ja_core_news_trf`](/models/ja#ja_core_news_trf), which uses the `basic` +pretokenizer instead of `mecab` to limit the number of dependencies required for +the pipeline. Thanks to Hiroshi Matsuda and the spaCy Japanese community for +their contributions! + +### Pipeline and language updates {id="pipeline-updates"} + +- All Universal Dependencies training data has been updated to v2.8. +- The Catalan data, tokenizer and lemmatizer have been updated, thanks to Carlos + Rodriguez, Carme Armentano and the Barcelona Supercomputing Center! +- The transformer pipelines are trained using spacy-transformers v1.1, with + improved IO and more options for + [model config and output](/api/architectures#TransformerModel). +- Trailing whitespace has been added as a `tok2vec` feature, improving the + performance for many components, especially fine-grained tagging and sentence + segmentation. +- The English attribute ruler patterns have been overhauled to improve + `Token.pos` and `Token.morph`. + +spaCy v3.2 also features a new Irish lemmatizer, support for `noun_chunks` in +Portuguese, improved `noun_chunks` for Spanish and additional updates for +Bulgarian, Catalan, Sinhala, Tagalog, Tigrinya and Vietnamese. + +## Notes about upgrading from v3.1 {id="upgrading"} + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with spaCy v3.0 or v3.1, you will +see a warning telling you that the pipeline may be incompatible. This doesn't +necessarily have to be true, but we recommend running your pipelines against +your test suite or evaluation data to make sure there are no unexpected results. +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.1.0,<3.2.0", ++ "spacy_version": ">=3.2.0,<3.3.0", +``` + +### Updating v3.1 configs + +To update a config from spaCy v3.1 with the new v3.2 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```bash +$ python -m spacy init fill-config config-v3.1.cfg config-v3.2.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). + +## Notes about upgrading from spacy-transformers v1.0 {id="upgrading-transformers"} + +When you're loading a transformer pipeline package trained with +[`spacy-transformers`](https://github.com/explosion/spacy-transformers) v1.0 +after upgrading to `spacy-transformers` v1.1, you'll see a warning telling you +that the pipeline may be incompatible. `spacy-transformers` v1.1 should be able +to import v1.0 `transformer` components into the new internal format with no +change in performance, but here we'd also recommend running your test suite to +verify that the pipeline still performs as expected. + +If you save your pipeline with [`nlp.to_disk`](/api/language#to_disk), it will +be saved in the new v1.1 format and should be fully compatible with +`spacy-transformers` v1.1. Once you've confirmed the performance, you can update +the requirements in [`meta.json`](/api/data-formats#meta): + +```diff + "requirements": [ +- "spacy-transformers>=1.0.3,<1.1.0" ++ "spacy-transformers>=1.1.2,<1.2.0" + ] +``` + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). diff --git a/website/docs/usage/v3-3.mdx b/website/docs/usage/v3-3.mdx new file mode 100644 index 0000000..fd211e6 --- /dev/null +++ b/website/docs/usage/v3-3.mdx @@ -0,0 +1,247 @@ +--- +title: What's New in v3.3 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {id="features",hidden="true"} + +spaCy v3.3 improves the speed of core pipeline components, adds a new trainable +lemmatizer, and introduces trained pipelines for Finnish, Korean and Swedish. + +### Speed improvements {id="speed"} + +v3.3 includes a slew of speed improvements: + +- Speed up parser and NER by using constant-time head lookups. +- Support unnormalized softmax probabilities in `spacy.Tagger.v2` to speed up + inference for tagger, morphologizer, senter and trainable lemmatizer. +- Speed up parser projectivization functions. +- Replace `Ragged` with faster `AlignmentArray` in `Example` for training. +- Improve `Matcher` speed. +- Improve serialization speed for empty `Doc.spans`. + +For longer texts, the trained pipeline speeds improve **15%** or more in +prediction. We benchmarked `en_core_web_md` (same components as in v3.2) and +`de_core_news_md` (with the new trainable lemmatizer) across a range of text +sizes on Linux (Intel Xeon W-2265) and OS X (M1) to compare spaCy v3.2 vs. v3.3: + +**Intel Xeon W-2265** + +| Model | Avg. Words/Doc | v3.2 Words/Sec | v3.3 Words/Sec | Diff | +| :----------------------------------------------- | -------------: | -------------: | -------------: | -----: | +| [`en_core_web_md`](/models/en#en_core_web_md) | 100 | 17292 | 17441 | 0.86% | +| (=same components) | 1000 | 15408 | 16024 | 4.00% | +| | 10000 | 12798 | 15346 | 19.91% | +| [`de_core_news_md`](/models/de/#de_core_news_md) | 100 | 20221 | 19321 | -4.45% | +| (+v3.3 trainable lemmatizer) | 1000 | 17480 | 17345 | -0.77% | +| | 10000 | 14513 | 17036 | 17.38% | + +**Apple M1** + +| Model | Avg. Words/Doc | v3.2 Words/Sec | v3.3 Words/Sec | Diff | +| ------------------------------------------------ | -------------: | -------------: | -------------: | -----: | +| [`en_core_web_md`](/models/en#en_core_web_md) | 100 | 18272 | 18408 | 0.74% | +| (=same components) | 1000 | 18794 | 19248 | 2.42% | +| | 10000 | 15144 | 17513 | 15.64% | +| [`de_core_news_md`](/models/de/#de_core_news_md) | 100 | 19227 | 19591 | 1.89% | +| (+v3.3 trainable lemmatizer) | 1000 | 20047 | 20628 | 2.90% | +| | 10000 | 15921 | 18546 | 16.49% | + +### Trainable lemmatizer {id="trainable-lemmatizer"} + +The new [trainable lemmatizer](/api/edittreelemmatizer) component uses +[edit trees](https://explosion.ai/blog/edit-tree-lemmatizer) to transform tokens +into lemmas. Try out the trainable lemmatizer with the +[training quickstart](/usage/training#quickstart)! + +### displaCy support for overlapping spans and arcs {id="displacy"} + +displaCy now supports overlapping spans with a new +[`span`](/usage/visualizers#span) style and multiple arcs with different labels +between the same tokens for [`dep`](/usage/visualizers#dep) visualizations. + +Overlapping spans can be visualized for any spans key in `doc.spans`: + +```python +import spacy +from spacy import displacy +from spacy.tokens import Span + +nlp = spacy.blank("en") +text = "Welcome to the Bank of China." +doc = nlp(text) +doc.spans["custom"] = [Span(doc, 3, 6, "ORG"), Span(doc, 5, 6, "GPE")] +displacy.serve(doc, style="span", options={"spans_key": "custom"}) +``` + + +
Welcome to the BankORG of ChinaGPE.
+ + +## Additional features and improvements + +- Config comparisons with [`spacy debug diff-config`](/api/cli#debug-diff). +- Span suggester debugging with + [`SpanCategorizer.set_candidates`](/api/spancategorizer#set_candidates). +- Big endian support with + [`thinc-bigendian-ops`](https://github.com/andrewsi-z/thinc-bigendian-ops) and + updates to make `floret`, `murmurhash`, Thinc and spaCy endian neutral. +- Initial support for Lower Sorbian and Upper Sorbian. +- Language updates for English, French, Italian, Japanese, Korean, Norwegian, + Russian, Slovenian, Spanish, Turkish, Ukrainian and Vietnamese. +- New noun chunks for Finnish. + +## Trained pipelines {id="pipelines"} + +### New trained pipelines {id="new-pipelines"} + +v3.3 introduces new CPU/CNN pipelines for Finnish, Korean and Swedish, which use +the new trainable lemmatizer and +[floret vectors](https://github.com/explosion/floret). Due to the use +[Bloom embeddings](https://explosion.ai/blog/bloom-embeddings) and subwords, the +pipelines have compact vectors with no out-of-vocabulary words. + +| Package | Language | UPOS | Parser LAS | NER F | +| ----------------------------------------------- | -------- | ---: | ---------: | ----: | +| [`fi_core_news_sm`](/models/fi#fi_core_news_sm) | Finnish | 92.5 | 71.9 | 75.9 | +| [`fi_core_news_md`](/models/fi#fi_core_news_md) | Finnish | 95.9 | 78.6 | 80.6 | +| [`fi_core_news_lg`](/models/fi#fi_core_news_lg) | Finnish | 96.2 | 79.4 | 82.4 | +| [`ko_core_news_sm`](/models/ko#ko_core_news_sm) | Korean | 86.1 | 65.6 | 71.3 | +| [`ko_core_news_md`](/models/ko#ko_core_news_md) | Korean | 94.7 | 80.9 | 83.1 | +| [`ko_core_news_lg`](/models/ko#ko_core_news_lg) | Korean | 94.7 | 81.3 | 85.3 | +| [`sv_core_news_sm`](/models/sv#sv_core_news_sm) | Swedish | 95.0 | 75.9 | 74.7 | +| [`sv_core_news_md`](/models/sv#sv_core_news_md) | Swedish | 96.3 | 78.5 | 79.3 | +| [`sv_core_news_lg`](/models/sv#sv_core_news_lg) | Swedish | 96.3 | 79.1 | 81.1 | + +### Pipeline updates {id="pipeline-updates"} + +The following languages switch from lookup or rule-based lemmatizers to the new +trainable lemmatizer: Danish, Dutch, German, Greek, Italian, Lithuanian, +Norwegian, Polish, Portuguese and Romanian. The overall lemmatizer accuracy +improves for all of these pipelines, but be aware that the types of errors may +look quite different from the lookup-based lemmatizers. If you'd prefer to +continue using the previous lemmatizer, you can +[switch from the trainable lemmatizer to a non-trainable lemmatizer](/models#design-modify). + +
+ +| Model | v3.2 Lemma Acc | v3.3 Lemma Acc | +| ----------------------------------------------- | -------------: | -------------: | +| [`da_core_news_md`](/models/da#da_core_news_md) | 84.9 | 94.8 | +| [`de_core_news_md`](/models/de#de_core_news_md) | 73.4 | 97.7 | +| [`el_core_news_md`](/models/el#el_core_news_md) | 56.5 | 88.9 | +| [`fi_core_news_md`](/models/fi#fi_core_news_md) | - | 86.2 | +| [`it_core_news_md`](/models/it#it_core_news_md) | 86.6 | 97.2 | +| [`ko_core_news_md`](/models/ko#ko_core_news_md) | - | 90.0 | +| [`lt_core_news_md`](/models/lt#lt_core_news_md) | 71.1 | 84.8 | +| [`nb_core_news_md`](/models/nb#nb_core_news_md) | 76.7 | 97.1 | +| [`nl_core_news_md`](/models/nl#nl_core_news_md) | 81.5 | 94.0 | +| [`pl_core_news_md`](/models/pl#pl_core_news_md) | 87.1 | 93.7 | +| [`pt_core_news_md`](/models/pt#pt_core_news_md) | 76.7 | 96.9 | +| [`ro_core_news_md`](/models/ro#ro_core_news_md) | 81.8 | 95.5 | +| [`sv_core_news_md`](/models/sv#sv_core_news_md) | - | 95.5 | + +
+ +In addition, the vectors in the English pipelines are deduplicated to improve +the pruned vectors in the `md` models and reduce the `lg` model size. + +## Notes about upgrading from v3.2 {id="upgrading"} + +### Span comparisons + +Span comparisons involving ordering (`<`, `<=`, `>`, `>=`) now take all span +attributes into account (start, end, label, and KB ID) so spans may be sorted in +a slightly different order. + +### Whitespace annotation + +During training, annotation on whitespace tokens is handled in the same way as +annotation on non-whitespace tokens in order to allow custom whitespace +annotation. + +### Doc.from_docs + +[`Doc.from_docs`](/api/doc#from_docs) now includes `Doc.tensor` by default and +supports excludes with an `exclude` argument in the same format as +`Doc.to_bytes`. The supported exclude fields are `spans`, `tensor` and +`user_data`. + +Docs including `Doc.tensor` may be quite a bit larger in RAM, so to exclude +`Doc.tensor` as in v3.2: + +```diff +-merged_doc = Doc.from_docs(docs) ++merged_doc = Doc.from_docs(docs, exclude=["tensor"]) +``` + +### Using trained pipelines with floret vectors + +If you're running a new trained pipeline for Finnish, Korean or Swedish on new +texts and working with `Doc` objects, you shouldn't notice any difference with +floret vectors vs. default vectors. + +If you use vectors for similarity comparisons, there are a few differences, +mainly because a floret pipeline doesn't include any kind of frequency-based +word list similar to the list of in-vocabulary vector keys with default vectors. + +- If your workflow iterates over the vector keys, you should use an external + word list instead: + + ```diff + - lexemes = [nlp.vocab[orth] for orth in nlp.vocab.vectors] + + lexemes = [nlp.vocab[word] for word in external_word_list] + ``` + +- `Vectors.most_similar` is not supported because there's no fixed list of + vectors to compare your vectors to. + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.2.0,<3.3.0", ++ "spacy_version": ">=3.2.0,<3.4.0", +``` + +### Updating v3.2 configs + +To update a config from spaCy v3.2 with the new v3.3 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```bash +$ python -m spacy init fill-config config-v3.2.cfg config-v3.3.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). + +To see the speed improvements for the +[`Tagger` architecture](/api/architectures#Tagger), edit your config to switch +from `spacy.Tagger.v1` to `spacy.Tagger.v2` and then run `init fill-config`. diff --git a/website/docs/usage/v3-4.mdx b/website/docs/usage/v3-4.mdx new file mode 100644 index 0000000..c06db9d --- /dev/null +++ b/website/docs/usage/v3-4.mdx @@ -0,0 +1,143 @@ +--- +title: What's New in v3.4 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {id="features",hidden="true"} + +spaCy v3.4 brings typing and speed improvements along with new vectors for +English CNN pipelines and new trained pipelines for Croatian. This release also +includes prebuilt linux aarch64 wheels for all spaCy dependencies distributed by +Explosion. + +### Typing improvements {id="typing"} + +spaCy v3.4 supports pydantic v1.9 and mypy 0.950+ through extensive updates to +types in Thinc v8.1. + +### Speed improvements {id="speed"} + +- For the parser, use C `saxpy`/`sgemm` provided by the `Ops` implementation in + order to use Accelerate through `thinc-apple-ops`. +- Improved speed of vector lookups. +- Improved speed for `Example.get_aligned_parse` and `Example.get_aligned`. + +## Additional features and improvements + +- Min/max `{n,m}` operator for `Matcher` patterns. +- Language updates: + - Improve tokenization for Cyrillic combining diacritics. + - Improve English tokenizer exceptions for contractions with + this/that/these/those. +- Updated `spacy project clone` to try both `main` and `master` branches by + default. +- Added confidence threshold for named entity linker. +- Improved handling of Typer optional default values for `init_config_cli`. +- Added cycle detection in parser projectivization methods. +- Added counts for NER labels in `debug data`. +- Support for adding NVTX ranges to `TrainablePipe` components. +- Support env variable `SPACY_NUM_BUILD_JOBS` to specify the number of build + jobs to run in parallel with `pip`. + +## Trained pipelines {id="pipelines"} + +### New trained pipelines {id="new-pipelines"} + +v3.4 introduces new CPU/CNN pipelines for Croatian, which use the trainable +lemmatizer and [floret vectors](https://github.com/explosion/floret). Due to the +use of [Bloom embeddings](https://explosion.ai/blog/bloom-embeddings) and +subwords, the pipelines have compact vectors with no out-of-vocabulary words. + +| Package | UPOS | Parser LAS | NER F | +| ----------------------------------------------- | ---: | ---------: | ----: | +| [`hr_core_news_sm`](/models/hr#hr_core_news_sm) | 96.6 | 77.5 | 76.1 | +| [`hr_core_news_md`](/models/hr#hr_core_news_md) | 97.3 | 80.1 | 81.8 | +| [`hr_core_news_lg`](/models/hr#hr_core_news_lg) | 97.5 | 80.4 | 83.0 | + +### Pipeline updates {id="pipeline-updates"} + +All CNN pipelines have been extended with whitespace augmentation. + +The English CNN pipelines have new word vectors: + +| Package | Model Version | TAG | Parser LAS | NER F | +| --------------------------------------------- | ------------- | ---: | ---------: | ----: | +| [`en_core_web_md`](/models/en#en_core_web_md) | v3.3.0 | 97.3 | 90.1 | 84.6 | +| [`en_core_web_md`](/models/en#en_core_web_md) | v3.4.0 | 97.2 | 90.3 | 85.5 | +| [`en_core_web_lg`](/models/en#en_core_web_lg) | v3.3.0 | 97.4 | 90.1 | 85.3 | +| [`en_core_web_lg`](/models/en#en_core_web_lg) | v3.4.0 | 97.3 | 90.2 | 85.6 | + +## Notes about upgrading from v3.3 {id="upgrading"} + +### Doc.has_vector + +`Doc.has_vector` now matches `Token.has_vector` and `Span.has_vector`: it +returns `True` if at least one token in the doc has a vector rather than +checking only whether the vocab contains vectors. + +### Using trained pipelines with floret vectors + +If you're using a trained pipeline for Croatian, Finnish, Korean or Swedish with +new texts and working with `Doc` objects, you shouldn't notice any difference +between floret vectors and default vectors. + +If you use vectors for similarity comparisons, there are a few differences, +mainly because a floret pipeline doesn't include any kind of frequency-based +word list similar to the list of in-vocabulary vector keys with default vectors. + +- If your workflow iterates over the vector keys, you should use an external + word list instead: + + ```diff + - lexemes = [nlp.vocab[orth] for orth in nlp.vocab.vectors] + + lexemes = [nlp.vocab[word] for word in external_word_list] + ``` + +- `Vectors.most_similar` is not supported because there's no fixed list of + vectors to compare your vectors to. + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.3.0,<3.4.0", ++ "spacy_version": ">=3.3.0,<3.5.0", +``` + +### Updating v3.3 configs + +To update a config from spaCy v3.3 with the new v3.4 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```bash +$ python -m spacy init fill-config config-v3.3.cfg config-v3.4.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). diff --git a/website/docs/usage/v3-5.mdx b/website/docs/usage/v3-5.mdx new file mode 100644 index 0000000..3ca64f8 --- /dev/null +++ b/website/docs/usage/v3-5.mdx @@ -0,0 +1,230 @@ +--- +title: What's New in v3.5 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {id="features",hidden="true"} + +spaCy v3.5 introduces three new CLI commands, `apply`, `benchmark` and +`find-threshold`, adds fuzzy matching, provides improvements to our entity +linking functionality, and includes a range of language updates and bug fixes. + +### New CLI commands {id="cli"} + +#### apply CLI + +The [`apply` CLI](/api/cli#apply) can be used to apply a pipeline to one or more +`.txt`, `.jsonl` or `.spacy` input files, saving the annotated docs in a single +`.spacy` file. + +```bash +$ spacy apply en_core_web_sm my_texts/ output.spacy +``` + +#### benchmark CLI + +The [`benchmark` CLI](/api/cli#benchmark) has been added to extend the existing +`evaluate` functionality with a wider range of profiling subcommands. + +The `benchmark accuracy` CLI is introduced as an alias for `evaluate`. The new +`benchmark speed` CLI performs warmup rounds before measuring the speed in words +per second on batches of randomly shuffled documents from the provided data. + +```bash +$ spacy benchmark speed my_pipeline data.spacy +``` + +The output is the mean performance using batches (`nlp.pipe`) with a 95% +confidence interval, e.g., profiling `en_core_web_sm` on CPU: + +```none +Outliers: 2.0%, extreme outliers: 0.0% +Mean: 18904.1 words/s (95% CI: -256.9 +244.1) +``` + +#### find-threshold CLI + +The [`find-threshold` CLI](/api/cli#find-threshold) runs a series of trials +across threshold values from `0.0` to `1.0` and identifies the best threshold +for the provided score metric. + +The following command runs 20 trials for the `spancat` component in +`my_pipeline`, recording the `spans_sc_f` score for each value of the threshold +`[components.spancat.threshold]` from `0.0` to `1.0`: + +```bash +$ spacy find-threshold my_pipeline data.spacy spancat threshold spans_sc_f --n_trials 20 +``` + +The `find-threshold` CLI can be used with `textcat_multilabel`, `spancat` and +custom components with thresholds that are applied while predicting or scoring. + +### Fuzzy matching {id="fuzzy"} + +New `FUZZY` operators support [fuzzy matching](/usage/rule-based-matching#fuzzy) +with the `Matcher`. By default, the `FUZZY` operator allows a Levenshtein edit +distance of 2 and up to 30% of the pattern string length. `FUZZY1`..`FUZZY9` can +be used to specify the exact number of allowed edits. + +```python +# Match lowercase with fuzzy matching (allows up to 3 edits) +pattern = [{"LOWER": {"FUZZY": "definitely"}}] + +# Match custom attribute values with fuzzy matching (allows up to 3 edits) +pattern = [{"_": {"country": {"FUZZY": "Kyrgyzstan"}}}] + +# Match with exact Levenshtein edit distance limits (allows up to 4 edits) +pattern = [{"_": {"country": {"FUZZY4": "Kyrgyzstan"}}}] +``` + +Note that `FUZZY` uses Levenshtein edit distance rather than Damerau-Levenshtein +edit distance, so a transposition like `teh` for `the` counts as two edits, one +insertion and one deletion. + +If you'd prefer an alternate fuzzy matching algorithm, you can provide your own +custom method to the `Matcher` or as a config option for an entity ruler and +span ruler. + +### FUZZY and REGEX with lists {id="fuzzy-regex-lists"} + +The `FUZZY` and `REGEX` operators are also now supported for lists with `IN` and +`NOT_IN`: + +```python +pattern = [{"TEXT": {"FUZZY": {"IN": ["awesome", "cool", "wonderful"]}}}] +pattern = [{"TEXT": {"REGEX": {"NOT_IN": ["^awe(some)?$", "^wonder(ful)?"]}}}] +``` + +### Entity linking generalization {id="el"} + +The knowledge base used for entity linking is now easier to customize and has a +new default implementation [`InMemoryLookupKB`](/api/inmemorylookupkb). + +### Additional features and improvements {id="additional-features-and-improvements"} + +- Language updates: + - Extended support for Slovenian + - Fixed lookup fallback for French and Catalan lemmatizers + - Switch Russian and Ukrainian lemmatizers to `pymorphy3` + - Support for editorial punctuation in Ancient Greek + - Update to Russian tokenizer exceptions + - Small fix for Dutch stop words +- Allow up to `typer` v0.7.x, `mypy` 0.990 and `typing_extensions` v4.4.x. +- New `spacy.ConsoleLogger.v3` with expanded progress + [tracking](/api/top-level#ConsoleLogger). +- Improved scoring behavior for `textcat` with `spacy.textcat_scorer.v2` and + `spacy.textcat_multilabel_scorer.v2`. +- Updates so that downstream components can train properly on a frozen `tok2vec` + or `transformer` layer. +- Allow interpolation of variables in directory names in projects. +- Support for local file system [remotes](/usage/projects#remote) for projects. +- Improve UX around `displacy.serve` when the default port is in use. +- Optional `before_update` callback that is invoked at the start of each + [training step](/api/data-formats#config-training). +- Improve performance of `SpanGroup` and fix typing issues for `SpanGroup` and + `Span` objects. +- Patch a + [security vulnerability](https://github.com/advisories/GHSA-gw9q-c7gh-j9vm) in + extracting tar files. +- Add equality definition for `Vectors`. +- Ensure `Vocab.to_disk` respects the exclude setting for `lookups` and + `vectors`. +- Correctly handle missing annotations in the edit tree lemmatizer. + +### Trained pipeline updates {id="pipelines"} + +- The CNN pipelines add `IS_SPACE` as a `tok2vec` feature for `tagger` and + `morphologizer` components to improve tagging of non-whitespace vs. whitespace + tokens. +- The transformer pipelines require `spacy-transformers` v1.2, which uses the + exact alignment from `tokenizers` for fast tokenizers instead of the heuristic + alignment from `spacy-alignments`. For all trained pipelines except + `ja_core_news_trf`, the alignments between spaCy tokens and transformer tokens + may be slightly different. More details about the `spacy-transformers` changes + in the + [v1.2.0 release notes](https://github.com/explosion/spacy-transformers/releases/tag/v1.2.0). + +## Notes about upgrading from v3.4 {id="upgrading"} + +### Validation of textcat values {id="textcat-validation"} + +An error is now raised when unsupported values are given as input to train a +`textcat` or `textcat_multilabel` model - ensure that values are `0.0` or `1.0` +as explained in the [docs](/api/textcategorizer#assigned-attributes). + +### Using the default knowledge base + +As `KnowledgeBase` is now an abstract class, you should call the constructor of +the new `InMemoryLookupKB` instead when you want to use spaCy's default KB +implementation: + +```diff +- kb = KnowledgeBase() ++ kb = InMemoryLookupKB() +``` + +If you've written a custom KB that inherits from `KnowledgeBase`, you'll need to +implement its abstract methods, or alternatively inherit from `InMemoryLookupKB` +instead. + +### Updated scorers for tokenization and textcat {id="scores"} + +We fixed a bug that inflated the `token_acc` scores in v3.0-v3.4. The reported +`token_acc` will drop from v3.4 to v3.5, but if `token_p/r/f` stay the same, +your tokenization performance has not changed from v3.4. + +For new `textcat` or `textcat_multilabel` configs, the new default `v2` scorers: + +- ignore `threshold` for `textcat`, so the reported `cats_p/r/f` may increase + slightly in v3.5 even though the underlying predictions are unchanged +- report the performance of only the **final** `textcat` or `textcat_multilabel` + component in the pipeline by default +- allow custom scorers to be used to score multiple `textcat` and + `textcat_multilabel` components with `Scorer.score_cats` by restricting the + evaluation to the component's provided labels + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.4.0,<3.5.0", ++ "spacy_version": ">=3.4.0,<3.6.0", +``` + +### Updating v3.4 configs + +To update a config from spaCy v3.4 with the new v3.5 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```cli +$ python -m spacy init fill-config config-v3.4.cfg config-v3.5.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). diff --git a/website/docs/usage/v3-6.mdx b/website/docs/usage/v3-6.mdx new file mode 100644 index 0000000..eda46b3 --- /dev/null +++ b/website/docs/usage/v3-6.mdx @@ -0,0 +1,143 @@ +--- +title: What's New in v3.6 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {id="features",hidden="true"} + +spaCy v3.6 adds the new [`SpanFinder`](/api/spanfinder) component to the core +spaCy library and new trained pipelines for Slovenian. + +### SpanFinder {id="spanfinder"} + +The [`SpanFinder`](/api/spanfinder) component identifies potentially +overlapping, unlabeled spans by identifying span start and end tokens. It is +intended for use in combination with a component like +[`SpanCategorizer`](/api/spancategorizer) that may further filter or label the +spans. See our +[Spancat blog post](https://explosion.ai/blog/spancat#span-finder) for a more +detailed introduction to the span finder. + +To train a pipeline with `span_finder` + `spancat`, remember to add +`span_finder` (and its `tok2vec` or `transformer` if required) to +`[training.annotating_components]` so that the `spancat` component can be +trained directly from its predictions: + +```ini +[nlp] +pipeline = ["tok2vec","span_finder","spancat"] + +[training] +annotating_components = ["tok2vec","span_finder"] +``` + +In practice it can be helpful to initially train the `span_finder` separately +before [sourcing](/usage/processing-pipelines#sourced-components) it (along with +its `tok2vec`) into the `spancat` pipeline for further training. Otherwise the +memory usage can spike for `spancat` in the first few training steps if the +`span_finder` makes a large number of predictions. + +### Additional features and improvements {id="additional-features-and-improvements"} + +- Language updates: + - Add initial support for Malay. + - Update Latin defaults to support noun chunks, update lexical/tokenizer + settings and add example sentences. +- Support `spancat_singlelabel` in `spacy debug data` CLI. +- Add `doc.spans` rendering to `spacy evaluate` CLI displaCy output. +- Support custom token/lexeme attribute for vectors. +- Add option to return scores separately keyed by component name with + `spacy evaluate --per-component`, `Language.evaluate(per_component=True)` and + `Scorer.score(per_component=True)`. This is useful when the pipeline contains + more than one of the same component like `textcat` that may have overlapping + scores keys. +- Typing updates for `PhraseMatcher` and `SpanGroup`. + +## Trained pipelines {id="pipelines"} + +### New trained pipelines {id="new-pipelines"} + +v3.6 introduces new pipelines for Slovenian, which use the trainable lemmatizer +and [floret vectors](https://github.com/explosion/floret). + +| Package | UPOS | Parser LAS | NER F | +| ------------------------------------------------- | ---: | ---------: | ----: | +| [`sl_core_news_sm`](/models/sl#sl_core_news_sm) | 96.9 | 82.1 | 62.9 | +| [`sl_core_news_md`](/models/sl#sl_core_news_md) | 97.6 | 84.3 | 73.5 | +| [`sl_core_news_lg`](/models/sl#sl_core_news_lg) | 97.7 | 84.3 | 79.0 | +| [`sl_core_news_trf`](/models/sl#sl_core_news_trf) | 99.0 | 91.7 | 90.0 | + +### Pipeline updates {id="pipeline-updates"} + +The English pipelines have been updated to improve handling of contractions with +various apostrophes and to lemmatize "get" as a passive auxiliary. + +The Danish pipeline `da_core_news_trf` has been updated to use +[`vesteinn/DanskBERT`](https://huggingface.co/vesteinn/DanskBERT) with +performance improvements across the board. + +## Notes about upgrading from v3.5 {id="upgrading"} + +### SpanGroup spans are now required to be from the same doc {id="spangroup-spans"} + +When initializing a `SpanGroup`, there is a new check to verify that all added +spans refer to the current doc. Without this check, it was possible to run into +string store or other errors. + +One place this may crop up is when creating `Example` objects for training with +custom spans: + +```diff + doc = Doc(nlp.vocab, words=tokens) # predicted doc + example = Example.from_dict(doc, {"ner": iob_tags}) + # use the reference doc when creating reference spans +- span = Span(doc, 0, 5, "ORG") ++ span = Span(example.reference, 0, 5, "ORG") + example.reference.spans[spans_key] = [span] +``` + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.5.0,<3.6.0", ++ "spacy_version": ">=3.5.0,<3.7.0", +``` + +### Updating v3.5 configs + +To update a config from spaCy v3.5 with the new v3.6 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```cli +$ python -m spacy init fill-config config-v3.5.cfg config-v3.6.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). diff --git a/website/docs/usage/v3-7.mdx b/website/docs/usage/v3-7.mdx new file mode 100644 index 0000000..76fc953 --- /dev/null +++ b/website/docs/usage/v3-7.mdx @@ -0,0 +1,140 @@ +--- +title: What's New in v3.7 +teaser: New features and how to upgrade +menu: + - ['New Features', 'features'] + - ['Upgrading Notes', 'upgrading'] +--- + +## New features {id="features",hidden="true"} + +spaCy v3.7 adds support for Python 3.12, introduces the new standalone library +[Weasel](https://github.com/explosion/weasel) for project workflows, and updates +the transformer-based trained pipelines to use our new +[Curated Transformers](https://github.com/explosion/curated-transformers) +library. + +This release drops support for Python 3.6. + +### Weasel {id="weasel"} + +The [spaCy projects](/usage/projects) functionality has been moved into a new +standalone library [Weasel](https://github.com/explosion/weasel). This brings +minor changes to spaCy-specific settings in spaCy projects (see +[upgrading](#upgrading) below), but also makes it possible to use the same +workflow functionality outside of spaCy. + +All `spacy project` commands should run as before, just now they're using Weasel +under the hood. + + + +Remote storage for spaCy projects is not yet supported for Python 3.12. Use +Python 3.11 or earlier for remote storage. + + + +### Registered vectors {id="custom-vectors"} + +You can specify a custom registered vectors class under `[nlp.vectors]` in order +to use static vectors in formats other than the ones supported by +[`Vectors`](/api/vectors). To implement your custom vectors, extend the abstract +class [`BaseVectors`](/api/basevectors). See an example using +[BPEmb subword embeddings](/usage/embeddings-transformers#custom-vectors). + +### Additional features and improvements {id="additional-features-and-improvements"} + +- Add support for Python 3.12. +- Extend to Thinc v8.2. +- Extend `transformers` extra to `spacy-transformers` v1.3. +- Add `--spans-key` option for CLI evaluation with `spacy benchmark accuracy`. +- Load the CLI module lazily for `spacy.info`. +- Add type stubs for for `spacy.training.example`. +- Warn for unsupported pattern keys in dependency matcher. +- `Language.replace_listeners`: Pass the replaced listener and the `tok2vec` + pipe to the callback in order to support `spacy-curated-transformers`. +- Always use `tqdm` with `disable=None` in order to disable output in + non-interactive environments. +- Language updates: + - Add left and right pointing angle brackets as punctuation to ancient Greek. + - Update example sentences for Turkish. +- Package setup updates: + - Update NumPy build constraints for NumPy 1.25+. For Python 3.9+, it is no + longer necessary to set build constraints while building binary wheels. + - Refactor Cython profiling in order to disable profiling for Python 3.12 in + the package setup, since Cython does not currently support profiling for + Python 3.12. + +## Trained pipelines {id="pipelines"} + +### Pipeline updates {id="pipeline-updates"} + +The transformer-based `trf` pipelines have been updated to use our new +[Curated Transformers](https://github.com/explosion/curated-transformers) +library using the Thinc model wrappers and pipeline component from +[spaCy Curated Transformers](https://github.com/explosion/spacy-curated-transformers). + +## Notes about upgrading from v3.6 {id="upgrading"} + +This release drops support for Python 3.6, drops mypy checks for Python 3.7 and +removes the `ray` extra. In addition there are several minor changes for spaCy +projects described in the following section. + +### Backwards incompatibilities for spaCy Projects {id="upgrading-projects"} + +`spacy project` has a few backwards incompatibilities due to the transition to +the standalone library [Weasel](https://github.com/explosion/weasel), which is +not as tightly coupled to spaCy. Weasel produces warnings when it detects older +spaCy-specific settings in your environment or project config. + +- Support for the `spacy_version` configuration key has been dropped. +- Support for the `check_requirements` configuration key has been dropped due to + the deprecation of `pkg_resources`. +- The `SPACY_CONFIG_OVERRIDES` environment variable is no longer checked. You + can set configuration overrides using `WEASEL_CONFIG_OVERRIDES`. +- Support for `SPACY_PROJECT_USE_GIT_VERSION` environment variable has been + dropped. +- Error codes are now Weasel-specific and do not follow spaCy error codes. + +### Pipeline package version compatibility {id="version-compat"} + +> #### Using legacy implementations +> +> In spaCy v3, you'll still be able to load and reference legacy implementations +> via [`spacy-legacy`](https://github.com/explosion/spacy-legacy), even if the +> components or architectures change and newer versions are available in the +> core library. + +When you're loading a pipeline package trained with an earlier version of spaCy +v3, you will see a warning telling you that the pipeline may be incompatible. +This doesn't necessarily have to be true, but we recommend running your +pipelines against your test suite or evaluation data to make sure there are no +unexpected results. + +If you're using one of the [trained pipelines](/models) we provide, you should +run [`spacy download`](/api/cli#download) to update to the latest version. To +see an overview of all installed packages and their compatibility, you can run +[`spacy validate`](/api/cli#validate). + +If you've trained your own custom pipeline and you've confirmed that it's still +working as expected, you can update the spaCy version requirements in the +[`meta.json`](/api/data-formats#meta): + +```diff +- "spacy_version": ">=3.6.0,<3.7.0", ++ "spacy_version": ">=3.6.0,<3.8.0", +``` + +### Updating v3.6 configs + +To update a config from spaCy v3.6 with the new v3.7 settings, run +[`init fill-config`](/api/cli#init-fill-config): + +```cli +$ python -m spacy init fill-config config-v3.6.cfg config-v3.7.cfg +``` + +In many cases ([`spacy train`](/api/cli#train), +[`spacy.load`](/api/top-level#spacy.load)), the new defaults will be filled in +automatically, but you'll need to fill in the new settings to run +[`debug config`](/api/cli#debug) and [`debug data`](/api/cli#debug-data). diff --git a/website/docs/usage/v3.mdx b/website/docs/usage/v3.mdx new file mode 100644 index 0000000..7e7548d --- /dev/null +++ b/website/docs/usage/v3.mdx @@ -0,0 +1,1184 @@ +--- +title: What's New in v3.0 +teaser: New features, backwards incompatibilities and migration guide +menu: + - ['Summary', 'summary'] + - ['New Features', 'features'] + - ['Backwards Incompatibilities', 'incompat'] + - ['Migrating from v2.x', 'migrating'] +--- + +## Summary {id="summary",hidden="true"} + +> #### 📖 Looking for the old docs? +> +> To help you make the transition from v2.x to v3.0, we've uploaded the old +> website to [**v2.spacy.io**](https://v2.spacy.io/docs). + + + +
+ +spaCy v3.0 features all new **transformer-based pipelines** that bring spaCy's +accuracy right up to the current **state-of-the-art**. You can use any +pretrained transformer to train your own pipelines, and even share one +transformer between multiple components with **multi-task learning**. Training +is now fully configurable and extensible, and you can define your own custom +models using **PyTorch**, **TensorFlow** and other frameworks. The new spaCy +projects system lets you describe whole **end-to-end workflows** in a single +file, giving you an easy path from prototype to production, and making it easy +to clone and adapt best-practice projects for your own use cases. + +
+ + + +- [Summary](#summary) +- [New features](#features) +- [Transformer-based pipelines](#features-transformers) +- [Training & config system](#features-training) +- [Custom models](#features-custom-models) +- [End-to-end project workflows](#features-projects) +- [Parallel training with Ray](#features-parallel-training) +- [New built-in components](#features-pipeline-components) +- [New custom component API](#features-components) +- [Dependency matching](#features-dep-matcher) +- [Python type hints](#features-types) +- [New methods & attributes](#new-methods) +- [New & updated documentation](#new-docs) +- [Backwards incompatibilities](#incompat) +- [Migrating from spaCy v2.x](#migrating) + + + + + +## New Features {id="features"} + +This section contains an overview of the most important **new features and +improvements**. The [API docs](/api) include additional deprecation notes. New +methods and functions that were introduced in this version are marked with the +tag 3. + + + + + + + + + + + +### Transformer-based pipelines {id="features-transformers"} + +> #### Example +> +> ```bash +> $ python -m spacy download en_core_web_trf +> ``` + +spaCy v3.0 features all new transformer-based pipelines that bring spaCy's +accuracy right up to the current **state-of-the-art**. You can use any +pretrained transformer to train your own pipelines, and even share one +transformer between multiple components with **multi-task learning**. spaCy's +transformer support interoperates with [PyTorch](https://pytorch.org) and the +[HuggingFace `transformers`](https://huggingface.co/transformers/) library, +giving you access to thousands of pretrained models for your pipelines. + +![Pipeline components listening to shared embedding component](/images/tok2vec-listener.svg) + + + +#### New trained transformer-based pipelines {id="features-transformers-pipelines"} + +> #### Notes on model capabilities +> +> The models are each trained with a **single transformer** shared across the +> pipeline, which requires it to be trained on a single corpus. For +> [English](/models/en) and [Chinese](/models/zh), we used the OntoNotes 5 +> corpus, which has annotations across several tasks. For [French](/models/fr), +> [Spanish](/models/es) and [German](/models/de), we didn't have a suitable +> corpus that had both syntactic and entity annotations, so the transformer +> models for those languages do not include NER. + +| Package | Language | Transformer | Tagger | Parser | NER | +| ------------------------------------------------ | -------- | --------------------------------------------------------------------------------------------- | -----: | -----: | ---: | +| [`en_core_web_trf`](/models/en#en_core_web_trf) | English | [`roberta-base`](https://huggingface.co/roberta-base) | 97.8 | 95.2 | 89.9 | +| [`de_dep_news_trf`](/models/de#de_dep_news_trf) | German | [`bert-base-german-cased`](https://huggingface.co/bert-base-german-cased) | 99.0 | 95.8 | - | +| [`es_dep_news_trf`](/models/es#es_dep_news_trf) | Spanish | [`bert-base-spanish-wwm-cased`](https://huggingface.co/dccuchile/bert-base-spanish-wwm-cased) | 98.2 | 94.6 | - | +| [`fr_dep_news_trf`](/models/fr#fr_dep_news_trf) | French | [`camembert-base`](https://huggingface.co/camembert-base) | 95.7 | 94.4 | - | +| [`zh_core_web_trf`](/models/zh#zh_core_news_trf) | Chinese | [`bert-base-chinese`](https://huggingface.co/bert-base-chinese) | 92.5 | 76.6 | 75.4 | + + + +- **Usage:** [Embeddings & Transformers](/usage/embeddings-transformers), + [Training pipelines and models](/usage/training), + [Benchmarks](/usage/facts-figures#benchmarks) +- **API:** [`Transformer`](/api/transformer), + [`TransformerData`](/api/transformer#transformerdata), + [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) +- **Architectures: ** [TransformerModel](/api/architectures#TransformerModel), + [TransformerListener](/api/architectures#TransformerListener), + [Tok2VecTransformer](/api/architectures#Tok2VecTransformer) +- **Implementation:** + [`spacy-transformers`](https://github.com/explosion/spacy-transformers) + + + +### New training workflow and config system {id="features-training"} + +> #### Example +> +> ```ini +> [training] +> accumulate_gradient = 3 +> +> [training.optimizer] +> @optimizers = "Adam.v1" +> +> [training.optimizer.learn_rate] +> @schedules = "warmup_linear.v1" +> warmup_steps = 250 +> total_steps = 20000 +> initial_rate = 0.01 +> ``` + +spaCy v3.0 introduces a comprehensive and extensible system for **configuring +your training runs**. A single configuration file describes every detail of your +training run, with no hidden defaults, making it easy to rerun your experiments +and track changes. You can use the +[quickstart widget](/usage/training#quickstart) or the `init config` command to +get started. Instead of providing lots of arguments on the command line, you +only need to pass your `config.cfg` file to [`spacy train`](/api/cli#train). +Training config files include all **settings and hyperparameters** for training +your pipeline. Some settings can also be registered **functions** that you can +swap out and customize, making it easy to implement your own custom models and +architectures. + +![Illustration of pipeline lifecycle](/images/lifecycle.svg) + + + +- **Usage:** [Training pipelines and models](/usage/training) +- **Thinc:** [Thinc's config system](https://thinc.ai/docs/usage-config), + [`Config`](https://thinc.ai/docs/api-config#config) +- **CLI:** [`init config`](/api/cli#init-config), + [`init fill-config`](/api/cli#init-fill-config), [`train`](/api/cli#train), + [`pretrain`](/api/cli#pretrain), [`evaluate`](/api/cli#evaluate) +- **API:** [Config format](/api/data-formats#config), + [`registry`](/api/top-level#registry) + + + +### Custom models using any framework {id="features-custom-models"} + +> #### Example +> +> ```python +> from torch import nn +> from thinc.api import PyTorchWrapper +> +> torch_model = nn.Sequential( +> nn.Linear(32, 32), +> nn.ReLU(), +> nn.Softmax(dim=1) +> ) +> model = PyTorchWrapper(torch_model) +> ``` + +spaCy's new configuration system makes it easy to customize the neural network +models used by the different pipeline components. You can also implement your +own architectures via spaCy's machine learning library [Thinc](https://thinc.ai) +that provides various layers and utilities, as well as thin wrappers around +frameworks like **PyTorch**, **TensorFlow** and **MXNet**. Component models all +follow the same unified [`Model`](https://thinc.ai/docs/api-model) API and each +`Model` can also be used as a sublayer of a larger network, allowing you to +freely combine implementations from different frameworks into a single model. + +![Illustration of Pipe methods](/images/trainable_component.svg) + + + +- **Usage: ** [Layers and architectures](/usage/layers-architectures), + [Trainable component API](/usage/processing-pipelines#trainable-components), + [Trainable components and models](/usage/layers-architectures#components) +- **Thinc: ** + [Wrapping PyTorch, TensorFlow & MXNet](https://thinc.ai/docs/usage-frameworks), + [`Model` API](https://thinc.ai/docs/api-model) +- **API:** [Model architectures](/api/architectures), + [`TrainablePipe`](/api/pipe) + + + +### Manage end-to-end workflows with projects {id="features-projects"} + +> #### Example +> +> ```bash +> # Clone a project template +> $ python -m spacy project clone pipelines/tagger_parser_ud +> $ cd tagger_parser_ud +> # Download data assets +> $ python -m spacy project assets +> # Run a workflow +> $ python -m spacy project run all +> ``` + +spaCy projects let you manage and share **end-to-end spaCy workflows** for +different **use cases and domains**, and orchestrate training, packaging and +serving your custom pipelines. You can start off by cloning a pre-defined +project template, adjust it to fit your needs, load in your data, train a +pipeline, export it as a Python package, upload your outputs to a remote storage +and share your results with your team. + +![Illustration of project workflow and commands](/images/projects.svg) + +spaCy projects also make it easy to **integrate with other tools** in the data +science and machine learning ecosystem, including [DVC](/usage/projects#dvc) for +data version control, [Prodigy](/usage/projects#prodigy) for creating labelled +data, [Streamlit](/usage/projects#streamlit) for building interactive apps, +[FastAPI](/usage/projects#fastapi) for serving models in production, +[Ray](/usage/projects#ray) for parallel training, +[Weights & Biases](/usage/projects#wandb) for experiment tracking, and more! + + + +- **Usage:** [spaCy projects](/usage/projects), + [Training pipelines and models](/usage/training) +- **CLI:** [`project`](/api/cli#project), [`train`](/api/cli#train) +- **Templates:** [`projects`](https://github.com/explosion/projects) + + + + + +The easiest way to get started is to clone a [project template](/usage/projects) +and run it – for example, this end-to-end template that lets you train a +**part-of-speech tagger** and **dependency parser** on a Universal Dependencies +treebank. + + + +### Parallel and distributed training with Ray {id="features-parallel-training"} + +> #### Example +> +> ```bash +> $ pip install -U %%SPACY_PKG_NAME[ray]%%SPACY_PKG_FLAGS +> # Check that the CLI is registered +> $ python -m spacy ray --help +> # Train a pipeline +> $ python -m spacy ray train config.cfg --n-workers 2 +> ``` + +[Ray](https://ray.io/) is a fast and simple framework for building and running +**distributed applications**. You can use Ray to train spaCy on one or more +remote machines, potentially speeding up your training process. The Ray +integration is powered by a lightweight extension package, +[`spacy-ray`](https://github.com/explosion/spacy-ray), that automatically adds +the [`ray`](/api/cli#ray) command to your spaCy CLI if it's installed in the +same environment. You can then run [`spacy ray train`](/api/cli#ray-train) for +parallel training. + +![Illustration of setup](/images/spacy-ray.svg) + + + +- **Usage: ** + [Parallel and distributed training](/usage/training#parallel-training), + [spaCy Projects integration](/usage/projects#ray) +- **CLI:** [`ray`](/api/cli#ray), [`ray train`](/api/cli#ray-train) +- **Implementation:** [`spacy-ray`](https://github.com/explosion/spacy-ray) + + + +### New built-in pipeline components {id="features-pipeline-components"} + +spaCy v3.0 includes several new trainable and rule-based components that you can +add to your pipeline and customize for your use case: + +> #### Example +> +> ```python +> # pip install -U %%SPACY_PKG_NAME[lookups]%%SPACY_PKG_FLAGS +> nlp = spacy.blank("en") +> nlp.add_pipe("lemmatizer") +> ``` + +| Name | Description | +| ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`SentenceRecognizer`](/api/sentencerecognizer) | Trainable component for sentence segmentation. | +| [`Morphologizer`](/api/morphologizer) | Trainable component to predict morphological features. | +| [`Lemmatizer`](/api/lemmatizer) | Standalone component for rule-based and lookup lemmatization. | +| [`AttributeRuler`](/api/attributeruler) | Component for setting token attributes using match patterns. | +| [`Transformer`](/api/transformer) | Component for using [transformer models](/usage/embeddings-transformers) in your pipeline, accessing outputs and aligning tokens. Provided via [`spacy-transformers`](https://github.com/explosion/spacy-transformers). | +| [`TrainablePipe`](/api/pipe) | Base class for trainable pipeline components. | +| [`Multi-label TextCategorizer`](/api/textcategorizer) | Trainable component for multi-label text classification. | + + + +- **Usage:** [Processing pipelines](/usage/processing-pipelines) +- **API:** [Built-in pipeline components](/api#architecture-pipeline) +- **Implementation:** [`spacy/pipeline`](%%GITHUB_SPACY/spacy/pipeline) + + + +### New and improved pipeline component APIs {id="features-components"} + +> #### Example +> +> ```python +> @Language.component("my_component") +> def my_component(doc): +> return doc +> +> nlp.add_pipe("my_component") +> nlp.add_pipe("ner", source=other_nlp) +> nlp.analyze_pipes(pretty=True) +> ``` + +Defining, configuring, reusing, training and analyzing pipeline components is +now easier and more convenient. The `@Language.component` and +`@Language.factory` decorators let you register your component, define its +default configuration and meta data, like the attribute values it assigns and +requires. Any custom component can be included during training, and sourcing +components from existing trained pipelines lets you **mix and match custom +pipelines**. The `nlp.analyze_pipes` method outputs structured information about +the current pipeline and its components, including the attributes they assign, +the scores they compute during training and whether any required attributes +aren't set. + + + +- **Usage:** [Custom components](/usage/processing-pipelines#custom_components), + [Defining components for training](/usage/training#config-components) +- **API:** [`@Language.component`](/api/language#component), + [`@Language.factory`](/api/language#factory), + [`Language.add_pipe`](/api/language#add_pipe), + [`Language.analyze_pipes`](/api/language#analyze_pipes) +- **Implementation:** [`spacy/language.py`](%%GITHUB_SPACY/spacy/language.py) + + + +### Dependency matching {id="features-dep-matcher"} + +> #### Example +> +> ```python +> from spacy.matcher import DependencyMatcher +> +> matcher = DependencyMatcher(nlp.vocab) +> pattern = [ +> {"RIGHT_ID": "anchor_founded", "RIGHT_ATTRS": {"ORTH": "founded"}}, +> {"LEFT_ID": "anchor_founded", "REL_OP": ">", "RIGHT_ID": "subject", "RIGHT_ATTRS": {"DEP": "nsubj"}} +> ] +> matcher.add("FOUNDED", [pattern]) +> ``` + +The new [`DependencyMatcher`](/api/dependencymatcher) lets you match patterns +within the dependency parse using +[Semgrex](https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/semgraph/semgrex/SemgrexPattern.html) +operators. It follows the same API as the token-based [`Matcher`](/api/matcher). +A pattern added to the dependency matcher consists of a **list of +dictionaries**, with each dictionary describing a **token to match** and its +**relation to an existing token** in the pattern. + +![Dependency matcher pattern](/images/dep-match-diagram.svg) + + + +- **Usage:** + [Dependency matching](/usage/rule-based-matching#dependencymatcher), +- **API:** [`DependencyMatcher`](/api/dependencymatcher), +- **Implementation:** + [`spacy/matcher/dependencymatcher.pyx`](%%GITHUB_SPACY/spacy/matcher/dependencymatcher.pyx) + + + +### Type hints and type-based data validation {id="features-types"} + +> #### Example +> +> ```python +> from spacy.language import Language +> from pydantic import StrictBool +> +> @Language.factory("my_component") +> def create_my_component( +> nlp: Language, +> name: str, +> custom: StrictBool +> ): +> ... +> ``` + +spaCy v3.0 officially drops support for Python 2 and now requires **Python +3.6+**. This also means that the code base can take full advantage of +[type hints](https://docs.python.org/3/library/typing.html). spaCy's user-facing +API that's implemented in pure Python (as opposed to Cython) now comes with type +hints. The new version of spaCy's machine learning library +[Thinc](https://thinc.ai) also features extensive +[type support](https://thinc.ai/docs/usage-type-checking/), including custom +types for models and arrays, and a custom `mypy` plugin that can be used to +type-check model definitions. + +For data validation, spaCy v3.0 adopts +[`pydantic`](https://github.com/samuelcolvin/pydantic). It also powers the data +validation of Thinc's [config system](https://thinc.ai/docs/usage-config), which +lets you register **custom functions with typed arguments**, reference them in +your config and see validation errors if the argument values don't match. + + + +- **Usage: ** + [Component type hints and validation](/usage/processing-pipelines#type-hints), + [Training with custom code](/usage/training#custom-code) +- **Thinc: ** + [Type checking in Thinc](https://thinc.ai/docs/usage-type-checking), + [Thinc's config system](https://thinc.ai/docs/usage-config) + + + +### New methods, attributes and commands {id="new-methods"} + +The following methods, attributes and commands are new in spaCy v3.0. + +| Name | Description | +| ------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`Token.lex`](/api/token#attributes) | Access a token's [`Lexeme`](/api/lexeme). | +| [`Token.morph`](/api/token#attributes) | Access a token's morphological analysis. | +| [`Doc.spans`](/api/doc#spans) | Named span groups to store and access collections of potentially overlapping spans. Uses the new [`SpanGroup`](/api/spangroup) data structure. | +| [`Doc.has_annotation`](/api/doc#has_annotation) | Check whether a doc has annotation on a token attribute. | +| [`Language.select_pipes`](/api/language#select_pipes) | Context manager for enabling or disabling specific pipeline components for a block. | +| [`Language.disable_pipe`](/api/language#disable_pipe), [`Language.enable_pipe`](/api/language#enable_pipe) | Disable or enable a loaded pipeline component (but don't remove it). | +| [`Language.analyze_pipes`](/api/language#analyze_pipes) | [Analyze](/usage/processing-pipelines#analysis) components and their interdependencies. | +| [`Language.resume_training`](/api/language#resume_training) | Experimental: continue training a trained pipeline and initialize "rehearsal" for components that implement a `rehearse` method to prevent catastrophic forgetting. | +| [`@Language.factory`](/api/language#factory), [`@Language.component`](/api/language#component) | Decorators for [registering](/usage/processing-pipelines#custom-components) pipeline component factories and simple stateless component functions. | +| [`Language.has_factory`](/api/language#has_factory) | Check whether a component factory is registered on a language class. | +| [`Language.get_factory_meta`](/api/language#get_factory_meta), [`Language.get_pipe_meta`](/api/language#get_factory_meta) | Get the [`FactoryMeta`](/api/language#factorymeta) with component metadata for a factory or instance name. | +| [`Language.config`](/api/language#config) | The [config](/usage/training#config) used to create the current `nlp` object. An instance of [`Config`](https://thinc.ai/docs/api-config#config) and can be saved to disk and used for training. | +| [`Language.components`](/api/language#attributes), [`Language.component_names`](/api/language#attributes) | All available components and component names, including disabled components that are not run as part of the pipeline. | +| [`Language.disabled`](/api/language#attributes) | Names of disabled components that are not run as part of the pipeline. | +| [`TrainablePipe.score`](/api/pipe#score) | Method on pipeline components that returns a dictionary of evaluation scores. | +| [`registry`](/api/top-level#registry) | Function registry to map functions to string names that can be referenced in [configs](/usage/training#config). | +| [`util.load_meta`](/api/top-level#util.load_meta), [`util.load_config`](/api/top-level#util.load_config) | Updated helpers for loading a pipeline's [`meta.json`](/api/data-formats#meta) and [`config.cfg`](/api/data-formats#config). | +| [`util.get_installed_models`](/api/top-level#util.get_installed_models) | Names of all pipeline packages installed in the environment. | +| [`init config`](/api/cli#init-config), [`init fill-config`](/api/cli#init-fill-config), [`debug config`](/api/cli#debug-config) | CLI commands for initializing, auto-filling and debugging [training configs](/usage/training). | +| [`init vectors`](/api/cli#init-vectors) | Convert word vectors for use with spaCy. | +| [`init labels`](/api/cli#init-labels) | Generate JSON files for the labels in the data to speed up training. | +| [`project`](/api/cli#project) | Suite of CLI commands for cloning, running and managing [spaCy projects](/usage/projects). | +| [`ray`](/api/cli#ray) | Suite of CLI commands for parallel training with [Ray](https://ray.io/), provided by the [`spacy-ray`](https://github.com/explosion/spacy-ray) extension package. | + +### New and updated documentation {id="new-docs"} + + + +
+ +To help you get started with spaCy v3.0 and the new features, we've added +several new or rewritten documentation pages, including a new usage guide on +[embeddings, transformers and transfer learning](/usage/embeddings-transformers), +a guide on [training pipelines and models](/usage/training) rewritten from +scratch, a page explaining the new [spaCy projects](/usage/projects) and updated +usage documentation on +[custom pipeline components](/usage/processing-pipelines#custom-components). +We've also added a bunch of new illustrations and new API reference pages +documenting spaCy's machine learning [model architectures](/api/architectures) +and the expected [data formats](/api/data-formats). API pages about +[pipeline components](/api/#architecture-pipeline) now include more information, +like the default config and implementation, and we've adopted a more detailed +format for documenting argument and return types. + +
+ +Library architecture + + + + + +- **Usage: ** [Embeddings & Transformers](/usage/embeddings-transformers), + [Training models](/usage/training), + [Layers & Architectures](/usage/layers-architectures), + [Projects](/usage/projects), + [Custom pipeline components](/usage/processing-pipelines#custom-components), + [Custom tokenizers](/usage/linguistic-features#custom-tokenizer), + [Morphology](/usage/linguistic-features#morphology), + [Lemmatization](/usage/linguistic-features#lemmatization), + [Mapping & Exceptions](/usage/linguistic-features#mappings-exceptions), + [Dependency matching](/usage/rule-based-matching#dependencymatcher) +- **API Reference: ** [Library architecture](/api), + [Model architectures](/api/architectures), [Data formats](/api/data-formats) +- **New Classes: ** [`Example`](/api/example), [`Tok2Vec`](/api/tok2vec), + [`Transformer`](/api/transformer), [`Lemmatizer`](/api/lemmatizer), + [`Morphologizer`](/api/morphologizer), + [`AttributeRuler`](/api/attributeruler), + [`SentenceRecognizer`](/api/sentencerecognizer), + [`DependencyMatcher`](/api/dependencymatcher), [`TrainablePipe`](/api/pipe), + [`Corpus`](/api/corpus), [`SpanGroup`](/api/spangroup), + + + +## Backwards Incompatibilities {id="incompat"} + +As always, we've tried to keep the breaking changes to a minimum and focus on +changes that were necessary to support the new features, fix problems or improve +usability. The following section lists the relevant changes to the user-facing +API. For specific examples of how to rewrite your code, check out the +[migration guide](#migrating). + + + +Note that spaCy v3.0 now requires **Python 3.6+**. + + + +### API changes {id="incompat-api"} + +- Pipeline package symlinks, the `link` command and shortcut names are now + deprecated. There can be many [different trained pipelines](/models) and not + just one "English model", so you should always use the full package name like + `en_core_web_sm` explicitly. +- A pipeline's `meta.json` is now only used to provide meta information like the + package name, author, license and labels. It's **not** used to construct the + processing pipeline anymore. This is all defined in the + [`config.cfg`](/api/data-formats#config), which also includes all settings + used to train the pipeline. +- The `train`, `pretrain` and `debug data` commands now only take a + `config.cfg`. +- [`Language.add_pipe`](/api/language#add_pipe) now takes the **string name** of + the component factory instead of the component function. +- **Custom pipeline components** now need to be decorated with the + [`@Language.component`](/api/language#component) or + [`@Language.factory`](/api/language#factory) decorator. +- The [`Language.update`](/api/language#update), + [`Language.evaluate`](/api/language#evaluate) and + [`TrainablePipe.update`](/api/pipe#update) methods now all take batches of + [`Example`](/api/example) objects instead of `Doc` and `GoldParse` objects, or + raw text and a dictionary of annotations. +- The `begin_training` methods have been renamed to `initialize` and now take a + function that returns a sequence of `Example` objects to initialize the model + instead of a list of tuples. +- [`Matcher.add`](/api/matcher#add) and + [`PhraseMatcher.add`](/api/phrasematcher#add) now only accept a list of + patterns as the second argument (instead of a variable number of arguments). + The `on_match` callback becomes an optional keyword argument. +- The `Doc` flags like `Doc.is_parsed` or `Doc.is_tagged` have been replaced by + [`Doc.has_annotation`](/api/doc#has_annotation). +- The `spacy.gold` module has been renamed to + [`spacy.training`](%%GITHUB_SPACY/spacy/training). +- The `PRON_LEMMA` symbol and `-PRON-` as an indicator for pronoun lemmas has + been removed. +- The `TAG_MAP` and `MORPH_RULES` in the language data have been replaced by the + more flexible [`AttributeRuler`](/api/attributeruler). +- The [`Lemmatizer`](/api/lemmatizer) is now a standalone pipeline component and + doesn't provide lemmas by default or switch automatically between lookup and + rule-based lemmas. You can now add it to your pipeline explicitly and set its + mode on initialization. +- Various keyword arguments across functions and methods are now explicitly + declared as **keyword-only** arguments. Those arguments are documented + accordingly across the API reference using the keyword-only tag. +- The `textcat` pipeline component is now only applicable for classification of + mutually exclusives classes - i.e. one predicted class per input sentence or + document. To perform multi-label classification, use the new + `textcat_multilabel` component instead. + +### Removed or renamed API {id="incompat-removed"} + +| Removed | Replacement | +| -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `Language.disable_pipes` | [`Language.select_pipes`](/api/language#select_pipes), [`Language.disable_pipe`](/api/language#disable_pipe), [`Language.enable_pipe`](/api/language#enable_pipe) | +| `Language.begin_training`, `Pipe.begin_training`, ... | [`Language.initialize`](/api/language#initialize), [`Pipe.initialize`](/api/pipe#initialize), ... | +| `Doc.is_tagged`, `Doc.is_parsed`, ... | [`Doc.has_annotation`](/api/doc#has_annotation) | +| `GoldParse` | [`Example`](/api/example) | +| `GoldCorpus` | [`Corpus`](/api/corpus) | +| `KnowledgeBase.load_bulk`, `KnowledgeBase.dump` | [`KnowledgeBase.from_disk`](/api/kb#from_disk), [`KnowledgeBase.to_disk`](/api/kb#to_disk) | +| `KnowledgeBase.get_candidates` | [`KnowledgeBase.get_alias_candidates`](/api/kb#get_alias_candidates) | +| `Matcher.pipe`, `PhraseMatcher.pipe` | not needed | +| `gold.offsets_from_biluo_tags`, `gold.spans_from_biluo_tags`, `gold.biluo_tags_from_offsets` | [`training.biluo_tags_to_offsets`](/api/top-level#biluo_tags_to_offsets), [`training.biluo_tags_to_spans`](/api/top-level#biluo_tags_to_spans), [`training.offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags) | +| `spacy init-model` | [`spacy init vectors`](/api/cli#init-vectors) | +| `spacy debug-data` | [`spacy debug data`](/api/cli#debug-data) | +| `spacy profile` | [`spacy debug profile`](/api/cli#debug-profile) | +| `spacy link`, `util.set_data_path`, `util.get_data_path` | not needed, symlinks are deprecated | + +The following methods, attributes and arguments were removed in v3.0. Most of +them have been **deprecated for a while** and many would previously raise +errors. Many of them were also mostly internals. If you've been working with +more recent versions of spaCy v2.x, it's **unlikely** that your code relied on +them. + +| Removed | Replacement | +| ----------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Doc.tokens_from_list` | [`Doc.__init__`](/api/doc#init) | +| `Doc.merge`, `Span.merge` | [`Doc.retokenize`](/api/doc#retokenize) | +| `Token.string`, `Span.string`, `Span.upper`, `Span.lower` | [`Span.text`](/api/span#attributes), [`Token.text`](/api/token#attributes) | +| `Language.tagger`, `Language.parser`, `Language.entity` | [`Language.get_pipe`](/api/language#get_pipe) | +| keyword-arguments like `vocab=False` on `to_disk`, `from_disk`, `to_bytes`, `from_bytes` | `exclude=["vocab"]` | +| `n_threads` argument on [`Tokenizer`](/api/tokenizer), [`Matcher`](/api/matcher), [`PhraseMatcher`](/api/phrasematcher) | `n_process` | +| `verbose` argument on [`Language.evaluate`](/api/language#evaluate) | logging (`DEBUG`) | +| `SentenceSegmenter` hook, `SimilarityHook` | [user hooks](/usage/processing-pipelines#custom-components-user-hooks), [`Sentencizer`](/api/sentencizer), [`SentenceRecognizer`](/api/sentencerecognizer) | + +## Migrating from v2.x {id="migrating"} + +### Downloading and loading trained pipelines {id="migrating-downloading-models"} + +Symlinks and shortcuts like `en` have been deprecated for a while, and are now +not supported anymore. There are [many different trained pipelines](/models) +with different capabilities and not just one "English model". In order to +download and load a package, you should always use its full name – for instance, +[`en_core_web_sm`](/models/en#en_core_web_sm). + +```diff +- python -m spacy download en ++ python -m spacy download en_core_web_sm +``` + +```diff +- nlp = spacy.load("en") ++ nlp = spacy.load("en_core_web_sm") +``` + +### Custom pipeline components and factories {id="migrating-pipeline-components"} + +Custom pipeline components now have to be registered explicitly using the +[`@Language.component`](/api/language#component) or +[`@Language.factory`](/api/language#factory) decorator. For simple functions +that take a `Doc` and return it, all you have to do is add the +`@Language.component` decorator to it and assign it a name: + +```diff {title="Stateless function components"} ++ from spacy.language import Language + ++ @Language.component("my_component") +def my_component(doc): + return doc +``` + +For class components that are initialized with settings and/or the shared `nlp` +object, you can use the `@Language.factory` decorator. Also make sure that that +the method used to initialize the factory has **two named arguments**: `nlp` +(the current `nlp` object) and `name` (the string name of the component +instance). + +```diff {title="Stateful class components"} ++ from spacy.language import Language + ++ @Language.factory("my_component") +class MyComponent: +- def __init__(self, nlp): ++ def __init__(self, nlp, name): + self.nlp = nlp + + def __call__(self, doc): + return doc +``` + +Instead of decorating your class, you could also add a factory function that +takes the arguments `nlp` and `name` and returns an instance of your component: + +```diff {title="Stateful class components with factory function"} ++ from spacy.language import Language + ++ @Language.factory("my_component") ++ def create_my_component(nlp, name): ++ return MyComponent(nlp) + +class MyComponent: + def __init__(self, nlp): + self.nlp = nlp + + def __call__(self, doc): + return doc +``` + +The `@Language.component` and `@Language.factory` decorators now take care of +adding an entry to the component factories, so spaCy knows how to load a +component back in from its string name. You won't have to write to +`Language.factories` manually anymore. + +```diff +- Language.factories["my_component"] = lambda nlp, **cfg: MyComponent(nlp) +``` + +#### Adding components to the pipeline {id="migrating-add-pipe"} + +The [`nlp.add_pipe`](/api/language#add_pipe) method now takes the **string +name** of the component factory instead of a callable component. This allows +spaCy to track and serialize components that have been added and their settings. + +```diff ++ @Language.component("my_component") +def my_component(doc): + return doc + +- nlp.add_pipe(my_component) ++ nlp.add_pipe("my_component") +``` + +[`nlp.add_pipe`](/api/language#add_pipe) now also returns the pipeline component +itself, so you can access its attributes. The +[`nlp.create_pipe`](/api/language#create_pipe) method is now mostly internals +and you typically shouldn't have to use it in your code. + +```diff +- parser = nlp.create_pipe("parser") +- nlp.add_pipe(parser) ++ parser = nlp.add_pipe("parser") +``` + +If you need to add a component from an existing trained pipeline, you can now +use the `source` argument on [`nlp.add_pipe`](/api/language#add_pipe). This will +check that the component is compatible, and take care of porting over all +config. During training, you can also reference existing trained components in +your [config](/usage/training#config-components) and decide whether or not they +should be updated with more data. + +> #### config.cfg (excerpt) +> +> ```ini +> [components.ner] +> source = "en_core_web_sm" +> component = "ner" +> ``` + +```diff +source_nlp = spacy.load("en_core_web_sm") +nlp = spacy.blank("en") +- ner = source_nlp.get_pipe("ner") +- nlp.add_pipe(ner) ++ nlp.add_pipe("ner", source=source_nlp) +``` + +#### Configuring pipeline components with settings {id="migrating-configure-pipe"} + +Because pipeline components are now added using their string names, you won't +have to instantiate the [component classes](/api/#architecture-pipeline) +directly anymore. To configure the component, you can now use the `config` +argument on [`nlp.add_pipe`](/api/language#add_pipe). + +> #### config.cfg (excerpt) +> +> ```ini +> [components.sentencizer] +> factory = "sentencizer" +> punct_chars = ["!", ".", "?"] +> ``` + +```diff +punct_chars = ["!", ".", "?"] +- sentencizer = Sentencizer(punct_chars=punct_chars) ++ sentencizer = nlp.add_pipe("sentencizer", config={"punct_chars": punct_chars}) +``` + +The `config` corresponds to the component settings in the +[`config.cfg`](/usage/training#config-components) and will overwrite the default +config defined by the components. + + + +Config values you pass to components **need to be JSON-serializable** and can't +be arbitrary Python objects. Otherwise, the settings you provide can't be +represented in the `config.cfg` and spaCy has no way of knowing how to re-create +your component with the same settings when you load the pipeline back in. If you +need to pass arbitrary objects to a component, use a +[registered function](/usage/processing-pipelines#example-stateful-components): + +```diff +- config = {"model": MyTaggerModel()} ++ config= {"model": {"@architectures": "MyTaggerModel"}} +tagger = nlp.add_pipe("tagger", config=config) +``` + + + +### Adding match patterns {id="migrating-matcher"} + +The [`Matcher.add`](/api/matcher#add), +[`PhraseMatcher.add`](/api/phrasematcher#add) and +[`DependencyMatcher.add`](/api/dependencymatcher#add) methods now only accept a +**list of patterns** as the second argument (instead of a variable number of +arguments). The `on_match` callback becomes an optional keyword argument. + +```diff +matcher = Matcher(nlp.vocab) +patterns = [[{"TEXT": "Google"}, {"TEXT": "Now"}], [{"TEXT": "GoogleNow"}]] +- matcher.add("GoogleNow", on_match, *patterns) ++ matcher.add("GoogleNow", patterns, on_match=on_match) +``` + +```diff +matcher = PhraseMatcher(nlp.vocab) +patterns = [nlp("health care reform"), nlp("healthcare reform")] +- matcher.add("HEALTH", on_match, *patterns) ++ matcher.add("HEALTH", patterns, on_match=on_match) +``` + +### Migrating attributes in tokenizer exceptions {id="migrating-tokenizer-exceptions"} + +Tokenizer exceptions are now only allowed to set `ORTH` and `NORM` values as +part of the token attributes. Exceptions for other attributes such as `TAG` and +`LEMMA` should be moved to an [`AttributeRuler`](/api/attributeruler) component: + +```diff +nlp = spacy.blank("en") +- nlp.tokenizer.add_special_case("don't", [{"ORTH": "do"}, {"ORTH": "n't", "LEMMA": "not"}]) ++ nlp.tokenizer.add_special_case("don't", [{"ORTH": "do"}, {"ORTH": "n't"}]) ++ ruler = nlp.add_pipe("attribute_ruler") ++ ruler.add(patterns=[[{"ORTH": "n't"}]], attrs={"LEMMA": "not"}) +``` + +### Migrating tag maps and morph rules {id="migrating-training-mappings-exceptions"} + +Instead of defining a `tag_map` and `morph_rules` in the language data, spaCy +v3.0 now manages mappings and exceptions with a separate and more flexible +pipeline component, the [`AttributeRuler`](/api/attributeruler). See the +[usage guide](/usage/linguistic-features#mappings-exceptions) for examples. If +you have tag maps and morph rules in the v2.x format, you can load them into the +attribute ruler before training using the `[initialize]` block of your config. + +### Using Lexeme Tables + +To use tables like `lexeme_prob` when training a model from scratch, you need to +add an entry to the `initialize` block in your config. Here's what that looks +like for the existing trained pipelines: + +```ini +[initialize.lookups] +@misc = "spacy.LookupsDataLoader.v1" +lang = ${nlp.lang} +tables = ["lexeme_norm"] +``` + +> #### What does the initialization do? +> +> The `[initialize]` block is used when +> [`nlp.initialize`](/api/language#initialize) is called (usually right before +> training). It lets you define data resources for initializing the pipeline in +> your `config.cfg`. After training, the rules are saved to disk with the +> exported pipeline, so your runtime model doesn't depend on local data. For +> details see the [config lifecycle](/usage/training/#config-lifecycle) and +> [initialization](/usage/training/#initialization) docs. + +```ini {title="config.cfg (excerpt)"} +[initialize.components.attribute_ruler] + +[initialize.components.attribute_ruler.tag_map] +@readers = "srsly.read_json.v1" +path = "./corpus/tag_map.json" +``` + +The `AttributeRuler` also provides two handy helper methods +[`load_from_tag_map`](/api/attributeruler#load_from_tag_map) and +[`load_from_morph_rules`](/api/attributeruler#load_from_morph_rules) that let +you load in your existing tag map or morph rules: + +```diff +nlp = spacy.blank("en") +- nlp.vocab.morphology.load_tag_map(YOUR_TAG_MAP) ++ ruler = nlp.add_pipe("attribute_ruler") ++ ruler.load_from_tag_map(YOUR_TAG_MAP) +``` + +### Migrating Doc flags {id="migrating-doc-flags"} + +The [`Doc`](/api/doc) flags `Doc.is_tagged`, `Doc.is_parsed`, `Doc.is_nered` and +`Doc.is_sentenced` are deprecated in v3.0 and replaced by +[`Doc.has_annotation`](/api/doc#has_annotation) method, which refers to the +token attribute symbols (the same symbols used in [`Matcher`](/api/matcher) +patterns): + +```diff +doc = nlp(text) +- doc.is_parsed ++ doc.has_annotation("DEP") +- doc.is_tagged ++ doc.has_annotation("TAG") +- doc.is_sentenced ++ doc.has_annotation("SENT_START") +- doc.is_nered ++ doc.has_annotation("ENT_IOB") +``` + +### Training pipelines and models {id="migrating-training"} + +To train your pipelines, you should now pretty much always use the +[`spacy train`](/api/cli#train) CLI. You shouldn't have to put together your own +training scripts anymore, unless you _really_ want to. The training commands now +use a [flexible config file](/usage/training#config) that describes all training +settings and hyperparameters, as well as your pipeline, components and +architectures to use. The `--code` argument lets you pass in code containing +[custom registered functions](/usage/training#custom-code) that you can +reference in your config. To get started, check out the +[quickstart widget](/usage/training#quickstart). + +#### Binary .spacy training data format {id="migrating-training-format"} + +spaCy v3.0 uses a new +[binary training data format](/api/data-formats#binary-training) created by +serializing a [`DocBin`](/api/docbin), which represents a collection of `Doc` +objects. This means that you can train spaCy pipelines using the same format it +outputs: annotated `Doc` objects. The binary format is extremely **efficient in +storage**, especially when packing multiple documents together. You can convert +your existing JSON-formatted data using the [`spacy convert`](/api/cli#convert) +command, which outputs `.spacy` files: + +```bash +$ python -m spacy convert ./training.json ./output +``` + +#### Training config {id="migrating-training-config"} + +The easiest way to get started with a training config is to use the +[`init config`](/api/cli#init-config) command or the +[quickstart widget](/usage/training#quickstart). You can define your +requirements, and it will auto-generate a starter config with the best-matching +default settings. + +```bash +$ python -m spacy init config ./config.cfg --lang en --pipeline tagger,parser +``` + +If you've exported a starter config from our +[quickstart widget](/usage/training#quickstart), you can use the +[`init fill-config`](/api/cli#init-fill-config) to fill it with all default +values. You can then use the auto-generated `config.cfg` for training: + +```diff +- python -m spacy train en ./output ./train.json ./dev.json +--pipeline tagger,parser --cnn-window 1 --bilstm-depth 0 ++ python -m spacy train ./config.cfg --output ./output +``` + + + +The easiest way to get started is to clone a [project template](/usage/projects) +and run it – for example, this end-to-end template that lets you train a +**part-of-speech tagger** and **dependency parser** on a Universal Dependencies +treebank. + + + +#### Modifying tokenizer settings + +If you were using a base model with `spacy train` to customize the tokenizer +settings in v2, your modifications can be provided in the +`[initialize.before_init]` callback. + +Write a registered callback that modifies the tokenizer settings and specify +this callback in your config: + +> #### config.cfg +> +> ```ini +> [initialize] +> +> [initialize.before_init] +> @callbacks = "customize_tokenizer" +> ``` + +```python {title="functions.py"} +from spacy.util import registry, compile_suffix_regex + +@registry.callbacks("customize_tokenizer") +def make_customize_tokenizer(): + def customize_tokenizer(nlp): + # remove a suffix + suffixes = list(nlp.Defaults.suffixes) + suffixes.remove("\\[") + suffix_regex = compile_suffix_regex(suffixes) + nlp.tokenizer.suffix_search = suffix_regex.search + + # add a special case + nlp.tokenizer.add_special_case("_SPECIAL_", [{"ORTH": "_SPECIAL_"}]) + return customize_tokenizer +``` + +When training, provide the function above with the `--code` option: + +```bash +$ python -m spacy train config.cfg --code ./functions.py +``` + +The train step requires the `--code` option with your registered functions from +the `[initialize]` block, but since those callbacks are only required during the +initialization step, you don't need to provide them with the final pipeline +package. However, to make it easier for others to replicate your training setup, +you can choose to package the initialization callbacks with the pipeline package +or to publish them separately. + +#### Training via the Python API {id="migrating-training-python"} + +For most use cases, you **shouldn't** have to write your own training scripts +anymore. Instead, you can use [`spacy train`](/api/cli#train) with a +[config file](/usage/training#config) and custom +[registered functions](/usage/training#custom-code) if needed. You can even +register callbacks that can modify the `nlp` object at different stages of its +lifecycle to fully customize it before training. + +If you do decide to use the [internal training API](/usage/training#api) from +Python, you should only need a few small modifications to convert your scripts +from spaCy v2.x to v3.x. The [`Example.from_dict`](/api/example#from_dict) +classmethod takes a reference `Doc` and a +[dictionary of annotations](/api/data-formats#dict-input), similar to the +"simple training style" in spaCy v2.x: + +```diff {title="Migrating Doc and GoldParse"} +doc = nlp.make_doc("Mark Zuckerberg is the CEO of Facebook") +entities = [(0, 15, "PERSON"), (30, 38, "ORG")] +- gold = GoldParse(doc, entities=entities) ++ example = Example.from_dict(doc, {"entities": entities}) +``` + +```diff {title="Migrating simple training style"} +text = "Mark Zuckerberg is the CEO of Facebook" +annotations = {"entities": [(0, 15, "PERSON"), (30, 38, "ORG")]} ++ doc = nlp.make_doc(text) ++ example = Example.from_dict(doc, annotations) +``` + +The [`Language.update`](/api/language#update), +[`Language.evaluate`](/api/language#evaluate) and +[`TrainablePipe.update`](/api/pipe#update) methods now all take batches of +[`Example`](/api/example) objects instead of `Doc` and `GoldParse` objects, or +raw text and a dictionary of annotations. + +```python {title="Training loop",highlight="5-8,12"} +TRAIN_DATA = [ + ("Who is Shaka Khan?", {"entities": [(7, 17, "PERSON")]}), + ("I like London.", {"entities": [(7, 13, "LOC")]}), +] +examples = [] +for text, annots in TRAIN_DATA: + examples.append(Example.from_dict(nlp.make_doc(text), annots)) +nlp.initialize(lambda: examples) +for i in range(20): + random.shuffle(examples) + for batch in minibatch(examples, size=8): + nlp.update(batch) +``` + +`Language.begin_training` and `TrainablePipe.begin_training` have been renamed +to [`Language.initialize`](/api/language#initialize) and +[`TrainablePipe.initialize`](/api/pipe#initialize), and the methods now take a +function that returns a sequence of `Example` objects to initialize the model +instead of a list of tuples. The data examples are used to **initialize the +models** of trainable pipeline components, which includes validating the +network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme. + +```diff +- nlp.begin_training() ++ nlp.initialize(lambda: examples) +``` + +#### Packaging trained pipelines {id="migrating-training-packaging"} + +The [`spacy package`](/api/cli#package) command now automatically builds the +installable `.tar.gz` sdist of the Python package, so you don't have to run this +step manually anymore. To disable the behavior, you can set `--build none`. You +can also choose to build a binary wheel (which installs more efficiently) by +setting `--build wheel`, or to build both the sdist and wheel by setting +`--build sdist,wheel`. + +```diff +python -m spacy package ./output ./packages +- cd /output/en_pipeline-0.0.0 +- python setup.py sdist +``` + +#### Data utilities and gold module {id="migrating-gold"} + +The `spacy.gold` module has been renamed to `spacy.training` and the conversion +utilities now follow the naming format of `x_to_y`. This mostly affects +internals, but if you've been using the span offset conversion utilities +[`offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags), +[`biluo_tags_to_offsets`](/api/top-level#biluo_tags_to_offsets) or +[`biluo_tags_to_spans`](/api/top-level#biluo_tags_to_spans), you'll have to +change your names and imports: + +```diff +- from spacy.gold import biluo_tags_from_offsets, offsets_from_biluo_tags, spans_from_biluo_tags ++ from spacy.training import offsets_to_biluo_tags, biluo_tags_to_offsets, biluo_tags_to_spans +``` + +#### Migration notes for plugin maintainers {id="migrating-plugins"} + +Thanks to everyone who's been contributing to the spaCy ecosystem by developing +and maintaining one of the many awesome [plugins and extensions](/universe). +We've tried to make it as easy as possible for you to upgrade your packages for +spaCy v3.0. The most common use case for plugins is providing pipeline +components and extension attributes. When migrating your plugin, double-check +the following: + +- Use the [`@Language.factory`](/api/language#factory) decorator to register + your component and assign it a name. This allows users to refer to your + components by name and serialize pipelines referencing them. Remove all manual + entries to the `Language.factories`. +- Make sure your component factories take at least two **named arguments**: + `nlp` (the current `nlp` object) and `name` (the instance name of the added + component so you can identify multiple instances of the same component). +- Update all references to [`nlp.add_pipe`](/api/language#add_pipe) in your docs + to use **string names** instead of the component functions. + +```python {highlight="1-5"} +from spacy.language import Language + +@Language.factory("my_component", default_config={"some_setting": False}) +def create_component(nlp: Language, name: str, some_setting: bool): + return MyCoolComponent(some_setting=some_setting) + + +class MyCoolComponent: + def __init__(self, some_setting): + self.some_setting = some_setting + + def __call__(self, doc): + # Do something to the doc + return doc +``` + +> #### Result in config.cfg +> +> ```ini +> [components.my_component] +> factory = "my_component" +> some_setting = true +> ``` + +```diff +import spacy +from your_plugin import MyCoolComponent + +nlp = spacy.load("en_core_web_sm") +- component = MyCoolComponent(some_setting=True) +- nlp.add_pipe(component) ++ nlp.add_pipe("my_component", config={"some_setting": True}) +``` + + + +The [`@Language.factory`](/api/language#factory) decorator takes care of letting +spaCy know that a component of that name is available. This means that your +users can add it to the pipeline using its **string name**. However, this +requires the decorator to be executed – so users will still have to **import +your plugin**. Alternatively, your plugin could expose an +[entry point](/usage/saving-loading#entry-points), which spaCy can read from. +This means that spaCy knows how to initialize `my_component`, even if your +package isn't imported. + + + +#### Using GPUs in Jupyter notebooks {id="jupyter-notebook-gpu"} + +In Jupyter notebooks, run [`prefer_gpu`](/api/top-level#spacy.prefer_gpu), +[`require_gpu`](/api/top-level#spacy.require_gpu) or +[`require_cpu`](/api/top-level#spacy.require_cpu) in the same cell as +[`spacy.load`](/api/top-level#spacy.load) to ensure that the model is loaded on +the correct device. + +Due to a bug related to `contextvars` (see the +[bug report](https://github.com/ipython/ipython/issues/11565)), the GPU settings +may not be preserved correctly across cells, resulting in models being loaded on +the wrong device or only partially on GPU. diff --git a/website/docs/usage/visualizers.mdx b/website/docs/usage/visualizers.mdx new file mode 100644 index 0000000..66d9af9 --- /dev/null +++ b/website/docs/usage/visualizers.mdx @@ -0,0 +1,440 @@ +--- +title: Visualizers +teaser: Visualize dependencies and entities in your browser or in a notebook +version: 2 +menu: + - ['Dependencies', 'dep'] + - ['Named Entities', 'ent'] + - ['Spans', 'span'] + - ['Jupyter Notebooks', 'jupyter'] + - ['Rendering HTML', 'html'] + - ['Web app usage', 'webapp'] +--- + +Visualizing a dependency parse or named entities in a text is not only a fun NLP +demo – it can also be incredibly helpful in speeding up development and +debugging your code and training process. That's why our popular visualizers, +[displaCy](https://explosion.ai/demos/displacy) and +[displaCy ENT](https://explosion.ai/demos/displacy-ent) are also an +official part of the core library. If you're running a +[Jupyter](https://jupyter.org) notebook, displaCy will detect this and return +the markup in a format [ready to be rendered and exported](#jupyter). + +The quickest way to visualize `Doc` is to use +[`displacy.serve`](/api/top-level#displacy.serve). This will spin up a simple +web server and let you view the result straight from your browser. displaCy can +either take a single `Doc` or a list of `Doc` objects as its first argument. +This lets you construct them however you like – using any pipeline or +modifications you like. If you're using [Streamlit](https://streamlit.io), check +out the [`spacy-streamlit`](https://github.com/explosion/spacy-streamlit) +package that helps you integrate spaCy visualizations into your apps! + +## Visualizing the dependency parse {id="dep"} + +The dependency visualizer, `dep`, shows part-of-speech tags and syntactic +dependencies. + +```python {title="Dependency example"} +import spacy +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") +doc = nlp("This is a sentence.") +displacy.serve(doc, style="dep") +``` + +![displaCy visualizer](/images/displacy.svg) + +The argument `options` lets you specify a dictionary of settings to customize +the layout, for example: + + + +There's currently a known issue with the `compact` mode for sentences with short +arrows and long dependency labels, that causes labels longer than the arrow to +wrap. So if you come across this problem, especially when using custom labels, +you'll have to increase the `distance` setting in the `options` to allow longer +arcs. + +Moreover, you might need to modify the `offset_x` argument depending on the shape +of your document. Otherwise, the left part of the document may overflow beyond the +container's border. + + + +| Argument | Description | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | +| `color` | Text color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#000000"`. ~~str~~ | +| `bg` | Background color. Can be provided in any CSS legal format as a string e.g.: `"#00ff00"`, `"rgb(0, 255, 0)"`, `"hsl(120, 100%, 50%)"` and `"green"` all correspond to the color green (without transparency). Defaults to `"#ffffff"`. ~~str~~ | +| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | +| `offset_x` | Spacing on left side of the SVG in px. You might need to tweak this setting for long texts. Defaults to `50`. ~~int~~ | + +For a list of all available options, see the +[`displacy` API documentation](/api/top-level#displacy_options). + +> #### Options example +> +> ```python +> options = {"compact": True, "bg": "#09a3d5", +> "color": "white", "font": "Source Sans Pro"} +> displacy.serve(doc, style="dep", options=options) +> ``` + +![displaCy visualizer (compact mode)](/images/displacy-compact.svg) + +### Visualizing long texts {id="dep-long-text",version="2.0.12"} + +Long texts can become difficult to read when displayed in one row, so it's often +better to visualize them sentence-by-sentence instead. As of v2.0.12, `displacy` +supports rendering both [`Doc`](/api/doc) and [`Span`](/api/span) objects, as +well as lists of `Doc`s or `Span`s. Instead of passing the full `Doc` to +`displacy.serve`, you can also pass in a list `doc.sents`. This will create one +visualization for each sentence. + +```python +import spacy +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") +text = """In ancient Rome, some neighbors live in three adjacent houses. In the center is the house of Senex, who lives there with wife Domina, son Hero, and several slaves, including head slave Hysterium and the musical's main character Pseudolus. A slave belonging to Hero, Pseudolus wishes to buy, win, or steal his freedom. One of the neighboring houses is owned by Marcus Lycus, who is a buyer and seller of beautiful women; the other belongs to the ancient Erronius, who is abroad searching for his long-lost children (stolen in infancy by pirates). One day, Senex and Domina go on a trip and leave Pseudolus in charge of Hero. Hero confides in Pseudolus that he is in love with the lovely Philia, one of the courtesans in the House of Lycus (albeit still a virgin).""" +doc = nlp(text) +sentence_spans = list(doc.sents) +displacy.serve(sentence_spans, style="dep") +``` + +## Visualizing the entity recognizer {id="ent"} + +The entity visualizer, `ent`, highlights named entities and their labels in a +text. + +```python {title="Named Entity example"} +import spacy +from spacy import displacy + +text = "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously." + +nlp = spacy.load("en_core_web_sm") +doc = nlp(text) +displacy.serve(doc, style="ent") +``` + + +
When Sebastian Thrun PERSON started working on self-driving cars at Google ORG in 2007 DATE, few people outside of the company took him seriously.
+ + +The entity visualizer lets you customize the following `options`: + +| Argument | Description | +| -------- | ------------------------------------------------------------------------------------------------------------- | +| `ents` | Entity types to highlight (`None` for all types). Defaults to `None`. ~~Optional[List[str]]~~ | +| `colors` | Color overrides. Entity types should be mapped to color names or values. Defaults to `{}`. ~~Dict[str, str]~~ | + +If you specify a list of `ents`, only those entity types will be rendered – for +example, you can choose to display `PERSON` entities. Internally, the visualizer +knows nothing about available entity types and will render whichever spans and +labels it receives. This makes it especially easy to work with custom entity +types. By default, displaCy comes with colors for all entity types used by +[trained spaCy pipelines](/models). If you're using custom entity types, you can +use the `colors` setting to add your own colors for them. + +> #### Options example +> +> ```python +> colors = {"ORG": "linear-gradient(90deg, #aa9cfc, #fc9ce7)"} +> options = {"ents": ["ORG"], "colors": colors} +> displacy.serve(doc, style="ent", options=options) +> ``` + + +
But Google ORG is starting from behind. The company made a late push into hardware, and Apple ORG’s Siri, available on iPhones, and Amazon ORG’s Alexa software, which runs on its Echo and Dot devices, have clear leads in consumer adoption.
+ + +The above example uses a little trick: Since the background color values are +added as the `background` style attribute, you can use any +[valid background value](https://tympanus.net/codrops/css_reference/background/) +or shorthand – including gradients and even images! + +### Adding titles to documents {id="ent-titles"} + +Rendering several large documents on one page can easily become confusing. To +add a headline to each visualization, you can add a `title` to its `user_data`. +User data is never touched or modified by spaCy. + +```python +doc = nlp("This is a sentence about Google.") +doc.user_data["title"] = "This is a title" +displacy.serve(doc, style="ent") +``` + +This feature is especially handy if you're using displaCy to compare performance +at different stages of a process, e.g. during training. Here you could use the +title for a brief description of the text example and the number of iterations. + +## Visualizing spans {id="span"} + +The span visualizer, `span`, highlights overlapping spans in a text. + +```python {title="Span example"} +import spacy +from spacy import displacy +from spacy.tokens import Span + +text = "Welcome to the Bank of China." + +nlp = spacy.blank("en") +doc = nlp(text) + +doc.spans["sc"] = [ + Span(doc, 3, 6, "ORG"), + Span(doc, 5, 6, "GPE"), +] + +displacy.serve(doc, style="span") +``` + + +
Welcome to the BankORG of ChinaGPE.
+ + +The span visualizer lets you customize the following `options`: + +| Argument | Description | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | Which spans key to render spans from. Default is `"sc"`. ~~str~~ | +| `templates` | Dictionary containing the keys `"span"`, `"slice"`, and `"start"`. These dictate how the overall span, a span slice, and the starting token will be rendered. ~~Optional[Dict[str, str]~~ | +| `kb_url_template` | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in ~~Optional[str]~~ | +| `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | + +Because spans can be stored across different keys in `doc.spans`, you need to +specify which one displaCy should use with `spans_key` (`sc` is the default). + +> #### Options example +> +> ```python +> doc.spans["custom"] = [Span(doc, 3, 6, "BANK")] +> options = {"spans_key": "custom"} +> displacy.serve(doc, style="span", options=options) +> ``` + + +
Welcome to the BankBANK of China.
+ + +## Using displaCy in Jupyter notebooks {id="jupyter"} + +displaCy is able to detect whether you're working in a +[Jupyter](https://jupyter.org) notebook, and will return markup that can be +rendered in a cell straight away. When you export your notebook, the +visualizations will be included as HTML. + +```python {title="Jupyter example"} +# Don't forget to install a trained pipeline, e.g.: python -m spacy download en + +# In[1]: +import spacy +from spacy import displacy + +# In[2]: +doc = nlp("Rats are various medium-sized, long-tailed rodents.") +displacy.render(doc, style="dep") + +# In[3]: +doc2 = nlp(LONG_NEWS_ARTICLE) +displacy.render(doc2, style="ent") +``` + + + +To explicitly enable or disable "Jupyter mode", you can use the `jupyter` +keyword argument – e.g. to return raw HTML in a notebook, or to force Jupyter +rendering if auto-detection fails. + + + +![displaCy visualizer in a Jupyter notebook](/images/displacy_jupyter.jpg) + +Internally, displaCy imports `display` and `HTML` from `IPython.display` +and returns a Jupyter HTML object. If you were doing it manually, it'd look like +this: + +```python +from IPython.display import display, HTML + +html = displacy.render(doc, style="dep") +display(HTML(html)) +``` + +## Rendering HTML {id="html"} + +If you don't need the web server and just want to generate the markup – for +example, to export it to a file or serve it in a custom way – you can use +[`displacy.render`](/api/top-level#displacy.render). It works the same way, but +returns a string containing the markup. + +```python {title="Example"} +import spacy +from spacy import displacy + +nlp = spacy.load("en_core_web_sm") +doc1 = nlp("This is a sentence.") +doc2 = nlp("This is another sentence.") +html = displacy.render([doc1, doc2], style="dep", page=True) +``` + +`page=True` renders the markup wrapped as a full HTML page. For minified and +more compact HTML markup, you can set `minify=True`. If you're rendering a +dependency parse, you can also export it as an `.svg` file. + +> #### What's SVG? +> +> Unlike other image formats, the SVG (Scalable Vector Graphics) uses XML markup +> that's easy to manipulate +> [using CSS](https://www.smashingmagazine.com/2014/11/styling-and-animating-svgs-with-css/) +> or +> [JavaScript](https://css-tricks.com/smil-is-dead-long-live-smil-a-guide-to-alternatives-to-smil-features/). +> Essentially, SVG lets you design with code, which makes it a perfect fit for +> visualizing dependency trees. SVGs can be embedded online in an `` tag, +> or inlined in an HTML document. They're also pretty easy to +> [convert](https://convertio.co/image-converter/). + +```python +svg = displacy.render(doc, style="dep") +output_path = Path("/images/sentence.svg") +output_path.open("w", encoding="utf-8").write(svg) +``` + + + +Since each visualization is generated as a separate SVG, exporting `.svg` files +only works if you're rendering **one single doc** at a time. (This makes sense – +after all, each visualization should be a standalone graphic.) So instead of +rendering all `Doc`s at once, loop over them and export them separately. + + + +### Example: Export SVG graphics of dependency parses {id="examples-export-svg"} + +```python {title="Example"} +import spacy +from spacy import displacy +from pathlib import Path + +nlp = spacy.load("en_core_web_sm") +sentences = ["This is an example.", "This is another one."] +for sent in sentences: + doc = nlp(sent) + svg = displacy.render(doc, style="dep", jupyter=False) + file_name = '-'.join([w.text for w in doc if not w.is_punct]) + ".svg" + output_path = Path("/images/" + file_name) + output_path.open("w", encoding="utf-8").write(svg) +``` + +The above code will generate the dependency visualizations as two files, +`This-is-an-example.svg` and `This-is-another-one.svg`. + +### Rendering data manually {id="manual-usage"} + +You can also use displaCy to manually render data. This can be useful if you +want to visualize output from other libraries, like [NLTK](http://www.nltk.org) +or +[SyntaxNet](https://github.com/tensorflow/models/tree/master/research/syntaxnet). +If you set `manual=True` on either `render()` or `serve()`, you can pass in data +in displaCy's format as a dictionary (instead of `Doc` objects). There are +helper functions for converting `Doc` objects to +[displaCy's format](/api/top-level#displacy_structures) for use with +`manual=True`: [`displacy.parse_deps`](/api/top-level#displacy.parse_deps), +[`displacy.parse_ents`](/api/top-level#displacy.parse_ents), and +[`displacy.parse_spans`](/api/top-level#displacy.parse_spans). + +> #### Example with parse function +> +> ```python +> doc = nlp("But Google is starting from behind.") +> ex = displacy.parse_ents(doc) +> html = displacy.render(ex, style="ent", manual=True) +> ``` + +> #### Example with raw data +> +> ```python +> ex = [{"text": "But Google is starting from behind.", +> "ents": [{"start": 4, "end": 10, "label": "ORG"}], +> "title": None}] +> html = displacy.render(ex, style="ent", manual=True) +> ``` + +```python {title="DEP input"} +{ + "words": [ + {"text": "This", "tag": "DT"}, + {"text": "is", "tag": "VBZ"}, + {"text": "a", "tag": "DT"}, + {"text": "sentence", "tag": "NN"} + ], + "arcs": [ + {"start": 0, "end": 1, "label": "nsubj", "dir": "left"}, + {"start": 2, "end": 3, "label": "det", "dir": "left"}, + {"start": 1, "end": 3, "label": "attr", "dir": "right"} + ] +} +``` + +```python {title="ENT input"} +{ + "text": "But Google is starting from behind.", + "ents": [{"start": 4, "end": 10, "label": "ORG"}], + "title": None +} +``` + +```python {title="ENT input with knowledge base links"} +{ + "text": "But Google is starting from behind.", + "ents": [{"start": 4, "end": 10, "label": "ORG", "kb_id": "Q95", "kb_url": "https://www.wikidata.org/entity/Q95"}], + "title": None +} +``` + +```python {title="SPANS input"} +{ + "text": "Welcome to the Bank of China.", + "spans": [ + {"start_token": 3, "end_token": 6, "label": "ORG"}, + {"start_token": 5, "end_token": 6, "label": "GPE"}, + ], + "tokens": ["Welcome", "to", "the", "Bank", "of", "China", "."], +} +``` + +## Using displaCy in a web application {id="webapp"} + +If you want to use the visualizers as part of a web application, for example to +create something like our [online demo](https://explosion.ai/demos/displacy), +it's not recommended to only wrap and serve the displaCy renderer. Instead, you +should only rely on the server to perform spaCy's processing capabilities, and +use a client-side implementation like +[`displaCy.js`](https://github.com/explosion/displacy) to render the +JSON-formatted output. + +> #### Why not return the HTML by the server? +> +> It's certainly possible to just have your server return the markup. But +> outputting raw, unsanitized HTML is risky and makes your app vulnerable to +> [cross-site scripting](https://en.wikipedia.org/wiki/Cross-site_scripting) +> (XSS). All your user needs to do is find a way to make spaCy return text like +> ``, which is pretty easy in NER mode. +> Instead of relying on the server to render and sanitize HTML, you can do this +> on the client in JavaScript. displaCy.js creates the markup as DOM nodes and +> will never insert raw HTML. + + + +Alternatively, if you're using [Streamlit](https://streamlit.io), check out the +[`spacy-streamlit`](https://github.com/explosion/spacy-streamlit) package that +helps you integrate spaCy visualizations into your apps. It includes a full +embedded visualizer, as well as individual components. + +![Screenshot of the spacy-streamlit package in Streamlit](/images/spacy-streamlit.png) + + diff --git a/website/meta/dynamicMeta.mjs b/website/meta/dynamicMeta.mjs new file mode 100644 index 0000000..95fd310 --- /dev/null +++ b/website/meta/dynamicMeta.mjs @@ -0,0 +1,14 @@ +import site from './site.json' assert { type: 'json' } + +export const domain = process.env.BRANCH || site.domain +export const siteUrl = `https://${domain}` +export const nightly = site.nightlyBranches.includes(domain) +export const legacy = site.legacy || !!+process.env.SPACY_LEGACY +export const binderBranch = domain +export const branch = nightly ? 'develop' : 'master' +export const replacements = { + GITHUB_SPACY: `https://github.com/explosion/spaCy/tree/${branch}`, + GITHUB_PROJECTS: `https://github.com/${site.projectsRepo}`, + SPACY_PKG_NAME: nightly ? 'spacy-nightly' : 'spacy', + SPACY_PKG_FLAGS: nightly ? ' --pre' : '', +} diff --git a/website/meta/languageSorted.tsx b/website/meta/languageSorted.tsx new file mode 100644 index 0000000..fe93bae --- /dev/null +++ b/website/meta/languageSorted.tsx @@ -0,0 +1,5 @@ +import models from './languages.json' + +export const languagesSorted = models.languages + .filter(({ models }) => models && models.length) + .sort((a, b) => a.name.localeCompare(b.name)) diff --git a/website/meta/languages.json b/website/meta/languages.json new file mode 100644 index 0000000..a824b7d --- /dev/null +++ b/website/meta/languages.json @@ -0,0 +1,551 @@ +{ + "languages": [ + { + "code": "af", + "name": "Afrikaans" + }, + { + "code": "am", + "name": "Amharic", + "has_examples": true + }, + { + "code": "ar", + "name": "Arabic", + "example": "هذه جملة", + "has_examples": true + }, + { + "code": "az", + "name": "Azerbaijani", + "has_examples": true + }, + { + "code": "bg", + "name": "Bulgarian", + "example": "Това е изречение", + "has_examples": true + }, + { + "code": "bn", + "name": "Bengali", + "has_examples": true + }, + { + "code": "bo", + "name": "Tibetan", + "example": "འདི་ཚིག་གྲུབ་རེད།", + "has_examples": true + }, + { + "code": "ca", + "name": "Catalan", + "example": "Això és una frase.", + "has_examples": true, + "models": ["ca_core_news_sm", "ca_core_news_md", "ca_core_news_lg", "ca_core_news_trf"] + }, + { + "code": "cs", + "name": "Czech", + "has_examples": true + }, + { + "code": "da", + "name": "Danish", + "example": "Dette er en sætning.", + "has_examples": true, + "models": ["da_core_news_sm", "da_core_news_md", "da_core_news_lg", "da_core_news_trf"] + }, + { + "code": "de", + "name": "German", + "models": ["de_core_news_sm", "de_core_news_md", "de_core_news_lg", "de_dep_news_trf"], + "example": "Dies ist ein Satz.", + "has_examples": true + }, + { + "code": "dsb", + "name": "Lower Sorbian", + "has_examples": true + }, + { + "code": "el", + "name": "Greek", + "models": ["el_core_news_sm", "el_core_news_md", "el_core_news_lg"], + "example": "Αυτή είναι μια πρόταση.", + "has_examples": true + }, + { + "code": "en", + "name": "English", + "models": ["en_core_web_sm", "en_core_web_md", "en_core_web_lg", "en_core_web_trf"], + "example": "This is a sentence.", + "has_examples": true + }, + { + "code": "es", + "name": "Spanish", + "models": ["es_core_news_sm", "es_core_news_md", "es_core_news_lg", "es_dep_news_trf"], + "example": "Esto es una frase.", + "has_examples": true + }, + { + "code": "et", + "name": "Estonian" + }, + { + "code": "eu", + "name": "Basque", + "has_examples": true + }, + { + "code": "fa", + "name": "Persian", + "has_examples": true + }, + { + "code": "fi", + "name": "Finnish", + "has_examples": true, + "models": ["fi_core_news_sm", "fi_core_news_md", "fi_core_news_lg"] + }, + { + "code": "fo", + "name": "Faroese" + }, + { + "code": "fr", + "name": "French", + "models": ["fr_core_news_sm", "fr_core_news_md", "fr_core_news_lg", "fr_dep_news_trf"], + "example": "C'est une phrase.", + "has_examples": true + }, + { + "code": "ga", + "name": "Irish" + }, + { + "code": "grc", + "name": "Ancient Greek", + "has_examples": true + }, + { + "code": "gu", + "name": "Gujarati", + "has_examples": true + }, + { + "code": "he", + "name": "Hebrew", + "example": "זהו משפט.", + "has_examples": true + }, + { + "code": "hi", + "name": "Hindi", + "example": "यह एक वाक्य है।", + "has_examples": true + }, + { + "code": "hr", + "name": "Croatian", + "has_examples": true, + "models": ["hr_core_news_sm", "hr_core_news_md", "hr_core_news_lg"] + }, + { + "code": "hsb", + "name": "Upper Sorbian", + "has_examples": true + }, + { + "code": "hu", + "name": "Hungarian", + "example": "Ez egy mondat.", + "has_examples": true + }, + { + "code": "hy", + "name": "Armenian", + "has_examples": true + }, + { + "code": "id", + "name": "Indonesian", + "example": "Ini adalah sebuah kalimat.", + "has_examples": true + }, + { + "code": "is", + "name": "Icelandic" + }, + { + "code": "it", + "name": "Italian", + "models": ["it_core_news_sm", "it_core_news_md", "it_core_news_lg"], + "example": "Questa è una frase.", + "has_examples": true + }, + { + "code": "ja", + "name": "Japanese", + "models": ["ja_core_news_sm", "ja_core_news_md", "ja_core_news_lg", "ja_core_news_trf"], + "dependencies": [ + { + "name": "SudachiPy", + "url": "https://github.com/WorksApplications/SudachiPy" + } + ], + "example": "これは文章です。", + "has_examples": true + }, + { + "code": "kn", + "name": "Kannada", + "has_examples": true + }, + { + "code": "ko", + "name": "Korean", + "dependencies": [ + { + "name": "mecab-ko", + "url": "https://bitbucket.org/eunjeon/mecab-ko/src/master/README.md" + }, + { + "name": "mecab-ko-dic", + "url": "https://bitbucket.org/eunjeon/mecab-ko-dic" + }, + { + "name": "natto-py", + "url": "https://github.com/buruzaemon/natto-py" + } + ], + "example": "이것은 문장입니다.", + "has_examples": true, + "models": ["ko_core_news_sm", "ko_core_news_md", "ko_core_news_lg"] + }, + { + "code": "ky", + "name": "Kyrgyz", + "example": "Адамга эң кыйыны — күн сайын адам болуу", + "has_examples": true + }, + { + "code": "la", + "name": "Latin", + "example": "In principio creavit Deus caelum et terram.", + "has_examples": true + }, + { + "code": "lb", + "name": "Luxembourgish", + "has_examples": true + }, + { + "code": "lg", + "name": "Luganda", + "has_examples": true + }, + { + "code": "lij", + "name": "Ligurian", + "example": "Sta chì a l'é unna fraxe.", + "has_examples": true + }, + { + "code": "lt", + "name": "Lithuanian", + "has_examples": true, + "models": ["lt_core_news_sm", "lt_core_news_md", "lt_core_news_lg"] + }, + { + "code": "lv", + "name": "Latvian" + }, + { + "code": "mk", + "name": "Macedonian", + "models": ["mk_core_news_sm", "mk_core_news_md", "mk_core_news_lg"] + }, + { + "code": "ml", + "name": "Malayalam", + "has_examples": true + }, + { + "code": "mr", + "name": "Marathi" + }, + { + "code": "ms", + "name": "Malay", + "has_examples": true + }, + { + "code": "nb", + "name": "Norwegian Bokmål", + "example": "Dette er en setning.", + "has_examples": true, + "models": ["nb_core_news_sm", "nb_core_news_md", "nb_core_news_lg"] + }, + { + "code": "ne", + "name": "Nepali", + "has_examples": true + }, + { + "code": "nl", + "name": "Dutch", + "models": ["nl_core_news_sm", "nl_core_news_md", "nl_core_news_lg"], + "example": "Dit is een zin.", + "has_examples": true + }, + { + "code": "nn", + "name": "Norwegian Nynorsk", + "example": "Det er ein meir enn i same periode i fjor.", + "has_examples": true + }, + { + "code": "pl", + "name": "Polish", + "example": "To jest zdanie.", + "has_examples": true, + "models": ["pl_core_news_sm", "pl_core_news_md", "pl_core_news_lg"] + }, + { + "code": "pt", + "name": "Portuguese", + "models": ["pt_core_news_sm", "pt_core_news_md", "pt_core_news_lg"], + "example": "Esta é uma frase.", + "has_examples": true + }, + { + "code": "ro", + "name": "Romanian", + "example": "Aceasta este o propoziție.", + "has_examples": true, + "models": ["ro_core_news_sm", "ro_core_news_md", "ro_core_news_lg"] + }, + { + "code": "ru", + "name": "Russian", + "has_examples": true, + "dependencies": [ + { + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" + } + ], + "models": ["ru_core_news_sm", "ru_core_news_md", "ru_core_news_lg"] + }, + { + "code": "sa", + "name": "Sanskrit", + "has_examples": true + }, + { + "code": "si", + "name": "Sinhala", + "example": "මෙය වාක්‍යයකි.", + "has_examples": true + }, + { + "code": "sk", + "name": "Slovak", + "has_examples": true + }, + { + "code": "sl", + "name": "Slovenian", + "example": "France Prešeren je umrl 8. februarja 1849 v Kranju", + "has_examples": true, + "models": ["sl_core_news_sm", "sl_core_news_md", "sl_core_news_lg", "sl_core_news_trf"] + }, + { + "code": "sq", + "name": "Albanian", + "example": "Kjo është një fjali.", + "has_examples": true + }, + { + "code": "sr", + "name": "Serbian", + "has_examples": true + }, + { + "code": "sv", + "name": "Swedish", + "has_examples": true, + "models": ["sv_core_news_sm", "sv_core_news_md", "sv_core_news_lg"] + }, + { + "code": "ta", + "name": "Tamil", + "has_examples": true + }, + { + "code": "te", + "name": "Telugu", + "example": "ఇది ఒక వాక్యం.", + "has_examples": true + }, + { + "code": "th", + "name": "Thai", + "dependencies": [ + { + "name": "pythainlp", + "url": "https://github.com/wannaphongcom/pythainlp" + } + ], + "example": "นี่คือประโยค", + "has_examples": true + }, + { + "code": "ti", + "name": "Tigrinya", + "has_examples": true + }, + { + "code": "tl", + "name": "Tagalog" + }, + { + "code": "tn", + "name": "Setswana", + "has_examples": true + }, + { + "code": "tr", + "name": "Turkish", + "example": "Bu bir cümledir.", + "has_examples": true + }, + { + "code": "tt", + "name": "Tatar", + "has_examples": true + }, + { + "code": "uk", + "name": "Ukrainian", + "has_examples": true, + "models": ["uk_core_news_sm", "uk_core_news_md", "uk_core_news_lg", "uk_core_news_trf"], + "dependencies": [ + { + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" + }, + { + "name": "pymorphy3-dicts-uk", + "url": "https://github.com/no-plagiarism/pymorphy3-dicts" + } + ] + }, + { + "code": "ur", + "name": "Urdu", + "example": "یہ ایک جملہ ہے", + "has_examples": true + }, + { + "code": "vi", + "name": "Vietnamese", + "dependencies": [ + { + "name": "Pyvi", + "url": "https://github.com/trungtv/pyvi" + } + ] + }, + { + "code": "xx", + "name": "Multi-language", + "models": ["xx_ent_wiki_sm", "xx_sent_ud_sm"], + "example": "This is a sentence about Facebook." + }, + { + "code": "yo", + "name": "Yoruba", + "has_examples": true + }, + { + "code": "zh", + "name": "Chinese", + "models": ["zh_core_web_sm", "zh_core_web_md", "zh_core_web_lg", "zh_core_web_trf"], + "dependencies": [ + { + "name": "Jieba", + "url": "https://github.com/fxsjy/jieba" + }, + { + "name": "spacy-pkuseg", + "url": "https://github.com/explosion/spacy-pkuseg" + } + ], + "example": "这是一个用于示例的句子。", + "has_examples": true + }, + { + "code": "kmr", + "name": "Kurdish Kurmanji", + "example": "Ev hevokek e", + "has_examples": true + } + ], + "licenses": [ + { + "id": "CC BY 4.0", + "url": "https://creativecommons.org/licenses/by/4.0/" + }, + { + "id": "CC BY-SA", + "url": "https://creativecommons.org/licenses/by-sa/3.0/" + }, + { + "id": "CC BY-SA 3.0", + "url": "https://creativecommons.org/licenses/by-sa/3.0/" + }, + { + "id": "CC BY-SA 4.0", + "url": "https://creativecommons.org/licenses/by-sa/4.0/" + }, + { + "id": "CC BY-NC", + "url": "https://creativecommons.org/licenses/by-nc/3.0/" + }, + { + "id": "CC BY-NC 3.0", + "url": "https://creativecommons.org/licenses/by-nc/3.0/" + }, + { + "id": "CC BY-NC 4.0", + "url": "https://creativecommons.org/licenses/by-nc/4.0/" + }, + { + "id": "CC-BY-NC-SA 3.0", + "url": "https://creativecommons.org/licenses/by-nc-sa/3.0/" + }, + { + "id": "GPL", + "url": "https://www.gnu.org/licenses/gpl.html" + }, + { + "id": "GPU GPL 3.0", + "url": "https://www.gnu.org/licenses/gpl-3.0.en.html" + }, + { + "id": "LGPL", + "url": "https://www.gnu.org/licenses/lgpl.html" + }, + { + "id": "MIT", + "url": "https://opensource.org/licenses/MIT" + }, + { + "id": "LGPL-LR", + "url": "https://github.com/UniversalDependencies/UD_French-Sequoia/blob/master/LICENSE.txt" + } + ] +} diff --git a/website/meta/recordLanguages.tsx b/website/meta/recordLanguages.tsx new file mode 100644 index 0000000..6555a46 --- /dev/null +++ b/website/meta/recordLanguages.tsx @@ -0,0 +1,7 @@ +import models from './languages.json' + +const recordLanguages = Object.fromEntries( + models.languages.map((language, index) => [language.code, language]) +) + +export default recordLanguages diff --git a/website/meta/recordSections.tsx b/website/meta/recordSections.tsx new file mode 100644 index 0000000..4dd1718 --- /dev/null +++ b/website/meta/recordSections.tsx @@ -0,0 +1,5 @@ +import siteMetadata from './site.json' + +const recordSections = Object.fromEntries(siteMetadata.sections.map((s) => [s.id, s])) + +export default recordSections diff --git a/website/meta/recordUniverse.tsx b/website/meta/recordUniverse.tsx new file mode 100644 index 0000000..588b0b9 --- /dev/null +++ b/website/meta/recordUniverse.tsx @@ -0,0 +1,9 @@ +import universe from './universe.json' + +export const recordUniverseCategories = Object.fromEntries( + universe.categories.flatMap((category) => category.items.map((item) => [item.id, item])) +) + +export const recordUniverseResources = Object.fromEntries( + universe.resources.map((resource) => [resource.id, resource]) +) diff --git a/website/meta/sidebarFlat.tsx b/website/meta/sidebarFlat.tsx new file mode 100644 index 0000000..b9f94de --- /dev/null +++ b/website/meta/sidebarFlat.tsx @@ -0,0 +1,5 @@ +import sidebars from './sidebars.json' + +export const sidebarUsageFlat = sidebars + .find((sidebar) => sidebar.section === 'usage') + .items.flatMap((item) => item.items) diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json new file mode 100644 index 0000000..74a9eac --- /dev/null +++ b/website/meta/sidebars.json @@ -0,0 +1,388 @@ +[ + { + "section": "usage", + "items": [ + { + "label": "Get started", + "items": [ + { + "text": "Installation", + "url": "/usage" + }, + { + "text": "Models & Languages", + "url": "/usage/models" + }, + { + "text": "Facts & Figures", + "url": "/usage/facts-figures" + }, + { + "text": "spaCy 101", + "url": "/usage/spacy-101" + }, + { + "text": "New in v3.7", + "url": "/usage/v3-7" + }, + { + "text": "New in v3.6", + "url": "/usage/v3-6" + }, + { + "text": "New in v3.5", + "url": "/usage/v3-5" + } + ] + }, + { + "label": "Guides", + "items": [ + { + "text": "Linguistic Features", + "url": "/usage/linguistic-features" + }, + { + "text": "Rule-based Matching", + "url": "/usage/rule-based-matching" + }, + { + "text": "Processing Pipelines", + "url": "/usage/processing-pipelines" + }, + { + "text": "Embeddings & Transformers", + "url": "/usage/embeddings-transformers" + }, + { + "text": "Large Language Models", + "url": "/usage/large-language-models" + }, + { + "text": "Training Models", + "url": "/usage/training" + }, + { + "text": "Layers & Model Architectures", + "url": "/usage/layers-architectures" + }, + { + "text": "spaCy Projects", + "url": "/usage/projects" + }, + { + "text": "Saving & Loading", + "url": "/usage/saving-loading" + }, + { + "text": "Memory Management", + "url": "/usage/memory-management" + }, + { + "text": "Visualizers", + "url": "/usage/visualizers" + } + ] + }, + { + "label": "Resources", + "items": [ + { + "text": "Project Templates", + "url": "https://github.com/explosion/projects" + }, + { + "text": "v2.x Documentation", + "url": "https://v2.spacy.io" + }, + { + "text": "Custom Solutions", + "url": "https://explosion.ai/custom-solutions" + } + ] + } + ] + }, + { + "section": "models", + "items": [ + { + "label": "Models", + "items": [ + { + "text": "Overview", + "url": "/models" + } + ] + }, + { + "label": "Trained Pipelines", + "items": [] + } + ] + }, + { + "section": "api", + "items": [ + { + "label": "Overview", + "items": [ + { + "text": "Library Architecture", + "url": "/api" + }, + { + "text": "Model Architectures", + "url": "/api/architectures" + }, + { + "text": "Data Formats", + "url": "/api/data-formats" + }, + { + "text": "Command Line", + "url": "/api/cli" + }, + { + "text": "Functions", + "url": "/api/top-level" + } + ] + }, + { + "label": "Containers", + "items": [ + { + "text": "Doc", + "url": "/api/doc" + }, + { + "text": "DocBin", + "url": "/api/docbin" + }, + { + "text": "Example", + "url": "/api/example" + }, + { + "text": "Language", + "url": "/api/language" + }, + { + "text": "Lexeme", + "url": "/api/lexeme" + }, + { + "text": "Span", + "url": "/api/span" + }, + { + "text": "SpanGroup", + "url": "/api/spangroup" + }, + { + "text": "Token", + "url": "/api/token" + } + ] + }, + { + "label": "Pipeline", + "items": [ + { + "text": "AttributeRuler", + "url": "/api/attributeruler" + }, + { + "text": "CoreferenceResolver", + "url": "/api/coref" + }, + { + "text": "CuratedTransformer", + "url": "/api/curatedtransformer" + }, + { + "text": "DependencyParser", + "url": "/api/dependencyparser" + }, + { + "text": "EditTreeLemmatizer", + "url": "/api/edittreelemmatizer" + }, + { + "text": "EntityLinker", + "url": "/api/entitylinker" + }, + { + "text": "EntityRecognizer", + "url": "/api/entityrecognizer" + }, + { + "text": "EntityRuler", + "url": "/api/entityruler" + }, + { + "text": "Large Language Models", + "url": "/api/large-language-models" + }, + { + "text": "Lemmatizer", + "url": "/api/lemmatizer" + }, + { + "text": "Morphologizer", + "url": "/api/morphologizer" + }, + { + "text": "SentenceRecognizer", + "url": "/api/sentencerecognizer" + }, + { + "text": "Sentencizer", + "url": "/api/sentencizer" + }, + { + "text": "SpanCategorizer", + "url": "/api/spancategorizer" + }, + { + "text": "SpanFinder", + "url": "/api/spanfinder" + }, + { + "text": "SpanResolver", + "url": "/api/span-resolver" + }, + { + "text": "SpanRuler", + "url": "/api/spanruler" + }, + { + "text": "Tagger", + "url": "/api/tagger" + }, + { + "text": "TextCategorizer", + "url": "/api/textcategorizer" + }, + { + "text": "Tok2Vec", + "url": "/api/tok2vec" + }, + { + "text": "Tokenizer", + "url": "/api/tokenizer" + }, + { + "text": "TrainablePipe", + "url": "/api/pipe" + }, + { + "text": "Transformer", + "url": "/api/transformer" + }, + { + "text": "Other Functions", + "url": "/api/pipeline-functions" + } + ] + }, + { + "label": "Matchers", + "items": [ + { + "text": "DependencyMatcher", + "url": "/api/dependencymatcher" + }, + { + "text": "Matcher", + "url": "/api/matcher" + }, + { + "text": "PhraseMatcher", + "url": "/api/phrasematcher" + } + ] + }, + { + "label": "Other", + "items": [ + { + "text": "Attributes", + "url": "/api/attributes" + }, + { + "text": "BaseVectors", + "url": "/api/basevectors" + }, + { + "text": "Corpus", + "url": "/api/corpus" + }, + { + "text": "InMemoryLookupKB", + "url": "/api/inmemorylookupkb" + }, + { + "text": "KnowledgeBase", + "url": "/api/kb" + }, + { + "text": "Lookups", + "url": "/api/lookups" + }, + { + "text": "MorphAnalysis", + "url": "/api/morphology#morphanalysis" + }, + { + "text": "Morphology", + "url": "/api/morphology" + }, + { + "text": "Scorer", + "url": "/api/scorer" + }, + { + "text": "StringStore", + "url": "/api/stringstore" + }, + { + "text": "Vectors", + "url": "/api/vectors" + }, + { + "text": "Vocab", + "url": "/api/vocab" + } + ] + }, + { + "label": "Cython", + "items": [ + { + "text": "Architecture", + "url": "/api/cython" + }, + { + "text": "Classes", + "url": "/api/cython-classes" + }, + { + "text": "Structs", + "url": "/api/cython-structs" + } + ] + }, + { + "label": "Legacy", + "items": [ + { + "text": "Legacy functions", + "url": "/api/legacy" + } + ] + } + ] + } +] diff --git a/website/meta/site.json b/website/meta/site.json new file mode 100644 index 0000000..5aac9f5 --- /dev/null +++ b/website/meta/site.json @@ -0,0 +1,86 @@ +{ + "title": "spaCy", + "description": "spaCy is a free open-source library for Natural Language Processing in Python. It features NER, POS tagging, dependency parsing, word vectors and more.", + "slogan": "Industrial-strength Natural Language Processing in Python", + "domain": "spacy.io", + "nightlyBranches": ["nightly.spacy.io"], + "legacy": false, + "email": "contact@explosion.ai", + "company": "Explosion", + "companyUrl": "https://explosion.ai", + "repo": "explosion/spaCy", + "modelsRepo": "explosion/spacy-models", + "projectsRepo": "explosion/projects/tree/v3", + "social": { + "twitter": "spacy_io", + "github": "explosion" + }, + "theme": "#09a3d5", + "newsletter": { + "user": "spacy.us12", + "id": "83b0498b1e7fa3c91ce68c3f1", + "list": "ecc82e0493" + }, + "docSearch": { + "appId": "Y1LB128RON", + "indexName": "spacy" + }, + "binderUrl": "explosion/spacy-io-binder", + "binderVersion": "3.7", + "sections": [ + { "id": "usage", "title": "Usage Documentation", "theme": "blue" }, + { "id": "models", "title": "Models Documentation", "theme": "blue" }, + { "id": "api", "title": "API Documentation", "theme": "green" }, + { "id": "universe", "title": "Universe", "theme": "purple" } + ], + "navigation": [ + { "text": "Usage", "url": "/usage" }, + { "text": "Models", "url": "/models" }, + { "text": "API", "url": "/api" }, + { "text": "Universe", "url": "/universe" } + ], + "footer": [ + { + "label": "spaCy", + "items": [ + { "text": "Usage", "url": "/usage" }, + { "text": "Models", "url": "/models" }, + { "text": "API Reference", "url": "/api" }, + { "text": "Online Course", "url": "https://course.spacy.io" }, + { + "text": "Custom Solutions", + "url": "https://explosion.ai/custom-solutions" + } + ] + }, + { + "label": "Community", + "items": [ + { "text": "Universe", "url": "/universe" }, + { + "text": "GitHub Discussions", + "url": "https://github.com/explosion/spaCy/discussions" + }, + { "text": "Issue Tracker", "url": "https://github.com/explosion/spaCy/issues" }, + { + "text": "Stack Overflow", + "url": "http://stackoverflow.com/questions/tagged/spacy" + }, + { + "text": "Merchandise", + "url": "https://explosion.ai/merch" + } + ] + }, + { + "label": "Connect", + "items": [ + { "text": "Bluesky", "url": "https://bsky.app/profile/explosion-ai.bsky.social" }, + { "text": "GitHub", "url": "https://github.com/explosion/spaCy" }, + { "text": "Live Stream", "url": "https://www.youtube.com/playlist?list=PLBmcuObd5An5_iAxNYLJa_xWmNzsYce8c" }, + { "text": "YouTube", "url": "https://youtube.com/c/ExplosionAI" }, + { "text": "Blog", "url": "https://explosion.ai/blog" } + ] + } + ] +} diff --git a/website/meta/type-annotations.json b/website/meta/type-annotations.json new file mode 100644 index 0000000..0ffcbfb --- /dev/null +++ b/website/meta/type-annotations.json @@ -0,0 +1,52 @@ +{ + "Doc": "/api/doc", + "Token": "/api/token", + "Span": "/api/span", + "SpanGroup": "/api/spangroup", + "Lexeme": "/api/lexeme", + "Example": "/api/example", + "Alignment": "/api/example#alignment-object", + "Vocab": "/api/vocab", + "StringStore": "/api/stringstore", + "Lookups": "/api/lookups", + "Table": "/api/lookups#table", + "Vectors": "/api/vectors", + "Language": "/api/language", + "Defaults": "/api/language#defaults", + "Scorer": "/api/scorer", + "DocBin": "/api/docbin", + "FactoryMeta": "/api/language#factorymeta", + "Tokenizer": "/api/tokenizer", + "MorphAnalysis": "/api/morphology#morphanalysis", + "KnowledgeBase": "/api/kb", + "Candidate": "/api/kb#candidate", + "Matcher": "/api/matcher", + "PhraseMatcher": "/api/phrasematcher", + "TransformerData": "/api/transformer#transformerdata", + "FullTransformerBatch": "/api/transformer#fulltransformerbatch", + "Corpus": "/api/corpus", + "JsonlCorpus": "/api/corpus#jsonlcorpus", + "LexemeC": "/api/cython-structs#lexemec", + "TokenC": "/api/cython-structs#tokenc", + "Config": "https://thinc.ai/docs/api-config#config", + "Optimizer": "https://thinc.ai/docs/api-optimizers", + "Model": "https://thinc.ai/docs/api-model", + "Ragged": "https://thinc.ai/docs/api-types#ragged", + "Padded": "https://thinc.ai/docs/api-types#padded", + "Ints2d": "https://thinc.ai/docs/api-types#types", + "Floats2d": "https://thinc.ai/docs/api-types#types", + "Floats3d": "https://thinc.ai/docs/api-types#types", + "FloatsXd": "https://thinc.ai/docs/api-types#types", + "Array1d": "https://thinc.ai/docs/api-types#types", + "Array2d": "https://thinc.ai/docs/api-types#types", + "Ops": "https://thinc.ai/docs/api-backends#ops", + "cymem.Pool": "https://github.com/explosion/cymem", + "preshed.BloomFilter": "https://github.com/explosion/preshed", + "transformers.BatchEncoding": "https://huggingface.co/transformers/main_classes/tokenizer.html#transformers.BatchEncoding", + "transformers.file_utils.ModelOutput": "https://huggingface.co/transformers/main_classes/output.html#modeloutput", + "torch.Tensor": "https://pytorch.org/docs/stable/tensors.html", + "numpy.ndarray": "https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html", + "Match": "https://docs.python.org/3/library/re.html#match-objects", + "Pattern": "https://docs.python.org/3/library/re.html#regular-expression-objects", + "Path": "https://docs.python.org/3/library/pathlib.html" +} diff --git a/website/meta/universe.json b/website/meta/universe.json new file mode 100644 index 0000000..1f55d96 --- /dev/null +++ b/website/meta/universe.json @@ -0,0 +1,5817 @@ +{ + "resources": [ + { + "id": "TeNs", + "title": "Temporal Expressions Normalization spaCy", + "thumb": "https://github-production-user-asset-6210df.s3.amazonaws.com/40547052/433595900-fae3c9d9-7181-4d8b-8b49-e6dc4fca930b.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20250414%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20250414T235545Z&X-Amz-Expires=300&X-Amz-Signature=e21d3c06300ceb15fa1dadd7cb60081cc9f1b35e5a7bfd07f6e8b90dd7fad9d0&X-Amz-SignedHeaders=host", + "url": "https://pypi.org/project/temporal-normalization-spacy/", + "slogan": "A temporal expression normalization plugin for Romanian using rule-based methods and DBpedia mappings.", + "description": "**[Temporal Expressions Normalization spaCy (TeNs)](https://github.com/iliedorobat/timespan-normalization-spacy)** is a powerful pipeline component for spaCy that seamlessly identifies and parses date entities in text. It leverages the **[Temporal Expressions Normalization Framework]( https://github.com/iliedorobat/timespan-normalization)** to recognize a wide variety of date formats using an extensive set of regular expressions (RegEx), ensuring robust and adaptable date extraction across diverse textual sources.\n\nUnlike conventional solutions that primarily focus on well-structured date formats, TeNs excels in handling real-world text by **identifying** not only standard date representations but also **abbreviated, informal, or even misspelled temporal expressions.** This makes it particularly effective for processing noisy or unstructured data, such as historical records, user-generated content, and scanned documents with OCR inaccuracies.", + "github": "iliedorobat/timespan-normalization-spacy", + "pip": "temporal-normalization-spacy", + "code_example": [ + "import subprocess", + "", + "import spacy", + "", + "from temporal_normalization.commons.print_utils import console", + "from temporal_normalization.index import create_normalized_component, TemporalNormalization # noqa: F401", + "", + "", + "try:", + " # Load the spaCy model if it has already been downloaded", + " nlp = spacy.load('ro_core_news_sm')", + "except OSError:", + " console.warning(f'Started downloading ro_core_news_sm...')", + " # Download the Romanian model if it wasn't already downloaded", + " subprocess.run(['python', '-m', 'spacy', 'download', 'ro_core_news_sm'])", + " # Load the spaCy model", + " nlp = spacy.load('ro_core_news_sm')", + "", + "# Add 'temporal_normalization' component to the spaCy pipeline", + "nlp.add_pipe('temporal_normalization', last=True)", + "doc = nlp('Sec al II-lea a.ch. - I d.ch reprezintă o perioadă de mari schimbări.')", + "", + "# Display information about the identified and normalized dates in the text.", + "for entity in doc.ents:", + " edges = entity._.time_series.edges", + "", + " print('Start Edge:')", + " print(edges.start.serialize('\\t'))", + " print()", + "", + " print('End Edge:')", + " print(edges.end.serialize('\\t'))", + " print()", + "", + " print('Periods:')", + " for period in entity._.time_series.periods:", + " print(period.serialize('\\t'))", + " print()", + " print('---------------------')" + ], + "code_language": "python", + "author": "Ilie Cristian Dorobat", + "author_links": { + "github": "iliedorobat", + "website": "https://iliedorobat.ro/" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "temporal", + "normalization", + "date", + "romanian", + "temporal-expression", + "dbpedia" + ] + }, + { + "id": "spacy-vscode", + "title": "spaCy Visual Studio Code Extension", + "thumb": "https://raw.githubusercontent.com/explosion/spacy-vscode/main/icon.png", + "slogan": "Work with spaCy's config files in VS Code", + "description": "The spaCy VS Code Extension provides additional tooling and features for working with spaCy's config files. Version 1.0.0 includes hover descriptions for registry functions, variables, and section names within the config as an installable extension.", + "url": "https://marketplace.visualstudio.com/items?itemName=Explosion.spacy-extension", + "github": "explosion/spacy-vscode", + "code_language": "python", + "author": "Explosion", + "author_links": { + "twitter": "@explosion_ai", + "github": "explosion" + }, + "category": [ + "extension" + ], + "tags": [] + }, + { + "id": "constituent_treelib", + "title": "Constituent Treelib", + "slogan": "Extract constituents with ease!", + "description": "Constituent Treelib (CTL) is a lightweight Python library built on top of benepar (Berkeley Neural Parser) as well as the two well-known NLP frameworks spaCy and NLTK. CTL offers you a convenient way to parse sentences into constituent trees, modify them according to their structure, as well as visualize and export them into various file formats. In addition, you can extract phrases according to their phrasal categories (which can be used e.g., as features for various NLP tasks), validate already parsed sentences in bracket notation or convert them back into sentences.", + "github": "Halvani/Constituent-Treelib", + "pip": "constituent-treelib", + "code_example": [ + "from constituent_treelib import ConstituentTree, Language", + "# Define the language for the sentence as well as for the spaCy and benepar models", + "language = Language.English", + "# Define which specific SpaCy model should be used (default is Medium)", + "spacy_model_size = ConstituentTree.SpacyModelSize.Medium", + "# Create the pipeline (note, the required models will be downloaded and installed automatically)", + "nlp = ConstituentTree.create_pipeline(language, spacy_model_size)", + "# Your sentence", + "sentence = 'We try to explicitly describe the geometry of the edges of the images.'", + "# Create the tree from where we are going to extract the desired noun phrases", + "tree = ConstituentTree(sentence, nlp)", + "all_phrases = tree.extract_all_phrases(min_words_in_phrases=1)", + "print(all_phrases)", + "# {'PP': ['of the edges of the images', 'of the images'], 'NP': ['We', 'the geometry of the edges of the images', 'the geometry', 'the edges of the images', 'the edges', 'the images'], 'S': ['We try to explicitly describe the geometry of the edges of the images .', 'to explicitly describe the geometry of the edges of the images'], 'VP': ['try to explicitly describe the geometry of the edges of the images', 'to explicitly describe the geometry of the edges of the images', 'describe the geometry of the edges of the images'], 'ADVP': ['explicitly']}" + ], + "code_language": "python", + "url": "https://github.com/Halvani/Constituent-Treelib", + "thumb": "https://github.com/Halvani/Constituent-Treelib/blob/main/assets/images/promo_tree.svg", + "author": "Oren Halvani", + "author_links": { + "github": "Halvani", + "website": "https://www.linkedin.com/in/orenhalvani" + }, + "category": [ + "apis", + "standalone", + "visualizers" + ], + "tags": [ + "apis", + "deployment", + "constituency ", + "parsing" + ] + }, + { + "id": "sayswho", + "title": "SaysWho", + "slogan": "Quote identification, attribution and resolution", + "description": "A Python package for identifying and attributing quotes in text. It uses a combination of spaCy functionality, logic and grammar to find quotes and their speakers, then uses the spaCy coreferencing model to better clarify who is speaking. Currently English only.", + "github": "afriedman412/sayswho", + "pip": "sayswho", + "code_language": "python", + "author": "Andy Friedman", + "author_links": { + "twitter": "@steadynappin", + "github": "afriedman412" + }, + "code_example": [ + "from sayswho import SaysWho", + "text = open(\"path/to/your/text_file.txt\").read()", + "sw = SaysWho()", + "sw.attribute(text)", + "sw.expand_match() # see quote/cluster matches", + "sw.render_to_html() # output your text, quotes and cluster matches to an html file called \"temp.html\"" + ], + "category": [ + "standalone" + ], + "tags": [ + "attribution", + "coref", + "text-processing" + ] + }, + { + "id": "parsigs", + "title": "parsigs", + "slogan": "Structuring prescriptions text made simple using spaCy", + "description": "Parsigs is an open-source project that aims to extract the relevant dosage information from prescriptions text without compromising the patient's privacy.\n\nNotice you also need to install the model in order to use the package: `pip install https://huggingface.co/royashcenazi/en_parsigs/resolve/main/en_parsigs-any-py3-none-any.whl`", + "github": "royashcenazi/parsigs", + "pip": "parsigs", + "code_language": "python", + "author": "Roy Ashcenazi", + "code_example": [ + "# You'll need to install the trained model, see instructions in the description section", + "from parsigs.parse_sig_api import StructuredSig, SigParser", + "sig_parser = SigParser()", + "", + "sig = 'Take 1 tablet of ibuprofen 200mg 3 times every day for 3 weeks'", + "parsed_sig = sig_parser.parse(sig)" + ], + "author_links": { + "github": "royashcenazi" + }, + "category": [ + "model", + "research", + "biomedical" + ], + "tags": [ + "sigs", + "prescription", + "pharma" + ] + }, + { + "id": "latincy", + "title": "LatinCy", + "thumb": "https://raw.githubusercontent.com/diyclassics/la_core_web_lg/main/latincy-logo.png", + "slogan": "Synthetic trained spaCy pipelines for Latin NLP", + "description": "Set of trained general purpose Latin-language 'core' pipelines for use with spaCy. The models are trained on a large amount of available Latin data, including all five of the Latin Universal Dependency treebanks, which have been preprocessed to be compatible with each other.", + "url": "https://huggingface.co/latincy", + "code_example": [ + "# pip install https://huggingface.co/latincy/la_core_web_lg/resolve/main/la_core_web_lg-any-py3-none-any.whl", + "import spacy", + "nlp = spacy.load('la_core_web_lg')", + "doc = nlp('Haec narrantur a poetis de Perseo')", + "", + "print(f'{doc[0].text}, {doc[0].norm_}, {doc[0].lemma_}, {doc[0].pos_}')", + "", + "# > Haec, haec, hic, DET" + ], + "code_language": "python", + "author": "Patrick J. Burns", + "author_links": { + "twitter": "@diyclassics", + "github": "diyclassics", + "website": "https://diyclassics.github.io/" + }, + "category": [ + "pipeline", + "research" + ], + "tags": [ + "latin" + ] + }, + { + "id": "odycy", + "title": "OdyCy", + "slogan": "General-purpose language pipelines for premodern Greek.", + "description": "Academically validated modular NLP pipelines for premodern Greek. odyCy achieves state of the art performance on multiple tasks on unseen test data from the Universal Dependencies Perseus treebank, and performs second best on the PROIEL treebank’s test set on even more tasks. In addition performance also seems relatively stable across the two evaluation datasets in comparison with other NLP pipelines. OdyCy is being used at the Center for Humanities Computing for preprocessing and analyzing Ancient Greek corpora for New Testament research, meaning that you can expect consistent maintenance and improvements.", + "github": "centre-for-humanities-computing/odyCy", + "code_example": [ + "# To install the high-accuracy transformer-based pipeline", + "# pip install https://huggingface.co/chcaa/grc_odycy_joint_trf/resolve/main/grc_odycy_joint_trf-any-py3-none-any.whl", + "import spacy", + "", + "nlp = spacy.load('grc_odycy_joint_trf')", + "", + "doc = nlp('τὴν γοῦν Ἀττικὴν ἐκ τοῦ ἐπὶ πλεῖστον διὰ τὸ λεπτόγεων ἀστασίαστον οὖσαν ἄνθρωποι ᾤκουν οἱ αὐτοὶ αἰεί.')" + ], + "code_language": "python", + "url": "https://centre-for-humanities-computing.github.io/odyCy/", + "thumb": "https://raw.githubusercontent.com/centre-for-humanities-computing/odyCy/7b94fec60679d06272dca88a4dcfe0f329779aea/docs/_static/logo.svg", + "image": "https://github.com/centre-for-humanities-computing/odyCy/raw/main/docs/_static/logo_with_text_below.svg", + "author": "Jan Kostkan, Márton Kardos (Center for Humanities Computing, Aarhus University)", + "author_links": { + "github": "centre-for-humanities-computing", + "website": "https://chc.au.dk/" + }, + "category": [ + "pipeline", + "standalone", + "research" + ], + "tags": [ + "ancient Greek" + ] + }, + { + "id": "spacy-wasm", + "title": "spacy-wasm", + "slogan": "spaCy in the browser using WebAssembly", + "description": "Run spaCy directly in the browser with WebAssembly. Using Pyodide, the application loads the spaCy model and renders the text prompt with displaCy.", + "url": "https://spacy-wasm.vercel.app/", + "github": "SyedAhkam/spacy-wasm", + "code_language": "python", + "author": "Syed Ahkam", + "author_links": { + "twitter": "@SyedAhkam1", + "github": "SyedAhkam" + }, + "category": [ + "visualizers" + ], + "tags": [ + "visualization", + "deployment" + ] + }, + { + "id": "spacysee", + "title": "spaCysee", + "slogan": "Visualize spaCy's Dependency Parsing, POS tagging, and morphological analysis", + "description": "A project that helps you visualize your spaCy docs in Jupyter notebooks. Each of the dependency tags, POS tags and morphological features are clickable. Clicking on a tag will bring up the relevant documentation for that tag.", + "github": "moxley01/spacysee", + "pip": "spacysee", + "code_example": [ + "import spacy", + "from spacysee import render", + "", + "nlp = spacy.load('en_core_web_sm')", + "doc = nlp('This is a neat way to visualize your spaCy docs')", + "render(doc, width='500', height='500')" + ], + "code_language": "python", + "thumb": "https://www.mattoxley.com/static/images/spacysee_logo.svg", + "image": "https://www.mattoxley.com/static/images/spacysee_logo.svg", + "author": "Matt Oxley", + "author_links": { + "twitter": "matt0xley", + "github": "moxley01", + "website": "https://mattoxley.com" + }, + "category": [ + "visualizers" + ], + "tags": [ + "visualization" + ] + }, + { + "id": "grecy", + "title": "greCy", + "slogan": "Ancient Greek pipelines for spaCy", + "description": "greCy offers state-of-the-art pipelines for ancient Greek NLP. It installs language models available in various sizes, some of them containing either word vectors or the aristoBERTo transformer.", + "github": "jmyerston/greCy", + "pip": "grecy", + "code_example": [ + "python -m grecy install grc_proiel_trf", + "", + "#After installing grc_proiel_trf or any other model", + "import spacy", + "", + "nlp = spacy.load('grc_proiel_trf')", + "doc = nlp('δοκῶ μοι περὶ ὧν πυνθάνεσθε οὐκ ἀμελέτητος εἶναι')", + "", + "for token in doc:", + " print(f'{token.text}, lemma: {token.lemma_}, pos: {token.pos_}, dep: {token.dep_}')" + ], + "code_language": "python", + "thumb": "https://jacobo-syntax.hf.space/media/03a5317fa660c142e41dd2870b4273ce4e668e6fcdee0a276891f563.png", + "author": "Jacobo Myerston", + "author_links": { + "twitter": "@jcbmyrstn", + "github": "jmyerston", + "website": "https://huggingface.co/spaces/Jacobo/syntax" + }, + "category": [ + "pipeline", + "research", + "models" + ], + "tags": [ + "ancient Greek" + ] + }, + { + "id": "solipcysme", + "title": "solipCysme", + "slogan": "spaCy pipeline for french fictions and first person point of view texts.", + "description": "__solipCysme__ is a pipeline for french language, designed for the analysis of fictions and first person point of view texts, with a focus on personal pronouns.", + "github": "thjbdvlt/solipCysme", + "code_example": [ + "pip install https://huggingface.co/thjbdvlt/fr_solipcysme/resolve/main/fr_solipcysme-any-py3-none-any.whl", + "", + "import spacy", + "", + "nlp = spacy.load('fr_solipcysme')", + "for i in nlp(", + "'la MACHINE à (b)rouiller le temps s'est peut-être déraillée..?'", + "):", + " print(", + " i, ", + " i.norm_, ", + " i.pos_, ", + " i.morph, ", + " i.lemma_, ", + " i.dep_, ", + " i._.tokentype,", + " i._.vv_pos,", + " i._.vv_morph", + " )" + ], + "code_language": "python", + "author": "thjbdvlt", + "author_links": { + "github": "thjbdvlt" + }, + "category": [ + "pipeline", + "research", + "models" + ], + "tags": [ + "french" + ] + }, + { + "id": "spacy-cleaner", + "title": "spacy-cleaner", + "slogan": "Easily clean text with spaCy!", + "description": "**spacy-cleaner** utilises spaCy `Language` models to replace, remove, and \n mutate spaCy tokens. Cleaning actions available are:\n\n* Remove/replace stopwords.\n* Remove/replace punctuation.\n* Remove/replace numbers.\n* Remove/replace emails.\n* Remove/replace URLs.\n* Perform lemmatisation.\n\nSee our [docs](https://ce11an.github.io/spacy-cleaner/) for more information.", + "github": "Ce11an/spacy-cleaner", + "pip": "spacy-cleaner", + "code_example": [ + "import spacy", + "import spacy_cleaner", + "from spacy_cleaner.processing import removers, replacers, mutators", + "", + "model = spacy.load(\"en_core_web_sm\")", + "pipeline = spacy_cleaner.Pipeline(", + " model,", + " removers.remove_stopword_token,", + " replacers.replace_punctuation_token,", + " mutators.mutate_lemma_token,", + ")", + "", + "texts = [\"Hello, my name is Cellan! I love to swim!\"]", + "", + "pipeline.clean(texts)", + "# ['hello _IS_PUNCT_ Cellan _IS_PUNCT_ love swim _IS_PUNCT_']" + ], + "code_language": "python", + "url": "https://ce11an.github.io/spacy-cleaner/", + "image": "https://raw.githubusercontent.com/Ce11an/spacy-cleaner/main/docs/assets/images/spacemen.png", + "author": "Cellan Hall", + "author_links": { + "twitter": "Ce11an", + "github": "Ce11an", + "website": "https://www.linkedin.com/in/cellan-hall/" + }, + "category": [ + "extension" + ], + "tags": [ + "text-processing" + ] + }, + { + "id": "Zshot", + "title": "Zshot", + "slogan": "Zero and Few shot named entity & relationships recognition", + "github": "ibm/zshot", + "pip": "zshot", + "code_example": [ + "import spacy", + "from zshot import PipelineConfig, displacy", + "from zshot.linker import LinkerRegen", + "from zshot.mentions_extractor import MentionsExtractorSpacy", + "from zshot.utils.data_models import Entity", + "", + "nlp = spacy.load('en_core_web_sm')", + "# zero shot definition of entities", + "nlp_config = PipelineConfig(", + " mentions_extractor=MentionsExtractorSpacy(),", + " linker=LinkerRegen(),", + " entities=[", + " Entity(name='Paris',", + " description='Paris is located in northern central France, in a north-bending arc of the river Seine'),", + " Entity(name='IBM',", + " description='International Business Machines Corporation (IBM) is an American multinational technology corporation headquartered in Armonk, New York'),", + " Entity(name='New York', description='New York is a city in U.S. state'),", + " Entity(name='Florida', description='southeasternmost U.S. state'),", + " Entity(name='American',", + " description='American, something of, from, or related to the United States of America, commonly known as the United States or America'),", + " Entity(name='Chemical formula',", + " description='In chemistry, a chemical formula is a way of presenting information about the chemical proportions of atoms that constitute a particular chemical compound or molecul'),", + " Entity(name='Acetamide',", + " description='Acetamide (systematic name: ethanamide) is an organic compound with the formula CH3CONH2. It is the simplest amide derived from acetic acid. It finds some use as a plasticizer and as an industrial solvent.'),", + " Entity(name='Armonk',", + " description='Armonk is a hamlet and census-designated place (CDP) in the town of North Castle, located in Westchester County, New York, United States.'),", + " Entity(name='Acetic Acid',", + " description='Acetic acid, systematically named ethanoic acid, is an acidic, colourless liquid and organic compound with the chemical formula CH3COOH'),", + " Entity(name='Industrial solvent',", + " description='Acetamide (systematic name: ethanamide) is an organic compound with the formula CH3CONH2. It is the simplest amide derived from acetic acid. It finds some use as a plasticizer and as an industrial solvent.'),", + " ]", + ")", + "nlp.add_pipe('zshot', config=nlp_config, last=True)", + "", + "text = 'International Business Machines Corporation (IBM) is an American multinational technology corporation' \\", + " ' headquartered in Armonk, New York, with operations in over 171 countries.'", + "", + "doc = nlp(text)", + "displacy.serve(doc, style='ent')" + ], + "thumb": "https://ibm.github.io/zshot/img/graph.png", + "url": "https://ibm.github.io/zshot/", + "author": "IBM Research", + "author_links": { + "github": "ibm", + "twitter": "IBMResearch", + "website": "https://research.ibm.com/labs/ireland/" + }, + "category": [ + "scientific", + "models", + "research" + ] + }, + { + "id": "concepcy", + "title": "concepCy", + "slogan": "A multilingual knowledge graph in spaCy", + "description": "A spaCy wrapper for ConceptNet, a freely-available semantic network designed to help computers understand the meaning of words.", + "github": "JulesBelveze/concepcy", + "pip": "concepcy", + "code_example": [ + "import spacy", + "import concepcy", + "", + "nlp = spacy.load('en_core_web_sm')", + "# Using default concepCy configuration", + "nlp.add_pipe('concepcy')", + "", + "doc = nlp('WHO is a lovely company')", + "", + "# Access all the 'RelatedTo' relations from the Doc", + "for word, relations in doc._.relatedto.items():", + " print(f'Word: {word}\n{relations}')", + "", + "# Access the 'RelatedTo' relations word by word", + "for token in doc:", + " print(f'Word: {token}\n{token._.relatedto}')" + ], + "category": [ + "pipeline" + ], + "image": "https://github.com/JulesBelveze/concepcy/blob/main/figures/concepcy.png", + "tags": [ + "semantic", + "ConceptNet" + ], + "author": "Jules Belveze", + "author_links": { + "github": "JulesBelveze", + "website": "https://www.linkedin.com/in/jules-belveze/" + } + }, + { + "id": "spacyfishing", + "title": "spaCy fishing", + "slogan": "Named entity disambiguation and linking on Wikidata in spaCy with Entity-Fishing.", + "description": "A spaCy wrapper of Entity-Fishing for named entity disambiguation and linking against a Wikidata knowledge base.", + "github": "Lucaterre/spacyfishing", + "pip": "spacyfishing", + "code_example": [ + "import spacy", + "text = 'Victor Hugo and Honoré de Balzac are French writers who lived in Paris.'", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe('entityfishing')", + "doc = nlp(text)", + "for span in doc.ents:", + " print((ent.text, ent.label_, ent._.kb_qid, ent._.url_wikidata, ent._.nerd_score))", + "# ('Victor Hugo', 'PERSON', 'Q535', 'https://www.wikidata.org/wiki/Q535', 0.972)", + "# ('Honoré de Balzac', 'PERSON', 'Q9711', 'https://www.wikidata.org/wiki/Q9711', 0.9724)", + "# ('French', 'NORP', 'Q121842', 'https://www.wikidata.org/wiki/Q121842', 0.3739)", + "# ('Paris', 'GPE', 'Q90', 'https://www.wikidata.org/wiki/Q90', 0.5652)", + "## Set parameter `extra_info` to `True` and check also span._.description, span._.src_description, span._.normal_term, span._.other_ids" + ], + "category": [ + "models", + "pipeline" + ], + "image": "https://raw.githubusercontent.com/Lucaterre/spacyfishing/main/docs/spacyfishing-logo-resized.png", + "tags": [ + "NER", + "NEL" + ], + "author": "Lucas Terriel", + "author_links": { + "twitter": "TerreLuca", + "github": "Lucaterre" + } + }, + { + "id": "aim-spacy", + "title": "Aim-spaCy", + "slogan": "Aim-spaCy is an Aim-based spaCy experiment tracker.", + "description": "Aim-spaCy helps to easily collect, store and explore training logs for spaCy, including: hyper-parameters, metrics and displaCy visualizations", + "github": "aimhubio/aim-spacy", + "pip": "aim-spacy", + "code_example": [ + "https://github.com/aimhubio/aim-spacy/tree/master/examples" + ], + "code_language": "python", + "url": "https://aimstack.io/spacy", + "thumb": "https://user-images.githubusercontent.com/13848158/172912427-ee9327ea-3cd8-47fa-8427-6c0d36cd831f.png", + "image": "https://user-images.githubusercontent.com/13848158/136364717-0939222c-55b6-44f0-ad32-d9ab749546e4.png", + "author": "AimStack", + "author_links": { + "twitter": "aimstackio", + "github": "aimhubio", + "website": "https://aimstack.io" + }, + "category": [ + "visualizers" + ], + "tags": [ + "experiment-tracking", + "visualization" + ] + }, + { + "id": "spacy-report", + "title": "spacy-report", + "slogan": "Generates interactive reports for spaCy models.", + "description": "The goal of spacy-report is to offer static reports for spaCy models that help users make better decisions on how the models can be used.", + "github": "koaning/spacy-report", + "pip": "spacy-report", + "thumb": "https://github.com/koaning/spacy-report/raw/main/icon.png", + "image": "https://raw.githubusercontent.com/koaning/spacy-report/main/gif.gif", + "code_example": [ + "python -m spacy report textcat training/model-best/ corpus/train.spacy corpus/dev.spacy" + ], + "category": [ + "visualizers", + "research" + ], + "author": "Vincent D. Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning", + "website": "https://koaning.io" + } + }, + { + "id": "scrubadub_spacy", + "title": "scrubadub_spacy", + "category": [ + "pipeline" + ], + "slogan": "Remove personally identifiable information from text using spaCy.", + "description": "scrubadub removes personally identifiable information from text. scrubadub_spacy is an extension that uses spaCy NLP models to remove personal information from text.", + "github": "LeapBeyond/scrubadub_spacy", + "pip": "scrubadub-spacy", + "url": "https://github.com/LeapBeyond/scrubadub_spacy", + "code_language": "python", + "author": "Leap Beyond", + "author_links": { + "github": "LeapBeyond", + "website": "https://leapbeyond.ai" + }, + "code_example": [ + "import scrubadub, scrubadub_spacy", + "scrubber = scrubadub.Scrubber()", + "scrubber.add_detector(scrubadub_spacy.detectors.SpacyEntityDetector)", + "print(scrubber.clean(\"My name is Alex, I work at LifeGuard in London, and my eMail is alex@lifeguard.com btw. my super secret twitter login is username: alex_2000 password: g-dragon180888\"))", + "# My name is {{NAME}}, I work at {{ORGANIZATION}} in {{LOCATION}}, and my eMail is {{EMAIL}} btw. my super secret twitter login is username: {{USERNAME}} password: {{PASSWORD}}" + ] + }, + { + "id": "spacy-setfit-textcat", + "title": "spacy-setfit-textcat", + "category": [ + "research" + ], + "tags": [ + "SetFit", + "Few-Shot" + ], + "slogan": "spaCy Project: Experiments with SetFit & Few-Shot Classification", + "description": "This project is an experiment with spaCy and few-shot text classification using SetFit", + "github": "pmbaumgartner/spacy-setfit-textcat", + "url": "https://github.com/pmbaumgartner/spacy-setfit-textcat", + "code_language": "python", + "author": "Peter Baumgartner", + "author_links": { + "twitter": "pmbaumgartner", + "github": "pmbaumgartner", + "website": "https://www.peterbaumgartner.com/" + }, + "code_example": [ + "https://colab.research.google.com/drive/1CvGEZC0I9_v8gWrBxSJQ4Z8JGPJz-HYb?usp=sharing" + ] + }, + { + "id": "spacy-experimental", + "title": "spacy-experimental", + "category": [ + "extension" + ], + "slogan": "Cutting-edge experimental spaCy components and features", + "description": "This package includes experimental components and features for spaCy v3.x, for example model architectures, pipeline components and utilities.", + "github": "explosion/spacy-experimental", + "pip": "spacy-experimental", + "url": "https://github.com/explosion/spacy-experimental", + "code_language": "python", + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai/" + }, + "code_example": [ + "python -m pip install -U pip setuptools wheel", + "python -m pip install spacy-experimental" + ] + }, + { + "id": "spacypdfreader", + "title": "spacypdfreader", + "category": [ + "pipeline" + ], + "tags": [ + "PDF" + ], + "slogan": "Easy PDF to text to spaCy text extraction in Python.", + "description": "*spacypdfreader* is a Python library that allows you to convert PDF files directly into *spaCy* `Doc` objects. The library provides several built in parsers or bring your own parser. `Doc` objects are annotated with several custom attributes including: `token._.page_number`, `doc._.page_range`, `doc._.first_page`, `doc._.last_page`, `doc._.pdf_file_name`, and `doc._.page(int)`.", + "github": "SamEdwardes/spacypdfreader", + "pip": "spacypdfreader", + "url": "https://samedwardes.github.io/spacypdfreader/", + "code_language": "python", + "author": "Sam Edwardes", + "author_links": { + "twitter": "TheReaLSamlam", + "github": "SamEdwardes", + "website": "https://samedwardes.com" + }, + "code_example": [ + "import spacy", + "from spacypdfreader.spacypdfreader import pdf_reader", + "", + "nlp = spacy.load('en_core_web_sm')", + "doc = pdf_reader('tests/data/test_pdf_01.pdf', nlp)", + "", + "# Get the page number of any token.", + "print(doc[0]._.page_number) # 1", + "print(doc[-1]._.page_number) # 4", + "", + "# Get page meta data about the PDF document.", + "print(doc._.pdf_file_name) # 'tests/data/test_pdf_01.pdf'", + "print(doc._.page_range) # (1, 4)", + "print(doc._.first_page) # 1", + "print(doc._.last_page) # 4", + "", + "# Get all of the text from a specific PDF page.", + "print(doc._.page(4)) # 'able to display the destination page (unless...'" + ] + }, + { + "id": "nlpcloud", + "title": "NLPCloud.io", + "slogan": "Production-ready API for spaCy models in production", + "description": "A highly-available hosted API to easily deploy and use spaCy models in production. Supports NER, POS tagging, dependency parsing, and tokenization.", + "github": "nlpcloud", + "pip": "nlpcloud", + "code_example": [ + "import nlpcloud", + "", + "client = nlpcloud.Client('en_core_web_lg', '4eC39HqLyjWDarjtT1zdp7dc')", + "client.entities('John Doe is a Go Developer at Google')", + "# [{'end': 8, 'start': 0, 'text': 'John Doe', 'type': 'PERSON'}, {'end': 25, 'start': 13, 'text': 'Go Developer', 'type': 'POSITION'}, {'end': 35,'start': 30, 'text': 'Google', 'type': 'ORG'}]" + ], + "thumb": "https://avatars.githubusercontent.com/u/77671902", + "image": "https://nlpcloud.io/assets/images/logo.svg", + "code_language": "python", + "author": "NLPCloud.io", + "author_links": { + "github": "nlpcloud", + "twitter": "cloud_nlp", + "website": "https://nlpcloud.io" + }, + "category": [ + "apis", + "nonpython", + "standalone" + ], + "tags": [ + "api", + "deploy", + "production" + ] + }, + { + "id": "eMFDscore", + "title": "eMFDscore : Extended Moral Foundation Dictionary Scoring for Python", + "slogan": "Extended Moral Foundation Dictionary Scoring for Python", + "description": "eMFDscore is a library for the fast and flexible extraction of various moral information metrics from textual input data. eMFDscore is built on spaCy for faster execution and performs minimal preprocessing consisting of tokenization, syntactic dependency parsing, lower-casing, and stopword/punctuation/whitespace removal. eMFDscore lets users score documents with multiple Moral Foundations Dictionaries, provides various metrics for analyzing moral information, and extracts moral patient, agent, and attribute words related to entities.", + "github": "medianeuroscience/emfdscore", + "code_example": [ + "from emfdscore.scoring import score_docs", + "import pandas as pd", + "template_input = pd.read_csv('emfdscore/template_input.csv', header=None)", + "DICT_TYPE = 'emfd'", + "PROB_MAP = 'single'", + "SCORE_METHOD = 'bow'", + "OUT_METRICS = 'vice-virtue'", + "OUT_CSV_PATH = 'single-vv.csv'", + "df = score_docs(template_input,DICT_TYPE,PROB_MAP,SCORE_METHOD,OUT_METRICS,num_docs)" + ], + "code_language": "python", + "author": "Media Neuroscience Lab", + "author_links": { + "github": "medianeuroscience", + "twitter": "medianeuro" + }, + "category": [ + "research", + "teaching" + ], + "tags": [ + "morality", + "dictionary", + "sentiment" + ] + }, + { + "id": "skweak", + "title": "skweak", + "slogan": "Weak supervision for NLP", + "description": "`skweak` brings the power of weak supervision to NLP tasks, and in particular sequence labelling and text classification. Instead of annotating documents by hand, `skweak` allows you to define *labelling functions* to automatically label your documents, and then aggregate their results using a statistical model that estimates the accuracy and confusions of each labelling function.", + "github": "NorskRegnesentral/skweak", + "pip": "skweak", + "code_example": [ + "import spacy, re", + "from skweak import heuristics, gazetteers, aggregation, utils", + "", + "# LF 1: heuristic to detect occurrences of MONEY entities", + "def money_detector(doc):", + " for tok in doc[1:]:", + " if tok.text[0].isdigit() and tok.nbor(-1).is_currency:", + " yield tok.i-1, tok.i+1, 'MONEY'", + "lf1 = heuristics.FunctionAnnotator('money', money_detector)", + "", + "# LF 2: detection of years with a regex", + "lf2= heuristics.TokenConstraintAnnotator ('years', lambda tok: re.match('(19|20)\\d{2}$', tok.text), 'DATE')", + "", + "# LF 3: a gazetteer with a few names", + "NAMES = [('Barack', 'Obama'), ('Donald', 'Trump'), ('Joe', 'Biden')]", + "trie = gazetteers.Trie(NAMES)", + "lf3 = gazetteers.GazetteerAnnotator('presidents', {'PERSON':trie})", + "", + "# We create a corpus (here with a single text)", + "nlp = spacy.load('en_core_web_sm')", + "doc = nlp('Donald Trump paid $750 in federal income taxes in 2016')", + "", + "# apply the labelling functions", + "doc = lf3(lf2(lf1(doc)))", + "", + "# and aggregate them", + "hmm = aggregation.HMM('hmm', ['PERSON', 'DATE', 'MONEY'])", + "hmm.fit_and_aggregate([doc])", + "", + "# we can then visualise the final result (in Jupyter)", + "utils.display_entities(doc, 'hmm')" + ], + "code_language": "python", + "url": "https://github.com/NorskRegnesentral/skweak", + "thumb": "https://raw.githubusercontent.com/NorskRegnesentral/skweak/main/data/skweak_logo_thumbnail.jpg", + "image": "https://raw.githubusercontent.com/NorskRegnesentral/skweak/main/data/skweak_logo.jpg", + "author": "Pierre Lison", + "author_links": { + "twitter": "plison2", + "github": "plison", + "website": "https://www.nr.no/~plison" + }, + "category": [ + "pipeline", + "standalone", + "research", + "training" + ], + "tags": [], + "spacy_version": 3 + }, + { + "id": "numerizer", + "title": "numerizer", + "slogan": "Convert natural language numerics into ints and floats.", + "description": "A SpaCy extension for Docs, Spans and Tokens that converts numerical words and quantitative named entities into numeric strings.", + "github": "jaidevd/numerizer", + "pip": "numerizer", + "code_example": [ + "from spacy import load", + "import numerizer", + "nlp = load('en_core_web_sm') # or any other model", + "doc = nlp('The Hogwarts Express is at platform nine and three quarters')", + "doc._.numerize()", + "# {nine and three quarters: '9.75'}" + ], + "author": "Jaidev Deshpande", + "author_links": { + "github": "jaidevd", + "twitter": "jaidevd" + }, + "category": [ + "standalone" + ] + }, + { + "id": "spacy-dbpedia-spotlight", + "title": "DBpedia Spotlight for SpaCy", + "slogan": "Use DBpedia Spotlight to link entities inside SpaCy", + "description": "This library links SpaCy with [DBpedia Spotlight](https://www.dbpedia-spotlight.org/). You can easily get the DBpedia entities from your documents, using the public web service or by using your own instance of DBpedia Spotlight. The `doc.ents` are populated with the entities and all their details (URI, type, ...).", + "github": "MartinoMensio/spacy-dbpedia-spotlight", + "pip": "spacy-dbpedia-spotlight", + "code_example": [ + "import spacy_dbpedia_spotlight", + "# load your model as usual", + "nlp = spacy.load('en_core_web_lg')", + "# add the pipeline stage", + "nlp.add_pipe('dbpedia_spotlight')", + "# get the document", + "doc = nlp('The president of USA is calling Boris Johnson to decide what to do about coronavirus')", + "# see the entities", + "print('Entities', [(ent.text, ent.label_, ent.kb_id_) for ent in doc.ents])", + "# inspect the raw data from DBpedia spotlight", + "print(doc.ents[0]._.dbpedia_raw_result)" + ], + "category": [ + "models", + "pipeline" + ], + "author": "Martino Mensio", + "author_links": { + "twitter": "MartinoMensio", + "github": "MartinoMensio", + "website": "https://martinomensio.github.io" + } + }, + { + "id": "spacy-textblob", + "title": "spacytextblob", + "slogan": "A TextBlob sentiment analysis pipeline component for spaCy.", + "thumb": "https://github.com/SamEdwardes/spacytextblob/raw/main/docs/static/img/logo-thumb-square-250x250.png", + "description": "spacytextblob is a pipeline component that enables sentiment analysis using the [TextBlob](https://github.com/sloria/TextBlob) library. It will add the additional extension `._.blob` to `Doc`, `Span`, and `Token` objects.", + "github": "SamEdwardes/spacytextblob", + "pip": "spacytextblob", + "code_example": [ + "# the following installations are required", + "# python -m textblob.download_corpora", + "# python -m spacy download en_core_web_sm", + "", + "import spacy", + "from spacytextblob.spacytextblob import SpacyTextBlob", + "", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe('spacytextblob')", + "text = 'I had a really horrible day. It was the worst day ever! But every now and then I have a really good day that makes me happy.'", + "doc = nlp(text)", + "doc._.blob.polarity # Polarity: -0.125", + "doc._.blob.subjectivity # Subjectivity: 0.9", + "doc._.blob.sentiment_assessments.assessments # Assessments: [(['really', 'horrible'], -1.0, 1.0, None), (['worst', '!'], -1.0, 1.0, None), (['really', 'good'], 0.7, 0.6000000000000001, None), (['happy'], 0.8, 1.0, None)]", + "doc._.blob.ngrams() # [WordList(['I', 'had', 'a']), WordList(['had', 'a', 'really']), WordList(['a', 'really', 'horrible']), WordList(['really', 'horrible', 'day']), WordList(['horrible', 'day', 'It']), WordList(['day', 'It', 'was']), WordList(['It', 'was', 'the']), WordList(['was', 'the', 'worst']), WordList(['the', 'worst', 'day']), WordList(['worst', 'day', 'ever']), WordList(['day', 'ever', 'But']), WordList(['ever', 'But', 'every']), WordList(['But', 'every', 'now']), WordList(['every', 'now', 'and']), WordList(['now', 'and', 'then']), WordList(['and', 'then', 'I']), WordList(['then', 'I', 'have']), WordList(['I', 'have', 'a']), WordList(['have', 'a', 'really']), WordList(['a', 'really', 'good']), WordList(['really', 'good', 'day']), WordList(['good', 'day', 'that']), WordList(['day', 'that', 'makes']), WordList(['that', 'makes', 'me']), WordList(['makes', 'me', 'happy'])]" + ], + "code_language": "python", + "url": "https://spacytextblob.netlify.app/", + "author": "Sam Edwardes", + "author_links": { + "twitter": "TheReaLSamlam", + "github": "SamEdwardes", + "website": "https://samedwardes.com" + }, + "category": [ + "pipeline" + ], + "tags": [ + "sentiment", + "textblob" + ], + "spacy_version": 3 + }, + { + "id": "spacy-sentence-bert", + "title": "spaCy - sentence-transformers", + "slogan": "Pipelines for pretrained sentence-transformers (BERT, RoBERTa, XLM-RoBERTa & Co.) directly within spaCy", + "description": "This library lets you use the embeddings from [sentence-transformers](https://github.com/UKPLab/sentence-transformers) of Docs, Spans and Tokens directly from spaCy. Most models are for the english language but three of them are multilingual.", + "github": "MartinoMensio/spacy-sentence-bert", + "pip": "spacy-sentence-bert", + "code_example": [ + "import spacy_sentence_bert", + "# load one of the models listed at https://github.com/MartinoMensio/spacy-sentence-bert/", + "nlp = spacy_sentence_bert.load_model('en_roberta_large_nli_stsb_mean_tokens')", + "# get two documents", + "doc_1 = nlp('Hi there, how are you?')", + "doc_2 = nlp('Hello there, how are you doing today?')", + "# use the similarity method that is based on the vectors, on Doc, Span or Token", + "print(doc_1.similarity(doc_2[0:7]))" + ], + "category": [ + "models", + "pipeline" + ], + "author": "Martino Mensio", + "author_links": { + "twitter": "MartinoMensio", + "github": "MartinoMensio", + "website": "https://martinomensio.github.io" + } + }, + { + "id": "spacy-streamlit", + "title": "spacy-streamlit", + "slogan": "spaCy building blocks for Streamlit apps", + "github": "explosion/spacy-streamlit", + "description": "This package contains utilities for visualizing spaCy models and building interactive spaCy-powered apps with [Streamlit](https://streamlit.io). It includes various building blocks you can use in your own Streamlit app, like visualizers for **syntactic dependencies**, **named entities**, **text classification**, **semantic similarity** via word vectors, token attributes, and more.", + "pip": "spacy-streamlit", + "category": [ + "visualizers" + ], + "thumb": "https://i.imgur.com/mhEjluE.jpg", + "image": "https://user-images.githubusercontent.com/13643239/85388081-f2da8700-b545-11ea-9bd4-e303d3c5763c.png", + "code_example": [ + "import spacy_streamlit", + "", + "models = [\"en_core_web_sm\", \"en_core_web_md\"]", + "default_text = \"Sundar Pichai is the CEO of Google.\"", + "spacy_streamlit.visualize(models, default_text)" + ], + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + } + }, + { + "id": "spaczz", + "title": "spaczz", + "slogan": "Fuzzy matching and more for spaCy.", + "description": "Spaczz provides fuzzy matching and multi-token regex matching functionality for spaCy. Spaczz's components have similar APIs to their spaCy counterparts and spaczz pipeline components can integrate into spaCy pipelines where they can be saved/loaded as models.", + "github": "gandersen101/spaczz", + "pip": "spaczz", + "code_example": [ + "import spacy", + "from spaczz.matcher import FuzzyMatcher", + "", + "nlp = spacy.blank(\"en\")", + "text = \"\"\"Grint Anderson created spaczz in his home at 555 Fake St,", + "Apt 5 in Nashv1le, TN 55555-1234 in the US.\"\"\" # Spelling errors intentional.", + "doc = nlp(text)", + "", + "matcher = FuzzyMatcher(nlp.vocab)", + "matcher.add(\"NAME\", [nlp(\"Grant Andersen\")])", + "matcher.add(\"GPE\", [nlp(\"Nashville\")])", + "matches = matcher(doc)", + "", + "for match_id, start, end, ratio in matches:", + " print(match_id, doc[start:end], ratio)" + ], + "code_language": "python", + "url": "https://spaczz.readthedocs.io/en/latest/", + "author": "Grant Andersen", + "author_links": { + "twitter": "gandersen101", + "github": "gandersen101" + }, + "category": [ + "pipeline" + ], + "tags": [ + "fuzzy-matching", + "regex" + ] + }, + { + "id": "spacy-universal-sentence-encoder", + "title": "spaCy - Universal Sentence Encoder", + "slogan": "Make use of Google's Universal Sentence Encoder directly within spaCy", + "description": "This library lets you use Universal Sentence Encoder embeddings of Docs, Spans and Tokens directly from TensorFlow Hub", + "github": "MartinoMensio/spacy-universal-sentence-encoder", + "pip": "spacy-universal-sentence-encoder", + "code_example": [ + "import spacy_universal_sentence_encoder", + "# load one of the models: ['en_use_md', 'en_use_lg', 'xx_use_md', 'xx_use_lg']", + "nlp = spacy_universal_sentence_encoder.load_model('en_use_lg')", + "# get two documents", + "doc_1 = nlp('Hi there, how are you?')", + "doc_2 = nlp('Hello there, how are you doing today?')", + "# use the similarity method that is based on the vectors, on Doc, Span or Token", + "print(doc_1.similarity(doc_2[0:7]))" + ], + "category": [ + "models", + "pipeline" + ], + "author": "Martino Mensio", + "author_links": { + "twitter": "MartinoMensio", + "github": "MartinoMensio", + "website": "https://martinomensio.github.io" + } + }, + { + "id": "whatlies", + "title": "whatlies", + "slogan": "Make interactive visualisations to figure out 'what lies' in word embeddings.", + "description": "This small library offers tools to make visualisation easier of both word embeddings as well as operations on them. It has support for spaCy prebuilt models as a first class citizen but also offers support for sense2vec. There's a convenient API to perform linear algebra as well as support for popular transformations like PCA/UMAP/etc.", + "github": "koaning/whatlies", + "pip": "whatlies", + "thumb": "https://i.imgur.com/rOkOiLv.png", + "image": "https://raw.githubusercontent.com/koaning/whatlies/master/docs/gif-two.gif", + "code_example": [ + "from whatlies import EmbeddingSet", + "from whatlies.language import SpacyLanguage", + "", + "lang = SpacyLanguage('en_core_web_md')", + "words = ['cat', 'dog', 'fish', 'kitten', 'man', 'woman', 'king', 'queen', 'doctor', 'nurse']", + "", + "emb = lang[words]", + "emb.plot_interactive(x_axis='man', y_axis='woman')" + ], + "category": [ + "visualizers", + "research" + ], + "author": "Vincent D. Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning", + "website": "https://koaning.io" + } + }, + { + "id": "bertopic", + "title": "BERTopic", + "slogan": "Leveraging BERT and c-TF-IDF to create easily interpretable topics.", + "description": "BERTopic is a topic modeling technique that leverages embedding models and c-TF-IDF to create dense clusters allowing for easily interpretable topics whilst keeping important words in the topic descriptions. BERTopic supports guided, (semi-) supervised, hierarchical, and dynamic topic modeling.", + "github": "maartengr/bertopic", + "pip": "bertopic", + "thumb": "https://i.imgur.com/Rx2LfBm.png", + "image": "https://raw.githubusercontent.com/MaartenGr/BERTopic/master/images/topic_visualization.gif", + "code_example": [ + "import spacy", + "from bertopic import BERTopic", + "from sklearn.datasets import fetch_20newsgroups", + "", + "docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']", + "nlp = spacy.load('en_core_web_md', exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])", + "", + "topic_model = BERTopic(embedding_model=nlp)", + "topics, probs = topic_model.fit_transform(docs)", + "", + "fig = topic_model.visualize_topics()", + "fig.show()" + ], + "category": [ + "visualizers", + "training" + ], + "author": "Maarten Grootendorst", + "author_links": { + "twitter": "maartengr", + "github": "maartengr", + "website": "https://maartengrootendorst.com" + } + }, + { + "id": "tokenwiser", + "title": "tokenwiser", + "slogan": "Connect vowpal-wabbit & scikit-learn models to spaCy to run simple classification benchmarks. Comes with many utility functions for spaCy pipelines.", + "github": "koaning/tokenwiser", + "pip": "tokenwiser", + "thumb": "https://koaning.github.io/tokenwiser/token.png", + "image": "https://koaning.github.io/tokenwiser/logo-tokw.png", + "code_example": [ + "import spacy", + "", + "from sklearn.pipeline import make_pipeline", + "from sklearn.feature_extraction.text import CountVectorizer", + "from sklearn.linear_model import LogisticRegression", + "", + "from tokenwiser.component import attach_sklearn_categoriser", + "", + "X = [", + " 'i really like this post',", + " 'thanks for that comment',", + " 'i enjoy this friendly forum',", + " 'this is a bad post',", + " 'i dislike this article',", + " 'this is not well written'", + "]", + "", + "y = ['pos', 'pos', 'pos', 'neg', 'neg', 'neg']", + "", + "# Note that we're training a pipeline here via a single-batch `.fit()` method", + "pipe = make_pipeline(CountVectorizer(), LogisticRegression()).fit(X, y)", + "", + "nlp = spacy.load('en_core_web_sm')", + "# This is where we attach our pre-trained model as a pipeline step.", + "attach_sklearn_categoriser(nlp, pipe_name='silly_sentiment', estimator=pipe)" + ], + "category": [ + "pipeline", + "training" + ], + "author": "Vincent D. Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning", + "website": "https://koaning.io" + } + }, + { + "id": "Klayers", + "title": "Klayers", + "category": [ + "pipeline" + ], + "tags": [ + "AWS" + ], + "slogan": "spaCy as a AWS Lambda Layer", + "description": "A collection of Python Packages as AWS Lambda(λ) Layers", + "github": "keithrozario/Klayers", + "pip": "", + "url": "https://github.com/keithrozario/Klayers", + "code_language": "python", + "author": "Keith Rozario", + "author_links": { + "twitter": "keithrozario", + "github": "keithrozario", + "website": "https://www.keithrozario.com" + }, + "code_example": [ + "# SAM Template", + "MyLambdaFunction:", + " Type: AWS::Serverless::Function", + " Handler: 02_pipeline/spaCy.main", + " Description: Name Entity Extraction", + " Runtime: python3.8", + " Layers:", + " - arn:aws:lambda:${self:provider.region}:113088814899:layer:Klayers-python37-spacy:18" + ] + }, + { + "type": "education", + "id": "video-spacys-ner-model-alt", + "title": "Named Entity Recognition (NER) using spaCy", + "slogan": "", + "description": "In this video, I show you how to do named entity recognition using the spaCy library for Python.", + "youtube": "Gn_PjruUtrc", + "author": "Applied Language Technology", + "author_links": { + "twitter": "HelsinkiNLP", + "github": "Applied-Language-Technology", + "website": "https://applied-language-technology.mooc.fi/" + }, + "category": [ + "videos" + ] + }, + { + "id": "HuSpaCy", + "title": "HuSpaCy", + "category": [ + "models" + ], + "tags": [ + "Hungarian" + ], + "slogan": "HuSpaCy: industrial-strength Hungarian natural language processing", + "description": "HuSpaCy is a spaCy model and a library providing industrial-strength Hungarian language processing facilities.", + "github": "huspacy/huspacy", + "pip": "huspacy", + "url": "https://github.com/huspacy/huspacy", + "code_language": "python", + "author": "SzegedAI", + "author_links": { + "github": "https://szegedai.github.io/", + "website": "https://u-szeged.hu/english" + }, + "code_example": [ + "# Load the model using huspacy", + "import huspacy", + "", + "nlp = huspacy.load()", + "", + "# Load the mode using spacy.load()", + "import spacy", + "", + "nlp = spacy.load(\"hu_core_news_lg\")", + "", + "# Load the model directly as a module", + "import hu_core_news_lg", + "", + "nlp = hu_core_news_lg.load()\n", + "# Either way you get the same model and can start processing texts.", + "doc = nlp(\"Csiribiri csiribiri zabszalma - négy csillag közt alszom ma.\")" + ] + }, + { + "id": "spacy-stanza", + "title": "spacy-stanza", + "slogan": "Use the latest Stanza (StanfordNLP) research models directly in spaCy", + "description": "This package wraps the Stanza (formerly StanfordNLP) library, so you can use Stanford's models as a spaCy pipeline. Using this wrapper, you'll be able to use the following annotations, computed by your pretrained `stanza` model:\n\n- Statistical tokenization (reflected in the `Doc` and its tokens)\n - Lemmatization (`token.lemma` and `token.lemma_`)\n - Part-of-speech tagging (`token.tag`, `token.tag_`, `token.pos`, `token.pos_`)\n - Dependency parsing (`token.dep`, `token.dep_`, `token.head`)\n - Named entity recognition (`doc.ents`, `token.ent_type`, `token.ent_type_`, `token.ent_iob`, `token.ent_iob_`)\n - Sentence segmentation (`doc.sents`)", + "github": "explosion/spacy-stanza", + "pip": "spacy-stanza", + "thumb": "https://i.imgur.com/myhLjMJ.png", + "code_example": [ + "import stanza", + "import spacy_stanza", + "", + "stanza.download(\"en\")", + "nlp = spacy_stanza.load_pipeline(\"en\")", + "", + "doc = nlp(\"Barack Obama was born in Hawaii. He was elected president in 2008.\")", + "for token in doc:", + " print(token.text, token.lemma_, token.pos_, token.dep_, token.ent_type_)", + "print(doc.ents)" + ], + "category": [ + "pipeline", + "standalone", + "models", + "research" + ], + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai" + } + }, + { + "id": "spacy-udpipe", + "title": "spacy-udpipe", + "slogan": "Use the latest UDPipe models directly in spaCy", + "description": "This package wraps the fast and efficient UDPipe language-agnostic NLP pipeline (via its Python bindings), so you can use UDPipe pre-trained models as a spaCy pipeline for 50+ languages out-of-the-box. Inspired by spacy-stanza, this package offers slightly less accurate models that are in turn much faster.", + "github": "TakeLab/spacy-udpipe", + "pip": "spacy-udpipe", + "code_example": [ + "import spacy_udpipe", + "", + "spacy_udpipe.download(\"en\") # download English model", + "", + "text = \"Wikipedia is a free online encyclopedia, created and edited by volunteers around the world.\"", + "nlp = spacy_udpipe.load(\"en\")", + "", + "doc = nlp(text)", + "for token in doc:", + " print(token.text, token.lemma_, token.pos_, token.dep_)" + ], + "category": [ + "pipeline", + "standalone", + "models", + "research" + ], + "author": "TakeLab", + "author_links": { + "github": "TakeLab", + "website": "https://takelab.fer.hr/" + } + }, + { + "id": "spacy-server", + "title": "spaCy Server", + "slogan": "🦜 Containerized HTTP API for spaCy NLP", + "description": "For developers who need programming language agnostic NLP, spaCy Server is a containerized HTTP API that provides industrial-strength natural language processing. Unlike other servers, our server is fast, idiomatic, and well documented.", + "github": "neelkamath/spacy-server", + "code_example": [ + "docker run --rm -dp 8080:8080 neelkamath/spacy-server", + "curl http://localhost:8080/ner -H 'Content-Type: application/json' -d '{\"sections\": [\"My name is John Doe. I grew up in California.\"]}'" + ], + "code_language": "shell", + "url": "https://hub.docker.com/r/neelkamath/spacy-server", + "author": "Neel Kamath", + "author_links": { + "github": "neelkamath", + "website": "https://neelkamath.com" + }, + "category": [ + "apis" + ], + "tags": [ + "docker" + ] + }, + { + "id": "nlp-architect", + "title": "NLP Architect", + "slogan": "Python lib for exploring Deep NLP & NLU by Intel AI", + "github": "NervanaSystems/nlp-architect", + "pip": "nlp-architect", + "thumb": "https://i.imgur.com/vMideRx.png", + "category": [ + "standalone", + "research" + ], + "tags": [ + "pytorch" + ] + }, + { + "id": "Chatterbot", + "title": "ChatterBot", + "slogan": "A machine-learning based conversational dialog engine for creating chat bots", + "github": "gunthercox/ChatterBot", + "pip": "chatterbot", + "thumb": "https://i.imgur.com/eyAhwXk.jpg", + "code_example": [ + "from chatterbot import ChatBot", + "from chatterbot.trainers import ListTrainer", + "# Create a new chat bot named Charlie", + "chatbot = ChatBot('Charlie')", + "trainer = ListTrainer(chatbot)", + "trainer.train([", + "'Hi, can I help you?',", + "'Sure, I would like to book a flight to Iceland.',", + "'Your flight has been booked.'", + "])", + "", + "response = chatbot.get_response('I would like to book a flight.')" + ], + "author": "Gunther Cox", + "author_links": { + "github": "gunthercox" + }, + "category": [ + "conversational", + "standalone" + ], + "tags": [ + "chatbots" + ] + }, + { + "id": "alibi", + "title": "alibi", + "slogan": "Algorithms for monitoring and explaining machine learning models ", + "github": "SeldonIO/alibi", + "pip": "alibi", + "thumb": "https://i.imgur.com/YkzQHRp.png", + "code_example": [ + "from alibi.explainers import AnchorTabular", + "explainer = AnchorTabular(predict_fn, feature_names)", + "explainer.fit(X_train)", + "explainer.explain(x)" + ], + "author": "Seldon", + "category": [ + "standalone", + "research" + ] + }, + { + "id": "spacymoji", + "slogan": "Emoji handling and meta data as a spaCy pipeline component", + "github": "ines/spacymoji", + "description": "spaCy extension and pipeline component for adding emoji meta data to `Doc` objects. Detects emoji consisting of one or more unicode characters, and can optionally merge multi-char emoji (combined pictures, emoji with skin tone modifiers) into one token. Human-readable emoji descriptions are added as a custom attribute, and an optional lookup table can be provided for your own descriptions. The extension sets the custom `Doc`, `Token` and `Span` attributes `._.is_emoji`, `._.emoji_desc`, `._.has_emoji` and `._.emoji`.", + "pip": "spacymoji", + "category": [ + "pipeline" + ], + "tags": [ + "emoji", + "unicode" + ], + "thumb": "https://i.imgur.com/XOTYIgn.jpg", + "code_example": [ + "import spacy", + "from spacymoji import Emoji", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "nlp.add_pipe(\"emoji\", first=True)", + "doc = nlp(\"This is a test 😻 👍🏿\")", + "", + "assert doc._.has_emoji is True", + "assert doc[2:5]._.has_emoji is True", + "assert doc[0]._.is_emoji is False", + "assert doc[4]._.is_emoji is True", + "assert doc[5]._.emoji_desc == \"thumbs up dark skin tone\"", + "assert len(doc._.emoji) == 2", + "assert doc._.emoji[1] == (\"👍🏿\", 5, \"thumbs up dark skin tone\")" + ], + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + } + }, + { + "id": "spacy-layout", + "slogan": "Process PDFs, Word documents and more with spaCy", + "github": "explosion/spacy-layout", + "description": "This plugin integrates with [Docling](https://ds4sd.github.io/docling/) to bring structured processing of PDFs, Word documents and other input formats to your spaCy pipeline. It outputs clean, structured data in a text-based format and outputs spaCy's familiar `Doc` objects that let you access labelled text spans like sections, headings, or footnotes.\n\nThis workflow makes it easy to apply powerful NLP techniques to your documents, including linguistic analysis, named entity recognition, text classification and more. It's also great for implementing chunking for RAG pipelines.", + "pip": "spacy-layout", + "category": [ + "pipeline" + ], + "code_example": [ + "import spacy", + "from spacy_layout import spaCyLayout", + "", + "nlp = spacy.blank(\"en\")", + "layout = spaCyLayout(nlp)", + "", + "# Process a document and create a spaCy Doc object", + "doc = layout(\"./starcraft.pdf\")", + "", + "# The text-based contents of the document", + "print(doc.text)", + "# Document layout including pages and page sizes", + "print(doc._.layout)", + "", + "# Layout spans for different sections", + "for span in doc.spans[\"layout\"]:", + " # Document section and token and character offsets into the text", + " print(span.text, span.start, span.end, span.start_char, span.end_char)", + " # Section type, e.g. \"text\", \"title\", \"section_header\" etc.", + " print(span.label_)", + " # Layout features of the section, including bounding box", + " print(span._.layout)", + " # Closest heading to the span (accuracy depends on document structure)", + " print(span._.heading)" + ], + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + } + }, + { + "id": "spacyopentapioca", + "title": "spaCyOpenTapioca", + "slogan": "Named entity linking on Wikidata in spaCy via OpenTapioca", + "description": "A spaCy wrapper of OpenTapioca for named entity linking on Wikidata", + "github": "UB-Mannheim/spacyopentapioca", + "pip": "spacyopentapioca", + "code_example": [ + "import spacy", + "nlp = spacy.blank('en')", + "nlp.add_pipe('opentapioca')", + "doc = nlp('Christian Drosten works in Germany.')", + "for span in doc.ents:", + " print((span.text, span.kb_id_, span.label_, span._.description, span._.score))", + "# ('Christian Drosten', 'Q1079331', 'PERSON', 'German virologist and university teacher', 3.6533377082098895)", + "# ('Germany', 'Q183', 'LOC', 'sovereign state in Central Europe', 2.1099332471902863)", + "## Check also span._.types, span._.aliases, span._.rank" + ], + "category": [ + "models", + "pipeline" + ], + "tags": [ + "NER", + "NEL" + ], + "author": "Renat Shigapov", + "author_links": { + "twitter": "_shigapov", + "github": "shigapov" + } + }, + { + "id": "spacy_readability", + "slogan": "Add text readability meta data to Doc objects", + "description": "spaCy v2.0 pipeline component for calculating readability scores of of text. Provides scores for Flesh-Kincaid grade level, Flesh-Kincaid reading ease, and Dale-Chall.", + "github": "mholtzscher/spacy_readability", + "pip": "spacy-readability", + "code_example": [ + "import spacy", + "from spacy_readability import Readability", + "", + "nlp = spacy.load('en')", + "read = Readability(nlp)", + "nlp.add_pipe(read, last=True)", + "doc = nlp(\"I am some really difficult text to read because I use obnoxiously large words.\")", + "doc._.flesch_kincaid_grade_level", + "doc._.flesch_kincaid_reading_ease", + "doc._.dale_chall" + ], + "author": "Michael Holtzscher", + "author_links": { + "github": "mholtzscher" + }, + "category": [ + "pipeline" + ] + }, + { + "id": "spacy_cld", + "title": "spaCy-CLD", + "slogan": "Add language detection to your spaCy pipeline using CLD2", + "description": "spaCy-CLD operates on `Doc` and `Span` spaCy objects. When called on a `Doc` or `Span`, the object is given two attributes: `languages` (a list of up to 3 language codes) and `language_scores` (a dictionary mapping language codes to confidence scores between 0 and 1).\n\nspacy-cld is a little extension that wraps the [PYCLD2](https://github.com/aboSamoor/pycld2) Python library, which in turn wraps the [Compact Language Detector 2](https://github.com/CLD2Owners/cld2) C library originally built at Google for the Chromium project. CLD2 uses character n-grams as features and a Naive Bayes classifier to identify 80+ languages from Unicode text strings (or XML/HTML). It can detect up to 3 different languages in a given document, and reports a confidence score (reported in with each language.", + "github": "nickdavidhaynes/spacy-cld", + "pip": "spacy_cld", + "code_example": [ + "import spacy", + "from spacy_cld import LanguageDetector", + "", + "nlp = spacy.load('en')", + "language_detector = LanguageDetector()", + "nlp.add_pipe(language_detector)", + "doc = nlp('This is some English text.')", + "", + "doc._.languages # ['en']", + "doc._.language_scores['en'] # 0.96" + ], + "author": "Nicholas D Haynes", + "author_links": { + "github": "nickdavidhaynes" + }, + "category": [ + "pipeline" + ] + }, + { + "id": "spacy-iwnlp", + "slogan": "German lemmatization with IWNLP", + "description": "This package uses the [spaCy 2.0 extensions](https://spacy.io/usage/processing-pipelines#extensions) to add [IWNLP-py](https://github.com/Liebeck/iwnlp-py) as German lemmatizer directly into your spaCy pipeline.", + "github": "Liebeck/spacy-iwnlp", + "pip": "spacy-iwnlp", + "code_example": [ + "import spacy", + "from spacy_iwnlp import spaCyIWNLP", + "", + "nlp = spacy.load('de')", + "iwnlp = spaCyIWNLP(lemmatizer_path='data/IWNLP.Lemmatizer_20170501.json')", + "nlp.add_pipe(iwnlp)", + "doc = nlp('Wir mögen Fußballspiele mit ausgedehnten Verlängerungen.')", + "for token in doc:", + " print('POS: {}\tIWNLP:{}'.format(token.pos_, token._.iwnlp_lemmas))" + ], + "author": "Matthias Liebeck", + "author_links": { + "github": "Liebeck" + }, + "category": [ + "pipeline" + ], + "tags": [ + "lemmatizer", + "german" + ] + }, + { + "id": "spacy-sentiws", + "slogan": "German sentiment scores with SentiWS", + "description": "This package uses the [spaCy 2.0 extensions](https://spacy.io/usage/processing-pipelines#extensions) to add [SentiWS](http://wortschatz.uni-leipzig.de/en/download) as German sentiment score directly into your spaCy pipeline.", + "github": "Liebeck/spacy-sentiws", + "pip": "spacy-sentiws", + "code_example": [ + "import spacy", + "from spacy_sentiws import spaCySentiWS", + "", + "nlp = spacy.load('de_core_news_sm')", + "nlp.add_pipe('sentiws', config={'sentiws_path': 'data/sentiws'})", + "doc = nlp('Die Dummheit der Unterwerfung blüht in hübschen Farben.')", + "", + "for token in doc:", + " print('{}, {}, {}'.format(token.text, token._.sentiws, token.pos_))" + ], + "author": "Matthias Liebeck", + "author_links": { + "github": "Liebeck" + }, + "category": [ + "pipeline" + ], + "tags": [ + "sentiment", + "german" + ] + }, + { + "id": "spacy-lefff", + "slogan": "POS and French lemmatization with Lefff", + "description": "spacy v2.0 extension and pipeline component for adding a French POS and lemmatizer based on [Lefff](https://hal.inria.fr/inria-00521242/).", + "github": "sammous/spacy-lefff", + "pip": "spacy-lefff", + "code_example": [ + "import spacy", + "from spacy_lefff import LefffLemmatizer, POSTagger", + "", + "nlp = spacy.load('fr')", + "pos = POSTagger()", + "french_lemmatizer = LefffLemmatizer(after_melt=True)", + "nlp.add_pipe(pos, name='pos', after='parser')", + "nlp.add_pipe(french_lemmatizer, name='lefff', after='pos')", + "doc = nlp(u\"Paris est une ville très chère.\")", + "for d in doc:", + " print(d.text, d.pos_, d._.melt_tagger, d._.lefff_lemma, d.tag_, d.lemma_)" + ], + "author": "Sami Moustachir", + "author_links": { + "github": "sammous" + }, + "category": [ + "pipeline" + ], + "tags": [ + "pos", + "lemmatizer", + "french" + ] + }, + { + "id": "lemmy", + "title": "Lemmy", + "slogan": "A Danish lemmatizer", + "description": "Lemmy is a lemmatizer for Danish 🇩🇰 . It comes already trained on Dansk Sprognævns (DSN) word list (‘fuldformliste’) and the Danish Universal Dependencies and is ready for use. Lemmy also supports training on your own dataset. The model currently included in Lemmy was evaluated on the Danish Universal Dependencies dev dataset and scored an accruacy > 99%.\n\nYou can use Lemmy as a spaCy extension, more specifcally a spaCy pipeline component. This is highly recommended and makes the lemmas easily accessible from the spaCy tokens. Lemmy makes use of POS tags to predict the lemmas. When wired up to the spaCy pipeline, Lemmy has the benefit of using spaCy’s builtin POS tagger.", + "github": "sorenlind/lemmy", + "pip": "lemmy", + "code_example": [ + "import da_custom_model as da # name of your spaCy model", + "import lemmy.pipe", + "nlp = da.load()", + "", + "# create an instance of Lemmy's pipeline component for spaCy", + "pipe = lemmy.pipe.load()", + "", + "# add the comonent to the spaCy pipeline.", + "nlp.add_pipe(pipe, after='tagger')", + "", + "# lemmas can now be accessed using the `._.lemma` attribute on the tokens", + "nlp(\"akvariernes\")[0]._.lemma" + ], + "thumb": "https://i.imgur.com/RJVFRWm.jpg", + "author": "Søren Lind Kristiansen", + "author_links": { + "github": "sorenlind" + }, + "category": [ + "pipeline" + ], + "tags": [ + "lemmatizer", + "danish" + ] + }, + { + "id": "augmenty", + "title": "Augmenty", + "slogan": "The cherry on top of your NLP pipeline", + "description": "Augmenty is an augmentation library based on spaCy for augmenting texts. Augmenty differs from other augmentation libraries in that it corrects (as far as possible) the token, sentence and document labels under the augmentation.", + "github": "kennethenevoldsen/augmenty", + "pip": "augmenty", + "code_example": [ + "import spacy", + "import augmenty", + "", + "nlp = spacy.load('en_core_web_md')", + "", + "docs = nlp.pipe(['Augmenty is a great tool for text augmentation'])", + "", + "ent_dict = {'ORG': [['spaCy'], ['spaCy', 'Universe']]}", + "entity_augmenter = augmenty.load('ents_replace.v1',", + " ent_dict = ent_dict, level=1)", + "", + "for doc in augmenty.docs(docs, augmenter=entity_augmenter, nlp=nlp):", + " print(doc)" + ], + "thumb": "https://github.com/KennethEnevoldsen/augmenty/blob/master/img/icon.png?raw=true", + "author": "Kenneth Enevoldsen", + "author_links": { + "github": "kennethenevoldsen", + "website": "https://www.kennethenevoldsen.com" + }, + "category": [ + "training", + "research" + ], + "tags": [ + "training", + "research", + "augmentation" + ] + }, + { + "id": "dacy", + "title": "DaCy", + "slogan": "An efficient Pipeline for Danish NLP", + "description": "DaCy is a Danish preprocessing pipeline trained in SpaCy. It has achieved State-of-the-Art performance on Named entity recognition, part-of-speech tagging and dependency parsing for Danish. This repository contains material for using the DaCy, reproducing the results and guides on usage of the package. Furthermore, it also contains a series of behavioural test for biases and robustness of Danish NLP pipelines.", + "github": "centre-for-humanities-computing/DaCy", + "pip": "dacy", + "code_example": [ + "import dacy", + "print(dacy.models()) # get a list of dacy models", + "nlp = dacy.load('medium') # load your spacy pipeline", + "", + "# DaCy also includes functionality for adding other Danish models to the pipeline", + "# For instance you can add the BertTone model for classification of sentiment polarity to the pipeline:", + "nlp = add_berttone_polarity(nlp)" + ], + "thumb": "https://github.com/centre-for-humanities-computing/DaCy/blob/main/img/icon_no_title.png?raw=true", + "author": "Centre for Humanities Computing Aarhus", + "author_links": { + "github": "centre-for-humanities-computing", + "website": "https://chcaa.io/#/" + }, + "category": [ + "pipeline" + ], + "tags": [ + "pipeline", + "danish" + ] + }, + { + "id": "spacy-wrap", + "title": "spaCy-wrap", + "slogan": "For Wrapping fine-tuned transformers in spaCy pipelines", + "description": "spaCy-wrap is a wrapper library for spaCy for including fine-tuned transformers from Huggingface in your spaCy pipeline allowing inclusion of existing models within existing workflows.", + "github": "kennethenevoldsen/spacy-wrap", + "pip": "spacy_wrap", + "code_example": [ + "import spacy", + "import spacy_wrap", + "", + "nlp = spacy.blank('en')", + "config = {", + " 'doc_extension_trf_data': 'clf_trf_data', # document extention for the forward pass", + " 'doc_extension_prediction': 'sentiment', # document extention for the prediction", + " 'labels': ['negative', 'neutral', 'positive'],", + " 'model': {", + " 'name': 'cardiffnlp/twitter-roberta-base-sentiment', # the model name or path of huggingface model", + "},", + "}", + "", + "transformer = nlp.add_pipe('classification_transformer', config=config)", + "transformer.model.initialize()", + "", + "doc = nlp('spaCy is a wonderful tool')", + "", + "print(doc._.clf_trf_data)", + "# TransformerData(wordpieces=...", + "print(doc._.sentiment)", + "# 'positive'", + "print(doc._.sentiment_prob)", + "# {'prob': array([0.004, 0.028, 0.969], dtype=float32), 'labels': ['negative', 'neutral', 'positive']}" + ], + "thumb": "https://raw.githubusercontent.com/KennethEnevoldsen/spacy-wrap/main/docs/_static/icon.png", + "author": "Kenneth Enevoldsen", + "author_links": { + "github": "KennethEnevoldsen", + "website": "https://www.kennethenevoldsen.com" + }, + "category": [ + "pipeline", + "models", + "training" + ], + "tags": [ + "pipeline", + "models", + "transformers" + ] + }, + { + "id": "asent", + "title": "Asent", + "slogan": "Fast, flexible and transparent sentiment analysis", + "description": "Asent is a rule-based sentiment analysis library for Python made using spaCy. It is inspired by VADER, but uses a more modular ruleset, that allows the user to change e.g. the method for finding negations. Furthermore it includes visualisers to visualize the model predictions, making the model easily interpretable.", + "github": "kennethenevoldsen/asent", + "pip": "asent", + "code_example": [ + "import spacy", + "import asent", + "", + "# load spacy pipeline", + "nlp = spacy.blank('en')", + "nlp.add_pipe('sentencizer')", + "", + "# add the rule-based sentiment model", + "nlp.add_pipe('asent_en_v1')", + "", + "# try an example", + "text = 'I am not very happy, but I am also not especially sad'", + "doc = nlp(text)", + "", + "# print polarity of document, scaled to be between -1, and 1", + "print(doc._.polarity)", + "# neg=0.0 neu=0.631 pos=0.369 compound=0.7526", + "", + "# Naturally, a simple score can be quite unsatisfying, thus Asent implements a series of visualizer to interpret the results:", + "asent.visualize(doc, style='prediction')", + " # or", + "asent.visualize(doc[:5], style='analysis')" + ], + "thumb": "https://github.com/KennethEnevoldsen/asent/raw/main/docs/img/logo_black_font.png?raw=true", + "author": "Kenneth Enevoldsen", + "author_links": { + "github": "KennethEnevoldsen", + "website": "https://www.kennethenevoldsen.com" + }, + "category": [ + "pipeline", + "models" + ], + "tags": [ + "pipeline", + "models", + "sentiment" + ] + }, + { + "id": "textdescriptives", + "title": "TextDescriptives", + "slogan": "Extraction of descriptive stats, readability, and syntactic complexity measures", + "description": "Pipeline component for spaCy v.3 that calculates descriptive statistics, readability metrics, and syntactic complexity (dependency distance).", + "github": "HLasse/TextDescriptives", + "pip": "textdescriptives", + "code_example": [ + "import spacy", + "import textdescriptives as td", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe('textdescriptives')", + "doc = nlp('This is a short test text')", + "doc._.readability # access some of the values", + "td.extract_df(doc) # extract all metrics to DataFrame" + ], + "author": "Lasse Hansen, Kenneth Enevoldsen, Ludvig Olsen", + "author_links": { + "github": "HLasse" + }, + "category": [ + "pipeline" + ], + "tags": [ + "pipeline", + "readability", + "syntactic complexity", + "descriptive statistics" + ] + }, + { + "id": "neuralcoref", + "slogan": "State-of-the-art coreference resolution based on neural nets and spaCy", + "description": "This coreference resolution module is based on the super fast [spaCy](https://spacy.io/) parser and uses the neural net scoring model described in [Deep Reinforcement Learning for Mention-Ranking Coreference Models](http://cs.stanford.edu/people/kevclark/resources/clark-manning-emnlp2016-deep.pdf) by Kevin Clark and Christopher D. Manning, EMNLP 2016. Since ✨Neuralcoref v2.0, you can train the coreference resolution system on your own dataset — e.g., another language than English! — **provided you have an annotated dataset**. Note that to use neuralcoref with spaCy > 2.1.0, you'll have to install neuralcoref from source.", + "github": "huggingface/neuralcoref", + "thumb": "https://i.imgur.com/j6FO9O6.jpg", + "code_example": [ + "import spacy", + "import neuralcoref", + "", + "nlp = spacy.load('en')", + "neuralcoref.add_to_pipe(nlp)", + "doc1 = nlp('My sister has a dog. She loves him.')", + "print(doc1._.coref_clusters)", + "", + "doc2 = nlp('Angela lives in Boston. She is quite happy in that city.')", + "for ent in doc2.ents:", + " print(ent._.coref_cluster)" + ], + "author": "Hugging Face", + "author_links": { + "github": "huggingface" + }, + "category": [ + "standalone", + "conversational", + "models" + ], + "tags": [ + "coref" + ] + }, + { + "id": "neuralcoref-vizualizer", + "title": "Neuralcoref Visualizer", + "slogan": "State-of-the-art coreference resolution based on neural nets and spaCy", + "description": "In short, coreference is the fact that two or more expressions in a text – like pronouns or nouns – link to the same person or thing. It is a classical Natural language processing task, that has seen a revival of interest in the past two years as several research groups applied cutting-edge deep-learning and reinforcement-learning techniques to it. It is also one of the key building blocks to building conversational Artificial intelligences.", + "url": "https://huggingface.co/coref/", + "image": "https://i.imgur.com/3yy4Qyf.png", + "thumb": "https://i.imgur.com/j6FO9O6.jpg", + "github": "huggingface/neuralcoref", + "category": [ + "visualizers", + "conversational" + ], + "tags": [ + "coref", + "chatbots" + ], + "author": "Hugging Face", + "author_links": { + "github": "huggingface" + } + }, + { + "id": "matcher-explorer", + "title": "Rule-based Matcher Explorer", + "slogan": "Test spaCy's rule-based Matcher by creating token patterns interactively", + "description": "Test spaCy's rule-based `Matcher` by creating token patterns interactively and running them over your text. Each token can set multiple attributes like text value, part-of-speech tag or boolean flags. The token-based view lets you explore how spaCy processes your text – and why your pattern matches, or why it doesn't. For more details on rule-based matching, see the [documentation](https://spacy.io/usage/rule-based-matching).", + "image": "https://explosion.ai/assets/img/demos/matcher.png", + "thumb": "https://i.imgur.com/rPK4AGt.jpg", + "url": "https://explosion.ai/demos/matcher", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "displacy", + "title": "displaCy", + "slogan": "A modern syntactic dependency visualizer", + "description": "Visualize spaCy's guess at the syntactic structure of a sentence. Arrows point from children to heads, and are labelled by their relation type.", + "url": "https://explosion.ai/demos/displacy", + "thumb": "https://i.imgur.com/nxDcHaL.jpg", + "image": "https://explosion.ai/assets/img/demos/displacy.png", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "displacy-ent", + "title": "displaCy ENT", + "slogan": "A modern named entity visualizer", + "description": "Visualize spaCy's guess at the named entities in the document. You can filter the displayed types, to only show the annotations you're interested in.", + "url": "https://explosion.ai/demos/displacy-ent", + "thumb": "https://i.imgur.com/A77Ecbs.jpg", + "image": "https://explosion.ai/assets/img/demos/displacy-ent.png", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "explacy", + "slogan": "A small tool that explains spaCy parse results", + "github": "tylerneylon/explacy", + "thumb": "https://i.imgur.com/V1hCWmn.jpg", + "image": "https://raw.githubusercontent.com/tylerneylon/explacy/master/img/screenshot.png", + "code_example": [ + "import spacy", + "import explacy", + "", + "nlp = spacy.load('en')", + "explacy.print_parse_info(nlp, 'The salad was surprisingly tasty.')" + ], + "author": "Tyler Neylon", + "author_links": { + "github": "tylerneylon" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "deplacy", + "slogan": "CUI-based Tree Visualizer for Universal Dependencies and Immediate Catena Analysis", + "description": "Simple dependency visualizer for [spaCy](https://spacy.io/), [UniDic2UD](https://pypi.org/project/unidic2ud), [Stanza](https://stanfordnlp.github.io/stanza/), [NLP-Cube](https://github.com/Adobe/NLP-Cube), [Trankit](https://github.com/nlp-uoregon/trankit), etc.", + "github": "KoichiYasuoka/deplacy", + "image": "https://i.imgur.com/6uOI4Op.png", + "code_example": [ + "import spacy", + "import deplacy", + "", + "nlp=spacy.load('en_core_web_sm')", + "doc=nlp('I saw a horse yesterday which had no name.')", + "deplacy.render(doc)" + ], + "author": "Koichi Yasuoka", + "author_links": { + "github": "KoichiYasuoka" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "scattertext", + "slogan": "Beautiful visualizations of how language differs among document types", + "description": "A tool for finding distinguishing terms in small-to-medium-sized corpora, and presenting them in a sexy, interactive scatter plot with non-overlapping term labels. Exploratory data analysis just got more fun.", + "github": "JasonKessler/scattertext", + "image": "https://jasonkessler.github.io/2012conventions0.0.2.2.png", + "code_example": [ + "import spacy", + "", + "from scattertext import SampleCorpora, produce_scattertext_explorer", + "from scattertext import produce_scattertext_html", + "from scattertext.CorpusFromPandas import CorpusFromPandas", + "", + "nlp = spacy.load('en_core_web_sm')", + "convention_df = SampleCorpora.ConventionData2012.get_data()", + "corpus = CorpusFromPandas(convention_df,", + " category_col='party',", + " text_col='text',", + " nlp=nlp).build()", + "", + "html = produce_scattertext_html(corpus,", + " category='democrat',", + " category_name='Democratic',", + " not_category_name='Republican',", + " minimum_term_frequency=5,", + " width_in_pixels=1000)", + "open('./simple.html', 'wb').write(html.encode('utf-8'))", + "print('Open ./simple.html in Chrome or Firefox.')" + ], + "author": "Jason Kessler", + "author_links": { + "github": "JasonKessler", + "twitter": "jasonkessler" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "rasa", + "title": "Rasa", + "slogan": "Turn natural language into structured data", + "description": "Machine learning tools for developers to build, improve, and deploy contextual chatbots and assistants. Powered by open source.", + "github": "RasaHQ/rasa", + "pip": "rasa", + "thumb": "https://i.imgur.com/TyZnpwL.png", + "url": "https://rasa.com/", + "author": "Rasa", + "author_links": { + "github": "RasaHQ" + }, + "category": [ + "conversational" + ], + "tags": [ + "chatbots" + ] + }, + { + "id": "mindmeld", + "title": "MindMeld - Conversational AI platform", + "slogan": "Conversational AI platform for deep-domain voice interfaces and chatbots", + "description": "The MindMeld Conversational AI platform is among the most advanced AI platforms for building production-quality conversational applications. It is a Python-based machine learning framework which encompasses all of the algorithms and utilities required for this purpose. (https://github.com/cisco/mindmeld)", + "github": "cisco/mindmeld", + "pip": "mindmeld", + "thumb": "https://www.mindmeld.com/img/mindmeld-logo.png", + "category": [ + "conversational", + "ner" + ], + "tags": [ + "chatbots" + ], + "author": "Cisco", + "author_links": { + "github": "cisco/mindmeld", + "website": "https://www.mindmeld.com/" + } + }, + { + "id": "torchtext", + "title": "torchtext", + "slogan": "Data loaders and abstractions for text and NLP", + "github": "pytorch/text", + "pip": "torchtext", + "thumb": "https://i.imgur.com/WFkxuPo.png", + "code_example": [ + ">>> pos = data.TabularDataset(", + "... path='data/pos/pos_wsj_train.tsv', format='tsv',", + "... fields=[('text', data.Field()),", + "... ('labels', data.Field())])", + "...", + ">>> sentiment = data.TabularDataset(", + "... path='data/sentiment/train.json', format='json',", + "... fields={'sentence_tokenized': ('text', data.Field(sequential=True)),", + "... 'sentiment_gold': ('labels', data.Field(sequential=False))})" + ], + "category": [ + "standalone", + "research" + ], + "tags": [ + "pytorch" + ] + }, + { + "id": "allennlp", + "title": "AllenNLP", + "slogan": "An open-source NLP research library, built on PyTorch and spaCy", + "description": "AllenNLP is a new library designed to accelerate NLP research, by providing a framework that supports modern deep learning workflows for cutting-edge language understanding problems. AllenNLP uses spaCy as a preprocessing component. You can also use Allen NLP to develop spaCy pipeline components, to add annotations to the `Doc` object.", + "github": "allenai/allennlp", + "pip": "allennlp", + "thumb": "https://i.imgur.com/U8opuDN.jpg", + "url": "http://allennlp.org", + "author": " Allen Institute for Artificial Intelligence", + "author_links": { + "github": "allenai", + "twitter": "allenai_org", + "website": "http://allenai.org" + }, + "category": [ + "standalone", + "research" + ] + }, + { + "id": "scispacy", + "title": "scispaCy", + "slogan": "A full spaCy pipeline and models for scientific/biomedical documents", + "github": "allenai/scispacy", + "pip": "scispacy", + "thumb": "https://i.imgur.com/dJQSclW.png", + "url": "https://allenai.github.io/scispacy/", + "author": " Allen Institute for Artificial Intelligence", + "author_links": { + "github": "allenai", + "twitter": "allenai_org", + "website": "http://allenai.org" + }, + "category": [ + "scientific", + "models", + "research", + "biomedical" + ] + }, + { + "id": "textacy", + "slogan": "NLP, before and after spaCy", + "description": "`textacy` is a Python library for performing a variety of natural language processing (NLP) tasks, built on the high-performance `spacy` library. With the fundamentals – tokenization, part-of-speech tagging, dependency parsing, etc. – delegated to another library, `textacy` focuses on the tasks that come before and follow after.", + "github": "chartbeat-labs/textacy", + "pip": "textacy", + "url": "https://github.com/chartbeat-labs/textacy", + "author": "Burton DeWilde", + "author_links": { + "github": "bdewilde", + "twitter": "bjdewilde" + }, + "category": [ + "standalone" + ] + }, + { + "id": "textpipe", + "slogan": "clean and extract metadata from text", + "description": "`textpipe` is a Python package for converting raw text in to clean, readable text and extracting metadata from that text. Its functionalities include transforming raw text into readable text by removing HTML tags and extracting metadata such as the number of words and named entities from the text.", + "github": "textpipe/textpipe", + "pip": "textpipe", + "author": "Textpipe Contributors", + "author_links": { + "github": "textpipe", + "website": "https://github.com/textpipe/textpipe/blob/master/CONTRIBUTORS.md" + }, + "category": [ + "standalone" + ], + "tags": [ + "text-processing", + "named-entity-recognition" + ], + "thumb": "https://avatars0.githubusercontent.com/u/40492530", + "code_example": [ + "from textpipe import doc, pipeline", + "sample_text = 'Sample text! '", + "document = doc.Doc(sample_text)", + "print(document.clean)", + "'Sample text!'", + "print(document.language)", + "# 'en'", + "print(document.nwords)", + "# 2", + "", + "pipe = pipeline.Pipeline(['CleanText', 'NWords'])", + "print(pipe(sample_text))", + "# {'CleanText': 'Sample text!', 'NWords': 2}" + ] + }, + { + "id": "mordecai", + "slogan": "Full text geoparsing using spaCy, Geonames and Keras", + "description": "Extract the place names from a piece of text, resolve them to the correct place, and return their coordinates and structured geographic information.", + "github": "openeventdata/mordecai", + "pip": "mordecai", + "thumb": "https://i.imgur.com/gPJ9upa.jpg", + "code_example": [ + "from mordecai import Geoparser", + "geo = Geoparser()", + "geo.geoparse(\"I traveled from Oxford to Ottawa.\")" + ], + "author": "Andy Halterman", + "author_links": { + "github": "ahalterman", + "twitter": "ahalterman" + }, + "category": [ + "standalone", + "scientific" + ] + }, + { + "id": "kindred", + "title": "Kindred", + "slogan": "Biomedical relation extraction using spaCy", + "description": "Kindred is a package for relation extraction in biomedical texts. Given some training data, it can build a model to identify relations between entities (e.g. drugs, genes, etc) in a sentence.", + "github": "jakelever/kindred", + "pip": "kindred", + "code_example": [ + "import kindred", + "", + "trainCorpus = kindred.bionlpst.load('2016-BB3-event-train')", + "devCorpus = kindred.bionlpst.load('2016-BB3-event-dev')", + "predictionCorpus = devCorpus.clone()", + "predictionCorpus.removeRelations()", + "classifier = kindred.RelationClassifier()", + "classifier.train(trainCorpus)", + "classifier.predict(predictionCorpus)", + "f1score = kindred.evaluate(devCorpus, predictionCorpus, metric='f1score')" + ], + "author": "Jake Lever", + "author_links": { + "github": "jakelever" + }, + "category": [ + "standalone", + "scientific" + ] + }, + { + "id": "sense2vec", + "slogan": "Use NLP to go beyond vanilla word2vec", + "description": "sense2vec ([Trask et. al](https://arxiv.org/abs/1511.06388), 2015) is a nice twist on [word2vec](https://en.wikipedia.org/wiki/Word2vec) that lets you learn more interesting, detailed and context-sensitive word vectors. For an interactive example of the technology, see our [sense2vec demo](https://explosion.ai/demos/sense2vec) that lets you explore semantic similarities across all Reddit comments of 2015.", + "github": "explosion/sense2vec", + "pip": "sense2vec==1.0.0a1", + "thumb": "https://i.imgur.com/awfdhX6.jpg", + "image": "https://explosion.ai/assets/img/demos/sense2vec.png", + "url": "https://explosion.ai/demos/sense2vec", + "code_example": [ + "import spacy", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "s2v = nlp.add_pipe(\"sense2vec\")", + "s2v.from_disk(\"/path/to/s2v_reddit_2015_md\")", + "", + "doc = nlp(\"A sentence about natural language processing.\")", + "assert doc[3:6].text == \"natural language processing\"", + "freq = doc[3:6]._.s2v_freq", + "vector = doc[3:6]._.s2v_vec", + "most_similar = doc[3:6]._.s2v_most_similar(3)", + "# [(('machine learning', 'NOUN'), 0.8986967),", + "# (('computer vision', 'NOUN'), 0.8636297),", + "# (('deep learning', 'NOUN'), 0.8573361)]" + ], + "category": [ + "pipeline", + "standalone", + "visualizers" + ], + "tags": [ + "vectors" + ], + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai" + } + }, + { + "id": "spacyr", + "slogan": "An R wrapper for spaCy", + "github": "quanteda/spacyr", + "cran": "spacyr", + "code_example": [ + "library(\"spacyr\")", + "spacy_initialize()", + "", + "txt <- c(d1 = \"spaCy excels at large-scale information extraction tasks.\",", + " d2 = \"Mr. Smith goes to North Carolina.\")", + "", + "# process documents and obtain a data.table", + "parsedtxt <- spacy_parse(txt)" + ], + "code_language": "r", + "author": "Kenneth Benoit & Aki Matsuo", + "category": [ + "nonpython" + ] + }, + { + "id": "cleannlp", + "title": "CleanNLP", + "slogan": "A tidy data model for NLP in R", + "description": "The cleanNLP package is designed to make it as painless as possible to turn raw text into feature-rich data frames. the package offers four backends that can be used for parsing text: `tokenizers`, `udpipe`, `spacy` and `corenlp`.", + "github": "statsmaths/cleanNLP", + "cran": "cleanNLP", + "author": "Taylor B. Arnold", + "author_links": { + "github": "statsmaths" + }, + "category": [ + "nonpython" + ] + }, + { + "id": "spacy-cpp", + "slogan": "C++ wrapper library for spaCy", + "description": "The goal of spacy-cpp is to expose the functionality of spaCy to C++ applications, and to provide an API that is similar to that of spaCy, enabling rapid development in Python and simple porting to C++.", + "github": "d99kris/spacy-cpp", + "code_example": [ + "Spacy::Spacy spacy;", + "auto nlp = spacy.load(\"en_core_web_sm\");", + "auto doc = nlp.parse(\"This is a sentence.\");", + "for (auto& token : doc.tokens())", + " std::cout << token.text() << \" [\" << token.pos_() << \"]\\n\";" + ], + "code_language": "cpp", + "author": "Kristofer Berggren", + "author_links": { + "github": "d99kris" + }, + "category": [ + "nonpython" + ] + }, + { + "id": "ruby-spacy", + "title": "ruby-spacy", + "slogan": "Wrapper module for using spaCy from Ruby via PyCall", + "description": "ruby-spacy is a wrapper module for using spaCy from the Ruby programming language via PyCall. This module aims to make it easy and natural for Ruby programmers to use spaCy.", + "github": "yohasebe/ruby-spacy", + "code_example": [ + "require \"ruby-spacy\"", + "require \"terminal-table\"", + "nlp = Spacy::Language.new(\"en_core_web_sm\")", + "doc = nlp.read(\"Apple is looking at buying U.K. startup for $1 billion\")", + "headings = [\"text\", \"lemma\", \"pos\", \"tag\", \"dep\"]", + "rows = []", + "doc.each do |token|", + " rows << [token.text, token.lemma, token.pos, token.tag, token.dep]", + "end", + "table = Terminal::Table.new rows: rows, headings: headings", + "puts table" + ], + "code_language": "ruby", + "url": "https://rubygems.org/gems/ruby-spacy", + "author": "Yoichiro Hasebe", + "author_links": { + "github": "yohasebe", + "twitter": "yohasebe" + }, + "category": [ + "nonpython" + ], + "tags": [ + "ruby" + ] + }, + { + "id": "spacy_api", + "slogan": "Server/client to load models in a separate, dedicated process", + "github": "kootenpv/spacy_api", + "pip": "spacy_api", + "code_example": [ + "from spacy_api import Client", + "", + "spacy_client = Client() # default args host/port", + "doc = spacy_client.single(\"How are you\")" + ], + "author": "Pascal van Kooten", + "author_links": { + "github": "kootenpv" + }, + "category": [ + "apis" + ] + }, + { + "id": "spacy-api-docker", + "slogan": "spaCy REST API, wrapped in a Docker container", + "github": "jgontrum/spacy-api-docker", + "url": "https://hub.docker.com/r/jgontrum/spacyapi/", + "thumb": "https://i.imgur.com/NRnDKyj.jpg", + "code_example": [ + "version: '2'", + "", + "services:", + " spacyapi:", + " image: jgontrum/spacyapi:en_v2", + " ports:", + " - \"127.0.0.1:8080:80\"", + " restart: always" + ], + "code_language": "docker", + "author": "Johannes Gontrum", + "author_links": { + "github": "jgontrum" + }, + "category": [ + "apis" + ] + }, + { + "id": "spacy-nlp", + "slogan": " Expose spaCy NLP text parsing to Node.js (and other languages) via Socket.IO", + "github": "kengz/spacy-nlp", + "thumb": "https://i.imgur.com/w41VSr7.jpg", + "code_example": [ + "const spacyNLP = require(\"spacy-nlp\")", + "// default port 6466", + "// start the server with the python client that exposes spacyIO (or use an existing socketIO server at IOPORT)", + "var serverPromise = spacyNLP.server({ port: process.env.IOPORT });", + "// Loading spacy may take up to 15s" + ], + "code_language": "javascript", + "author": "Wah Loon Keng", + "author_links": { + "github": "kengz" + }, + "category": [ + "apis", + "nonpython" + ] + }, + { + "id": "prodigy", + "title": "Prodigy", + "slogan": "Radically efficient machine teaching, powered by active learning", + "description": "Prodigy is an annotation tool so efficient that data scientists can do the annotation themselves, enabling a new level of rapid iteration. Whether you're working on entity recognition, intent detection or image classification, Prodigy can help you train and evaluate your models faster. Stream in your own examples or real-world data from live APIs, update your model in real-time and chain models together to build more complex systems.", + "thumb": "https://i.imgur.com/UVRtP6g.jpg", + "image": "https://i.imgur.com/Dt5vrY6.png", + "url": "https://prodi.gy", + "code_example": [ + "prodigy dataset ner_product \"Improve PRODUCT on Reddit data\"", + "✨ Created dataset 'ner_product'.", + "", + "prodigy ner.teach ner_product en_core_web_sm ~/data.jsonl --label PRODUCT", + "✨ Starting the web server on port 8080..." + ], + "code_language": "bash", + "category": [ + "standalone", + "training" + ], + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai" + } + }, + { + "id": "dragonfire", + "title": "Dragonfire", + "slogan": "An open-source virtual assistant for Ubuntu based Linux distributions", + "github": "DragonComputer/Dragonfire", + "thumb": "https://i.imgur.com/5fqguKS.jpg", + "image": "https://raw.githubusercontent.com/DragonComputer/Dragonfire/master/docs/img/demo.gif", + "author": "Dragon Computer", + "author_links": { + "github": "DragonComputer", + "website": "http://dragon.computer" + }, + "category": [ + "standalone" + ] + }, + { + "id": "prefect", + "title": "Prefect", + "slogan": "Workflow management system designed for modern infrastructure", + "github": "PrefectHQ/prefect", + "pip": "prefect", + "thumb": "https://i.imgur.com/oLTwr0e.png", + "code_example": [ + "from prefect import Flow", + "from prefect.tasks.spacy.spacy_tasks import SpacyNLP", + "import spacy", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "", + "with Flow(\"Natural Language Processing\") as flow:", + " doc = SpacyNLP(text=\"This is some text\", nlp=nlp)", + "", + "flow.run()" + ], + "author": "Prefect", + "author_links": { + "website": "https://prefect.io" + }, + "category": [ + "standalone" + ] + }, + { + "id": "graphbrain", + "title": "Graphbrain", + "slogan": "Automated meaning extraction and text understanding", + "description": "Graphbrain is an Artificial Intelligence open-source software library and scientific research tool. Its aim is to facilitate automated meaning extraction and text understanding, as well as the exploration and inference of knowledge.", + "github": "graphbrain/graphbrain", + "pip": "graphbrain", + "thumb": "https://i.imgur.com/cct9W1E.png", + "author": "Graphbrain", + "category": [ + "standalone" + ] + }, + { + "type": "education", + "id": "nostarch-nlp-python", + "title": "Natural Language Processing Using Python", + "slogan": "No Starch Press, 2020", + "description": "Natural Language Processing Using Python is an introduction to natural language processing (NLP), the task of converting human language into data that a computer can process. The book uses spaCy, a leading Python library for NLP, to guide readers through common NLP tasks related to generating and understanding human language with code. It addresses problems like understanding a user's intent, continuing a conversation with a human, and maintaining the state of a conversation.", + "cover": "https://i.imgur.com/w0iycjl.jpg", + "url": "https://nostarch.com/NLPPython", + "author": "Yuli Vasiliev", + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "oreilly-python-ds", + "title": "Introduction to Machine Learning with Python: A Guide for Data Scientists", + "slogan": "O'Reilly, 2016", + "description": "Machine learning has become an integral part of many commercial applications and research projects, but this field is not exclusive to large companies with extensive research teams. If you use Python, even as a beginner, this book will teach you practical ways to build your own machine learning solutions. With all the data available today, machine learning applications are limited only by your imagination.", + "cover": "https://covers.oreillystatic.com/images/0636920030515/lrg.jpg", + "url": "http://shop.oreilly.com/product/0636920030515.do", + "author": "Andreas Müller, Sarah Guido", + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "text-analytics-python", + "title": "Text Analytics with Python", + "slogan": "Apress / Springer, 2016", + "description": "*Text Analytics with Python* teaches you the techniques related to natural language processing and text analytics, and you will gain the skills to know which technique is best suited to solve a particular problem. You will look at each technique and algorithm with both a bird's eye view to understand how it can be used as well as with a microscopic view to understand the mathematical concepts and to implement them to solve your own problems.", + "github": "dipanjanS/text-analytics-with-python", + "cover": "https://i.imgur.com/AOmzZu8.png", + "url": "https://www.amazon.com/Text-Analytics-Python-Real-World-Actionable/dp/148422387X", + "author": "Dipanjan Sarkar", + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "practical-ml-python", + "title": "Practical Machine Learning with Python", + "slogan": "Apress, 2017", + "description": "Master the essential skills needed to recognize and solve complex problems with machine learning and deep learning. Using real-world examples that leverage the popular Python machine learning ecosystem, this book is your perfect companion for learning the art and science of machine learning to become a successful practitioner. The concepts, techniques, tools, frameworks, and methodologies used in this book will teach you how to think, design, build, and execute machine learning systems and projects successfully.", + "github": "dipanjanS/practical-machine-learning-with-python", + "cover": "https://i.imgur.com/5F4mkt7.jpg", + "url": "https://www.amazon.com/Practical-Machine-Learning-Python-Problem-Solvers/dp/1484232062", + "author": "Dipanjan Sarkar, Raghav Bali, Tushar Sharma", + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "packt-nlp-computational-linguistics", + "title": "Natural Language Processing and Computational Linguistics", + "slogan": "Packt, 2018", + "description": "This book shows you how to use natural language processing, and computational linguistics algorithms, to make inferences and gain insights about data you have. These algorithms are based on statistical machine learning and artificial intelligence techniques. The tools to work with these algorithms are available to you right now - with Python, and tools like Gensim and spaCy.", + "cover": "https://i.imgur.com/aleMf1Y.jpg", + "url": "https://www.amazon.com/Natural-Language-Processing-Computational-Linguistics-ebook/dp/B07BWH779J", + "author": "Bhargav Srinivasa-Desikan", + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "mastering-spacy", + "title": "Mastering spaCy", + "slogan": "Packt, 2021", + "description": "This is your ultimate spaCy book. Master the crucial skills to use spaCy components effectively to create real-world NLP applications with spaCy. Explaining linguistic concepts such as dependency parsing, POS-tagging and named entity extraction with many examples, this book will help you to conquer computational linguistics with spaCy. The book further focuses on ML topics with Keras and Tensorflow. You'll cover popular topics, including intent recognition, sentiment analysis and context resolution; and use them on popular datasets and interpret the results. A special hands-on section on chatbot design is included.", + "github": "PacktPublishing/Mastering-spaCy", + "cover": "https://tinyimg.io/i/aWEm0dh.jpeg", + "url": "https://www.amazon.com/Mastering-spaCy-end-end-implementing/dp/1800563353", + "author": "Duygu Altinok", + "author_links": { + "github": "DuyguA", + "website": "https://www.linkedin.com/in/duygu-altinok-4021389a" + }, + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "applied-nlp-in-enterprise", + "title": "Applied Natural Language Processing in the Enterprise: Teaching Machines to Read, Write, and Understand", + "slogan": "O'Reilly, 2021", + "description": "Natural language processing (NLP) is one of the hottest topics in AI today. Having lagged behind other deep learning fields such as computer vision for years, NLP only recently gained mainstream popularity. Even though Google, Facebook, and OpenAI have open sourced large pretrained language models to make NLP easier, many organizations today still struggle with developing and productionizing NLP applications. This hands-on guide helps you learn the field quickly.", + "github": "nlpbook/nlpbook", + "cover": "https://i.imgur.com/6RxLBvf.jpg", + "url": "https://www.amazon.com/dp/149206257X", + "author": "Ankur A. Patel", + "author_links": { + "github": "aapatel09", + "website": "https://www.ankurapatel.io" + }, + "category": [ + "books" + ] + }, + { + "type": "education", + "id": "introduction-into-spacy-3", + "title": "Introduction to spaCy 3", + "slogan": "A free course for beginners by Dr. W.J.B. Mattingly", + "url": "http://spacy.pythonhumanities.com/", + "thumb": "https://spacy.pythonhumanities.com/_static/freecodecamp_small.jpg", + "author": "Dr. W.J.B. Mattingly", + "category": [ + "courses" + ] + }, + { + "type": "education", + "id": "spacy-course", + "title": "Advanced NLP with spaCy", + "slogan": "A free online course", + "description": "In this free interactive course, you'll learn how to use spaCy to build advanced natural language understanding systems, using both rule-based and machine learning approaches.", + "url": "https://course.spacy.io", + "image": "https://i.imgur.com/JC00pHW.jpg", + "thumb": "https://i.imgur.com/5RXLtrr.jpg", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + }, + "category": [ + "courses" + ] + }, + { + "type": "education", + "id": "applt-course", + "title": "Applied Language Technology", + "slogan": "NLP for newcomers using spaCy and Stanza", + "description": "These learning materials provide an introduction to applied language technology for audiences who are unfamiliar with language technology and programming. The learning materials assume no previous knowledge of the Python programming language.", + "url": "https://applied-language-technology.mooc.fi", + "image": "https://www.mv.helsinki.fi/home/thiippal/images/applt-preview.jpg", + "thumb": "https://www.mv.helsinki.fi/home/thiippal/images/applt-logo.png", + "author": "Tuomo Hiippala", + "author_links": { + "twitter": "tuomo_h", + "github": "thiippal", + "website": "https://www.mv.helsinki.fi/home/thiippal/" + }, + "category": [ + "courses" + ] + }, + + { + "type": "education", + "id": "video-spacys-ner-model", + "title": "spaCy's NER model", + "slogan": "Incremental parsing with bloom embeddings and residual CNNs", + "description": "spaCy v2.0's Named Entity Recognition system features a sophisticated word embedding strategy using subword features and \"Bloom\" embeddings, a deep convolutional neural network with residual connections, and a novel transition-based approach to named entity parsing. The system is designed to give a good balance of efficiency, accuracy and adaptability. In this talk, I sketch out the components of the system, explaining the intuition behind the various choices. I also give a brief introduction to the named entity recognition problem, with an overview of what else Explosion AI is working on, and why.", + "youtube": "sqDHBH9IjRU", + "author": "Matthew Honnibal", + "author_links": { + "twitter": "honnibal", + "github": "honnibal", + "website": "https://explosion.ai" + }, + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-new-nlp-solutions", + "title": "Building new NLP solutions with spaCy and Prodigy", + "slogan": "PyData Berlin 2018", + "description": "In this talk, I will discuss how to address some of the most likely causes of failure for new Natural Language Processing (NLP) projects. My main recommendation is to take an iterative approach: don't assume you know what your pipeline should look like, let alone your annotation schemes or model architectures.", + "author": "Matthew Honnibal", + "author_links": { + "twitter": "honnibal", + "github": "honnibal", + "website": "https://explosion.ai" + }, + "youtube": "jpWqz85F_4Y", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-modern-nlp-in-python", + "title": "Modern NLP in Python", + "slogan": "PyData DC 2016", + "description": "Academic and industry research in Natural Language Processing (NLP) has progressed at an accelerating pace over the last several years. Members of the Python community have been hard at work moving cutting-edge research out of papers and into open source, \"batteries included\" software libraries that can be applied to practical problems. We'll explore some of these tools for modern NLP in Python.", + "author": "Patrick Harrison", + "youtube": "6zm9NC9uRkk", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-course", + "title": "Advanced NLP with spaCy · A free online course", + "description": "spaCy is a modern Python library for industrial-strength Natural Language Processing. In this free and interactive online course, you'll learn how to use spaCy to build advanced natural language understanding systems, using both rule-based and machine learning approaches.", + "url": "https://course.spacy.io/en", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines" + }, + "youtube": "THduWAnG97k", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-course-de", + "title": "Modernes NLP mit spaCy · Ein Gratis-Onlinekurs", + "description": "spaCy ist eine moderne Python-Bibliothek für industriestarkes Natural Language Processing. In diesem kostenlosen und interaktiven Onlinekurs lernst du, mithilfe von spaCy fortgeschrittene Systeme für die Analyse natürlicher Sprache zu entwickeln und dabei sowohl regelbasierte Verfahren, als auch moderne Machine-Learning-Technologie einzusetzen.", + "url": "https://course.spacy.io/de", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines" + }, + "youtube": "K1elwpgDdls", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-course-es", + "title": "NLP avanzado con spaCy · Un curso en línea gratis", + "description": "spaCy es un paquete moderno de Python para hacer Procesamiento de Lenguaje Natural de potencia industrial. En este curso en línea, interactivo y gratuito, aprenderás a usar spaCy para construir sistemas avanzados de comprensión de lenguaje natural usando enfoques basados en reglas y en machine learning.", + "url": "https://course.spacy.io/es", + "author": "Camila Gutiérrez", + "author_links": { + "twitter": "Mariacamilagl30" + }, + "youtube": "RNiLVCE5d4k", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-1", + "title": "Intro to NLP with spaCy (1)", + "slogan": "Episode 1: Data exploration", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "WnGPv6HnBok", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-2", + "title": "Intro to NLP with spaCy (2)", + "slogan": "Episode 2: Rule-based Matching", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "KL4-Mpgbahw", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-3", + "title": "Intro to NLP with spaCy (3)", + "slogan": "Episode 2: Evaluation", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "4V0JDdohxAk", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-4", + "title": "Intro to NLP with spaCy (4)", + "slogan": "Episode 4: Named Entity Recognition", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "IqOJU1-_Fi0", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-5", + "title": "Intro to NLP with spaCy (5)", + "slogan": "Episode 5: Rules vs. Machine Learning", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "f4sqeLRzkPg", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-intro-to-nlp-episode-6", + "title": "Intro to NLP with spaCy (6)", + "slogan": "Episode 6: Moving to spaCy v3", + "description": "In this new video series, data science instructor Vincent Warmerdam gets started with spaCy, an open-source library for Natural Language Processing in Python. His mission: building a system to automatically detect programming languages in large volumes of text. Follow his process from the first idea to a prototype all the way to data collection and training a statistical named entity recogntion model from scratch.", + "author": "Vincent Warmerdam", + "author_links": { + "twitter": "fishnets88", + "github": "koaning" + }, + "youtube": "k77RrmMaKEI", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-irl-entity-linking", + "title": "Entity Linking functionality in spaCy", + "slogan": "spaCy IRL 2019", + "url": "https://www.youtube.com/playlist?list=PLBmcuObd5An4UC6jvK_-eSl6jCvP1gwXc", + "author": "Sofie Van Landeghem", + "author_links": { + "twitter": "OxyKodit", + "github": "svlandeg" + }, + "youtube": "PW3RJM8tDGo", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-irl-lemmatization", + "title": "Rethinking rule-based lemmatization", + "slogan": "spaCy IRL 2019", + "url": "https://www.youtube.com/playlist?list=PLBmcuObd5An4UC6jvK_-eSl6jCvP1gwXc", + "author": "Guadalupe Romero", + "author_links": { + "twitter": "_guadiromero", + "github": "guadi1994" + }, + "youtube": "88zcQODyuko", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "video-spacy-irl-scispacy", + "title": "ScispaCy: A spaCy pipeline & models for scientific & biomedical text", + "slogan": "spaCy IRL 2019", + "url": "https://www.youtube.com/playlist?list=PLBmcuObd5An4UC6jvK_-eSl6jCvP1gwXc", + "author": "Mark Neumann", + "author_links": { + "twitter": "MarkNeumannnn", + "github": "DeNeutoy" + }, + "youtube": "2_HSKDALwuw", + "category": [ + "videos" + ] + }, + { + "type": "education", + "id": "podcast-nlp-highlights", + "title": "NLP Highlights #78: Where do corpora come from?", + "slogan": "January 2019", + "description": "Most NLP projects rely crucially on the quality of annotations used for training and evaluating models. In this episode, Matt and Ines of Explosion AI tell us how Prodigy can improve data annotation and model development workflows. Prodigy is an annotation tool implemented as a python library, and it comes with a web application and a command line interface. A developer can define input data streams and design simple annotation interfaces. Prodigy can help break down complex annotation decisions into a series of binary decisions, and it provides easy integration with spaCy models. Developers can specify how models should be modified as new annotations come in in an active learning framework.", + "soundcloud": "559200912", + "thumb": "https://i.imgur.com/hOBQEzc.jpg", + "url": "https://soundcloud.com/nlp-highlights/78-where-do-corpora-come-from-with-matt-honnibal-and-ines-montani", + "author": "Matt Gardner, Waleed Ammar (Allen AI)", + "author_links": { + "website": "https://soundcloud.com/nlp-highlights" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "podcast-init", + "title": "Podcast.__init__ #87: spaCy with Matthew Honnibal", + "slogan": "December 2017", + "description": "As the amount of text available on the internet and in businesses continues to increase, the need for fast and accurate language analysis becomes more prominent. This week Matthew Honnibal, the creator of spaCy, talks about his experiences researching natural language processing and creating a library to make his findings accessible to industry.", + "iframe": "https://www.pythonpodcast.com/wp-content/plugins/podlove-podcasting-plugin-for-wordpress/lib/modules/podlove_web_player/player_v4/dist/share.html?episode=https://www.pythonpodcast.com/?podlove_player4=176", + "iframe_height": 200, + "thumb": "https://i.imgur.com/rpo6BuY.png", + "url": "https://www.podcastinit.com/episode-87-spacy-with-matthew-honnibal/", + "author": "Tobias Macey", + "author_links": { + "website": "https://www.podcastinit.com" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "podcast-init2", + "title": "Podcast.__init__ #256: An Open Source Toolchain For NLP From Explosion AI", + "slogan": "March 2020", + "description": "The state of the art in natural language processing is a constantly moving target. With the rise of deep learning, previously cutting edge techniques have given way to robust language models. Through it all the team at Explosion AI have built a strong presence with the trifecta of spaCy, Thinc, and Prodigy to support fast and flexible data labeling to feed deep learning models and performant and scalable text processing. In this episode founder and open source author Matthew Honnibal shares his experience growing a business around cutting edge open source libraries for the machine learning developent process.", + "iframe": "https://cdn.podlove.org/web-player/share.html?episode=https%3A%2F%2Fwww.pythonpodcast.com%2F%3Fpodlove_player4%3D614", + "iframe_height": 200, + "thumb": "https://i.imgur.com/rpo6BuY.png", + "url": "https://www.pythonpodcast.com/explosion-ai-natural-language-processing-episode-256/", + "author": "Tobias Macey", + "author_links": { + "website": "https://www.podcastinit.com" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "talk-python-podcast", + "title": "Talk Python #202: Building a software business", + "slogan": "March 2019", + "description": "One core question around open source is how do you fund it? Well, there is always that PayPal donate button. But that's been a tremendous failure for many projects. Often the go-to answer is consulting. But what if you don't want to trade time for money? You could take things up a notch and change the equation, exchanging value for money. That's what Ines Montani and her co-founder did when they started Explosion AI with spaCy as the foundation.", + "thumb": "https://i.imgur.com/q1twuK8.png", + "url": "https://talkpython.fm/episodes/show/202/building-a-software-business", + "soundcloud": "588364857", + "author": "Michael Kennedy", + "author_links": { + "website": "https://talkpython.fm/" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "twimlai-podcast", + "title": "TWiML & AI: Practical NLP with spaCy and Prodigy", + "slogan": "May 2019", + "description": "\"Ines and I caught up to discuss her various projects, including the aforementioned spaCy, an open-source NLP library built with a focus on industry and production use cases. In our conversation, Ines gives us an overview of the spaCy Library, a look at some of the use cases that excite her, and the Spacy community and contributors. We also discuss her work with Prodigy, an annotation service tool that uses continuous active learning to train models, and finally, what other exciting projects she is working on.\"", + "thumb": "https://i.imgur.com/ng2F5gK.png", + "url": "https://twimlai.com/twiml-talk-262-practical-natural-language-processing-with-spacy-and-prodigy-w-ines-montani", + "iframe": "https://html5-player.libsyn.com/embed/episode/id/9691514/height/90/theme/custom/thumbnail/no/preload/no/direction/backward/render-playlist/no/custom-color/3e85b1/", + "iframe_height": 90, + "author": "Sam Charrington", + "author_links": { + "website": "https://twimlai.com" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "analytics-vidhya", + "title": "DataHack Radio #23: The Brains behind spaCy", + "slogan": "June 2019", + "description": "\"What would you do if you had the chance to pick the brains behind one of the most popular Natural Language Processing (NLP) libraries of our era? A library that has helped usher in the current boom in NLP applications and nurtured tons of NLP scientists? Well – you invite the creators on our popular DataHack Radio podcast and let them do the talking! We are delighted to welcome Ines Montani and Matt Honnibal, the developers of spaCy – a powerful and advanced library for NLP.\"", + "thumb": "https://i.imgur.com/3zJKZ1P.jpg", + "url": "https://www.analyticsvidhya.com/blog/2019/06/datahack-radio-ines-montani-matthew-honnibal-brains-behind-spacy/", + "soundcloud": "630741825", + "author": "Analytics Vidhya", + "author_links": { + "website": "https://www.analyticsvidhya.com", + "twitter": "analyticsvidhya" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "practical-ai-podcast", + "title": "Practical AI: Modern NLP with spaCy", + "slogan": "December 2019", + "description": "\"spaCy is awesome for NLP! It’s easy to use, has widespread adoption, is open source, and integrates the latest language models. Ines Montani and Matthew Honnibal (core developers of spaCy and co-founders of Explosion) join us to discuss the history of the project, its capabilities, and the latest trends in NLP. We also dig into the practicalities of taking NLP workflows to production. You don’t want to miss this episode!\"", + "thumb": "https://i.imgur.com/jn8Bcdw.png", + "url": "https://changelog.com/practicalai/68", + "author": "Daniel Whitenack & Chris Benson", + "author_links": { + "website": "https://changelog.com/practicalai", + "twitter": "PracticalAIFM" + }, + "category": [ + "podcasts" + ] + }, + { + "type": "education", + "id": "video-entity-linking", + "title": "Training a custom entity linking mode with spaCy", + "author": "Sofie Van Landeghem", + "author_links": { + "twitter": "OxyKodit", + "github": "svlandeg" + }, + "youtube": "8u57WSXVpmw", + "category": [ + "videos" + ] + }, + { + "id": "self-attentive-parser", + "title": "Berkeley Neural Parser", + "slogan": "Constituency Parsing with a Self-Attentive Encoder (ACL 2018)", + "description": "A Python implementation of the parsers described in *\"Constituency Parsing with a Self-Attentive Encoder\"* from ACL 2018.", + "url": "https://arxiv.org/abs/1805.01052", + "github": "nikitakit/self-attentive-parser", + "pip": "benepar", + "code_example": [ + "import benepar, spacy", + "nlp = spacy.load('en_core_web_md')", + "nlp.add_pipe('benepar', config={'model': 'benepar_en3'})", + "doc = nlp('The time for action is now. It is never too late to do something.')", + "sent = list(doc.sents)[0]", + "print(sent._.parse_string)", + "# (S (NP (NP (DT The) (NN time)) (PP (IN for) (NP (NN action)))) (VP (VBZ is) (ADVP (RB now))) (. .))", + "print(sent._.labels)", + "# ('S',)", + "print(list(sent._.children)[0])", + "# The time for action" + ], + "author": "Nikita Kitaev", + "author_links": { + "github": "nikitakit", + "website": "http://kitaev.io" + }, + "category": [ + "research", + "pipeline" + ] + }, + { + "id": "spacy-graphql", + "title": "spacy-graphql", + "slogan": "Query spaCy's linguistic annotations using GraphQL", + "github": "ines/spacy-graphql", + "description": "A very simple and experimental app that lets you query spaCy's linguistic annotations using [GraphQL](https://graphql.org/). The API currently supports most token attributes, named entities, sentences and text categories (if available as `doc.cats`, i.e. if you added a text classifier to a model). The `meta` field will return the model meta data. Models are only loaded once and kept in memory.", + "url": "https://explosion.ai/demos/spacy-graphql", + "category": [ + "apis" + ], + "tags": [ + "graphql" + ], + "thumb": "https://i.imgur.com/xC7zpTO.png", + "code_example": [ + "{", + " nlp(text: \"Zuckerberg is the CEO of Facebook.\", model: \"en_core_web_sm\") {", + " meta {", + " lang", + " description", + " }", + " doc {", + " text", + " tokens {", + " text", + " pos_", + " }", + " ents {", + " text", + " label_", + " }", + " }", + " }", + "}" + ], + "code_language": "json", + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + } + }, + { + "id": "spacy-js", + "title": "spacy-js", + "slogan": "JavaScript API for spaCy with Python REST API", + "github": "ines/spacy-js", + "description": "JavaScript interface for accessing linguistic annotations provided by spaCy. This project is mostly experimental and was developed for fun to play around with different ways of mimicking spaCy's Python API.\n\nThe results will still be computed in Python and made available via a REST API. The JavaScript API resembles spaCy's Python API as closely as possible (with a few exceptions, as the values are all pre-computed and it's tricky to express complex recursive relationships).", + "code_language": "javascript", + "code_example": [ + "const spacy = require('spacy');", + "", + "(async function() {", + " const nlp = spacy.load('en_core_web_sm');", + " const doc = await nlp('This is a text about Facebook.');", + " for (let ent of doc.ents) {", + " console.log(ent.text, ent.label);", + " }", + " for (let token of doc) {", + " console.log(token.text, token.pos, token.head.text);", + " }", + "})();" + ], + "author": "Ines Montani", + "author_links": { + "twitter": "_inesmontani", + "github": "ines", + "website": "https://ines.io" + }, + "category": [ + "nonpython" + ], + "tags": [ + "javascript" + ] + }, + { + "id": "spacy-wordnet", + "title": "spacy-wordnet", + "slogan": "WordNet meets spaCy", + "description": "`spacy-wordnet` creates annotations that easily allow the use of WordNet and [WordNet Domains](http://wndomains.fbk.eu/) by using the [NLTK WordNet interface](http://www.nltk.org/howto/wordnet.html)", + "github": "recognai/spacy-wordnet", + "tags": [ + "wordnet", + "synsets" + ], + "thumb": "https://i.imgur.com/ud4C7cj.png", + "code_example": [ + "import spacy", + "from spacy_wordnet.wordnet_annotator import WordnetAnnotator ", + "", + "# Load a spaCy model (supported languages are \"es\" and \"en\") ", + "nlp = spacy.load('en_core_web_sm')", + "# spaCy 3.x", + "nlp.add_pipe(\"spacy_wordnet\", after='tagger')", + "# spaCy 2.x", + "# nlp.add_pipe(WordnetAnnotator(nlp.lang), after='tagger')", + "token = nlp('prices')[0]", + "", + "# WordNet object links spaCy token with NLTK WordNet interface by giving access to", + "# synsets and lemmas ", + "token._.wordnet.synsets()", + "token._.wordnet.lemmas()", + "", + "# And automatically add info about WordNet domains", + "token._.wordnet.wordnet_domains()" + ], + "author": "recognai", + "author_links": { + "github": "recognai", + "twitter": "recogn_ai", + "website": "https://recogn.ai" + }, + "category": [ + "pipeline" + ] + }, + { + "id": "spacy-conll", + "title": "spacy_conll", + "slogan": "Parsing from and to CoNLL-U format with `spacy`, `spacy-stanza` and `spacy-udpipe`", + "description": "This module allows you to parse text into CoNLL-U format or read ConLL-U into a spaCy `Doc`. You can use it as a command line tool, or embed it in your own scripts by adding it as a custom pipeline component to a `spacy`, `spacy-stanza` or `spacy-udpipe` pipeline. It also provides an easy-to-use function to quickly initialize any spaCy-wrapped parser. CoNLL-related properties are added to `Doc` elements, `Span` sentences, and `Token` objects.", + "code_example": [ + "from spacy_conll import init_parser", + "", + "", + "# Initialise English parser, already including the ConllFormatter as a pipeline component.", + "# Indicate that we want to get the CoNLL headers in the string output.", + "# `use_gpu` and `verbose` are specific to stanza. These keywords arguments are passed onto their Pipeline() initialisation", + "nlp = init_parser(\"en\",", + " \"stanza\",", + " parser_opts={\"use_gpu\": True, \"verbose\": False},", + " include_headers=True)", + "# Parse a given string", + "doc = nlp(\"A cookie is a baked or cooked food that is typically small, flat and sweet. It usually contains flour, sugar and some type of oil or fat.\")", + "", + "# Get the CoNLL representation of the whole document, including headers", + "conll = doc._.conll_str", + "print(conll)" + ], + "code_language": "python", + "author": "Bram Vanroy", + "author_links": { + "github": "BramVanroy", + "twitter": "BramVanroy", + "website": "http://bramvanroy.be" + }, + "github": "BramVanroy/spacy_conll", + "category": [ + "standalone", + "pipeline" + ], + "tags": [ + "linguistics", + "computational linguistics", + "conll", + "conll-u" + ] + }, + { + "id": "ludwig", + "title": "Ludwig", + "slogan": "A code-free deep learning toolbox", + "description": "Ludwig makes it easy to build deep learning models for many applications, including NLP ones. It uses spaCy for tokenizing text in different languages.", + "pip": "ludwig", + "github": "uber/ludwig", + "thumb": "https://i.imgur.com/j1sORgD.png", + "url": "http://ludwig.ai", + "author": "Piero Molino @ Uber AI", + "author_links": { + "github": "w4nderlust", + "twitter": "w4nderlus7", + "website": "http://w4nderlu.st" + }, + "category": [ + "standalone", + "research" + ] + }, + { + "id": "pic2phrase_bot", + "title": "pic2phrase_bot: Photo Description Generator", + "slogan": "A bot that generates descriptions to submitted photos, in a human-like manner.", + "description": "pic2phrase_bot runs inside Telegram messenger and can be used to generate a phrase describing a submitted photo, employing computer vision, web scraping, and syntactic dependency analysis powered by spaCy.", + "thumb": "https://i.imgur.com/ggVI02O.jpg", + "image": "https://i.imgur.com/z1yhWQR.jpg", + "url": "https://telegram.me/pic2phrase_bot", + "author": "Yuli Vasiliev", + "author_links": { + "twitter": "VasilievYuli" + }, + "category": [ + "standalone", + "conversational" + ] + }, + { + "id": "pyInflect", + "slogan": "A Python module for word inflections", + "description": "This package uses the [spaCy 2.0 extensions](https://spacy.io/usage/processing-pipelines#extensions) to add word inflections to the system.", + "github": "bjascob/pyInflect", + "pip": "pyinflect", + "code_example": [ + "import spacy", + "import pyinflect", + "", + "nlp = spacy.load('en_core_web_sm')", + "doc = nlp('This is an example.')", + "doc[3].tag_ # NN", + "doc[3]._.inflect('NNS') # examples" + ], + "author": "Brad Jascob", + "author_links": { + "github": "bjascob" + }, + "category": [ + "pipeline" + ], + "tags": [ + "inflection" + ] + }, + { + "id": "lemminflect", + "slogan": "A Python module for English lemmatization and inflection", + "description": "LemmInflect uses a dictionary approach to lemmatize English words and inflect them into forms specified by a user supplied [Universal Dependencies](https://universaldependencies.org/u/pos/) or [Penn Treebank](https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html) tag. The library works with out-of-vocabulary (OOV) words by applying neural network techniques to classify word forms and choose the appropriate morphing rules. The system acts as a standalone module or as an extension to spaCy.", + "github": "bjascob/LemmInflect", + "pip": "lemminflect", + "thumb": "https://raw.githubusercontent.com/bjascob/LemmInflect/master/docs/img/icons8-citrus-80.png", + "code_example": [ + "import spacy", + "import lemminflect", + "", + "nlp = spacy.load('en_core_web_sm')", + "doc = nlp('I am testing this example.')", + "doc[2]._.lemma() # 'test'", + "doc[4]._.inflect('NNS') # 'examples'" + ], + "author": "Brad Jascob", + "author_links": { + "github": "bjascob" + }, + "category": [ + "pipeline" + ], + "tags": [ + "inflection", + "lemmatizer" + ] + }, + { + "id": "amrlib", + "slogan": "A python library that makes AMR parsing, generation and visualization simple.", + "description": "amrlib is a python module and spaCy add-in for Abstract Meaning Representation (AMR). The system can parse sentences to AMR graphs or generate text from existing graphs. It includes a GUI for visualization and experimentation.", + "github": "bjascob/amrlib", + "pip": "amrlib", + "code_example": [ + "import spacy", + "import amrlib", + "amrlib.setup_spacy_extension()", + "nlp = spacy.load('en_core_web_sm')", + "doc = nlp('This is a test of the spaCy extension. The test has multiple sentences.')", + "graphs = doc._.to_amr()", + "for graph in graphs:", + " print(graph)" + ], + "author": "Brad Jascob", + "author_links": { + "github": "bjascob" + }, + "category": [ + "pipeline" + ] + }, + { + "id": "classyclassification", + "title": "Classy Classification", + "slogan": "Have you ever struggled with needing a spaCy TextCategorizer but didn't have the time to train one from scratch? Classy Classification is the way to go!", + "description": "Have you ever struggled with needing a [spaCy TextCategorizer](https://spacy.io/api/textcategorizer) but didn't have the time to train one from scratch? Classy Classification is the way to go! For few-shot classification using [sentence-transformers](https://github.com/UKPLab/sentence-transformers) or [spaCy models](https://spacy.io/usage/models), provide a dictionary with labels and examples, or just provide a list of labels for zero shot-classification with [Huggingface zero-shot classifiers](https://huggingface.co/models?pipeline_tag=zero-shot-classification).", + "github": "davidberenstein1957/classy-classification", + "pip": "classy-classification", + "thumb": "https://raw.githubusercontent.com/davidberenstein1957/classy-classification/master/logo.png", + "code_example": [ + "import spacy", + "", + "data = {", + " \"furniture\": [\"This text is about chairs.\",", + " \"Couches, benches and televisions.\",", + " \"I really need to get a new sofa.\"],", + " \"kitchen\": [\"There also exist things like fridges.\",", + " \"I hope to be getting a new stove today.\",", + " \"Do you also have some ovens.\"]", + "}", + "", + "# see github repo for examples on sentence-transformers and Huggingface", + "nlp = spacy.load('en_core_web_md')", + "nlp.add_pipe(\"classy_classification\", ", + " config={", + " \"data\": data,", + " \"model\": \"spacy\"", + " }", + ")", + "", + "print(nlp(\"I am looking for kitchen appliances.\")._.cats)", + "# Output:", + "#", + "# [{\"label\": \"furniture\", \"score\": 0.21}, {\"label\": \"kitchen\", \"score\": 0.79}]" + ], + "author": "David Berenstein", + "author_links": { + "github": "davidberenstein1957", + "website": "https://www.linkedin.com/in/david-berenstein-1bab11105/" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "classification", + "zero-shot", + "few-shot", + "sentence-transformers", + "huggingface" + ], + "spacy_version": 3 + }, + { + "id": "conciseconcepts", + "title": "Concise Concepts", + "slogan": "Concise Concepts uses few-shot NER based on word embedding similarity to get you going with easy!", + "description": "When wanting to apply NER to concise concepts, it is really easy to come up with examples, but it takes some effort to train an entire pipeline. Concise Concepts uses few-shot NER based on word embedding similarity to get you going with easy!", + "github": "davidberenstein1957/concise-concepts", + "pip": "concise-concepts", + "thumb": "https://raw.githubusercontent.com/davidberenstein1957/concise-concepts/master/img/logo.png", + "image": "https://raw.githubusercontent.com/davidberenstein1957/concise-concepts/master/img/example.png", + "code_example": [ + "import spacy", + "from spacy import displacy", + "", + "data = {", + " \"fruit\": [\"apple\", \"pear\", \"orange\"],", + " \"vegetable\": [\"broccoli\", \"spinach\", \"tomato\"],", + " \"meat\": [\"beef\", \"pork\", \"fish\", \"lamb\"]", + "}", + "", + "text = \"\"\"", + " Heat the oil in a large pan and add the Onion, celery and carrots.", + " Then, cook over a medium–low heat for 10 minutes, or until softened.", + " Add the courgette, garlic, red peppers and oregano and cook for 2–3 minutes.", + " Later, add some oranges and chickens.\"\"\"", + "", + "# use any model that has internal spacy embeddings", + "nlp = spacy.load('en_core_web_lg')", + "nlp.add_pipe(\"concise_concepts\", ", + " config={\"data\": data}", + ")", + "doc = nlp(text)", + "", + "options = {\"colors\": {\"fruit\": \"darkorange\", \"vegetable\": \"limegreen\", \"meat\": \"salmon\"},", + " \"ents\": [\"fruit\", \"vegetable\", \"meat\"]}", + "", + "displacy.render(doc, style=\"ent\", options=options)" + ], + "author": "David Berenstein", + "author_links": { + "github": "davidberenstein1957", + "website": "https://www.linkedin.com/in/david-berenstein-1bab11105/" + }, + "category": [ + "pipeline" + ], + "tags": [ + "ner", + "few-shot", + "gensim" + ], + "spacy_version": 3 + }, + { + "id": "crosslingualcoreference", + "title": "Crosslingual Coreference", + "slogan": "One multi-lingual coreference model to rule them all!", + "description": "Coreference is amazing but the data required for training a model is very scarce. In our case, the available training for non-English languages also data proved to be poorly annotated. Crosslingual Coreference therefore uses the assumption a trained model with English data and cross-lingual embeddings should work for other languages with similar sentence structure. Verified to work quite well for at least (EN, NL, DK, FR, DE).", + "github": "davidberenstein1957/crosslingual-coreference", + "pip": "crosslingual-coreference", + "thumb": "https://raw.githubusercontent.com/davidberenstein1957/crosslingual-coreference/master/img/logo.png", + "image": "https://raw.githubusercontent.com/davidberenstein1957/crosslingual-coreference/master/img/example_total.png", + "code_example": [ + "import spacy", + "", + "text = \"\"\"", + " Do not forget about Momofuku Ando!", + " He created instant noodles in Osaka.", + " At that location, Nissin was founded.", + " Many students survived by eating these noodles, but they don't even know him.\"\"\"", + "", + "# use any model that has internal spacy embeddings", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe(", + " \"xx_coref\", config={\"chunk_size\": 2500, \"chunk_overlap\": 2, \"device\": 0})", + ")", + "", + "doc = nlp(text)", + "", + "print(doc._.coref_clusters)", + "# Output", + "#", + "# [[[4, 5], [7, 7], [27, 27], [36, 36]],", + "# [[12, 12], [15, 16]],", + "# [[9, 10], [27, 28]],", + "# [[22, 23], [31, 31]]]", + "print(doc._.resolved_text)", + "# Output", + "#", + "# Do not forget about Momofuku Ando!", + "# Momofuku Ando created instant noodles in Osaka.", + "# At Osaka, Nissin was founded.", + "# Many students survived by eating instant noodles,", + "# but Many students don't even know Momofuku Ando." + ], + "author": "David Berenstein", + "author_links": { + "github": "davidberenstein1957", + "website": "https://www.linkedin.com/in/david-berenstein-1bab11105/" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "coreference", + "multi-lingual", + "cross-lingual", + "allennlp" + ], + "spacy_version": 3 + }, + { + "id": "adeptaugmentations", + "title": "Adept Augmentations", + "slogan": " A Python library aimed at dissecting and augmenting NER training data for a few-shot scenario.", + "description": "EntitySwapAugmenter takes either a `datasets.Dataset` or a `spacy.tokens.DocBin`. Additionally, it is optional to provide a set of labels. It initially creates a knowledge base of entities belonging to a certain label. When running `augmenter.augment()` for N runs, it then creates N new sentences with random swaps of the original entities with an entity of the same corresponding label from the knowledge base.\n\nFor example, assuming that we have knowledge base for `PERSONS`, `LOCATIONS` and `PRODUCTS`. We can then create additional data for the sentence \"Momofuko Ando created instant noodles in Osaka.\" using `augmenter.augment(N=2)`, resulting in \"David created instant noodles in Madrid.\" or \"Tom created Adept Augmentations in the Netherlands\".", + "github": "argilla-io/adept-augmentations", + "pip": "adept-augmentations", + "thumb": "https://raw.githubusercontent.com/argilla-io/adept-augmentations/main/logo.png", + "code_example": [ + "from adept_augmentations import EntitySwapAugmenter", + "import spacy", + "from spacy.tokens import Doc, DocBin", + "nlp = spacy.blank(\"en\")", + "", + "# Create some example golden data", + "example_data = [", + " (\"Apple is looking at buying U.K. startup for $1 billion\", [(0, 5, \"ORG\"), (27, 31, \"LOC\"), (44, 54, \"MONEY\")]),", + " (\"Microsoft acquires GitHub for $7.5 billion\", [(0, 9, \"ORG\"), (19, 25, \"ORG\"), (30, 42, \"MONEY\")]),", + "]", + "", + "# Create a new DocBin", + "nlp = spacy.blank(\"en\")", + "docs = []", + "for entry in example_data:", + " doc = Doc(nlp.vocab, words=entry[0].split())", + " doc.ents = [doc.char_span(ent[0], ent[1], label=ent[2]) for ent in entry[1]]", + " docs.append(doc)", + "golden_dataset = DocBin(docs=docs)", + "", + "# Augment Data", + "augmented_dataset = EntitySwapAugmenter(golden_dataset).augment(4)", + "for doc in augmented_dataset.get_docs(nlp.vocab):", + " print(doc.text)", + "", + "# GitHub is looking at buying U.K. startup for $ 7.5 billion", + "# Microsoft is looking at buying U.K. startup for $ 1 billion", + "# Microsoft is looking at buying U.K. startup for $ 7.5 billion", + "# GitHub is looking at buying U.K. startup for $ 1 billion", + "# Microsoft acquires Apple for $ 7.5 billion", + "# Apple acquires Microsoft for $ 1 billion", + "# Microsoft acquires Microsoft for $ 7.5 billion", + "# GitHub acquires GitHub for $ 1 billion" + ], + "author": "David Berenstein", + "author_links": { + "github": "davidberenstein1957", + "website": "https://www.linkedin.com/in/david-berenstein-1bab11105/" + }, + "category": [ + "standalone" + ], + "tags": [ + "ner", + "few-shot", + "augmentation", + "datasets", + "training" + ], + "spacy_version": 3 + }, + { + "id": "spacysetfit", + "title": "spaCy-SetFit", + "slogan": "An an easy and intuitive approach to use SetFit in combination with spaCy.", + "description": "spaCy-SetFit is a Python library that extends spaCy's text categorization capabilities by incorporating SetFit for few-shot classification. It allows you to train a text categorizer using a intuitive dictionary. \n\nThe library integrates with spaCy's pipeline architecture, enabling easy integration and configuration of the text categorizer component. You can provide a training dataset containing inlier and outlier examples, and spaCy-SetFit will use the paraphrase-MiniLM-L3-v2 model for training the text categorizer with SetFit. Once trained, you can use the categorizer to classify new text and obtain category probabilities.", + "github": "davidberenstein1957/spacy-setfit", + "pip": "spacy-setfit", + "thumb": "https://raw.githubusercontent.com/davidberenstein1957/spacy-setfit/main/logo.png", + "code_example": [ + "import spacy", + "", + "# Create some example data", + "train_dataset = {", + " \"inlier\": [", + " \"Text about furniture\",", + " \"Couches, benches and televisions.\",", + " \"I really need to get a new sofa.\"", + " ],", + " \"outlier\": [", + " \"Text about kitchen equipment\",", + " \"This text is about politics\",", + " \"Comments about AI and stuff.\"", + " ]", + "}", + "", + "# Load the spaCy language model:", + "nlp = spacy.load(\"en_core_web_sm\")", + "", + "# Add the \"spacy_setfit\" pipeline component to the spaCy model, and configure it with SetFit parameters:", + "nlp.add_pipe(\"spacy_setfit\", config={", + " \"pretrained_model_name_or_path\": \"paraphrase-MiniLM-L3-v2\",", + " \"setfit_trainer_args\": {", + " \"train_dataset\": train_dataset", + " }", + "})", + "doc = nlp(\"I really need to get a new sofa.\")", + "doc.cats", + "# {'inlier': 0.902350975129, 'outlier': 0.097649024871}" + ], + "author": "David Berenstein", + "author_links": { + "github": "davidberenstein1957", + "website": "https://www.linkedin.com/in/david-berenstein-1bab11105/" + }, + "category": [ + "pipeline" + ], + "tags": [ + "few-shot", + "SetFit", + "training" + ], + "spacy_version": 3 + }, + { + "id": "blackstone", + "title": "Blackstone", + "slogan": "A spaCy pipeline and model for NLP on unstructured legal text", + "description": "Blackstone is a spaCy model and library for processing long-form, unstructured legal text. Blackstone is an experimental research project from the [Incorporated Council of Law Reporting for England and Wales'](https://iclr.co.uk/) research lab, [ICLR&D](https://research.iclr.co.uk/).", + "github": "ICLRandD/Blackstone", + "pip": "blackstone", + "thumb": "https://iclr.s3-eu-west-1.amazonaws.com/assets/iclrand/Blackstone/thumb.png", + "url": "https://research.iclr.co.uk", + "author": " ICLR&D", + "author_links": { + "github": "ICLRandD", + "twitter": "ICLRanD", + "website": "https://research.iclr.co.uk" + }, + "category": [ + "scientific", + "models", + "research" + ] + }, + { + "id": "NGym", + "title": "NeuralGym", + "slogan": "A little Windows GUI for training models with spaCy", + "description": "NeuralGym is a Python application for Windows with a graphical user interface to train models with spaCy. Run the application, select an output folder, a training data file in spaCy's data format, a spaCy model or blank model and press 'Start'.", + "github": "d5555/NeuralGym", + "url": "https://github.com/d5555/NeuralGym", + "image": "https://github.com/d5555/NeuralGym/raw/master/NGym.png", + "thumb": "https://github.com/d5555/NeuralGym/raw/master/NGym/web.png", + "author": "d5555", + "category": [ + "training" + ], + "tags": [ + "windows" + ] + }, + { + "id": "holmes", + "title": "Holmes", + "slogan": "Information extraction from English and German texts based on predicate logic", + "github": "explosion/holmes-extractor", + "url": "https://github.com/explosion/holmes-extractor", + "description": "Holmes is a Python 3 library that supports a number of use cases involving information extraction from English and German texts, including chatbot, structural extraction, topic matching and supervised document classification. There is a [website demonstrating intelligent search based on topic matching](https://holmes-demo.explosion.services).", + "pip": "holmes-extractor", + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "chatbots", + "text-processing" + ], + "thumb": "https://raw.githubusercontent.com/explosion/holmes-extractor/master/docs/holmes_thumbnail.png", + "code_example": [ + "import holmes_extractor as holmes", + "holmes_manager = holmes.Manager(model='en_core_web_lg')", + "holmes_manager.register_search_phrase('A big dog chases a cat')", + "holmes_manager.start_chatbot_mode_console()" + ], + "author": "Richard Paul Hudson", + "author_links": { + "github": "richardpaulhudson" + } + }, + { + "id": "coreferee", + "title": "Coreferee", + "slogan": "Coreference resolution for multiple languages", + "github": "explosion/coreferee", + "url": "https://github.com/explosion/coreferee", + "description": "Coreferee is a pipeline plugin that performs coreference resolution for English, French, German and Polish. It is designed so that it is easy to add support for new languages and optimised for limited training data. It uses a mixture of neural networks and programmed rules. Please note you will need to [install models](https://github.com/explosion/coreferee#getting-started) before running the code example.", + "pip": "coreferee", + "category": [ + "pipeline", + "models", + "standalone" + ], + "tags": [ + "coreference-resolution", + "anaphora" + ], + "code_example": [ + "import coreferee, spacy", + "nlp = spacy.load('en_core_web_trf')", + "nlp.add_pipe('coreferee')", + "doc = nlp('Although he was very busy with his work, Peter had had enough of it. He and his wife decided they needed a holiday. They travelled to Spain because they loved the country very much.')", + "doc._.coref_chains.print()", + "# Output:", + "#", + "# 0: he(1), his(6), Peter(9), He(16), his(18)", + "# 1: work(7), it(14)", + "# 2: [He(16); wife(19)], they(21), They(26), they(31)", + "# 3: Spain(29), country(34)", + "#", + "print(doc._.coref_chains.resolve(doc[31]))", + "# Output:", + "#", + "# [Peter, wife]" + ], + "author": "Richard Paul Hudson", + "author_links": { + "github": "richardpaulhudson" + } + }, + { + "id": "spacy-transformers", + "title": "spacy-transformers", + "slogan": "spaCy pipelines for pretrained BERT, XLNet and GPT-2", + "description": "This package provides spaCy model pipelines that wrap [Hugging Face's `transformers`](https://github.com/huggingface/transformers) package, so you can use them in spaCy. The result is convenient access to state-of-the-art transformer architectures, such as BERT, GPT-2, XLNet, etc.", + "github": "explosion/spacy-transformers", + "url": "https://explosion.ai/blog/spacy-transformers", + "pip": "spacy-transformers", + "category": [ + "pipeline", + "models", + "research" + ], + "code_example": [ + "import spacy", + "", + "nlp = spacy.load(\"en_core_web_trf\")", + "doc = nlp(\"Apple shares rose on the news. Apple pie is delicious.\")" + ], + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai" + } + }, + { + "id": "spacy-huggingface-hub", + "title": "spacy-huggingface-hub", + "slogan": "Push your spaCy pipelines to the Hugging Face Hub", + "description": "This package provides a CLI command for uploading any trained spaCy pipeline packaged with [`spacy package`](https://spacy.io/api/cli#package) to the [Hugging Face Hub](https://huggingface.co). It auto-generates all meta information for you, uploads a pretty README (requires spaCy v3.1+) and handles version control under the hood.", + "github": "explosion/spacy-huggingface-hub", + "thumb": "https://i.imgur.com/j6FO9O6.jpg", + "url": "https://github.com/explosion/spacy-huggingface-hub", + "pip": "spacy-huggingface-hub", + "category": [ + "pipeline", + "models" + ], + "author": "Explosion", + "author_links": { + "twitter": "explosion_ai", + "github": "explosion", + "website": "https://explosion.ai" + } + }, + { + "id": "spacy-clausie", + "title": "spacy-clausie", + "slogan": "Implementation of the ClausIE information extraction system for Python+spaCy", + "github": "mmxgn/spacy-clausie", + "url": "https://github.com/mmxgn/spacy-clausie", + "description": "ClausIE, a novel, clause-based approach to open information extraction, which extracts relations and their arguments from natural language text", + "category": [ + "pipeline", + "scientific", + "research" + ], + "code_example": [ + "import spacy", + "import claucy", + "", + "nlp = spacy.load(\"en\")", + "claucy.add_to_pipe(nlp)", + "", + "doc = nlp(\"AE died in Princeton in 1955.\")", + "", + "print(doc._.clauses)", + "# Output:", + "# <SV, AE, died, None, None, None, [in Princeton, in 1955]>", + "", + "propositions = doc._.clauses[0].to_propositions(as_text=True)", + "", + "print(propositions)", + "# Output:", + "# [AE died in Princeton in 1955, AE died in 1955, AE died in Princeton" + ], + "author": "Emmanouil Theofanis Chourdakis", + "author_links": { + "github": "mmxgn" + } + }, + { + "id": "ipymarkup", + "slogan": "NER, syntax markup visualizations", + "description": "Collection of NLP visualizations for NER and syntax tree markup. Similar to [displaCy](https://explosion.ai/demos/displacy) and [displaCy ENT](https://explosion.ai/demos/displacy-ent).", + "github": "natasha/ipymarkup", + "image": "https://github.com/natasha/ipymarkup/blob/master/table.png?raw=true", + "pip": "pip install ipymarkup", + "code_example": [ + "from ipymarkup import show_span_ascii_markup, show_dep_ascii_markup", + "", + "text = 'В мероприятии примут участие не только российские учёные, но и зарубежные исследователи, в том числе, Крис Хелмбрехт - управляющий директор и совладелец креативного агентства Kollektiv (Германия, США), Ннека Угбома - руководитель проекта Mushroom works (Великобритания), Гергей Ковач - политик и лидер субкультурной партии «Dog with two tails» (Венгрия), Георг Жено - немецкий режиссёр, один из создателей экспериментального театра «Театр.doc», Театра им. Йозефа Бойса (Германия).'", + "spans = [(102, 116, 'PER'), (186, 194, 'LOC'), (196, 199, 'LOC'), (202, 214, 'PER'), (254, 268, 'LOC'), (271, 283, 'PER'), (324, 342, 'ORG'), (345, 352, 'LOC'), (355, 365, 'PER'), (445, 455, 'ORG'), (456, 468, 'PER'), (470, 478, 'LOC')]", + "show_span_ascii_markup(text, spans)" + ], + "author": "Alexander Kukushkin", + "author_links": { + "github": "kuk" + }, + "category": [ + "visualizers" + ] + }, + { + "id": "negspacy", + "title": "negspaCy", + "slogan": "spaCy pipeline object for negating concepts in text based on the NegEx algorithm.", + "github": "jenojp/negspacy", + "url": "https://github.com/jenojp/negspacy", + "description": "negspacy is a spaCy pipeline component that evaluates whether Named Entities are negated in text. It adds an extension to 'Span' objects.", + "pip": "negspacy", + "category": [ + "pipeline", + "scientific" + ], + "tags": [ + "negation", + "text-processing" + ], + "thumb": "https://github.com/jenojp/negspacy/blob/master/docs/thumb.png?raw=true", + "image": "https://github.com/jenojp/negspacy/blob/master/docs/icon.png?raw=true", + "code_example": [ + "import spacy", + "from negspacy.negation import Negex", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "nlp.add_pipe(\"negex\", config={\"ent_types\":[\"PERSON\",\"ORG\"]})", + "", + "doc = nlp(\"She does not like Steve Jobs but likes Apple products.\")", + "for e in doc.ents:", + " print(e.text, e._.negex)" + ], + "author": "Jeno Pizarro", + "author_links": { + "github": "jenojp", + "twitter": "jenojp" + } + }, + { + "id": "ronec", + "title": "RONEC - Romanian Named Entity Corpus", + "slogan": "Named Entity Recognition corpus for Romanian language.", + "github": "dumitrescustefan/ronec", + "url": "https://github.com/dumitrescustefan/ronec", + "description": "The corpus holds 5127 sentences, annotated with 16 classes, with a total of 26376 annotated entities. The corpus comes into two formats: BRAT and CONLLUP.", + "category": [ + "standalone", + "models" + ], + "tags": [ + "ner", + "romanian" + ], + "thumb": "https://raw.githubusercontent.com/dumitrescustefan/ronec/master/res/thumb.png", + "code_example": [ + "# to train a new model on ronec", + "python3 convert_spacy.py ronec/conllup/ronec.conllup output", + "python3 -m spacy train ro models output/train_ronec.json output/train_ronec.json -p ent", + "", + "# download the Romanian NER model", + "python -m spacy download ro_ner", + "", + "# load the model and print entities for a simple sentence", + "import spacy", + "", + "nlp = spacy.load(\"ro_ner\")", + "doc = nlp(\"Popescu Ion a fost la Cluj\")", + "", + "for ent in doc.ents:", + "\tprint(ent.text, ent.start_char, ent.end_char, ent.label_)" + ], + "author": "Stefan Daniel Dumitrescu, Andrei-Marius Avram" + }, + { + "id": "Healthsea", + "title": "Healthsea", + "slogan": "Healthsea: an end-to-end spaCy pipeline for exploring health supplement effects", + "description": "This spaCy project trains an NER model and a custom Text Classification model with Clause Segmentation and Blinding capabilities to analyze supplement reviews and their potential effects on health.", + "github": "explosion/healthsea", + "thumb": "https://github.com/explosion/healthsea/blob/main/img/Jellyfish.png", + "category": [ + "pipeline", + "research" + ], + "code_example": [ + "import spacy", + "", + "nlp = spacy.load(\"en_healthsea\")", + "doc = nlp(\"This is great for joint pain.\")", + "", + "# Clause Segmentation & Blinding", + "print(doc._.clauses)", + "", + "> {", + "> \"split_indices\": [0, 7],", + "> \"has_ent\": true,", + "> \"ent_indices\": [4, 6],", + "> \"blinder\": \"_CONDITION_\",", + "> \"ent_name\": \"joint pain\",", + "> \"cats\": {", + "> \"POSITIVE\": 0.9824668169021606,", + "> \"NEUTRAL\": 0.017364952713251114,", + "> \"NEGATIVE\": 0.00002889777533710003,", + "> \"ANAMNESIS\": 0.0001394189748680219", + "> \"prediction_text\": [\"This\", \"is\", \"great\", \"for\", \"_CONDITION_\", \"!\"]", + "> }", + "", + "# Aggregated results", + "> {", + "> \"joint_pain\": {", + "> \"effects\": [\"POSITIVE\"],", + "> \"effect\": \"POSITIVE\",", + "> \"label\": \"CONDITION\",", + "> \"text\": \"joint pain\"", + "> }", + "> }" + ], + "author": "Edward Schmuhl", + "author_links": { + "github": "thomashacker", + "twitter": "aestheticedwar1", + "website": "https://explosion.ai/" + } + }, + { + "id": "presidio", + "title": "Presidio", + "slogan": "Context aware, pluggable and customizable data protection and PII data anonymization", + "description": "Presidio *(Origin from Latin praesidium ‘protection, garrison’)* helps to ensure sensitive text is properly managed and governed. It provides fast ***analytics*** and ***anonymization*** for sensitive text such as credit card numbers, names, locations, social security numbers, bitcoin wallets, US phone numbers and financial data. Presidio analyzes the text using predefined or custom recognizers to identify entities, patterns, formats, and checksums with relevant context.", + "url": "https://aka.ms/presidio", + "image": "https://raw.githubusercontent.com/microsoft/presidio/master/docs/assets/before-after.png", + "github": "microsoft/presidio", + "category": [ + "standalone" + ], + "thumb": "https://avatars0.githubusercontent.com/u/6154722", + "author": "Microsoft", + "author_links": { + "github": "microsoft" + } + }, + { + "id": "presidio-research", + "title": "Presidio Research", + "slogan": "Toolbox for developing and evaluating PII detectors, NER models for PII and generating fake PII data", + "description": "This package features data-science related tasks for developing new recognizers for Microsoft Presidio. It is used for the evaluation of the entire system, as well as for evaluating specific PII recognizers or PII detection models. Anyone interested in evaluating an existing Microsoft Presidio instance, a specific PII recognizer or to develop new models or logic for detecting PII could leverage the preexisting work in this package. Additionally, anyone interested in generating new data based on previous datasets (e.g. to increase the coverage of entity values) for Named Entity Recognition models could leverage the data generator contained in this package.", + "url": "https://aka.ms/presidio-research", + "github": "microsoft/presidio-research", + "category": [ + "standalone" + ], + "thumb": "https://avatars0.githubusercontent.com/u/6154722", + "author": "Microsoft", + "author_links": { + "github": "microsoft" + } + }, + { + "id": "python-sentence-boundary-disambiguation", + "title": "pySBD - python Sentence Boundary Disambiguation", + "slogan": "Rule-based sentence boundary detection that works out-of-the-box", + "github": "nipunsadvilkar/pySBD", + "description": "pySBD is 'real-world' sentence segmenter which extracts reasonable sentences when the format and domain of the input text are unknown. It is a rules-based algorithm based on [The Golden Rules](https://s3.amazonaws.com/tm-town-nlp-resources/golden_rules.txt) - a set of tests to check accuracy of segmenter in regards to edge case scenarios developed by [TM-Town](https://www.tm-town.com/) dev team. pySBD is python port of ruby gem [Pragmatic Segmenter](https://github.com/diasks2/pragmatic_segmenter).", + "pip": "pysbd", + "category": [ + "scientific" + ], + "tags": [ + "sentence segmentation" + ], + "code_example": [ + "from pysbd.utils import PySBDFactory", + "", + "nlp = spacy.blank('en')", + "# Caution: works with spaCy<=2.x.x", + "nlp.add_pipe(PySBDFactory(nlp))", + "", + "doc = nlp('My name is Jonas E. Smith. Please turn to p. 55.')", + "print(list(doc.sents))", + "# [My name is Jonas E. Smith., Please turn to p. 55.]" + ], + "author": "Nipun Sadvilkar", + "author_links": { + "twitter": "nipunsadvilkar", + "github": "nipunsadvilkar", + "website": "https://nipunsadvilkar.github.io" + } + }, + { + "id": "cookiecutter-spacy-fastapi", + "title": "cookiecutter-spacy-fastapi", + "slogan": "Docker-based cookiecutter for easy spaCy APIs using FastAPI", + "description": "Docker-based cookiecutter for easy spaCy APIs using FastAPI. The default endpoints expect batch requests with a list of Records in the Azure Search Cognitive Skill format. So out of the box, this cookiecutter can be setup as a Custom Cognitive Skill. For more on Azure Search and Cognitive Skills [see this page](https://docs.microsoft.com/en-us/azure/search/cognitive-search-custom-skill-interface).", + "url": "https://github.com/microsoft/cookiecutter-spacy-fastapi", + "image": "https://raw.githubusercontent.com/microsoft/cookiecutter-spacy-fastapi/master/images/cookiecutter-docs.png", + "github": "microsoft/cookiecutter-spacy-fastapi", + "category": [ + "apis" + ], + "thumb": "https://avatars0.githubusercontent.com/u/6154722", + "author": "Microsoft", + "author_links": { + "github": "microsoft" + } + }, + { + "id": "dframcy", + "title": "Dframcy", + "slogan": "Dataframe Integration with spaCy NLP", + "github": "yash1994/dframcy", + "description": "DframCy is a light-weight utility module to integrate Pandas Dataframe to spaCy's linguistic annotation and training tasks.", + "pip": "dframcy", + "category": [ + "pipeline", + "training" + ], + "tags": [ + "pandas" + ], + "code_example": [ + "import spacy", + "from dframcy import DframCy", + "", + "nlp = spacy.load('en_core_web_sm')", + "dframcy = DframCy(nlp)", + "doc = dframcy.nlp(u'Apple is looking at buying U.K. startup for $1 billion')", + "annotation_dataframe = dframcy.to_dataframe(doc)" + ], + "author": "Yash Patadia", + "author_links": { + "twitter": "PatadiaYash", + "github": "yash1994" + } + }, + { + "id": "spacy-pytextrank", + "title": "PyTextRank", + "slogan": "Py impl of TextRank for lightweight phrase extraction", + "description": "An implementation of TextRank in Python for use in spaCy pipelines which provides fast, effective phrase extraction from texts, along with extractive summarization. The graph algorithm works independent of a specific natural language and does not require domain knowledge. See (Mihalcea 2004) https://web.eecs.umich.edu/~mihalcea/papers/mihalcea.emnlp04.pdf", + "github": "DerwenAI/pytextrank", + "pip": "pytextrank", + "code_example": [ + "import spacy", + "import pytextrank", + "", + "# example text", + "text = \"\"\"Compatibility of systems of linear constraints over the set of natural numbers.", + "Criteria of compatibility of a system of linear Diophantine equations, strict inequations,", + "and nonstrict inequations are considered. Upper bounds for components of a minimal set of", + "solutions and algorithms of construction of minimal generating sets of solutions for all types", + "of systems are given. These criteria and the corresponding algorithms for constructing a minimal", + "supporting set of solutions can be used in solving all the considered types systems and systems of mixed types.\"\"\"", + "", + "# load a spaCy model, depending on language, scale, etc.", + "nlp = spacy.load(\"en_core_web_sm\")", + "# add PyTextRank to the spaCy pipeline", + "nlp.add_pipe(\"textrank\")", + "", + "doc = nlp(text)", + "# examine the top-ranked phrases in the document", + "for phrase in doc._.phrases:", + " print(phrase.text)", + " print(phrase.rank, phrase.count)", + " print(phrase.chunks)" + ], + "code_language": "python", + "url": "https://github.com/DerwenAI/pytextrank/wiki", + "thumb": "https://memegenerator.net/img/instances/66942896.jpg", + "image": "https://memegenerator.net/img/instances/66942896.jpg", + "author": "Paco Nathan", + "author_links": { + "twitter": "pacoid", + "github": "ceteri", + "website": "https://derwen.ai/paco" + }, + "category": [ + "pipeline" + ], + "tags": [ + "phrase extraction", + "ner", + "summarization", + "graph algorithms", + "textrank" + ] + }, + { + "id": "spacy_syllables", + "title": "Spacy Syllables", + "slogan": "Multilingual syllable annotations", + "description": "Spacy Syllables is a pipeline component that adds multilingual syllable annotations to Tokens. It uses Pyphen under the hood and has support for a long list of languages.", + "github": "sloev/spacy-syllables", + "pip": "spacy_syllables", + "code_example": [ + "import spacy", + "from spacy_syllables import SpacySyllables", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "nlp.add_pipe(\"syllables\", after=\"tagger\")", + "", + "assert nlp.pipe_names == [\"tok2vec\", \"tagger\", \"syllables\", \"parser\", \"attribute_ruler\", \"lemmatizer\", \"ner\"]", + "doc = nlp(\"terribly long\")", + "data = [(token.text, token._.syllables, token._.syllables_count) for token in doc]", + "assert data == [(\"terribly\", [\"ter\", \"ri\", \"bly\"], 3), (\"long\", [\"long\"], 1)]" + ], + "thumb": "https://raw.githubusercontent.com/sloev/spacy-syllables/master/logo.png", + "author": "Johannes Valbjørn", + "author_links": { + "github": "sloev" + }, + "category": [ + "pipeline" + ], + "tags": [ + "syllables", + "multilingual" + ] + }, + { + "id": "sentimental-onix", + "title": "Sentimental Onix", + "slogan": "Use onnx for sentiment models", + "description": "spaCy pipeline component for sentiment analysis using onnx", + "github": "sloev/sentimental-onix", + "pip": "sentimental-onix", + "code_example": [ + "# Download model:", + "# python -m sentimental_onix download en", + "import spacy", + "from sentimental_onix import pipeline", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "nlp.add_pipe(\"sentencizer\")", + "nlp.add_pipe(\"sentimental_onix\", after=\"sentencizer\")", + "", + "sentences = [", + " (sent.text, sent._.sentiment)", + " for doc in nlp.pipe(", + " [", + " \"i hate pasta on tuesdays\",", + " \"i like movies on wednesdays\",", + " \"i find your argument ridiculous\",", + " \"soda with straws are my favorite\",", + " ]", + " )", + " for sent in doc.sents", + "]", + "", + "assert sentences == [", + " (\"i hate pasta on tuesdays\", \"Negative\"),", + " (\"i like movies on wednesdays\", \"Positive\"),", + " (\"i find your argument ridiculous\", \"Negative\"),", + " (\"soda with straws are my favorite\", \"Positive\"),", + "]" + ], + "thumb": "https://raw.githubusercontent.com/sloev/sentimental-onix/master/.github/onix.webp", + "author": "Johannes Valbjørn", + "author_links": { + "github": "sloev" + }, + "category": [ + "pipeline" + ], + "tags": [ + "sentiment", + "english" + ] + }, + { + "id": "gobbli", + "title": "gobbli", + "slogan": "Deep learning for text classification doesn't have to be scary", + "description": "gobbli is a Python library which wraps several modern deep learning models in a uniform interface that makes it easy to evaluate feasibility and conduct analyses. It leverages the abstractive powers of Docker to hide nearly all dependency management and functional differences between models from the user. It also contains an interactive app for exploring text data and evaluating classification models. spaCy's base text classification models, as well as models integrated from `spacy-transformers`, are available in the collection of classification models. In addition, spaCy is used for data augmentation and document embeddings.", + "url": "https://github.com/rtiinternational/gobbli", + "github": "rtiinternational/gobbli", + "pip": "gobbli", + "thumb": "https://i.postimg.cc/NGpzhrdr/gobbli-lg.png", + "code_example": [ + "from gobbli.io import PredictInput, TrainInput", + "from gobbli.model.bert import BERT", + "", + "train_input = TrainInput(", + " X_train=['This is a training document.', 'This is another training document.'],", + " y_train=['0', '1'],", + " X_valid=['This is a validation sentence.', 'This is another validation sentence.'],", + " y_valid=['1', '0'],", + ")", + "", + "clf = BERT()", + "", + "# Set up classifier resources -- Docker image, etc.", + "clf.build()", + "", + "# Train model", + "train_output = clf.train(train_input)", + "", + "predict_input = PredictInput(", + " X=['Which class is this document?'],", + " labels=train_output.labels,", + " checkpoint=train_output.checkpoint,", + ")", + "", + "predict_output = clf.predict(predict_input)" + ], + "category": [ + "standalone" + ] + }, + { + "id": "spacy_fastlang", + "title": "Spacy FastLang", + "slogan": "Language detection done fast", + "description": "Fast language detection using FastText and Spacy.", + "github": "thomasthiebaud/spacy-fastlang", + "pip": "spacy_fastlang", + "code_example": [ + "import spacy_fastlang", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "nlp.add_pipe(\"language_detector\")", + "doc = nlp('Life is like a box of chocolates. You never know what you are gonna get.')", + "", + "assert doc._.language == 'en'", + "assert doc._.language_score >= 0.8" + ], + "author": "Thomas Thiebaud", + "author_links": { + "github": "thomasthiebaud" + }, + "category": [ + "pipeline" + ] + }, + { + "id": "mlflow", + "title": "MLflow", + "slogan": "An open source platform for the machine learning lifecycle", + "description": "MLflow is an open source platform to manage the ML lifecycle, including experimentation, reproducibility, deployment, and a central model registry. MLflow currently offers four components: Tracking, Projects, Models and Registry.", + "github": "mlflow/mlflow", + "pip": "mlflow", + "thumb": "https://www.mlflow.org/docs/latest/_static/MLflow-logo-final-black.png", + "image": "", + "url": "https://mlflow.org/", + "author": "Databricks", + "author_links": { + "github": "databricks", + "twitter": "databricks", + "website": "https://databricks.com/" + }, + "category": [ + "standalone", + "apis" + ], + "code_example": [ + "import mlflow", + "import mlflow.spacy", + "", + "# MLflow Tracking", + "nlp = spacy.load('my_best_model_path/output/model-best')", + "with mlflow.start_run(run_name='Spacy'):", + " mlflow.set_tag('model_flavor', 'spacy')", + " mlflow.spacy.log_model(spacy_model=nlp, artifact_path='model')", + " mlflow.log_metric(('accuracy', 0.72))", + " my_run_id = mlflow.active_run().info.run_id", + "", + "", + "# MLflow Models", + "model_uri = f'runs:/{my_run_id}/model'", + "nlp2 = mlflow.spacy.load_model(model_uri=model_uri)" + ] + }, + { + "id": "pyate", + "title": "PyATE", + "slogan": "Python Automated Term Extraction", + "description": "PyATE is a term extraction library written in Python using Spacy POS tagging with Basic, Combo Basic, C-Value, TermExtractor, and Weirdness.", + "github": "kevinlu1248/pyate", + "pip": "pyate", + "code_example": [ + "import spacy", + "import pyate", + "", + "nlp = spacy.load('en_core_web_sm')", + "nlp.add_pipe(\"combo_basic\") # or any of `basic`, `weirdness`, `term_extractor` or `cvalue`", + "# source: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1994795/", + "string = 'Central to the development of cancer are genetic changes that endow these “cancer cells” with many of the hallmarks of cancer, such as self-sufficient growth and resistance to anti-growth and pro-death signals. However, while the genetic changes that occur within cancer cells themselves, such as activated oncogenes or dysfunctional tumor suppressors, are responsible for many aspects of cancer development, they are not sufficient. Tumor promotion and progression are dependent on ancillary processes provided by cells of the tumor environment but that are not necessarily cancerous themselves. Inflammation has long been associated with the development of cancer. This review will discuss the reflexive relationship between cancer and inflammation with particular focus on how considering the role of inflammation in physiologic processes such as the maintenance of tissue homeostasis and repair may provide a logical framework for understanding the connection between the inflammatory response and cancer.'", + "", + "doc = nlp(string)", + "print(doc._.combo_basic.sort_values(ascending=False).head(5))", + "\"\"\"\"\"\"", + "dysfunctional tumor 1.443147", + "tumor suppressors 1.443147", + "genetic changes 1.386294", + "cancer cells 1.386294", + "dysfunctional tumor suppressors 1.298612", + "\"\"\"\"\"\"" + ], + "code_language": "python", + "url": "https://github.com/kevinlu1248/pyate", + "author": "Kevin Lu", + "author_links": { + "twitter": "kevinlu1248", + "github": "kevinlu1248", + "website": "https://github.com/kevinlu1248/pyate" + }, + "category": [ + "pipeline", + "research" + ], + "tags": [ + "term_extraction" + ] + }, + { + "id": "contextualSpellCheck", + "title": "Contextual Spell Check", + "slogan": "Contextual spell correction using BERT (bidirectional representations)", + "description": "This package currently focuses on Out of Vocabulary (OOV) word or non-word error (NWE) correction using BERT model. The idea of using BERT was to use the context when correcting NWE.", + "github": "R1j1t/contextualSpellCheck", + "pip": "contextualSpellCheck", + "code_example": [ + "import spacy", + "import contextualSpellCheck", + "", + "nlp = spacy.load('en_core_web_sm')", + "contextualSpellCheck.add_to_pipe(nlp)", + "doc = nlp('Income was $9.4 milion compared to the prior year of $2.7 milion.')", + "", + "print(doc._.performed_spellCheck) #Should be True", + "print(doc._.outcome_spellCheck) #Income was $9.4 million compared to the prior year of $2.7 million." + ], + "code_language": "python", + "url": "https://github.com/R1j1t/contextualSpellCheck", + "thumb": "https://user-images.githubusercontent.com/22280243/82760949-98e68480-9e14-11ea-952e-4738620fd9e3.png", + "image": "https://user-images.githubusercontent.com/22280243/82138959-2852cd00-9842-11ea-918a-49b2a7873ef6.png", + "author": "Rajat Goel", + "author_links": { + "github": "r1j1t", + "website": "https://github.com/R1j1t" + }, + "category": [ + "pipeline", + "conversational", + "research" + ], + "tags": [ + "spell check", + "correction", + "preprocessing", + "translation", + "correction" + ] + }, + { + "id": "texthero", + "title": "Texthero", + "slogan": "Text preprocessing, representation and visualization from zero to hero.", + "description": "Texthero is a python package to work with text data efficiently. It empowers NLP developers with a tool to quickly understand any text-based dataset and it provides a solid pipeline to clean and represent text data, from zero to hero.", + "github": "jbesomi/texthero", + "pip": "texthero", + "code_example": [ + "import texthero as hero", + "import pandas as pd", + "", + "df = pd.read_csv('https://github.com/jbesomi/texthero/raw/master/dataset/bbcsport.csv')", + "df['named_entities'] = hero.named_entities(df['text'])", + "df.head()" + ], + "code_language": "python", + "url": "https://texthero.org", + "thumb": "https://texthero.org/img/T.png", + "image": "https://texthero.org/docs/assets/texthero.png", + "author": "Jonathan Besomi", + "author_links": { + "github": "jbesomi", + "website": "https://besomi.ai" + }, + "category": [ + "standalone" + ] + }, + { + "id": "cov-bsv", + "title": "VA COVID-19 NLP BSV", + "slogan": "spaCy pipeline for COVID-19 surveillance.", + "github": "abchapman93/VA_COVID-19_NLP_BSV", + "description": "A spaCy rule-based pipeline for identifying positive cases of COVID-19 from clinical text. A version of this system was deployed as part of the US Department of Veterans Affairs biosurveillance response to COVID-19.", + "pip": "cov-bsv", + "code_example": [ + "import cov_bsv", + "", + "nlp = cov_bsv.load()", + "doc = nlp('Pt tested for COVID-19. His wife was recently diagnosed with novel coronavirus. SARS-COV-2: Detected')", + "", + "print(doc.ents)", + "print(doc._.cov_classification)", + "cov_bsv.visualize_doc(doc)" + ], + "category": [ + "pipeline", + "standalone", + "biomedical", + "scientific" + ], + "tags": [ + "clinical", + "epidemiology", + "covid-19", + "surveillance" + ], + "author": "Alec Chapman", + "author_links": { + "github": "abchapman93" + } + }, + { + "id": "medspacy", + "title": "medspaCy", + "thumb": "https://raw.githubusercontent.com/medspacy/medspacy/master/images/medspacy_logo.png", + "slogan": "A toolkit for clinical NLP with spaCy.", + "github": "medspacy/medspacy", + "description": "A toolkit for clinical NLP with spaCy. Features include sentence splitting, section detection, and asserting negation, family history, and uncertainty.", + "pip": "medspacy", + "code_example": [ + "import medspacy", + "from medspacy.ner import TargetRule", + "", + "nlp = medspacy.load()", + "print(nlp.pipe_names)", + "", + "nlp.get_pipe('target_matcher').add([TargetRule('stroke', 'CONDITION'), TargetRule('diabetes', 'CONDITION'), TargetRule('pna', 'CONDITION')])", + "doc = nlp('Patient has hx of stroke. Mother diagnosed with diabetes. No evidence of pna.')", + "", + "for ent in doc.ents:", + " print(ent, ent._.is_negated, ent._.is_family, ent._.is_historical)", + "medspacy.visualization.visualize_ent(doc)" + ], + "category": [ + "biomedical", + "scientific", + "research" + ], + "tags": [ + "clinical" + ], + "author": "medspacy", + "author_links": { + "github": "medspacy" + } + }, + { + "id": "rita-dsl", + "title": "RITA DSL", + "slogan": "Domain Specific Language for creating language rules", + "github": "zaibacu/rita-dsl", + "description": "A Domain Specific Language (DSL) for building language patterns. These can be later compiled into spaCy patterns, pure regex, or any other format", + "pip": "rita-dsl", + "thumb": "https://raw.githubusercontent.com/zaibacu/rita-dsl/master/docs/assets/logo-100px.png", + "code_language": "python", + "code_example": [ + "import spacy", + "from rita.shortcuts import setup_spacy", + "", + "rules = \"\"\"", + "cuts = {\"fitted\", \"wide-cut\"}", + "lengths = {\"short\", \"long\", \"calf-length\", \"knee-length\"}", + "fabric_types = {\"soft\", \"airy\", \"crinkled\"}", + "fabrics = {\"velour\", \"chiffon\", \"knit\", \"woven\", \"stretch\"}", + "", + "{IN_LIST(cuts)?, IN_LIST(lengths), WORD(\"dress\")}->MARK(\"DRESS_TYPE\")", + "{IN_LIST(lengths), IN_LIST(cuts), WORD(\"dress\")}->MARK(\"DRESS_TYPE\")", + "{IN_LIST(fabric_types)?, IN_LIST(fabrics)}->MARK(\"DRESS_FABRIC\")", + "\"\"\"", + "", + "nlp = spacy.load(\"en\")", + "setup_spacy(nlp, rules_string=rules)", + "r = nlp(\"She was wearing a short wide-cut dress\")", + "print(list([{\"label\": e.label_, \"text\": e.text} for e in r.ents]))" + ], + "category": [ + "standalone" + ], + "tags": [ + "dsl", + "language-patterns", + "language-rules", + "nlp" + ], + "author": "Šarūnas Navickas", + "author_links": { + "github": "zaibacu" + } + }, + { + "id": "PatternOmatic", + "title": "PatternOmatic", + "slogan": "Finds linguistic patterns effortlessly", + "description": "Discover spaCy's linguistic patterns matching a given set of String samples to be used by the spaCy's Rule Based Matcher", + "github": "revuel/PatternOmatic", + "pip": "PatternOmatic", + "code_example": [ + "from PatternOmatic.api import find_patterns", + "", + "samples = ['I am a cat!', 'You are a dog!', 'She is an owl!']", + "", + "patterns_found, _ = find_patterns(samples)", + "", + "print(f'Patterns found: {patterns_found}')" + ], + "code_language": "python", + "thumb": "https://svgshare.com/i/R3P.svg", + "image": "https://svgshare.com/i/R3P.svg", + "author": "Miguel Revuelta Espinosa", + "author_links": { + "github": "revuel" + }, + "category": [ + "scientific", + "research", + "standalone" + ], + "tags": [ + "Evolutionary Computation", + "Grammatical Evolution" + ] + }, + { + "id": "SpacyDotNet", + "title": "spaCy .NET Wrapper", + "slogan": "SpacyDotNet is a .NET Core compatible wrapper for spaCy, based on Python.NET", + "description": "This projects relies on [Python.NET](http://pythonnet.github.io/) to interop with spaCy. It's not meant to be a complete and exhaustive implementation of all spaCy features and [APIs](https://spacy.io/api). Although it should be enough for basic tasks, it's considered as a starting point if you need to build a complex project using spaCy in .NET Most of the basic features in _Spacy101_ are available. All `Container` classes are present (`Doc`, `Token`, `Span` and `Lexeme`) with their basic properties/methods running and also `Vocab` and `StringStore` in a limited form. Anyway, any developer should be ready to add the missing properties or classes in a very straightforward manner.", + "github": "AMArostegui/SpacyDotNet", + "thumb": "https://raw.githubusercontent.com/AMArostegui/SpacyDotNet/master/cslogo.png", + "code_example": [ + "var spacy = new Spacy();", + "", + "var nlp = spacy.Load(\"en_core_web_sm\");", + "var doc = nlp.GetDocument(\"Apple is looking at buying U.K. startup for $1 billion\");", + "", + "foreach (Token token in doc.Tokens)", + " Console.WriteLine($\"{token.Text} {token.Lemma} {token.PoS} {token.Tag} {token.Dep} {token.Shape} {token.IsAlpha} {token.IsStop}\");", + "", + "Console.WriteLine(\"\");", + "foreach (Span ent in doc.Ents)", + " Console.WriteLine($\"{ent.Text} {ent.StartChar} {ent.EndChar} {ent.Label}\");", + "", + "nlp = spacy.Load(\"en_core_web_md\");", + "var tokens = nlp.GetDocument(\"dog cat banana afskfsd\");", + "", + "Console.WriteLine(\"\");", + "foreach (Token token in tokens.Tokens)", + " Console.WriteLine($\"{token.Text} {token.HasVector} {token.VectorNorm}, {token.IsOov}\");", + "", + "tokens = nlp.GetDocument(\"dog cat banana\");", + "Console.WriteLine(\"\");", + "foreach (Token token1 in tokens.Tokens)", + "{", + " foreach (Token token2 in tokens.Tokens)", + " Console.WriteLine($\"{token1.Text} {token2.Text} {token1.Similarity(token2) }\");", + "}", + "", + "doc = nlp.GetDocument(\"I love coffee\");", + "Console.WriteLine(\"\");", + "Console.WriteLine(doc.Vocab.Strings[\"coffee\"]);", + "Console.WriteLine(doc.Vocab.Strings[3197928453018144401]);", + "", + "Console.WriteLine(\"\");", + "foreach (Token word in doc.Tokens)", + "{", + " var lexeme = doc.Vocab[word.Text];", + " Console.WriteLine($@\"{lexeme.Text} {lexeme.Orth} {lexeme.Shape} {lexeme.Prefix} {lexeme.Suffix} {lexeme.IsAlpha} {lexeme.IsDigit} {lexeme.IsTitle} {lexeme.Lang}\");", + "}" + ], + "code_language": "csharp", + "author": "Antonio Miras", + "author_links": { + "github": "AMArostegui" + }, + "category": [ + "nonpython" + ] + }, + { + "id": "ruts", + "title": "ruTS", + "slogan": "A library for statistics extraction from texts in Russian", + "description": "The library allows extracting the following statistics from a text: basic statistics, readability metrics, lexical diversity metrics, morphological statistics", + "github": "SergeyShk/ruTS", + "pip": "ruts", + "code_example": [ + "import spacy", + "import ruts", + "", + "nlp = spacy.load('ru_core_news_sm')", + "nlp.add_pipe('basic', last=True)", + "doc = nlp('мама мыла раму')", + "doc._.basic.get_stats()" + ], + "code_language": "python", + "thumb": "https://habrastorage.org/webt/6z/le/fz/6zlefzjavzoqw_wymz7v3pwgfp4.png", + "image": "https://clipartart.com/images/free-tree-roots-clipart-black-and-white-2.png", + "author": "Sergey Shkarin", + "author_links": { + "twitter": "shk_sergey", + "github": "SergeyShk" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "Text Analytics", + "Russian" + ] + }, + { + "id": "trunajod", + "title": "TRUNAJOD", + "slogan": "A text complexity library for text analysis built on spaCy", + "description": "With all the basic NLP capabilities provided by spaCy (dependency parsing, POS tagging, tokenizing), `TRUNAJOD` focuses on extracting measurements from texts that might be interesting for different applications and use cases.", + "github": "dpalmasan/TRUNAJOD2.0", + "pip": "trunajod", + "code_example": [ + "import spacy", + "from TRUNAJOD.entity_grid import EntityGrid", + "", + "nlp = spacy.load('es_core_news_sm', disable=['ner', 'textcat'])", + "example_text = (", + " 'El espectáculo del cielo nocturno cautiva la mirada y suscita preguntas'", + " 'sobre el universo, su origen y su funcionamiento. No es sorprendente que '", + " 'todas las civilizaciones y culturas hayan formado sus propias '", + " 'cosmologías. Unas relatan, por ejemplo, que el universo ha'", + " 'sido siempre tal como es, con ciclos que inmutablemente se repiten; '", + " 'otras explican que este universo ha tenido un principio, '", + " 'que ha aparecido por obra creadora de una divinidad.'", + ")", + "doc = nlp(example_text)", + "egrid = EntityGrid(doc)", + "print(egrid.get_egrid())" + ], + "code_language": "python", + "thumb": "https://raw.githubusercontent.com/dpalmasan/TRUNAJOD2.0/master/imgs/trunajod_thumb.png", + "image": "https://raw.githubusercontent.com/dpalmasan/TRUNAJOD2.0/master/imgs/trunajod_logo.png", + "author": "Diego Palma", + "author_links": { + "github": "dpalmasan" + }, + "category": [ + "research", + "standalone", + "scientific" + ], + "tags": [ + "Text Analytics", + "Coherence", + "Cohesion" + ] + }, + { + "id": "lingfeat", + "title": "LingFeat", + "slogan": "A Linguistic Feature Extraction (Text Analysis) Tool for Readability Assessment and Text Simplification", + "description": "LingFeat is a feature extraction library which currently extracts 255 linguistic features from English string input. Categories include syntax, semantics, discourse, and also traditional readability formulas. Published in EMNLP 2021.", + "github": "brucewlee/lingfeat", + "pip": "lingfeat", + "code_example": [ + "from lingfeat import extractor", + "", + "", + "text = 'TAEAN, South Chungcheong Province -- Just before sunup, Lee Young-ho, a seasoned fisherman with over 30 years of experience, silently waits for boats carrying blue crabs as the season for the seafood reaches its height. Soon afterward, small and big boats sail into Sinjin Port in Taean County, South Chungcheong Province, the second-largest source of blue crab after Incheon, accounting for 29 percent of total production of the country. A crane lifts 28 boxes filled with blue crabs weighing 40 kilograms each from the boat, worth about 10 million won ($8,500). “It has been a productive fall season for crabbing here. The water temperature is a very important factor affecting crab production. They hate cold water,” Lee said. The temperature of the sea off Taean appeared to have stayed at the level where crabs become active. If the sea temperature suddenly drops, crabs go into their winter dormancy mode, burrowing into the mud and sleeping through the cold months.'", + "", + "", + "#Pass text", + "LingFeat = extractor.pass_text(text)", + "", + "", + "#Preprocess text", + "LingFeat.preprocess()", + "", + "", + "#Extract features", + "#each method returns a dictionary of the corresponding features", + "#Advanced Semantic (AdSem) Features", + "WoKF = LingFeat.WoKF_() #Wikipedia Knowledge Features", + "WBKF = LingFeat.WBKF_() #WeeBit Corpus Knowledge Features", + "OSKF = LingFeat.OSKF_() #OneStopEng Corpus Knowledge Features", + "", + "#Discourse (Disco) Features", + "EnDF = LingFeat.EnDF_() #Entity Density Features", + "EnGF = LingFeat.EnGF_() #Entity Grid Features", + "", + "#Syntactic (Synta) Features", + "PhrF = LingFeat.PhrF_() #Noun/Verb/Adj/Adv/... Phrasal Features", + "TrSF = LingFeat.TrSF_() #(Parse) Tree Structural Features", + "POSF = LingFeat.POSF_() #Noun/Verb/Adj/Adv/... Part-of-Speech Features", + "", + "#Lexico Semantic (LxSem) Features", + "TTRF = LingFeat.TTRF_() #Type Token Ratio Features", + "VarF = LingFeat.VarF_() #Noun/Verb/Adj/Adv Variation Features", + "PsyF = LingFeat.PsyF_() #Psycholinguistic Difficulty of Words (AoA Kuperman)", + "WoLF = LingFeat.WorF_() #Word Familiarity from Frequency Count (SubtlexUS)", + "", + "Shallow Traditional (ShTra) Features", + "ShaF = LingFeat.ShaF_() #Shallow Features (e.g. avg number of tokens)", + "TraF = LingFeat.TraF_() #Traditional Formulas" + ], + "code_language": "python", + "thumb": "https://raw.githubusercontent.com/brucewlee/lingfeat/master/img/lingfeat_logo2.png", + "image": "https://raw.githubusercontent.com/brucewlee/lingfeat/master/img/lingfeat_logo.png", + "author": "Bruce W. Lee (이웅성)", + "author_links": { + "github": "brucewlee", + "website": "https://brucewlee.github.io/" + }, + "category": [ + "research", + "scientific" + ], + "tags": [ + "Readability", + "Simplification", + "Feature Extraction", + "Syntax", + "Discourse", + "Semantics", + "Lexical" + ] + }, + { + "id": "hmrb", + "title": "Hammurabi", + "slogan": "Python Rule Processing Engine 🏺", + "description": "Hammurabi works as a rule engine to parse input using a defined set of rules. It uses a simple and readable syntax to define complex rules to handle phrase matching. The syntax supports nested logical statements, regular expressions, reusable or side-loaded variables and match triggered callback functions to modularize your rules. The latest version works with both spaCy 2.X and 3.X. For more information check the documentation on [ReadTheDocs](https://hmrb.readthedocs.io/en/latest/).", + "github": "babylonhealth/hmrb", + "pip": "hmrb", + "code_example": [ + "import spacy", + "from hmrb.core import SpacyCore", + "", + "nlp = spacy.load(\"en_core_web_sm\")", + "sentences = \"I love gorillas. Peter loves gorillas. Jane loves Tarzan.\"", + "", + "def conj_be(subj: str) -> str:", + " if subj == \"I\":", + " return \"am\"", + " elif subj == \"you\":", + " return \"are\"", + " else:", + " return \"is\"", + "", + "@spacy.registry.callbacks(\"gorilla_callback\")", + "def gorilla_clb(seq: list, span: slice, data: dict) -> None:", + " subj = seq[span.start].text", + " be = conj_be(subj)", + " print(f\"{subj} {be} a gorilla person.\")", + "@spacy.registry.callbacks(\"lover_callback\")", + "def lover_clb(seq: list, span: slice, data: dict) -> None:", + " print(f\"{seq[span][-1].text} is a love interest of {seq[span.start].text}.\")", + "", + "grammar = \"\"\"", + " Law:", + " - callback: \"loves_gorilla\"", + " (", + " ((pos: \"PROPN\") or (pos: \"PRON\"))", + " (lemma: \"love\")", + " (lemma: \"gorilla\")", + " )", + " Law:", + " - callback: \"loves_someone\"", + " (", + " (pos: \"PROPN\")", + " (lower: \"loves\")", + " (pos: \"PROPN\")", + " )", + "\"\"\"", + "", + "@spacy.registry.augmenters(\"jsonify_span\")", + "def jsonify_span(span):", + " return [{\"lemma\": token.lemma_, \"pos\": token.pos_, \"lower\": token.lower_} for token in span]", + "", + "conf = {", + " \"rules\": grammar,", + " \"callbacks\": {", + " \"loves_gorilla\": \"callbacks.gorilla_callback\",", + " \"loves_someone\": \"callbacks.lover_callback\",", + " },", + " \"map_doc\": \"augmenters.jsonify_span\",", + " \"sort_length\": True,", + "}", + "", + "nlp.add_pipe(\"hmrb\", config=conf)", + "nlp(sentences)" + ], + "code_language": "python", + "thumb": "https://user-images.githubusercontent.com/6807878/118643685-cae6b880-b7d4-11eb-976e-066aec9505da.png", + "image": "https://user-images.githubusercontent.com/6807878/118643685-cae6b880-b7d4-11eb-976e-066aec9505da.png", + "author": "Kristian Boda", + "author_links": { + "github": "bodak", + "twitter": "bodak", + "website": "https://github.com/babylonhealth/" + }, + "category": [ + "pipeline", + "standalone", + "scientific", + "biomedical" + ], + "tags": [ + "babylonhealth", + "rule-engine", + "matcher" + ] + }, + { + "id": "forte", + "title": "Forte", + "slogan": "Forte is a toolkit for building Natural Language Processing pipelines, featuring cross-task interaction, adaptable data-model interfaces and composable pipelines.", + "description": "Forte provides a platform to assemble state-of-the-art NLP and ML technologies in a highly-composable fashion, including a wide spectrum of tasks ranging from Information Retrieval, Natural Language Understanding to Natural Language Generation.", + "github": "asyml/forte", + "pip": "forte.spacy stave torch", + "code_example": [ + "from fortex.spacy import SpacyProcessor", + "from forte.processors.stave import StaveProcessor", + "from forte import Pipeline", + "from forte.data.readers import StringReader", + "", + "pipeline = Pipeline()", + "pipeline.set_reader(StringReader())", + "pipeline.add(SpacyProcessor())", + "pipeline.add(StaveProcessor())", + "pipeline.run('Running SpaCy with Forte!')" + ], + "code_language": "python", + "url": "https://medium.com/casl-project/forte-building-modular-and-re-purposable-nlp-pipelines-cf5b5c5abbe9", + "thumb": "https://raw.githubusercontent.com/asyml/forte/master/docs/_static/img/forte_graphic.png", + "image": "https://raw.githubusercontent.com/asyml/forte/master/docs/_static/img/logo_h.png", + "author": "Petuum", + "author_links": { + "twitter": "PetuumInc", + "github": "asyml", + "website": "https://petuum.com" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "pipeline" + ] + }, + { + "id": "spacy-api-docker-v3", + "slogan": "spaCy v3 REST API, wrapped in a Docker container", + "github": "bbieniek/spacy-api-docker", + "url": "https://hub.docker.com/r/bbieniek/spacyapi/", + "thumb": "https://i.imgur.com/NRnDKyj.jpg", + "code_example": [ + "version: '3'", + "", + "services:", + " spacyapi:", + " image: bbieniek/spacyapi:en_v3", + " ports:", + " - \"127.0.0.1:8080:80\"", + " restart: always" + ], + "code_language": "docker", + "author": "Baltazar Bieniek", + "author_links": { + "github": "bbieniek" + }, + "category": [ + "apis" + ] + }, + { + "id": "phruzz_matcher", + "title": "phruzz-matcher", + "slogan": "Phrase matcher using RapidFuzz", + "description": "Combination of the RapidFuzz library with Spacy PhraseMatcher The goal of this component is to find matches when there were NO \"perfect matches\" due to typos or abbreviations between a Spacy doc and a list of phrases.", + "github": "mjvallone/phruzz-matcher", + "pip": "phruzz_matcher", + "code_example": [ + "import spacy", + "from spacy.language import Language", + "from phruzz_matcher.phrase_matcher import PhruzzMatcher", + "", + "famous_people = [", + " \"Brad Pitt\",", + " \"Demi Moore\",", + " \"Bruce Willis\",", + " \"Jim Carrey\",", + "]", + "", + "@Language.factory(\"phrase_matcher\")", + "def phrase_matcher(nlp: Language, name: str):", + " return PhruzzMatcher(nlp, famous_people, \"FAMOUS_PEOPLE\", 85)", + "", + "nlp = spacy.blank('es')", + "nlp.add_pipe(\"phrase_matcher\")", + "", + "doc = nlp(\"El otro día fui a un bar donde vi a brad pit y a Demi Moore, estaban tomando unas cervezas mientras charlaban de sus asuntos.\")", + "print(f\"doc.ents: {doc.ents}\")", + "", + "#OUTPUT", + "#doc.ents: (brad pit, Demi Moore)" + ], + "thumb": "https://avatars.githubusercontent.com/u/961296?v=4", + "image": "", + "code_language": "python", + "author": "Martin Vallone", + "author_links": { + "github": "mjvallone", + "twitter": "vallotin", + "website": "https://fiqus.coop/" + }, + "category": [ + "pipeline", + "research", + "standalone" + ], + "tags": [ + "spacy", + "python", + "nlp", + "ner" + ] + }, + { + "id": "WordDumb", + "title": "WordDumb", + "slogan": "A calibre plugin that generates Word Wise and X-Ray files.", + "description": "A calibre plugin that generates Word Wise and X-Ray files then sends them to Kindle. Supports KFX, AZW3 and MOBI eBooks. X-Ray supports 18 languages.", + "github": "xxyzz/WordDumb", + "code_language": "python", + "thumb": "https://raw.githubusercontent.com/xxyzz/WordDumb/master/starfish.svg", + "image": "https://user-images.githubusercontent.com/21101839/130245435-b874f19a-7785-4093-9975-81596efc42bb.png", + "author": "xxyzz", + "author_links": { + "github": "xxyzz" + }, + "category": [ + "standalone" + ] + }, + { + "id": "eng_spacysentiment", + "title": "eng_spacysentiment", + "slogan": "Simple sentiment analysis using spaCy pipelines", + "description": "Sentiment analysis for simple english sentences using pre-trained spaCy pipelines", + "github": "vishnunkumar/spacysentiment", + "pip": "eng-spacysentiment", + "code_example": [ + "import eng_spacysentiment", + "nlp = eng_spacysentiment.load()", + "text = \"Welcome to Arsenals official YouTube channel Watch as we take you closer and show you the personality of the club\"", + "doc = nlp(text)", + "print(doc.cats)", + "# {'positive': 0.29878824949264526, 'negative': 0.7012117505073547}" + ], + "thumb": "", + "image": "", + "code_language": "python", + "author": "Vishnu Nandakumar", + "author_links": { + "github": "Vishnunkumar", + "twitter": "vishnun_uchiha" + }, + "category": [ + "pipeline" + ], + "tags": [ + "pipeline", + "nlp", + "sentiment" + ] + }, + { + "id": "textnets", + "slogan": "Text analysis with networks", + "description": "textnets represents collections of texts as networks of documents and words. This provides novel possibilities for the visualization and analysis of texts.", + "github": "jboynyc/textnets", + "image": "https://user-images.githubusercontent.com/2187261/152641425-6c0fb41c-b8e0-44fb-a52a-7c1ba24eba1e.png", + "code_example": [ + "import textnets as tn", + "", + "corpus = tn.Corpus(tn.examples.moon_landing)", + "t = tn.Textnet(corpus.tokenized(), min_docs=1)", + "t.plot(label_nodes=True,", + " show_clusters=True,", + " scale_nodes_by=\"birank\",", + " scale_edges_by=\"weight\")" + ], + "author": "John Boy", + "author_links": { + "github": "jboynyc", + "twitter": "jboy" + }, + "category": [ + "visualizers", + "standalone" + ] + }, + { + "id": "tmtoolkit", + "slogan": "Text mining and topic modeling toolkit", + "description": "tmtoolkit is a set of tools for text mining and topic modeling with Python developed especially for the use in the social sciences, in journalism or related disciplines. It aims for easy installation, extensive documentation and a clear programming interface while offering good performance on large datasets by the means of vectorized operations (via NumPy) and parallel computation (using Python’s multiprocessing module and the loky package).", + "github": "WZBSocialScienceCenter/tmtoolkit", + "code_example": [ + "# Note: This requires these setup steps:", + "# pip install tmtoolkit[recommended]", + "# python -m tmtoolkit setup en", + "from tmtoolkit.corpus import Corpus, tokens_table, lemmatize, to_lowercase, dtm", + "from tmtoolkit.bow.bow_stats import tfidf, sorted_terms_table", + "# load built-in sample dataset and use 4 worker processes", + "corp = Corpus.from_builtin_corpus('en-News100', max_workers=4)", + "# investigate corpus as dataframe", + "toktbl = tokens_table(corp)", + "print(toktbl)", + "# apply some text normalization", + "lemmatize(corp)", + "to_lowercase(corp)", + "# build sparse document-token matrix (DTM)", + "# document labels identify rows, vocabulary tokens identify columns", + "mat, doc_labels, vocab = dtm(corp, return_doc_labels=True, return_vocab=True)", + "# apply tf-idf transformation to DTM", + "# operation is applied on sparse matrix and uses few memory", + "tfidf_mat = tfidf(mat)", + "# show top 5 tokens per document ranked by tf-idf", + "top_tokens = sorted_terms_table(tfidf_mat, vocab, doc_labels, top_n=5)", + "print(top_tokens)" + ], + "author": "Markus Konrad / WZB Social Science Center", + "author_links": { + "github": "internaut", + "twitter": "_knrd" + }, + "category": [ + "scientific", + "standalone" + ] + }, + { + "id": "edsnlp", + "title": "EDS-NLP", + "slogan": "spaCy components to extract information from clinical notes written in French.", + "description": "EDS-NLP provides a set of rule-based spaCy components to extract information for French clinical notes. It also features _qualifier_ pipelines that detect negations, speculations and family context, among other modalities. Check out the [demo](https://aphp.github.io/edsnlp/demo/)!", + "github": "aphp/edsnlp", + "pip": "edsnlp", + "code_example": [ + "import spacy", + "", + "nlp = spacy.blank(\"fr\")", + "", + "terms = dict(", + " covid=[\"covid\", \"coronavirus\"],", + ")", + "", + "# Sentencizer component, needed for negation detection", + "nlp.add_pipe(\"eds.sentences\")", + "# Matcher component", + "nlp.add_pipe(\"eds.matcher\", config=dict(terms=terms))", + "# Negation detection", + "nlp.add_pipe(\"eds.negation\")", + "", + "# Process your text in one call !", + "doc = nlp(\"Le patient est atteint de covid\")", + "", + "doc.ents", + "# Out: (covid,)", + "", + "doc.ents[0]._.negation", + "# Out: False" + ], + "code_language": "python", + "url": "https://aphp.github.io/edsnlp/", + "author": "AP-HP", + "author_links": { + "github": "aphp", + "website": "https://github.com/aphp" + }, + "category": [ + "biomedical", + "scientific", + "research", + "pipeline" + ], + "tags": [ + "clinical" + ] + }, + { + "id": "sent-pattern", + "title": "English Interpretation Sentence Pattern", + "slogan": "English interpretation for accurate translation from English to Japanese", + "description": "This package categorizes English sentences into one of five basic sentence patterns and identifies the subject, verb, object, and other components. The five basic sentence patterns are based on C. T. Onions's Advanced English Syntax and are frequently used when teaching English in Japan.", + "github": "lll-lll-lll-lll/sent-pattern", + "pip": "sent-pattern", + "author": "Shunpei Nakayama", + "author_links": { + "twitter": "ExZ79575296", + "github": "lll-lll-lll-lll" + }, + "category": [ + "pipeline" + ], + "tags": [ + "interpretation", + "ja" + ] + }, + { + "id": "spacy-partial-tagger", + "title": "spaCy - Partial Tagger", + "slogan": "Sequence Tagger for Partially Annotated Dataset in spaCy", + "description": "This is a library to build a CRF tagger with a partially annotated dataset in spaCy. You can build your own tagger only from dictionary.", + "github": "doccano/spacy-partial-tagger", + "pip": "spacy-partial-tagger", + "category": [ + "pipeline", + "training" + ], + "author": "Yasufumi Taniguchi", + "author_links": { + "github": "yasufumy" + } + }, + { + "id": "spacy-pythainlp", + "title": "spaCy-PyThaiNLP", + "slogan": "PyThaiNLP for spaCy", + "description": "This package wraps the PyThaiNLP library to add support for Thai to spaCy.", + "github": "PyThaiNLP/spaCy-PyThaiNLP", + "code_example": [ + "import spacy", + "import spacy_pythainlp.core", + "", + "nlp = spacy.blank('th')", + "nlp.add_pipe('pythainlp')", + "doc = nlp('ผมเป็นคนไทย แต่มะลิอยากไปโรงเรียนส่วนผมจะไปไหน ผมอยากไปเที่ยว')", + "", + "print(list(doc.sents))", + "# output: [ผมเป็นคนไทย แต่มะลิอยากไปโรงเรียนส่วนผมจะไปไหน , ผมอยากไปเที่ยว]" + ], + "code_language": "python", + "author": "Wannaphong Phatthiyaphaibun", + "author_links": { + "twitter": "@wannaphong_p", + "github": "wannaphong", + "website": "https://iam.wannaphong.com/" + }, + "category": [ + "pipeline", + "research" + ], + "tags": [ + "Thai" + ] + }, + { + "id": "vetiver", + "title": "Vetiver", + "slogan": "Version, share, deploy, and monitor models.", + "description": "The goal of vetiver is to provide fluent tooling to version, deploy, and monitor a trained model. Functions handle creating model objects, versioning models, predicting from a remote API endpoint, deploying Dockerfiles, and more.", + "github": "rstudio/vetiver-python", + "pip": "vetiver", + "code_example": [ + "import spacy", + "from vetiver import VetiverModel, VetiverAPI", + "", + "# If you use this model, you'll need to download it first:", + "# python -m spacy download en_core_web_md", + "nlp = spacy.load('en_core_web_md')", + "# Create deployable model object with your nlp Language object", + "v = VetiverModel(nlp, model_name = 'my_model')", + "# Try out your API endpoint locally", + "VetiverAPI(v).run()" + ], + "code_language": "python", + "url": "https://vetiver.rstudio.com/", + "thumb": "https://raw.githubusercontent.com/rstudio/vetiver-python/main/docs/figures/square-logo.svg", + "author": "Posit, PBC", + "author_links": { + "twitter": "posit_pbc", + "github": "rstudio", + "website": "https://posit.co/" + }, + "category": [ + "apis", + "standalone" + ], + "tags": [ + "apis", + "deployment" + ] + }, + { + "id": "span_marker", + "title": "SpanMarker", + "slogan": "Effortless state-of-the-art NER in spaCy", + "description": "The SpanMarker integration with spaCy allows you to seamlessly replace the default spaCy `\"ner\"` pipeline component with any [SpanMarker model available on the Hugging Face Hub](https://huggingface.co/models?library=span-marker). Through this, you can take advantage of the advanced Named Entity Recognition capabilities of SpanMarker within the familiar and powerful spaCy framework.\n\nBy default, the `span_marker` pipeline component uses a [SpanMarker model using RoBERTa-large trained on OntoNotes v5.0](https://huggingface.co/tomaarsen/span-marker-roberta-large-ontonotes5). This model reaches a competitive 91.54 F1, notably higher than the [85.5 and 89.8 F1](https://spacy.io/usage/facts-figures#section-benchmarks) from `en_core_web_lg` and `en_core_web_trf`, respectively. A short head-to-head between this SpanMarker model and the `trf` spaCy model has been posted [here](https://github.com/tomaarsen/SpanMarkerNER/pull/12).\n\nAdditionally, see [here](https://tomaarsen.github.io/SpanMarkerNER/notebooks/spacy_integration.html) for documentation on using SpanMarker with spaCy.", + "github": "tomaarsen/SpanMarkerNER", + "pip": "span_marker", + "code_example": [ + "import spacy", + "", + "nlp = spacy.load(\"en_core_web_sm\", exclude=[\"ner\"])", + "nlp.add_pipe(\"span_marker\", config={\"model\": \"tomaarsen/span-marker-roberta-large-ontonotes5\"})", + "", + "text = \"\"\"Cleopatra VII, also known as Cleopatra the Great, was the last active ruler of the \\", + "Ptolemaic Kingdom of Egypt. She was born in 69 BCE and ruled Egypt from 51 BCE until her \\", + "death in 30 BCE.\"\"\"", + "doc = nlp(text)", + "print([(entity, entity.label_) for entity in doc.ents])", + "# [(Cleopatra VII, \"PERSON\"), (Cleopatra the Great, \"PERSON\"), (the Ptolemaic Kingdom of Egypt, \"GPE\"),", + "# (69 BCE, \"DATE\"), (Egypt, \"GPE\"), (51 BCE, \"DATE\"), (30 BCE, \"DATE\")]" + ], + "code_language": "python", + "url": "https://tomaarsen.github.io/SpanMarkerNER", + "author": "Tom Aarsen", + "author_links": { + "github": "tomaarsen", + "website": "https://www.linkedin.com/in/tomaarsen" + }, + "category": [ + "pipeline", + "standalone", + "scientific" + ], + "tags": [ + "ner" + ] + }, + { + "id": "hobbit-spacy", + "title": "Hobbit spaCy", + "slogan": "NLP for Middle Earth", + "description": "Hobbit spaCy is a custom spaCy pipeline designed specifically for working with Middle Earth and texts from the world of J.R.R. Tolkien.", + "github": "wjbmattingly/hobbit-spacy", + "pip": "en-hobbit", + "code_example": [ + "import spacy", + "", + "nlp = spacy.load('en_hobbit')", + "doc = nlp('Frodo saw Glorfindel and Glóin; and in a corner alone Strider was sitting, clad in his old travel - worn clothes again')" + ], + "code_language": "python", + "thumb": "https://github.com/wjbmattingly/hobbit-spacy/blob/main/images/hobbit-thumbnail.png?raw=true", + "image": "https://github.com/wjbmattingly/hobbit-spacy/raw/main/images/hobbitspacy.png", + "author": "W.J.B. Mattingly", + "author_links": { + "twitter": "wjb_mattingly", + "github": "wjbmattingly", + "website": "https://wjbmattingly.com" + }, + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "spans", + "rules", + "ner" + ] + }, + { + "id": "rolegal", + "title": "A spaCy Package for Romanian Legal Document Processing", + "thumb": "https://raw.githubusercontent.com/senisioi/rolegal/main/img/paper200x200.jpeg", + "slogan": "rolegal: a spaCy Package for Noisy Romanian Legal Document Processing", + "description": "This is a spaCy language model for Romanian legal domain trained with floret 4-gram to 5-gram embeddings and `LEGAL` entity recognition. Useful for processing OCR-resulted noisy legal documents.", + "github": "senisioi/rolegal", + "pip": "ro-legal-fl", + "tags": [ + "legal", + "floret", + "ner", + "romanian" + ], + "code_example": [ + "import spacy", + "nlp = spacy.load(\"ro_legal_fl\")", + "", + "doc = nlp(\"Titlul III din LEGEA nr. 255 din 19 iulie 2013, publicată în MONITORUL OFICIAL\")", + "# legal entity identification", + "for entity in doc.ents:", + " print('entity: ', entity, '; entity type: ', entity.label_)", + "", + "# floret n-gram embeddings robust to typos", + "print(nlp('achizit1e public@').similarity(nlp('achiziții publice')))", + "# 0.7393895566928835", + "print(nlp('achizitii publice').similarity(nlp('achiziții publice')))", + "# 0.8996480808279399" + ], + "author": "Sergiu Nisioi", + "author_links": { + "github": "senisioi", + "website": "https://nlp.unibuc.ro/people/snisioi.html" + }, + "category": [ + "pipeline", + "training", + "models" + ] + }, + { + "id": "redfield-spacy-nodes", + "title": "Redfield NLP Nodes for KNIME", + "slogan": "Makes the functionality of the spaCy library available in KNIME Analytics Platform.", + "description": "This extension provides nodes that make the functionality of the spaCy library available in the [KNIME Analytics Platform](https://www.knime.com/).", + "github": "Redfield-AB/Spacy-Nodes", + "url": "https://redfield.ai/spacy-redfield/", + "thumb": "https://raw.githubusercontent.com/Redfield-AB/Spacy-Nodes/master/resource/redfield_logo_100x100.png", + "image": "https://raw.githubusercontent.com/Redfield-AB/Spacy-Nodes/master/resource/screen1.png", + "author": "Redfield AB", + "author_links": { + "twitter": "Redfield_AB", + "github": "Redfield-AB", + "website": "https://redfield.ai" + }, + "category": [ + "standalone" + ] + }, + { + "id": "quelquhui", + "title": "quelquhui", + "slogan": "Tokenizer for contemporary French", + "description": "A tokenizer for French that handles inword parentheses like in _(b)rouille_, inclusive language (won't split _relecteur.rice.s_,but will split _mais.maintenant_), hyphens (split _peut-on_, or _pouvons-vous_ but not _tubulu-pimpant_), apostrophes (split _j'arrive_ or _j'arrivons_, but not _aujourd'hui_ or _r'garder_), emoticons, text-emoji (_:happy:_), urls, mails and more.", + "github": "thjbdvlt/quelquhui", + "code_example": [ + "import spacy", + "import quelquhui", + "nlp = spacy.load('fr_core_news_lg')", + "nlp.tokenizer = quelquhui.Toquenizer(nlp.vocab)" + ], + "code_language": "python", + "author": "thjbdvlt", + "author_links": { + "github": "thjbdvlt" + }, + "category": [ + "pipeline" + ], + "tags": [ + "tokenizer", + "french" + ] + }, + { + "id": "gliner-spacy", + "title": "GLiNER spaCy Wrapper", + "slogan": "Integrating GLiNER's Advanced NER with spaCy", + "description": "GLiNER SpaCy Wrapper is a project that brings together GLiNER, a zero-shot Named Entity Recognition (NER) model, with spaCy's NLP capabilities. It provides an easy way to integrate GLiNER within the spaCy environment, thus enhancing NER tasks with GLiNER's features.", + "github": "theirstory/gliner-spacy", + "pip": "gliner-spacy", + "code_example": [ + "import spacy", + "", + "nlp = spacy.blank('en')", + "nlp.add_pipe('gliner_spacy')", + "text = 'This is a text about Bill Gates and Microsoft.'", + "doc = nlp(text)", + "", + "for ent in doc.ents:", + " print(ent.text, ent.label_)" + ], + "code_language": "python", + "url": "https://github.com/theirstory/gliner-spacy", + "author": "TheirStory", + "author_links": { + "website": "https://theirstory.io" + }, + "category": [ + "pipeline" + ], + "tags": [ + "NER" + ] + }, + { + "id": "presque", + "title": "presque", + "slogan": "Normalizer for contemporary French", + "description": "Normalizer for French with focus on online and informal communication, _peùUUUt-èTRE_ becomes _peut-être_, _voilaaaa_ becomes _voilà_. it also harmonizes inclusive language (the user can chose how): by default, _auteur-rice-s-x et relecteur.xrices_ becomes _auteur·ricexs et relecteur·ricexs_.", + "github": "thjbdvlt/presque", + "code_example": [ + "import spacy", + "import presque", + "", + "@spacy.Language.factory('presque_normalizer')", + "def create_presque_normalizer(nlp, name='presque_normalizer'):", + "return presque.Normalizer(nlp=nlp)", + "", + "nlp = spacy.load('fr_core_news_lg')", + "nlp.add_pipe('presque_normalizer', first=True)" + ], + "code_language": "python", + "author": "thjbdvlt", + "author_links": { + "github": "thjbdvlt" + }, + "category": [ + "pipeline" + ], + "tags": [ + "normalizer", + "french" + ] + }, + { + "id": "bagpipes-spacy", + "title": "Bagpipes spaCy", + "slogan": "A bag of custom spaCy pipes for various NLP tasks.", + "description": "Bagpipes spaCy is a versatile collection of custom spaCy pipeline components enhancing text processing capabilities. It includes functionalities such as phrase extraction, text normalization, triple detection, entity and sentence clustering, token clustering, and keyword extraction. These components augment NLP tasks with advanced processing and analysis features, offering a comprehensive toolkit for natural language data handling.", + "github": "wjbmattingly/bagpipes-spacy", + "pip": "bagpipes-spacy", + "code_example": [ + "import spacy", + "from bagpipes_spacy import PhrasesExtractor", + "nlp = spacy.load(\"en_core_web_md\")", + "nlp.add_pipe(\"phrases_extractor\")", + "text = 'Seconds later, he had climbed out onto a rather fine antique rug, brushing ash from the sleeves of his long pin-striped cloak, a lime-green bowler hat in his hand.'", + "doc = nlp(text)", + "print('Prepositional Phrases')", + "print(doc._.prep_phrases)", + "print('Noun Phrases')", + "print(doc._.noun_phrases)", + "print('Verb Phrases')", + "print(doc._.verb_phrases)", + "print('Adj Phrases')", + "print(doc._.adj_phrases)" + ], + "code_language": "python", + "url": "https://github.com/wjbmattingly/bagpipes-spacy", + "thumb": "https://github.com/wjbmattingly/bagpipes-spacy/raw/main/images/bagpipes-spacy-icon.png?raw=true", + "image": "https://github.com/wjbmattingly/bagpipes-spacy/raw/main/images/bagpipes-spacy-logo.png?raw=true", + "author": "W.J.B. Mattingly", + "author_links": { + "twitter": "wjb_mattingly", + "github": "wjbmattingly", + "website": "https://www.wjbmattingly.com" + }, + "category": [ + "pipeline" + ], + "tags": [ + "spacy", + "text processing", + "NLP", + "custom components" + ] + }, + { + "id": "number-spacy", + "title": "Number spaCy", + "slogan": "Enhancing Numeric Entity Recognition in Text with spaCy", + "description": "Number spaCy is a custom spaCy pipeline component that enhances the identification of number entities in text and fetches the parsed numeric values using spaCy's token extensions. It uses RegEx to identify number entities written in words and then leverages the [word2number](https://github.com/akshaynagpal/w2n) library to convert those words into structured numeric data. The output numeric value is stored in a custom entity extension: `._.number`. This lightweight component can be seamlessly added to an existing spaCy pipeline or integrated into a blank model. If using within an existing spaCy pipeline, ensure to insert it before the NER model.", + "github": "wjbmattingly/number-spacy", + "pip": "number-spacy", + "code_example": [ + "import spacy", + "from number_spacy import find_numbers", + "", + "nlp = spacy.blank('en')", + "nlp.add_pipe('find_numbers')", + "", + "doc = nlp('I have three apples. She gave me twenty-two more, and now I have twenty-five apples in total.')", + "", + "for ent in doc.ents:", + " if ent.label_ == 'NUMBER':", + " print(f'Text: {ent.text} -> Parsed Number: {ent._.number}')" + ], + "code_language": "python", + "url": "https://github.com/wjbmattingly/number-spacy", + "thumb": "https://github.com/wjbmattingly/number-spacy/raw/main/images/number-spacy-logo.png?raw=true", + "image": "https://github.com/wjbmattingly/number-spacy/raw/main/images/number-spacy-logo.png?raw=true", + "author": "W.J.B. Mattingly", + "author_links": { + "twitter": "wjb_mattingly", + "github": "wjbmattingly", + "website": "https://www.wjbmattingly.com" + }, + "category": [ + "pipeline" + ], + "tags": [ + "spacy", + "number", + "NLP", + "entity recognition" + ] + }, + { + "id": "spacy-annoy", + "title": "Spacy Annoy", + "slogan": "Integrating Spacy NLP and Annoy for Semantic Text Search with spaCy linguistic tags.", + "description": "Spacy Annoy offers a combination of Spacy's natural language processing (NLP) capabilities and Annoy's efficient similarity search algorithms. This Python class is tailored for analyzing and querying large text corpora, delivering results based on semantic similarity. Key features include contextual window chunking and controlled overlap with preservation of original context at the Doc level, allowing access to all original Spacy properties.", + "github": "wjbmattingly/spacy-annoy", + "pip": "spacy-annoy", + "code_example": [ + "from SpacyAnnoy import SpacyAnnoy", + "", + "# Initialize with a Spacy model name", + "sa = SpacyAnnoy('en_core_web_lg')", + "", + "texts = ['This is a text about sports', 'This is a text about dogs']*20", + "sa.load_docs(texts)", + "", + "sa.build_index(n_trees=10, metric='euclidean')", + "", + "# Query the index", + "results = sa.query_index('Dogs and cats.', depth=5)", + "", + "# Pretty print results", + "sa.pretty_print(results)", + "", + "# Accessing the Spacy span of the first result", + "first_result_span = results[0][0]" + ], + "author": "W.J.B. Mattingly", + "author_links": { + "twitter": "wjb_mattingly", + "github": "wjbmattingly", + "website": "https://wjbmattingly.com" + }, + "code_language": "python", + "url": "https://github.com/wjbmattingly/spacy-annoy", + "category": [ + "standalone" + ], + "tags": [ + "spacy", + "annoy", + "text analysis", + "semantic search" + ] + }, + { + "id": "spacy-whisper", + "title": "spaCy Whisper", + "slogan": "Seamless Integration of Whisper with spaCy NLP", + "description": "spaCy Whisper is a Python package designed for integrating Whisper transcriptions with spaCy's NLP capabilities. It provides users with the ability to process and analyze transcribed text using spaCy's features like tokenization, entity recognition, and part-of-speech tagging. Key features include word and segment level processing with custom attributes, as well as custom token, span, and document extensions, enriching the NLP analysis of transcribed texts.", + "github": "theirstory/spacy-whisper", + "pip": "spacy-whisper", + "code_example": [ + "from spacy_whisper import SpacyWhisper", + "import json", + "", + "# Load a Whisper Output (see repo for sample file):", + "with open('whisper_output.json', 'r', encoding='utf-8') as f:", + " whisper_output = json.load(f)", + "", + "# Initialize SpacyWhisper", + "sw = SpacyWhisper(lang='en', model='en_core_web_sm', segments_key='segments', word_level=True)", + "doc = sw.create_doc(whisper_output)", + "", + "# Access custom attributes", + "for token in doc:", + " print(token.text, token._.start_time, token._.end_time, token._.probability)" + ], + "code_language": "python", + "url": "https://github.com/theirstory/spacy-whisper", + "thumb": "https://github.com/theirstory/spacy-whisper/raw/main/images/spacy_whisper.jpeg", + "image": "https://github.com/theirstory/spacy-whisper/raw/main/images/spacy_whisper.jpeg", + "author": "TheirStory", + "author_links": { + "website": "https://theirstory.io" + }, + "category": [ + "standalone" + ], + "tags": [ + "spacy", + "whisper", + "transcription", + "nlp" + ] + }, + { + "id": "date-spacy", + "title": "Date spaCy", + "slogan": "Effortless Date Recognition in Text with spaCy", + "description": "Date spaCy is a spaCy pipeline component designed to identify and parse date entities in text effortlessly. It uses Regular Expressions (RegEx) to detect a wide range of date formats and leverages the 'dateparser' library for accurate conversion into structured datetime objects. Particularly useful in NLP tasks involving date information extraction, this component seamlessly integrates into existing or new spaCy pipelines. The tool assumes the current year for dates without a specified year, ensuring sensible defaults while maintaining flexibility. The parsed dates are stored in a custom entity extension, providing easy access and manipulation within spaCy's ecosystem. This makes Date spaCy a go-to solution for developers and data scientists dealing with temporal data in natural language.", + "github": "wjbmattingly/date-spacy", + "pip": "date-spacy", + "code_example": [ + "import spacy", + "from date_spacy import find_dates", + "", + "nlp = spacy.blank('en')", + "nlp.add_pipe('find_dates')", + "", + "doc = nlp(\"\"\"The event is scheduled for 25th August 2023.", + " We also have a meeting on 10 September and another one on the twelfth of October and a", + " final one on January fourth.\"\"\")", + "", + "for ent in doc.ents:", + " if ent.label_ == 'DATE':", + " print(f'Text: {ent.text} -> Parsed Date: {ent._.date}')" + ], + "code_language": "python", + "url": "https://github.com/wjbmattingly/date-spacy", + "thumb": "https://github.com/wjbmattingly/date-spacy/raw/main/images/date-spacy-logo.png?raw=true", + "image": "https://github.com/wjbmattingly/date-spacy/raw/main/images/date-spacy-logo.png?raw=true", + "author": "W.J.B. Mattingly", + "author_links": { + "twitter": "wjb_mattingly", + "github": "wjbmattingly", + "website": "https://wjbmattingly.com" + }, + "category": [ + "pipeline" + ], + "tags": [ + "dates", + "ner", + "nlp", + "spacy" + ] + } + ], + "categories": [ + { + "label": "Projects", + "items": [ + { + "id": "pipeline", + "title": "Pipeline", + "description": "Custom pipeline components and extensions" + }, + { + "id": "training", + "title": "Training", + "description": "Helpers and toolkits for training spaCy models" + }, + { + "id": "conversational", + "title": "Conversational", + "description": "Frameworks and utilities for working with conversational text, e.g. for chat bots" + }, + { + "id": "research", + "title": "Research", + "description": "Frameworks and utilities for developing better NLP models, especially using neural networks" + }, + { + "id": "scientific", + "title": "Scientific", + "description": "Frameworks and utilities for scientific text processing" + }, + { + "id": "biomedical", + "title": "Biomedical", + "description": "Frameworks and utilities for processing biomedical text" + }, + { + "id": "visualizers", + "title": "Visualizers", + "description": "Demos and tools to visualize NLP annotations or systems" + }, + { + "id": "apis", + "title": "Containers & APIs", + "description": "Infrastructure tools for managing or deploying spaCy" + }, + { + "id": "nonpython", + "title": "Non-Python", + "description": "Wrappers, bindings and implementations in other programming languages" + }, + { + "id": "standalone", + "title": "Standalone", + "description": "Self-contained libraries or tools that use spaCy under the hood" + }, + { + "id": "models", + "title": "Models", + "description": "Third-party pretrained models for different languages and domains" + } + ] + }, + { + "label": "Education", + "items": [ + { + "id": "books", + "title": "Books", + "description": "Books about or featuring spaCy" + }, + { + "id": "courses", + "title": "Courses", + "description": "Online courses and interactive tutorials" + }, + { + "id": "videos", + "title": "Videos", + "description": "Talks and tutorials in video format" + }, + { + "id": "podcasts", + "title": "Podcasts", + "description": "Episodes about spaCy or interviews with the spaCy team" + } + ] + } + ] +} diff --git a/website/netlify.toml b/website/netlify.toml new file mode 100644 index 0000000..a993959 --- /dev/null +++ b/website/netlify.toml @@ -0,0 +1,24 @@ +# Settings in the [build] context are global and are applied to +# all contexts unless otherwise overridden by more specific contexts. +[build] + # Directory that contains the deploy-ready HTML files and + # assets generated by the build. This is relative to the base + # directory if one has been set, or the root directory if + # a base has not been set. This sample publishes the directory + # located at the absolute path "root/project/build-output" + publish = "website/out" + + # Default build command. + command = "npm run build" + +[build.environment] +NETLIFY_NEXT_PLUGIN_SKIP = "true" + +[[plugins]] +package = "@netlify/plugin-nextjs" + +[[headers]] + for = "/*" + [headers.values] + X-Frame-Options = "DENY" + X-XSS-Protection = "1; mode=block" diff --git a/website/next-sitemap.config.mjs b/website/next-sitemap.config.mjs new file mode 100644 index 0000000..1d2a140 --- /dev/null +++ b/website/next-sitemap.config.mjs @@ -0,0 +1,10 @@ +import { siteUrl } from './meta/dynamicMeta.mjs' + +/** @type {import('next-sitemap').IConfig} */ +const config = { + siteUrl, + generateRobotsTxt: true, + autoLastmod: false, +} + +export default config diff --git a/website/next.config.mjs b/website/next.config.mjs new file mode 100644 index 0000000..5e2f8f8 --- /dev/null +++ b/website/next.config.mjs @@ -0,0 +1,41 @@ +import MDX from '@next/mdx' +import PWA from 'next-pwa' + +import remarkPlugins from './plugins/index.mjs' + +const withMDX = MDX({ + extension: /\.mdx?$/, + options: { + remarkPlugins, + providerImportSource: '@mdx-js/react', + }, + experimental: { + mdxRs: true, + }, +}) + +const withPWA = PWA({ + dest: 'public', + disable: process.env.NODE_ENV === 'development', +}) + +/** @type {import('next').NextConfig} */ +const nextConfig = withPWA( + withMDX({ + reactStrictMode: true, + swcMinify: true, + pageExtensions: ['js', 'jsx', 'ts', 'tsx', 'md', 'mdx'], + eslint: { + ignoreDuringBuilds: true, + }, + typescript: { + ignoreBuildErrors: true, + }, + images: { unoptimized: true }, + env: { + DOCSEARCH_API_KEY: process.env.DOCSEARCH_API_KEY + } + }) +) + +export default nextConfig diff --git a/website/package-lock.json b/website/package-lock.json new file mode 100644 index 0000000..7b89374 --- /dev/null +++ b/website/package-lock.json @@ -0,0 +1,21334 @@ +{ + "name": "next-blog", + "version": "0.1.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "next-blog", + "version": "0.1.0", + "dependencies": { + "@codemirror/lang-python": "^6.1.0", + "@docsearch/react": "^3.3.0", + "@jupyterlab/services": "^3.2.1", + "@lezer/highlight": "^1.1.3", + "@mapbox/rehype-prism": "^0.8.0", + "@mdx-js/loader": "^2.1.5", + "@mdx-js/react": "^2.1.5", + "@next/mdx": "^13.0.2", + "@rehooks/online-status": "^1.1.2", + "@types/node": "18.11.9", + "@types/react": "18.0.25", + "@types/react-dom": "18.0.8", + "@uiw/codemirror-themes": "^4.19.3", + "@uiw/react-codemirror": "^4.19.3", + "acorn": "^8.8.1", + "browser-monads": "^1.0.0", + "classnames": "^2.3.2", + "eslint": "8.27.0", + "eslint-config-next": "13.0.2", + "html-to-react": "^1.5.0", + "jinja-to-js": "^3.2.3", + "md-attr-parser": "^1.3.0", + "next": "13.0.2", + "next-mdx-remote": "^4.2.0", + "next-plausible": "^3.6.5", + "next-pwa": "^5.6.0", + "next-sitemap": "^3.1.32", + "node-fetch": "^2.6.7", + "parse-numeric-range": "^1.3.0", + "prettier": "^2.7.1", + "prismjs": "^1.29.0", + "prop-types": "^15.8.1", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-github-btn": "^1.4.0", + "react-inlinesvg": "^3.0.1", + "react-intersection-observer": "^9.4.0", + "remark": "^14.0.2", + "remark-gfm": "^3.0.1", + "remark-react": "^9.0.1", + "remark-smartypants": "^2.0.0", + "remark-unwrap-images": "^3.0.1", + "sass": "^1.56.1", + "typescript": "4.8.4", + "unist-util-visit": "^4.1.1", + "ws": "^8.11.0" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.7.2.tgz", + "integrity": "sha512-eclwUDC6qfApNnEfu1uWcL/rudQsn59tjEoUYZYE2JSXZrHLRjBUGMxiCoknobU2Pva8ejb0eRxpIYDtVVqdsw==", + "dependencies": { + "@algolia/autocomplete-shared": "1.7.2" + } + }, + "node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.7.2.tgz", + "integrity": "sha512-+RYEG6B0QiGGfRb2G3MtPfyrl0dALF3cQNTWBzBX6p5o01vCCGTTinAm2UKG3tfc2CnOMAtnPLkzNZyJUpnVJw==", + "dependencies": { + "@algolia/autocomplete-shared": "1.7.2" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.7.2.tgz", + "integrity": "sha512-QCckjiC7xXHIUaIL3ektBtjJ0w7tTA3iqKcAE/Hjn1lZ5omp7i3Y4e09rAr9ZybqirL7AbxCLLq0Ra5DDPKeug==" + }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.14.2.tgz", + "integrity": "sha512-FRweBkK/ywO+GKYfAWbrepewQsPTIEirhi1BdykX9mxvBPtGNKccYAxvGdDCumU1jL4r3cayio4psfzKMejBlA==", + "dependencies": { + "@algolia/cache-common": "4.14.2" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.14.2.tgz", + "integrity": "sha512-SbvAlG9VqNanCErr44q6lEKD2qoK4XtFNx9Qn8FK26ePCI8I9yU7pYB+eM/cZdS9SzQCRJBbHUumVr4bsQ4uxg==" + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.14.2.tgz", + "integrity": "sha512-HrOukWoop9XB/VFojPv1R5SVXowgI56T9pmezd/djh2JnVN/vXswhXV51RKy4nCpqxyHt/aGFSq2qkDvj6KiuQ==", + "dependencies": { + "@algolia/cache-common": "4.14.2" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.14.2.tgz", + "integrity": "sha512-WHtriQqGyibbb/Rx71YY43T0cXqyelEU0lB2QMBRXvD2X0iyeGl4qMxocgEIcbHyK7uqE7hKgjT8aBrHqhgc1w==", + "dependencies": { + "@algolia/client-common": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.14.2.tgz", + "integrity": "sha512-yBvBv2mw+HX5a+aeR0dkvUbFZsiC4FKSnfqk9rrfX+QrlNOKEhCG0tJzjiOggRW4EcNqRmaTULIYvIzQVL2KYQ==", + "dependencies": { + "@algolia/client-common": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.14.2.tgz", + "integrity": "sha512-43o4fslNLcktgtDMVaT5XwlzsDPzlqvqesRi4MjQz2x4/Sxm7zYg5LRYFol1BIhG6EwxKvSUq8HcC/KxJu3J0Q==", + "dependencies": { + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.14.2.tgz", + "integrity": "sha512-ACCoLi0cL8CBZ1W/2juehSltrw2iqsQBnfiu/Rbl9W2yE6o2ZUb97+sqN/jBqYNQBS+o0ekTMKNkQjHHAcEXNw==", + "dependencies": { + "@algolia/client-common": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.14.2.tgz", + "integrity": "sha512-L5zScdOmcZ6NGiVbLKTvP02UbxZ0njd5Vq9nJAmPFtjffUSOGEp11BmD2oMJ5QvARgx2XbX4KzTTNS5ECYIMWw==", + "dependencies": { + "@algolia/client-common": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.14.2.tgz", + "integrity": "sha512-/JGlYvdV++IcMHBnVFsqEisTiOeEr6cUJtpjz8zc0A9c31JrtLm318Njc72p14Pnkw3A/5lHHh+QxpJ6WFTmsA==" + }, + "node_modules/@algolia/logger-console": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.14.2.tgz", + "integrity": "sha512-8S2PlpdshbkwlLCSAB5f8c91xyc84VM9Ar9EdfE9UmX+NrKNYnWR1maXXVDQQoto07G1Ol/tYFnFVhUZq0xV/g==", + "dependencies": { + "@algolia/logger-common": "4.14.2" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.14.2.tgz", + "integrity": "sha512-CEh//xYz/WfxHFh7pcMjQNWgpl4wFB85lUMRyVwaDPibNzQRVcV33YS+63fShFWc2+42YEipFGH2iPzlpszmDw==", + "dependencies": { + "@algolia/requester-common": "4.14.2" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.14.2.tgz", + "integrity": "sha512-73YQsBOKa5fvVV3My7iZHu1sUqmjjfs9TteFWwPwDmnad7T0VTCopttcsM3OjLxZFtBnX61Xxl2T2gmG2O4ehg==" + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.14.2.tgz", + "integrity": "sha512-oDbb02kd1o5GTEld4pETlPZLY0e+gOSWjWMJHWTgDXbv9rm/o2cF7japO6Vj1ENnrqWvLBmW1OzV9g6FUFhFXg==", + "dependencies": { + "@algolia/requester-common": "4.14.2" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.14.2.tgz", + "integrity": "sha512-t89dfQb2T9MFQHidjHcfhh6iGMNwvuKUvojAj+JsrHAGbuSy7yE4BylhLX6R0Q1xYRoC4Vvv+O5qIw/LdnQfsQ==", + "dependencies": { + "@algolia/cache-common": "4.14.2", + "@algolia/logger-common": "4.14.2", + "@algolia/requester-common": "4.14.2" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "dependencies": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@ampproject/remapping/node_modules/@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dependencies": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "dependencies": { + "@babel/highlight": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.5.tgz", + "integrity": "sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.5.tgz", + "integrity": "sha512-UdOWmk4pNWTm/4DlPUl/Pt4Gz4rcEMb7CY0Y3eJl5Yz1vI8ZJGmHWaVE55LoxRjdpx0z259GE9U5STA9atUinQ==", + "dependencies": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.20.5", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-module-transforms": "^7.20.2", + "@babel/helpers": "^7.20.5", + "@babel/parser": "^7.20.5", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.20.5.tgz", + "integrity": "sha512-jl7JY2Ykn9S0yj4DQP82sYvPU+T3g0HFcWTqDLqiuA9tGRNIj9VfbtXGAYTTkyNEnQk1jkMGOdYka8aG/lulCA==", + "dependencies": { + "@babel/types": "^7.20.5", + "@jridgewell/gen-mapping": "^0.3.2", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.9.tgz", + "integrity": "sha512-yFQ0YCHoIqarl8BCRwBL8ulYUaZpz3bNsA7oFepAzee+8/+ImtADXNOmO5vJvsPff3qi+hvpkY/NYBTrBQgdNw==", + "dependencies": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.0.tgz", + "integrity": "sha512-0jp//vDGp9e8hZzBc6N/KwA5ZK3Wsm/pfm4CrY7vzegkVxc65SgSn6wYOnwHe9Js9HRQ1YTCKLGPzDtaS3RoLQ==", + "dependencies": { + "@babel/compat-data": "^7.20.0", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.21.3", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.20.5.tgz", + "integrity": "sha512-3RCdA/EmEaikrhayahwToF0fpweU/8o2p8vhc1c/1kftHOdTKuC65kik/TLc+qfbS8JKw4qqJbne4ovICDhmww==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-member-expression-to-functions": "^7.18.9", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.19.1", + "@babel/helper-split-export-declaration": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.20.5.tgz", + "integrity": "sha512-m68B1lkg3XDGX5yCvGO0kPx3v9WIYLnzjKfPcQiwntEQa5ZeRkPmo2X/ISJc8qxWGfwUr+kvZAeEzAwLec2r2w==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.2.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-define-polyfill-provider": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.3.tgz", + "integrity": "sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==", + "dependencies": { + "@babel/helper-compilation-targets": "^7.17.7", + "@babel/helper-plugin-utils": "^7.16.7", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + }, + "peerDependencies": { + "@babel/core": "^7.4.0-0" + } + }, + "node_modules/@babel/helper-define-polyfill-provider/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-environment-visitor": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz", + "integrity": "sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz", + "integrity": "sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w==", + "dependencies": { + "@babel/template": "^7.18.10", + "@babel/types": "^7.19.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.9.tgz", + "integrity": "sha512-RxifAh2ZoVU67PyKIO4AMi1wTenGfMR/O/ae0CCRqwgBAt5v7xjdtRw7UoSbsreKrQn5t7r89eruK/9JjYHuDg==", + "dependencies": { + "@babel/types": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.2.tgz", + "integrity": "sha512-zvBKyJXRbmK07XhMuujYoJ48B5yvvmM6+wcpv6Ivj4Yg6qO7NOZOSnvZN9CRl1zz1Z4cKf8YejmCMh8clOoOeA==", + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.20.2", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.19.1", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.1", + "@babel/types": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz", + "integrity": "sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.9.tgz", + "integrity": "sha512-dI7q50YKd8BAv3VEfgg7PS7yD3Rtbi2J1XMXaalXO0W0164hYLnh8zpjRS0mte9MfVp/tltvr/cfdXPvJr1opA==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-wrap-function": "^7.18.9", + "@babel/types": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.19.1", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.19.1.tgz", + "integrity": "sha512-T7ahH7wV0Hfs46SFh5Jz3s0B6+o8g3c+7TMxu7xKfmHikg7EAZ3I2Qk9LFhjxXq8sL7UkP5JflezNwoZa8WvWw==", + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-member-expression-to-functions": "^7.18.9", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.19.1", + "@babel/types": "^7.19.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz", + "integrity": "sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==", + "dependencies": { + "@babel/types": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz", + "integrity": "sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==", + "dependencies": { + "@babel/types": "^7.20.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.19.4", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz", + "integrity": "sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.19.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz", + "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.20.5.tgz", + "integrity": "sha512-bYMxIWK5mh+TgXGVqAtnu5Yn1un+v8DDZtqyzKRLUzrh70Eal2O3aZ7aPYiMADO4uKlkzOiRiZ6GX5q3qxvW9Q==", + "dependencies": { + "@babel/helper-function-name": "^7.19.0", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.20.6.tgz", + "integrity": "sha512-Pf/OjgfgFRW5bApskEz5pvidpim7tEDPlFtKcNRXWmfHGn9IEI2W2flqRQXTFb7gIPTyK++N6rVHuwKut4XK6w==", + "dependencies": { + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "dependencies": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.20.5.tgz", + "integrity": "sha512-r27t/cy/m9uKLXQNWWebeCUHgnAZq0CpG1OwKRxzJMP1vpSU4bSIK2hq+/cp0bQxetkXx38n09rNu8jVkcK/zA==", + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.9.tgz", + "integrity": "sha512-AHrP9jadvH7qlOj6PINbgSuphjQUAK7AOT7DPjBo9EHoLhQTnnK5u45e1Hd4DbSQEO9nqPWtQ89r+XEOWFScKg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9", + "@babel/plugin-proposal-optional-chaining": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.13.0" + } + }, + "node_modules/@babel/plugin-proposal-async-generator-functions": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.20.1.tgz", + "integrity": "sha512-Gh5rchzSwE4kC+o/6T8waD0WHEQIsDmjltY8WnWRXHUdH8axZhuH86Ov9M72YhJfDrZseQwuuWaaIT/TmePp3g==", + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-remap-async-to-generator": "^7.18.9", + "@babel/plugin-syntax-async-generators": "^7.8.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0" + } + }, + "node_modules/@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz", + "integrity": "sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.9.tgz", + "integrity": "sha512-128YbMpjCrP35IOExw2Fq+x55LMP42DzhOhX2aNNIdI9avSWl2PI0yuBWarr3RYpZBSPtabfadkH2yeRiMD61Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-object-rest-spread": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.2.tgz", + "integrity": "sha512-Ks6uej9WFK+fvIMesSqbAto5dD8Dz4VuuFvGJFKgIGSkJuRGcrwGECPA1fDgQK3/DbExBJpEkTeYeB8geIFCSQ==", + "dependencies": { + "@babel/compat-data": "^7.20.1", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.20.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.9.tgz", + "integrity": "sha512-v5nwt4IqBXihxGsW2QmCWMDS3B3bzGIk/EQVZz2ei7f3NJl8NzAJVvUmpDW5q1CRNY+Beb/k58UAH1Km1N411w==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-property-in-object": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.20.5.tgz", + "integrity": "sha512-Vq7b9dUA12ByzB4EjQTPo25sFhY+08pQDBSZRtUAkj7lb7jahaHR5igera16QZ+3my1nYR4dKsNdYj5IjPHilQ==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.20.5", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.20.0.tgz", + "integrity": "sha512-IUh1vakzNoWalR8ch/areW7qFopR2AEw03JlG7BbrDqmQ4X3q9uuipQwSGrUn7oGiemKjtSLDhNtQHzMHr1JdQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.19.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "dependencies": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.20.5.tgz", + "integrity": "sha512-WvpEIW9Cbj9ApF3yJCjIEEf1EiNJLtXagOrL5LNWEZOo3jv8pmPoYTSNJQvqej8OavVlgOoOPw6/htGZro6IkA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.20.2.tgz", + "integrity": "sha512-9rbPp0lCVVoagvtEyQKSo5L8oo0nQS/iif+lwlAz29MccX2642vWDlSZK+2T2buxbopotId2ld7zZAzRfz9j1g==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-replace-supers": "^7.19.1", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-classes/node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.9.tgz", + "integrity": "sha512-+i0ZU1bCDymKakLxn5srGHrsAPRELC2WIbzwjLhHW9SIE1cPYkLCL0NlnXMZaM1vhfgA2+M7hySk42VBvrkBRw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.2.tgz", + "integrity": "sha512-mENM+ZHrvEgxLTBXUiQ621rRXZes3KWUv6NdQlrnr1TkWVw+hUjQBZuP2X32qKlrlG2BzgR95gkuCRSkJl8vIw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.9.tgz", + "integrity": "sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "dependencies": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.18.8", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.8.tgz", + "integrity": "sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.9.tgz", + "integrity": "sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ==", + "dependencies": { + "@babel/helper-compilation-targets": "^7.18.9", + "@babel/helper-function-name": "^7.18.9", + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.9.tgz", + "integrity": "sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.19.6.tgz", + "integrity": "sha512-uG3od2mXvAtIFQIh0xrpLH6r5fpSQN04gIVovl+ODLdUMANokxQLZnPBHcjmv3GxRjnqwLuHvppjjcelqUFZvg==", + "dependencies": { + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.19.6.tgz", + "integrity": "sha512-8PIa1ym4XRTKuSsOUXqDG0YaOlEuTVvHMe5JCfgBMOtHvJKw/4NGovEGN33viISshG/rZNVrACiBmPQLvWN8xQ==", + "dependencies": { + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-simple-access": "^7.19.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.19.6.tgz", + "integrity": "sha512-fqGLBepcc3kErfR9R3DnVpURmckXP7gj7bAlrTQyBxrigFqszZCkFkcoxzCp2v32XmwXLvbw+8Yq9/b+QqksjQ==", + "dependencies": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-validator-identifier": "^7.19.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.20.5.tgz", + "integrity": "sha512-mOW4tTzi5iTLnw+78iEq3gr8Aoq4WNRGpmSlrogqaiCBoR1HFhpU4JkpQFOHfeYx3ReVIFWOQJS4aZBRvuZ6mA==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.20.5", + "@babel/helper-plugin-utils": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.5.tgz", + "integrity": "sha512-h7plkOmcndIUWXZFLgpbrh2+fXAi47zcUX7IrOQuZdLD0I0KvjJ6cvo3BEcAOsDOcZhVKGJqv07mkSqK0y2isQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.20.5.tgz", + "integrity": "sha512-kW/oO7HPBtntbsahzQ0qSE3tFvkFwnbozz3NWFhLGqH75vLEg+sCGngLlhVkePlCs3Jv0dBBHDzCHxNiFAQKCQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2", + "regenerator-transform": "^0.15.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.19.0.tgz", + "integrity": "sha512-RsuMk7j6n+r752EtzyScnWkQyuJdli6LdO5Klv8Yx0OfPVTcQkIUfS8clx5e9yHXzlnhOZF3CbQ8C2uP5j074w==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.9.tgz", + "integrity": "sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.9.tgz", + "integrity": "sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-escapes": { + "version": "7.18.10", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.10.tgz", + "integrity": "sha512-kKAdAI+YzPgGY/ftStBFXTI1LZFju38rYThnfMykS+IXy8BVx+res7s2fxf1l8I35DV2T97ezo6+SGrXz6B3iQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.20.2.tgz", + "integrity": "sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==", + "dependencies": { + "@babel/compat-data": "^7.20.1", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.9", + "@babel/plugin-proposal-async-generator-functions": "^7.20.1", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.9", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.9", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.20.2", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.9", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.20.0", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.20.2", + "@babel/plugin-transform-classes": "^7.20.2", + "@babel/plugin-transform-computed-properties": "^7.18.9", + "@babel/plugin-transform-destructuring": "^7.20.2", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.9", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.8", + "@babel/plugin-transform-function-name": "^7.18.9", + "@babel/plugin-transform-literals": "^7.18.9", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.19.6", + "@babel/plugin-transform-modules-commonjs": "^7.19.6", + "@babel/plugin-transform-modules-systemjs": "^7.19.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.19.1", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.20.1", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.19.0", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.9", + "@babel/plugin-transform-typeof-symbol": "^7.18.9", + "@babel/plugin-transform-unicode-escapes": "^7.18.10", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.20.2", + "babel-plugin-polyfill-corejs2": "^0.3.3", + "babel-plugin-polyfill-corejs3": "^0.6.0", + "babel-plugin-polyfill-regenerator": "^0.4.1", + "core-js-compat": "^3.25.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.1.tgz", + "integrity": "sha512-mrzLkl6U9YLF8qpqI7TB82PESyEGjm/0Ly91jG575eVxMMlb8fYfOXFZIJ8XfLrJZQbm7dlKry2bJmXBUEkdFg==", + "dependencies": { + "regenerator-runtime": "^0.13.10" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.20.1.tgz", + "integrity": "sha512-CGulbEDcg/ND1Im7fUNRZdGXmX2MTWVVZacQi/6DiKE5HNwZ3aVTm5PV4lO8HHz0B2h8WQyvKKjbX5XgTtydsg==", + "dependencies": { + "core-js-pure": "^3.25.1", + "regenerator-runtime": "^0.13.10" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.18.10", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.10.tgz", + "integrity": "sha512-TI+rCtooWHr3QJ27kJxfjutghu44DLnasDMwpDqCXVTal9RLp3RSYNh4NdBrRP2cQAoG9A8juOQl6P6oZG4JxA==", + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.10", + "@babel/types": "^7.18.10" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.20.5.tgz", + "integrity": "sha512-WM5ZNN3JITQIq9tFZaw1ojLU3WgWdtkxnhM1AegMS+PvHjkM5IXjmYEGY7yukz5XS4sJyEf2VzWjI8uAavhxBQ==", + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.20.5", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.20.5", + "@babel/types": "^7.20.5", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse/node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/types": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.20.5.tgz", + "integrity": "sha512-c9fst/h2/dcF7H+MJKZ2T0KjEQ8hY/BNnDk/H3XY8C4Aw/eWQXWn/lWntHF9ooUBnGmEvbfGrTgLWc+um0YDUg==", + "dependencies": { + "@babel/helper-string-parser": "^7.19.4", + "@babel/helper-validator-identifier": "^7.19.1", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@codemirror/autocomplete": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.4.0.tgz", + "integrity": "sha512-HLF2PnZAm1s4kGs30EiqKMgD7XsYaQ0XJnMR0rofEWQ5t5D60SfqpDIkIh1ze5tiEbyUWm8+VJ6W1/erVvBMIA==", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.6.0", + "@lezer/common": "^1.0.0" + }, + "peerDependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.1.2.tgz", + "integrity": "sha512-sO3jdX1s0pam6lIdeSJLMN3DQ6mPEbM4yLvyKkdqtmd/UDwhXA5+AwFJ89rRXm6vTeOXBsE5cAmlos/t7MJdgg==", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/lang-python": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/lang-python/-/lang-python-6.1.0.tgz", + "integrity": "sha512-a/JhyPYn5qz5T8WtAfZCuAZcfClgNVb7UZzdLr76bWUeG7Usd3Un5o8UQOkZ/5Xw+EM5YGHHG+T6ickMYkDcRQ==", + "dependencies": { + "@codemirror/autocomplete": "^6.3.2", + "@codemirror/language": "^6.0.0", + "@lezer/python": "^1.0.0" + } + }, + "node_modules/@codemirror/language": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.3.1.tgz", + "integrity": "sha512-MK+G1QKaGfSEUg9YEFaBkMBI6j1ge4VMBPZv9fDYotw7w695c42x5Ba1mmwBkesYnzYFBfte6Hh9TDcKa6xORQ==", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.1.0.tgz", + "integrity": "sha512-mdvDQrjRmYPvQ3WrzF6Ewaao+NWERYtpthJvoQ3tK3t/44Ynhk8ZGjTSL9jMEv8CgSMogmt75X8ceOZRDSXHtQ==", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/search": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.2.3.tgz", + "integrity": "sha512-V9n9233lopQhB1dyjsBK2Wc1i+8hcCqxl1wQ46c5HWWLePoe4FluV3TGHoZ04rBRlGjNyz9DTmpJErig8UE4jw==", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.1.4.tgz", + "integrity": "sha512-g+3OJuRylV5qsXuuhrc6Cvs1NQluNioepYMM2fhnpYkNk7NgX+j0AFuevKSVKzTDmDyt9+Puju+zPdHNECzCNQ==" + }, + "node_modules/@codemirror/theme-one-dark": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/theme-one-dark/-/theme-one-dark-6.1.0.tgz", + "integrity": "sha512-AiTHtFRu8+vWT9wWUWDM+cog6ZwgivJogB1Tm/g40NIpLwph7AnmxrSzWfvJN5fBVufsuwBxecQCNmdcR5D7Aw==", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/highlight": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.7.1.tgz", + "integrity": "sha512-kYtS+uqYw/q/0ytYxpkqE1JVuK5NsbmBklWYhwLFTKO9gVuTdh/kDEeZPKorbqHcJ+P+ucrhcsS1czVweOpT2g==", + "dependencies": { + "@codemirror/state": "^6.1.4", + "style-mod": "^4.0.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@corex/deepmerge": { + "version": "4.0.29", + "resolved": "https://registry.npmjs.org/@corex/deepmerge/-/deepmerge-4.0.29.tgz", + "integrity": "sha512-q/yVUnqckA8Do+EvAfpy7RLdumnBy9ZsducMUtZTvpdbJC7azEf1hGtnYYxm0QfphYxjwggv6XtH64prvS1W+A==" + }, + "node_modules/@docsearch/css": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.3.0.tgz", + "integrity": "sha512-rODCdDtGyudLj+Va8b6w6Y85KE85bXRsps/R4Yjwt5vueXKXZQKYw0aA9knxLBT6a/bI/GMrAcmCR75KYOM6hg==" + }, + "node_modules/@docsearch/react": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.3.0.tgz", + "integrity": "sha512-fhS5adZkae2SSdMYEMVg6pxI5a/cE+tW16ki1V0/ur4Fdok3hBRkmN/H8VvlXnxzggkQIIRIVvYPn00JPjen3A==", + "dependencies": { + "@algolia/autocomplete-core": "1.7.2", + "@algolia/autocomplete-preset-algolia": "1.7.2", + "@docsearch/css": "3.3.0", + "algoliasearch": "^4.0.0" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 19.0.0", + "react": ">= 16.8.0 < 19.0.0", + "react-dom": ">= 16.8.0 < 19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@eslint/eslintrc": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", + "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.4.0", + "globals": "^13.15.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.7", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.7.tgz", + "integrity": "sha512-kBbPWzN8oVMLb0hOUYXhmxggL/1cJE6ydvjDIGi9EnAGUyA7cLVKQg+d/Dsm+KZwx2czGHrCmMVLiyg8s5JPKw==", + "dependencies": { + "@humanwhocodes/object-schema": "^1.2.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", + "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==" + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.17", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz", + "integrity": "sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==", + "dependencies": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, + "node_modules/@jupyterlab/coreutils": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@jupyterlab/coreutils/-/coreutils-2.2.1.tgz", + "integrity": "sha512-XkGMBXqEAnENC4fK/L3uEqqxyNUtf4TI/1XNDln7d5VOPHQJSBTbYlBAZ0AQotn2qbs4WqmV6icxje2ITVedqQ==", + "dependencies": { + "@phosphor/algorithm": "^1.1.2", + "@phosphor/coreutils": "^1.3.0", + "@phosphor/disposable": "^1.1.2", + "@phosphor/signaling": "^1.2.2", + "ajv": "~5.1.6", + "comment-json": "^1.1.3", + "minimist": "~1.2.0", + "moment": "~2.21.0", + "path-posix": "~1.0.0", + "url-parse": "~1.4.3" + } + }, + "node_modules/@jupyterlab/coreutils/node_modules/ajv": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.1.6.tgz", + "integrity": "sha512-K/r7dMp3q7rKzhx6v6deMuxVuQCw0w/789F75BooHOOVBEXzejwUq3LwO4x41C/xzXNKSNzqoAAS48Sx2a2Qxg==", + "dependencies": { + "co": "^4.6.0", + "json-schema-traverse": "^0.3.0", + "json-stable-stringify": "^1.0.1" + } + }, + "node_modules/@jupyterlab/coreutils/node_modules/comment-json": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-1.1.3.tgz", + "integrity": "sha512-gGSHSON1HdpFDb3WpU6C1tdksZV0lpfzoihXK+CD0l4xQ2TvHbVuiZy8Dg414ZqbeRxt2sU8d9kC0srpM2+RzQ==", + "dependencies": { + "json-parser": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@jupyterlab/coreutils/node_modules/json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha512-4JD/Ivzg7PoW8NzdrBSr3UFwC9mHgvI7Z6z3QGBsSHgKaRTUDmyZAAKJo2UbG1kUVfS9WS8bi36N49U1xw43DA==" + }, + "node_modules/@jupyterlab/observables": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@jupyterlab/observables/-/observables-2.4.0.tgz", + "integrity": "sha512-M/fhAnPqd6F4Zwt4IIsvHCkJmwbSw1Tko/hUXgdUQG86lPsJiTOh98sB3qwV1gtzb9oFF+kH21XsHnQZ6Yl6Pw==", + "dependencies": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/coreutils": "^1.3.1", + "@phosphor/disposable": "^1.3.0", + "@phosphor/messaging": "^1.3.0", + "@phosphor/signaling": "^1.3.0" + } + }, + "node_modules/@jupyterlab/services": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@jupyterlab/services/-/services-3.2.1.tgz", + "integrity": "sha512-zCMruGGYxTe427ESK4YUO1V/liFOFYpebYPRsJ+j9CFdV+Hm760+nx4pFX6N6Z9TbS+5cs8BgZ+ebm8unRZrJg==", + "dependencies": { + "@jupyterlab/coreutils": "^2.2.1", + "@jupyterlab/observables": "^2.1.1", + "@phosphor/algorithm": "^1.1.2", + "@phosphor/coreutils": "^1.3.0", + "@phosphor/disposable": "^1.1.2", + "@phosphor/signaling": "^1.2.2" + } + }, + "node_modules/@lezer/common": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz", + "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng==" + }, + "node_modules/@lezer/highlight": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz", + "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.2.5.tgz", + "integrity": "sha512-f9319YG1A/3ysgUE3bqCHEd7g+3ZZ71MWlwEc42mpnLVYXgfJJgtu1XAyBB4Kz8FmqmnFe9caopDqKeMMMAU6g==", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/python": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@lezer/python/-/python-1.1.1.tgz", + "integrity": "sha512-ArUGh9kvdaOVu6IkSaYUS9WFQeMAFVWKRuZo6vexnxoeCLnxf0Y9DCFEAMMa7W9SQBGYE55OarSpPqSkdOXSCA==", + "dependencies": { + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@mapbox/hast-util-table-cell-style": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@mapbox/hast-util-table-cell-style/-/hast-util-table-cell-style-0.2.0.tgz", + "integrity": "sha512-gqaTIGC8My3LVSnU38IwjHVKJC94HSonjvFHDk8/aSrApL8v4uWgm8zJkK7MJIIbHuNOr/+Mv2KkQKcxs6LEZA==", + "dependencies": { + "unist-util-visit": "^1.4.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@mapbox/hast-util-table-cell-style/node_modules/unist-util-is": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-3.0.0.tgz", + "integrity": "sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==" + }, + "node_modules/@mapbox/hast-util-table-cell-style/node_modules/unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "dependencies": { + "unist-util-visit-parents": "^2.0.0" + } + }, + "node_modules/@mapbox/hast-util-table-cell-style/node_modules/unist-util-visit-parents": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", + "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", + "dependencies": { + "unist-util-is": "^3.0.0" + } + }, + "node_modules/@mapbox/rehype-prism": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.8.0.tgz", + "integrity": "sha512-bIz4a3oZ8g+pQBHSMMIxNpYOMX4yq4aZdpezoiCl9yJudh1Z9SRmHQqH6f+WtTBNOORzXMEGkvBze4PiH17wtA==", + "dependencies": { + "hast-util-to-string": "^1.0.4", + "mrm": "^3.0.9", + "refractor": "^3.4.0", + "unist-util-visit": "^2.0.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mapbox/rehype-prism/node_modules/unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mapbox/rehype-prism/node_modules/unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mapbox/rehype-prism/node_modules/unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mdx-js/loader": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/loader/-/loader-2.1.5.tgz", + "integrity": "sha512-oXjfTa/iAcMmW8DdQ+hQFodPCLqw5VKT2yoZkLwrZfPVVpUgKrI+5/ZePYq328xxtAUStZmR3ed0PWJrwd5Pkg==", + "dependencies": { + "@mdx-js/mdx": "^2.0.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "webpack": ">=4" + } + }, + "node_modules/@mdx-js/mdx": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-2.1.5.tgz", + "integrity": "sha512-zEG0lt+Bl/r5U6e0TOS7qDbsXICtemfAPquxWFsMbdzrvlWaqMGemLl+sjVpqlyaaiCiGVQBSGdCk0t1qXjkQg==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/mdx": "^2.0.0", + "estree-util-build-jsx": "^2.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "estree-util-to-js": "^1.1.0", + "estree-walker": "^3.0.0", + "hast-util-to-estree": "^2.0.0", + "markdown-extensions": "^1.0.0", + "periscopic": "^3.0.0", + "remark-mdx": "^2.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "unified": "^10.0.0", + "unist-util-position-from-estree": "^1.0.0", + "unist-util-stringify-position": "^3.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/@mdx-js/react": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-2.1.5.tgz", + "integrity": "sha512-3Az1I6SAWA9R38rYjz5rXBrGKeZhq96CSSyQtqY+maPj8stBsoUH5pNcmIixuGkufYsh8F5+ka2CVPo2fycWZw==", + "dependencies": { + "@types/mdx": "^2.0.0", + "@types/react": ">=16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "react": ">=16" + } + }, + "node_modules/@next/env": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/env/-/env-13.0.2.tgz", + "integrity": "sha512-Qb6WPuRriGIQ19qd6NBxpcrFOfj8ziN7l9eZUfwff5gl4zLXluqtuZPddYZM/oWjN53ZYcuRXzL+oowKyJeYtA==" + }, + "node_modules/@next/eslint-plugin-next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-13.0.2.tgz", + "integrity": "sha512-W+fIIIaFU7Kct7Okx91C7XDRGolv/w2RUenX2yZFeeNVcuVzDIKUcNmckrYbYcwrNQUSXmtwrs3g8xwast0YtA==", + "dependencies": { + "glob": "7.1.7" + } + }, + "node_modules/@next/mdx": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/mdx/-/mdx-13.0.2.tgz", + "integrity": "sha512-udtSlWoOLoHdJvtuGyj3ctXW4fogsQDWU2YXp4Sx4JdsS7VKvJkhTU0Df47nAJsQKCJhQ92h+j7rA+1Dulxxqg==", + "dependencies": { + "source-map": "^0.7.0" + }, + "peerDependencies": { + "@mdx-js/loader": ">=0.15.0", + "@mdx-js/react": "*" + } + }, + "node_modules/@next/swc-android-arm-eabi": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-13.0.2.tgz", + "integrity": "sha512-X54UQCTFyOGnJP//Z71dPPlp4BCYcQL2ncikKXQcPzVpqPs4C3m+tKC8ivBNH6edAXkppwsLRz1/yQwgSZ9Swg==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-android-arm64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-13.0.2.tgz", + "integrity": "sha512-1P00Kv8uKaLubqo7JzPrTqgFAzSOmfb8iwqJrOb9in5IvTRtNGlkR4hU0sXzqbQNM/+SaYxze6Z5ry1IDyb/cQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-arm64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.0.2.tgz", + "integrity": "sha512-1zGIOkInkOLRv0QQGZ+3wffYsyKI4vIy62LYTvDWUn7TAYqnmXwougp9NSLqDeagLwgsv2URrykyAFixA/YqxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.0.2.tgz", + "integrity": "sha512-ECDAjoMP1Y90cARaelS6X+k6BQx+MikAYJ8f/eaJrLur44NIOYc9HA/dgcTp5jenguY4yT8V+HCquLjAVle6fA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-freebsd-x64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-13.0.2.tgz", + "integrity": "sha512-2DcL/ofQdBnQX3IoI9sjlIAyLCD1oZoUBuhrhWbejvBQjutWrI0JTEv9uG69WcxWhVMm3BCsjv8GK2/68OKp7A==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm-gnueabihf": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-13.0.2.tgz", + "integrity": "sha512-Y3OQF1CSBSWW2vGkmvOIuOUNqOq8qX7f1ZpcKUVWP3/Uq++DZmVi9d18lgnSe1I3QFqc+nXWyun9ljsN83j0sw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.0.2.tgz", + "integrity": "sha512-mNyzwsFF6kwZYEjnGicx9ksDZYEZvyzEc1BtCu8vdZi/v8UeixQwCiAT6FyYX9uxMPEkzk8qiU0t0u9gvltsKw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.0.2.tgz", + "integrity": "sha512-M6SdYjWgRrY3tJBxz0663zCRPTu5BRONmxlftKWWHv9LjAJ59neTLaGj4rp0A08DkJglZIoCkLOzLrzST6TGag==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.0.2.tgz", + "integrity": "sha512-pi63RoxvG4ES1KS06Zpm0MATVIXTs/TIbLbdckeLoM40u1d3mQl/+hSSrLRSxzc2OtyL8fh92sM4gkJrQXAMAw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.0.2.tgz", + "integrity": "sha512-9Pv91gfYnDONgjtRm78n64b/c54+azeHtlnqBLTnIFWSMBDRl1/WDkhKWIj3fBGPLimtK7Tko3ULR3og9RRUPw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.0.2.tgz", + "integrity": "sha512-Nvewe6YZaizAkGHHprbMkYqQulBjZCHKBGKeFPwoPtOA+a2Qi4pZzc/qXFyC5/2A6Z0mr2U1zg9rd04WBYMwBw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.0.2.tgz", + "integrity": "sha512-ZUBYGZw5G3QrqDpRq1EWi3aHmvPZM8ijK5TFL6UbH16cYQ0JpANmuG2P66KB93Qe/lWWzbeAZk/tj1XqwoCuPA==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.0.2.tgz", + "integrity": "sha512-fA9uW1dm7C0mEYGcKlbmLcVm2sKcye+1kPxh2cM4jVR+kQQMtHWsjIzeSpe2grQLSDan06z4n6hbr8b1c3hA8w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@phosphor/algorithm": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", + "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" + }, + "node_modules/@phosphor/collections": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@phosphor/collections/-/collections-1.2.0.tgz", + "integrity": "sha512-T9/0EjSuY6+ga2LIFRZ0xupciOR3Qnyy8Q95lhGTC0FXZUFwC8fl9e8On6IcwasCszS+1n8dtZUWSIynfgdpzw==", + "dependencies": { + "@phosphor/algorithm": "^1.2.0" + } + }, + "node_modules/@phosphor/coreutils": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/coreutils/-/coreutils-1.3.1.tgz", + "integrity": "sha512-9OHCn8LYRcPU/sbHm5v7viCA16Uev3gbdkwqoQqlV+EiauDHl70jmeL7XVDXdigl66Dz0LI11C99XOxp+s3zOA==" + }, + "node_modules/@phosphor/disposable": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.1.tgz", + "integrity": "sha512-0NGzoTXTOizWizK/brKKd5EjJhuuEH4903tLika7q6wl/u0tgneJlTh7R+MBVeih0iNxtuJAfBa3IEY6Qmj+Sw==", + "dependencies": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/signaling": "^1.3.1" + } + }, + "node_modules/@phosphor/messaging": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@phosphor/messaging/-/messaging-1.3.0.tgz", + "integrity": "sha512-k0JE+BTMKlkM335S2AmmJxoYYNRwOdW5jKBqLgjJdGRvUQkM0+2i60ahM45+J23atGJDv9esKUUBINiKHFhLew==", + "dependencies": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/collections": "^1.2.0" + } + }, + "node_modules/@phosphor/signaling": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.1.tgz", + "integrity": "sha512-Eq3wVCPQAhUd9+gUGaYygMr+ov7dhSGblSBXiDzpZlSIfa8OVD4P3cCvYXr/acDTNmZ/gHTcSFO8/n3rDkeXzg==", + "dependencies": { + "@phosphor/algorithm": "^1.2.0" + } + }, + "node_modules/@rehooks/online-status": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@rehooks/online-status/-/online-status-1.1.2.tgz", + "integrity": "sha512-tuz6RtdgqkBMZr82UqTkzPCsHNx6tpEyhTZ5OD8WhDpU4/TLzljBCseuWLlbeQiygbM9F2A4Ea7P+wyjohJXBw==" + }, + "node_modules/@rollup/plugin-babel": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz", + "integrity": "sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q==", + "dependencies": { + "@babel/helper-module-imports": "^7.10.4", + "@rollup/pluginutils": "^3.1.0" + }, + "engines": { + "node": ">= 10.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0", + "@types/babel__core": "^7.1.9", + "rollup": "^1.20.0||^2.0.0" + }, + "peerDependenciesMeta": { + "@types/babel__core": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-node-resolve": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz", + "integrity": "sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==", + "dependencies": { + "@rollup/pluginutils": "^3.1.0", + "@types/resolve": "1.17.1", + "builtin-modules": "^3.1.0", + "deepmerge": "^4.2.2", + "is-module": "^1.0.0", + "resolve": "^1.19.0" + }, + "engines": { + "node": ">= 10.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0" + } + }, + "node_modules/@rollup/plugin-replace": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-2.4.2.tgz", + "integrity": "sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==", + "dependencies": { + "@rollup/pluginutils": "^3.1.0", + "magic-string": "^0.25.7" + }, + "peerDependencies": { + "rollup": "^1.20.0 || ^2.0.0" + } + }, + "node_modules/@rollup/pluginutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.1.0.tgz", + "integrity": "sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==", + "dependencies": { + "@types/estree": "0.0.39", + "estree-walker": "^1.0.1", + "picomatch": "^2.2.2" + }, + "engines": { + "node": ">= 8.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0" + } + }, + "node_modules/@rollup/pluginutils/node_modules/@types/estree": { + "version": "0.0.39", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", + "integrity": "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==" + }, + "node_modules/@rollup/pluginutils/node_modules/estree-walker": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-1.0.1.tgz", + "integrity": "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg==" + }, + "node_modules/@rushstack/eslint-patch": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz", + "integrity": "sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==" + }, + "node_modules/@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/@surma/rollup-plugin-off-main-thread": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz", + "integrity": "sha512-lR8q/9W7hZpMWweNiAKU7NQerBnzQQLvi8qnTDU/fxItPhtZVMbPV3lbCwjhIlNBe9Bbr5V+KHshvWmVSG9cxQ==", + "dependencies": { + "ejs": "^3.1.6", + "json5": "^2.2.0", + "magic-string": "^0.25.0", + "string.prototype.matchall": "^4.0.6" + } + }, + "node_modules/@surma/rollup-plugin-off-main-thread/node_modules/json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@swc/helpers": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.4.11.tgz", + "integrity": "sha512-rEUrBSGIoSFuYxwBYtlUFMlE2CwGhmW+w9355/5oduSw8e5h2+Tj4UrAGNNgP9915++wj5vkQo0UuOBqOAq4nw==", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dependencies": { + "defer-to-connect": "^1.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@types/acorn": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz", + "integrity": "sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/debug": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", + "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/eslint": { + "version": "8.4.10", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.10.tgz", + "integrity": "sha512-Sl/HOqN8NKPmhWo2VBEPm0nvHnu2LL3v9vKo8MEq0EtbJ4eVzGPl41VNPvn5E1i5poMk4/XD8UriLHpJvEP/Nw==", + "peer": true, + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz", + "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==", + "peer": true, + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.0.tgz", + "integrity": "sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ==" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.0.tgz", + "integrity": "sha512-3qvGd0z8F2ENTGr/GG1yViqfiKmRfrXVx5sJyHGFu3z7m5g5utCQtGp/g29JnjflhtQJBv1WDQukHiT58xPcYQ==", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "node_modules/@types/hast": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz", + "integrity": "sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" + }, + "node_modules/@types/json-schema": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==" + }, + "node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==" + }, + "node_modules/@types/mdast": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz", + "integrity": "sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", + "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==" + }, + "node_modules/@types/mdx": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.3.tgz", + "integrity": "sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==" + }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" + }, + "node_modules/@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==" + }, + "node_modules/@types/nlcst": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/nlcst/-/nlcst-1.0.0.tgz", + "integrity": "sha512-3TGCfOcy8R8mMQ4CNSNOe3PG66HttvjcLzCoOpvXvDtfWOTi+uT/rxeOKm/qEwbM4SNe1O/PjdiBK2YcTjU4OQ==", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/node": { + "version": "18.11.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", + "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + }, + "node_modules/@types/prop-types": { + "version": "15.7.5", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", + "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" + }, + "node_modules/@types/react": { + "version": "18.0.25", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.25.tgz", + "integrity": "sha512-xD6c0KDT4m7n9uD4ZHi02lzskaiqcBxf4zi+tXZY98a04wvc0hi/TcCPC2FOESZi51Nd7tlUeOJY8RofL799/g==", + "dependencies": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.0.8", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.8.tgz", + "integrity": "sha512-C3GYO0HLaOkk9dDAz3Dl4sbe4AKUGTCfFIZsz3n/82dPNN8Du533HzKatDxeUYWu24wJgMP1xICqkWk1YOLOIw==", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/resolve": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", + "integrity": "sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/scheduler": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", + "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" + }, + "node_modules/@types/trusted-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz", + "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==" + }, + "node_modules/@types/unist": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz", + "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==" + }, + "node_modules/@typescript-eslint/parser": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.42.1.tgz", + "integrity": "sha512-kAV+NiNBWVQDY9gDJDToTE/NO8BHi4f6b7zTsVAJoTkmB/zlfOpiEVBzHOKtlgTndCKe8vj9F/PuolemZSh50Q==", + "dependencies": { + "@typescript-eslint/scope-manager": "5.42.1", + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/typescript-estree": "5.42.1", + "debug": "^4.3.4" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.42.1.tgz", + "integrity": "sha512-QAZY/CBP1Emx4rzxurgqj3rUinfsh/6mvuKbLNMfJMMKYLRBfweus8brgXF8f64ABkIZ3zdj2/rYYtF8eiuksQ==", + "dependencies": { + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/visitor-keys": "5.42.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.42.1.tgz", + "integrity": "sha512-Qrco9dsFF5lhalz+lLFtxs3ui1/YfC6NdXu+RAGBa8uSfn01cjO7ssCsjIsUs484vny9Xm699FSKwpkCcqwWwA==", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.42.1.tgz", + "integrity": "sha512-qElc0bDOuO0B8wDhhW4mYVgi/LZL+igPwXtV87n69/kYC/7NG3MES0jHxJNCr4EP7kY1XVsRy8C/u3DYeTKQmw==", + "dependencies": { + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/visitor-keys": "5.42.1", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "semver": "^7.3.7", + "tsutils": "^3.21.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.42.1.tgz", + "integrity": "sha512-LOQtSF4z+hejmpUvitPlc4hA7ERGoj2BVkesOcG91HCn8edLGUXbTrErmutmPbl8Bo9HjAvOO/zBKQHExXNA2A==", + "dependencies": { + "@typescript-eslint/types": "5.42.1", + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@uiw/codemirror-extensions-basic-setup": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/codemirror-extensions-basic-setup/-/codemirror-extensions-basic-setup-4.19.3.tgz", + "integrity": "sha512-Itt/cgaw2g3facr52DSdkcKDZPTyW7M2YGjfSbjYC5Xaap0R31cClR09UZTEokxZmE4yYq7QIs3DTeURExxXzw==", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + }, + "peerDependencies": { + "@codemirror/autocomplete": ">=6.0.0", + "@codemirror/commands": ">=6.0.0", + "@codemirror/language": ">=6.0.0", + "@codemirror/lint": ">=6.0.0", + "@codemirror/search": ">=6.0.0", + "@codemirror/state": ">=6.0.0", + "@codemirror/view": ">=6.0.0" + } + }, + "node_modules/@uiw/codemirror-themes": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/codemirror-themes/-/codemirror-themes-4.19.3.tgz", + "integrity": "sha512-2mqrrYZnUL/hseEeIYC0DBP4igst/w5g5JT1+3v+yMyvwsvSAWxxN1Zeh5FFMfVTZWjpYujOuoJ4V507taorBw==", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + }, + "peerDependencies": { + "@codemirror/language": ">=6.0.0", + "@codemirror/state": ">=6.0.0", + "@codemirror/view": ">=6.0.0" + } + }, + "node_modules/@uiw/react-codemirror": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/react-codemirror/-/react-codemirror-4.19.3.tgz", + "integrity": "sha512-/Rg7U0CjPpsKd1vxve9he86MMQLVDz4N6fSBGOIx7R+6UPBoDcUOPlcmLRFUQvmP+H5Xyk5IBygeGPUR7IEPog==", + "dependencies": { + "@babel/runtime": "^7.18.6", + "@codemirror/commands": "^6.1.0", + "@codemirror/state": "^6.1.1", + "@codemirror/theme-one-dark": "^6.0.0", + "@uiw/codemirror-extensions-basic-setup": "4.19.3", + "codemirror": "^6.0.0" + }, + "peerDependencies": { + "@babel/runtime": ">=7.11.0", + "@codemirror/state": ">=6.0.0", + "@codemirror/theme-one-dark": ">=6.0.0", + "@codemirror/view": ">=6.0.0", + "codemirror": ">=6.0.0", + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", + "peer": true, + "dependencies": { + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", + "peer": true + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", + "peer": true + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", + "peer": true + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", + "peer": true, + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", + "peer": true + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", + "peer": true, + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", + "peer": true, + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", + "peer": true + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", + "peer": true, + "dependencies": { + "@webassemblyjs/ast": "1.11.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "peer": true + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "peer": true + }, + "node_modules/acorn": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-assertions": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz", + "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==", + "peer": true, + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "peerDependencies": { + "ajv": "^6.9.1" + } + }, + "node_modules/algoliasearch": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.14.2.tgz", + "integrity": "sha512-ngbEQonGEmf8dyEh5f+uOIihv4176dgbuOZspiuhmTTBRBuzWu3KCGHre6uHj5YyuC7pNvQGzB6ZNJyZi0z+Sg==", + "dependencies": { + "@algolia/cache-browser-local-storage": "4.14.2", + "@algolia/cache-common": "4.14.2", + "@algolia/cache-in-memory": "4.14.2", + "@algolia/client-account": "4.14.2", + "@algolia/client-analytics": "4.14.2", + "@algolia/client-common": "4.14.2", + "@algolia/client-personalization": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/logger-common": "4.14.2", + "@algolia/logger-console": "4.14.2", + "@algolia/requester-browser-xhr": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/requester-node-http": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/aria-query": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz", + "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==", + "dependencies": { + "@babel/runtime": "^7.10.2", + "@babel/runtime-corejs3": "^7.10.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/array-includes": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.6.tgz", + "integrity": "sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "get-intrinsic": "^1.1.3", + "is-string": "^1.0.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array-iterate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/array-iterate/-/array-iterate-2.0.1.tgz", + "integrity": "sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array.prototype.flat": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", + "integrity": "sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "es-shim-unscopables": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.flatmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz", + "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "es-shim-unscopables": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ast-types-flow": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", + "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==" + }, + "node_modules/astring": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.8.3.tgz", + "integrity": "sha512-sRpyiNrx2dEYIMmUXprS8nlpRg2Drs8m9ElX9vVEXaCB4XEAJhKfs7IcX0IwShjuOAjLR6wzIrgoptz1n19i1A==", + "bin": { + "astring": "bin/astring" + } + }, + "node_modules/async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "node_modules/at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/axe-core": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.5.1.tgz", + "integrity": "sha512-1exVbW0X1O/HSr/WMwnaweyqcWOgZgLiVxdLG34pvSQk4NlYQr9OUy0JLwuhFfuVNQzzqgH57eYzkFBCb3bIsQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/axobject-query": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz", + "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==" + }, + "node_modules/babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha512-XqYMR2dfdGMW+hd0IUZ2PwK+fGeFkOxZJ0wY+JaQAHzt1Zx8LcvpiZD2NiGkEG8qx0CfkAOr5xt76d1e8vG90g==", + "dependencies": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + } + }, + "node_modules/babel-code-frame/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/babel-code-frame/node_modules/js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha512-RjTcuD4xjtthQkaWH7dFlH85L+QaVtSoOyGdZ3g6HFhS9dFNDfLyqgm2NFe2X6cQpeFmt0452FJjFG5UameExg==" + }, + "node_modules/babel-code-frame/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/babel-loader": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.3.0.tgz", + "integrity": "sha512-H8SvsMF+m9t15HNLMipppzkC+Y2Yq+v3SonZyU70RBL/h1gxPkH08Ot8pEE9Z4Kd+czyWJClmFS8qzIP9OZ04Q==", + "dependencies": { + "find-cache-dir": "^3.3.1", + "loader-utils": "^2.0.0", + "make-dir": "^3.1.0", + "schema-utils": "^2.6.5" + }, + "engines": { + "node": ">= 8.9" + }, + "peerDependencies": { + "@babel/core": "^7.0.0", + "webpack": ">=2" + } + }, + "node_modules/babel-loader/node_modules/schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "dependencies": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 8.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/babel-plugin-polyfill-corejs2": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz", + "integrity": "sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q==", + "dependencies": { + "@babel/compat-data": "^7.17.7", + "@babel/helper-define-polyfill-provider": "^0.3.3", + "semver": "^6.1.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/babel-plugin-polyfill-corejs3": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.6.0.tgz", + "integrity": "sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.3", + "core-js-compat": "^3.25.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-regenerator": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.4.1.tgz", + "integrity": "sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/boxen": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "dependencies": { + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boxen/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/boxen/node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browser-monads": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-monads/-/browser-monads-1.0.0.tgz", + "integrity": "sha512-NgmpFaI9NtRKyzHYA+xVzPA5Bu+JgCBzgC3WJCXstlajCBB3fJTieBwImdG8JepK8GctHSnia866ceFLg1LLTQ==", + "dependencies": { + "nothing-mock": "^1.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "node_modules/builtin-modules": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", + "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/builtins": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz", + "integrity": "sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==" + }, + "node_modules/cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacheable-request/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cacheable-request/node_modules/lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001431", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz", + "integrity": "sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] + }, + "node_modules/capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==", + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "peer": true, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==" + }, + "node_modules/classnames": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" + }, + "node_modules/clean-webpack-plugin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/clean-webpack-plugin/-/clean-webpack-plugin-4.0.0.tgz", + "integrity": "sha512-WuWE1nyTNAyW5T7oNyys2EN0cfP2fdRxhxnIQWiAp0bMabPdHhoGxM8A6YL2GhqwgrPnnaemVE7nv5XJ2Fhh2w==", + "dependencies": { + "del": "^4.1.1" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "webpack": ">=4.0.0 <6.0.0" + } + }, + "node_modules/cli-boxes": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" + }, + "node_modules/cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dependencies": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/clone-response": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", + "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", + "dependencies": { + "mimic-response": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/codemirror": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz", + "integrity": "sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "node_modules/comment-json": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-2.4.2.tgz", + "integrity": "sha512-T+iXox779qsqneMYx/x5BZyz4xjCeQRmuNVzz8tko7qZUs3MlzpA3RAs+O1XsgcKToNBMIvfVzafGOeiU7RggA==", + "dependencies": { + "core-util-is": "^1.0.2", + "esprima": "^4.0.1", + "has-own-prop": "^2.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dependencies": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==" + }, + "node_modules/core-js-compat": { + "version": "3.26.1", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.26.1.tgz", + "integrity": "sha512-622/KzTudvXCDLRw70iHW4KKs1aGpcRcowGWyYJr2DEBfRrd6hNJybxSWJFuZYD4ma86xhrwDDHxmDaIq4EA8A==", + "dependencies": { + "browserslist": "^4.21.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-pure": { + "version": "3.26.0", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.26.0.tgz", + "integrity": "sha512-LiN6fylpVBVwT8twhhluD9TzXmZQQsr2I2eIKtWNbZI1XMfBT7CV18itaN6RA7EtQd/SDdRx/wzvAShX2HvhQA==", + "hasInstallScript": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "dependencies": { + "capture-stack-trace": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/crelt": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.5.tgz", + "integrity": "sha512-+BO9wPPi+DWTDcNYhr/W90myha8ptzftZT+LwcmUbbok0rcP/fequmFYCw8NMoH7pkAZQzU78b3kYrlua5a9eA==" + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/csstype": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", + "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==" + }, + "node_modules/damerau-levenshtein": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", + "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==" + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "dependencies": { + "mimic-response": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" + }, + "node_modules/deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" + }, + "node_modules/define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dependencies": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/del": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", + "dependencies": { + "@types/glob": "^7.1.1", + "globby": "^6.1.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/del/node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del/node_modules/globby": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "integrity": "sha512-KVbFv2TQtbzCoxAnfD6JcHZTYCzyliEaaeM/gH8qQdkKr5s0OP9scEgvdcngyk7AVdY6YVW/TJHd+lQ/Df3Daw==", + "dependencies": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del/node_modules/globby/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/del/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-indent": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", + "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz", + "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.1" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-prop": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-5.0.1.tgz", + "integrity": "sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==", + "engines": { + "node": ">=4.6.0" + } + }, + "node_modules/duplexer3": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz", + "integrity": "sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==" + }, + "node_modules/editorconfig": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", + "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", + "dependencies": { + "commander": "^2.19.0", + "lru-cache": "^4.1.5", + "semver": "^5.6.0", + "sigmund": "^1.0.1" + }, + "bin": { + "editorconfig": "bin/editorconfig" + } + }, + "node_modules/editorconfig/node_modules/lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dependencies": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "node_modules/editorconfig/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/editorconfig/node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + }, + "node_modules/ejs": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", + "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", + "peer": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", + "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-abstract": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.4.tgz", + "integrity": "sha512-0UtvRN79eMe2L+UNEF1BwRe364sj/DXhQ/k5FmivgoSdpM90b8Jc0mDzKMGo7QS0BVbOP/bTwBKNnDc9rNzaPA==", + "dependencies": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "function.prototype.name": "^1.1.5", + "get-intrinsic": "^1.1.3", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.2", + "object-keys": "^1.1.1", + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.4.3", + "safe-regex-test": "^1.0.0", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-module-lexer": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz", + "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==", + "peer": true + }, + "node_modules/es-shim-unscopables": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz", + "integrity": "sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w==", + "dependencies": { + "has": "^1.0.3" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.27.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.27.0.tgz", + "integrity": "sha512-0y1bfG2ho7mty+SiILVf9PfuRA49ek4Nc60Wmmu62QlobNR+CeXa4xXIJgcuwSQgZiWaPH+5BDsctpIW0PR/wQ==", + "dependencies": { + "@eslint/eslintrc": "^1.3.3", + "@humanwhocodes/config-array": "^0.11.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.1.1", + "eslint-utils": "^3.0.0", + "eslint-visitor-keys": "^3.3.0", + "espree": "^9.4.0", + "esquery": "^1.4.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.15.0", + "grapheme-splitter": "^1.0.4", + "ignore": "^5.2.0", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-sdsl": "^4.1.4", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "regexpp": "^3.2.0", + "strip-ansi": "^6.0.1", + "strip-json-comments": "^3.1.0", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-config-next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-13.0.2.tgz", + "integrity": "sha512-SrrHp+zBDYLjOFZdM5b9aW/pliK687Xxfa+qpDuL08Z04ReHhmz3L+maXaAqgrEVZHQximP7nh0El4yNDJW+CA==", + "dependencies": { + "@next/eslint-plugin-next": "13.0.2", + "@rushstack/eslint-patch": "^1.1.3", + "@typescript-eslint/parser": "^5.21.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-import-resolver-typescript": "^2.7.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-jsx-a11y": "^6.5.1", + "eslint-plugin-react": "^7.31.7", + "eslint-plugin-react-hooks": "^4.5.0" + }, + "peerDependencies": { + "eslint": "^7.23.0 || ^8.0.0", + "typescript": ">=3.3.1" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", + "dependencies": { + "debug": "^3.2.7", + "resolve": "^1.20.0" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint-import-resolver-typescript": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-2.7.1.tgz", + "integrity": "sha512-00UbgGwV8bSgUv34igBDbTOtKhqoRMy9bFjNehT40bXg6585PNIct8HhXZ0SybqB9rWtXj9crcku8ndDn/gIqQ==", + "dependencies": { + "debug": "^4.3.4", + "glob": "^7.2.0", + "is-glob": "^4.0.3", + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "*", + "eslint-plugin-import": "*" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/eslint-module-utils": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz", + "integrity": "sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA==", + "dependencies": { + "debug": "^3.2.7" + }, + "engines": { + "node": ">=4" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/eslint-module-utils/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/eslint-plugin-import": { + "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", + "dependencies": { + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", + "debug": "^2.6.9", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.7.3", + "has": "^1.0.3", + "is-core-module": "^2.8.1", + "is-glob": "^4.0.3", + "minimatch": "^3.1.2", + "object.values": "^1.1.5", + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^2 || ^3 || ^4 || ^5 || ^6 || ^7.2.0 || ^8" + } + }, + "node_modules/eslint-plugin-import/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/eslint-plugin-jsx-a11y": { + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", + "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", + "dependencies": { + "@babel/runtime": "^7.18.9", + "aria-query": "^4.2.2", + "array-includes": "^3.1.5", + "ast-types-flow": "^0.0.7", + "axe-core": "^4.4.3", + "axobject-query": "^2.2.0", + "damerau-levenshtein": "^1.0.8", + "emoji-regex": "^9.2.2", + "has": "^1.0.3", + "jsx-ast-utils": "^3.3.2", + "language-tags": "^1.0.5", + "minimatch": "^3.1.2", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=4.0" + }, + "peerDependencies": { + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" + } + }, + "node_modules/eslint-plugin-jsx-a11y/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/eslint-plugin-react": { + "version": "7.31.10", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.31.10.tgz", + "integrity": "sha512-e4N/nc6AAlg4UKW/mXeYWd3R++qUano5/o+t+wnWxIf+bLsOaH3a4q74kX3nDjYym3VBN4HyO9nEn1GcAqgQOA==", + "dependencies": { + "array-includes": "^3.1.5", + "array.prototype.flatmap": "^1.3.0", + "doctrine": "^2.1.0", + "estraverse": "^5.3.0", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.1.2", + "object.entries": "^1.1.5", + "object.fromentries": "^2.0.5", + "object.hasown": "^1.1.1", + "object.values": "^1.1.5", + "prop-types": "^15.8.1", + "resolve": "^2.0.0-next.3", + "semver": "^6.3.0", + "string.prototype.matchall": "^4.0.7" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz", + "integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==", + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/eslint-plugin-react/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-react/node_modules/resolve": { + "version": "2.0.0-next.4", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.4.tgz", + "integrity": "sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ==", + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/eslint-plugin-react/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/eslint-scope": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", + "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/eslint-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", + "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", + "dependencies": { + "eslint-visitor-keys": "^2.0.0" + }, + "engines": { + "node": "^10.0.0 || ^12.0.0 || >= 14.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + }, + "peerDependencies": { + "eslint": ">=5" + } + }, + "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "engines": { + "node": ">=10" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", + "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/espree": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", + "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "dependencies": { + "acorn": "^8.8.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-util-attach-comments": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-2.1.0.tgz", + "integrity": "sha512-rJz6I4L0GaXYtHpoMScgDIwM0/Vwbu5shbMeER596rB2D1EWF6+Gj0e0UKzJPZrpoOc87+Q2kgVFHfjAymIqmw==", + "dependencies": { + "@types/estree": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-build-jsx": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-2.2.0.tgz", + "integrity": "sha512-apsfRxF9uLrqosApvHVtYZjISPvTJ+lBiIydpC+9wE6cF6ssbhnjyQLqaIjgzGxvC2Hbmec1M7g91PoBayYoQQ==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "estree-walker": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-2.0.1.tgz", + "integrity": "sha512-rxZj1GkQhY4x1j/CSnybK9cGuMFQYFPLq0iNyopqf14aOVLFtMv7Esika+ObJWPWiOHuMOAHz3YkWoLYYRnzWQ==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-to-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-1.1.0.tgz", + "integrity": "sha512-490lbfCcpLk+ofK6HCgqDfYs4KAfq6QVvDw3+Bm1YoKRgiOjKiKYGAVQE1uwh7zVxBgWhqp4FDtp5SqunpUk1A==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-1.2.0.tgz", + "integrity": "sha512-wdsoqhWueuJKsh5hqLw3j8lwFqNStm92VcwtAOAny8g/KS/l5Y8RISjR4k5W6skCj3Nirag/WUCMS0Nfy3sgsg==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-walker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.1.tgz", + "integrity": "sha512-woY0RUD87WzMBUiZLx8NsYr23N5BKsOMZHhu2hoNRVh6NXGfoiT1KOL8G3UHlJAnEDGmfa5ubNA/AacfG+Kb0g==" + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "peer": true, + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha512-RztN09XglpYI7aBBrJCPW95jEH7YF1UEPOoX9yDhUTPdp7mK+CQvnLTuD10BNXZ3byLTu2uehZ8EcKT/4CGiFw==", + "dependencies": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa/node_modules/cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", + "dependencies": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/execa/node_modules/lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dependencies": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "node_modules/execa/node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa/node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/execa/node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + }, + "node_modules/exenv": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/exenv/-/exenv-1.2.2.tgz", + "integrity": "sha512-Z+ktTxTwv9ILfgKCk32OX3n/doe+OcLTRtqK9pcL+JsP3J1/VW8Uvl4ZjLlKqeW4rzK4oesDOGMEMRIZqtP4Iw==" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" + }, + "node_modules/fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/figures/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.1.tgz", + "integrity": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/avajs/find-cache-dir?sponsor=1" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "dependencies": { + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" + }, + "node_modules/fs-exists-sync": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz", + "integrity": "sha512-cR/vflFyPZtrN6b38ZyWxpWdhlXrzZEBawlpBQMq7033xVY7/kg0GDMBK5jg8lDYQckdJ5x/YC88lM3C7VMsLg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/function.prototype.name": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", + "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.0", + "functions-have-names": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-own-enumerable-property-symbols": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", + "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==" + }, + "node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/git-config-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/git-config-path/-/git-config-path-1.0.1.tgz", + "integrity": "sha512-KcJ2dlrrP5DbBnYIZ2nlikALfRhKzNSX0stvv3ImJ+fvC4hXKoV+U+74SV0upg+jlQZbrtQzc0bu6/Zh+7aQbg==", + "dependencies": { + "extend-shallow": "^2.0.1", + "fs-exists-sync": "^0.1.0", + "homedir-polyfill": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/git-username": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/git-username/-/git-username-1.0.0.tgz", + "integrity": "sha512-xm45KwBR6Eu1jO4umx/o2M84v9TC7tdOBuzLx8ayhdR9H1FBiiG9azz31uC0esDvaWVBTDINpJ5USomk+ja8OQ==", + "dependencies": { + "parse-github-url": "^1.0.2", + "remote-origin-url": "^1.0.0" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/github-buttons": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/github-buttons/-/github-buttons-2.22.1.tgz", + "integrity": "sha512-937dpW009lbV8p1gg1SoUvUnJBnfJKYU6CEJcsdoLnBzI4YP7Y8oT/M0O7WTQwfErTYLCG9t0W1huaexSLx7yA==" + }, + "node_modules/glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "peer": true + }, + "node_modules/global-dirs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.1.0.tgz", + "integrity": "sha512-MG6kdOUh/xBnyo9cJFeIKkLEc1AyFq42QTU4XiX51i2NEdxLxLWXIjEjmqKeSuKR7pAZjTqUVoT2b2huxVLgYQ==", + "dependencies": { + "ini": "1.3.7" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/global-dirs/node_modules/ini": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==" + }, + "node_modules/globals": { + "version": "13.17.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", + "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dependencies": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + }, + "node_modules/grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==" + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-ansi/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-bigints": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-own-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-own-prop/-/has-own-prop-2.0.0.tgz", + "integrity": "sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dependencies": { + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/hast-to-hyperscript": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/hast-to-hyperscript/-/hast-to-hyperscript-10.0.1.tgz", + "integrity": "sha512-dhIVGoKCQVewFi+vz3Vt567E4ejMppS1haBRL6TEmeLeJVB1i/FJIIg/e6s1Bwn0g5qtYojHEKvyGA+OZuyifw==", + "dependencies": { + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.3.0", + "unist-util-is": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-parse-selector": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", + "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-sanitize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-sanitize/-/hast-util-sanitize-4.0.0.tgz", + "integrity": "sha512-pw56+69jq+QSr/coADNvWTmBPDy+XsmwaF5KnUys4/wM1jt/fZdl7GPxhXXXYdXnz3Gj3qMkbUCH2uKjvX0MgQ==", + "dependencies": { + "@types/hast": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-estree": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-2.1.0.tgz", + "integrity": "sha512-Vwch1etMRmm89xGgz+voWXvVHba2iiMdGMKmaMfYt35rbVtFDq8JNwwAIvi8zHMkO6Gvqo9oTMwJTmzVRfXh4g==", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "estree-util-attach-comments": "^2.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "mdast-util-mdx-expression": "^1.0.0", + "mdast-util-mdxjs-esm": "^1.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.3.0", + "unist-util-position": "^4.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-string": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz", + "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz", + "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", + "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==", + "dependencies": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^1.0.0", + "hast-util-parse-selector": "^2.0.0", + "property-information": "^5.0.0", + "space-separated-tokens": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hastscript/node_modules/comma-separated-tokens": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", + "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hastscript/node_modules/property-information": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", + "integrity": "sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==", + "dependencies": { + "xtend": "^4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hastscript/node_modules/space-separated-tokens": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz", + "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/homedir-polyfill": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", + "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", + "dependencies": { + "parse-passwd": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "node_modules/html-to-react": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/html-to-react/-/html-to-react-1.5.0.tgz", + "integrity": "sha512-tjihXBgaJZRRYzmkrJZ/Qf9jFayilFYcb+sJxXXE2BVLk2XsNrGeuNCVvhXmvREULZb9dz6NFTBC96DTR/lQCQ==", + "dependencies": { + "domhandler": "^5.0", + "htmlparser2": "^8.0", + "lodash.camelcase": "^4.3.0" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.1.tgz", + "integrity": "sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "entities": "^4.3.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/idb": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/idb/-/idb-7.1.1.tgz", + "integrity": "sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==" + }, + "node_modules/ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/immutable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.1.0.tgz", + "integrity": "sha512-oNkuqVTA8jqG1Q6c+UglTOD1xhC1BtjKI7XkCXRkZHrN5m18/XsnUp8Q89GkQO/z+0WjonSvl0FLhDYftp46nQ==" + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha512-m7ZEHgtw69qOGw+jwxXkHlrlIPdTGkyh66zXZ1ajZbxkDBNjSY/LGbmjc7h0s2ELsUDTAhFr55TrPSSqJGPG0A==", + "engines": { + "node": ">=4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/inline-style-parser": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", + "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==" + }, + "node_modules/inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "dependencies": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "dependencies": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dependencies": { + "has-bigints": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dependencies": { + "ci-info": "^2.0.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/is-core-module": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-installed-globally": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "dependencies": { + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==" + }, + "node_modules/is-negative-zero": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-npm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-number-object": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "dependencies": { + "is-path-inside": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-in-cwd/node_modules/is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "dependencies": { + "path-is-inside": "^1.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-reference": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.0.tgz", + "integrity": "sha512-Eo1W3wUoHWoCoVM4GVl/a+K0IgiqE5aIo4kJABFyMum1ZORlPkC+UC357sSQUL5w5QCE5kCC9upl75b7+7CY/Q==", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", + "integrity": "sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, + "node_modules/is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jake": { + "version": "10.8.5", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", + "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/jinja-to-js": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/jinja-to-js/-/jinja-to-js-3.2.3.tgz", + "integrity": "sha512-ktEBxQG17fYaFcHThB719+EbePBx+AkkORQMyuP0UuLPS2zx8uJXP5CsItXjUUwMHFPj3hCRkyqEYzLbeklYgQ==" + }, + "node_modules/js-sdsl": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", + "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha512-CuUqjv0FUZIdXkHPI8MezCnFCdaTAacej1TZYulLoAg1h/PhwkdXFN4V/gzY4g+fMBCOV2xF+rp7t2XD2ns/NQ==" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "peer": true + }, + "node_modules/json-parser": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/json-parser/-/json-parser-1.1.5.tgz", + "integrity": "sha512-6wQrWqKdbvCQ4ccCM1+14Q2Z2XTXeUqwt3xEaGe/GpsObJRh1n1Z+5ZeHosmfSpN5bJKDfeNRy9jOE+wQRduOg==", + "deprecated": "`json-parser` is deprecated. Please use `comment-json` instead", + "dependencies": { + "esprima": "^2.7.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/json-parser/node_modules/esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "node_modules/json-stable-stringify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.2.tgz", + "integrity": "sha512-eunSSaEnxV12z+Z73y/j5N37/In40GK4GmsSy+tEHJMxknvqnA7/djeYtAgW0GsWHUfg+847WJjKaEylk2y09g==", + "dependencies": { + "jsonify": "^0.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==" + }, + "node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz", + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jsx-ast-utils": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", + "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", + "dependencies": { + "array-includes": "^3.1.5", + "object.assign": "^4.1.3" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dependencies": { + "json-buffer": "3.0.0" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/language-subtag-registry": { + "version": "0.3.22", + "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz", + "integrity": "sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==" + }, + "node_modules/language-tags": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.5.tgz", + "integrity": "sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ==", + "dependencies": { + "language-subtag-registry": "~0.3.2" + } + }, + "node_modules/latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dependencies": { + "package-json": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/libnpx": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/libnpx/-/libnpx-10.2.4.tgz", + "integrity": "sha512-BPc0D1cOjBeS8VIBKUu5F80s6njm0wbVt7CsGMrIcJ+SI7pi7V0uVPGpEMH9H5L8csOcclTxAXFE2VAsJXUhfA==", + "dependencies": { + "dotenv": "^5.0.1", + "npm-package-arg": "^6.0.0", + "rimraf": "^2.6.2", + "safe-buffer": "^5.1.0", + "update-notifier": "^2.3.0", + "which": "^1.3.0", + "y18n": "^4.0.0", + "yargs": "^14.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/ansi-align": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", + "integrity": "sha512-TdlOggdA/zURfMYa7ABC66j+oqfMew58KpJMbUlH3bcZP1b+cBHIHDDn5uH9INsxrHBPjsqM0tDB4jPTF/vgJA==", + "dependencies": { + "string-width": "^2.0.0" + } + }, + "node_modules/libnpx/node_modules/ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/boxen": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", + "integrity": "sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==", + "dependencies": { + "ansi-align": "^2.0.0", + "camelcase": "^4.0.0", + "chalk": "^2.0.1", + "cli-boxes": "^1.0.0", + "string-width": "^2.0.0", + "term-size": "^1.2.0", + "widest-line": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/ci-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", + "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==" + }, + "node_modules/libnpx/node_modules/cli-boxes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", + "integrity": "sha512-3Fo5wu8Ytle8q9iCzS4D2MWVL2X7JVWRiS1BnXbTFDhS9c/REkM9vd1AmabsoZoY5/dGi5TT9iKL8Kb6DeBRQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/libnpx/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/libnpx/node_modules/configstore": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.5.tgz", + "integrity": "sha512-nlOhI4+fdzoK5xmJ+NY+1gZK56bwEaWZr8fYuXohZ9Vkc1o3a4T/R3M+yE/w7x/ZVJ1zF8c+oaOvF0dztdUgmA==", + "dependencies": { + "dot-prop": "^4.2.1", + "graceful-fs": "^4.1.2", + "make-dir": "^1.0.0", + "unique-string": "^1.0.0", + "write-file-atomic": "^2.0.0", + "xdg-basedir": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/crypto-random-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", + "integrity": "sha512-GsVpkFPlycH7/fRR7Dhcmnoii54gV1nz7y4CWyeFS14N+JVBBhY+r8amRHE4BwSYal7BPTDp8isvAlCxyFt3Hg==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/dot-prop": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz", + "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==", + "dependencies": { + "is-obj": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/libnpx/node_modules/get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/global-dirs": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz", + "integrity": "sha512-NknMLn7F2J7aflwFOlGdNIuCDpN3VGoSoB+aap3KABFWbHVn1TCgFC+np23J8W2BiZbjfEw3BFBycSMv1AFblg==", + "dependencies": { + "ini": "^1.3.4" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/got": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", + "integrity": "sha512-Y/K3EDuiQN9rTZhBvPRWMLXIKdeD1Rj0nzunfoi0Yyn5WBEbzxXKU9Ub2X41oZBagVWOBU3MuDonFMgPWQFnwg==", + "dependencies": { + "create-error-class": "^3.0.0", + "duplexer3": "^0.1.4", + "get-stream": "^3.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "safe-buffer": "^5.0.1", + "timed-out": "^4.0.0", + "unzip-response": "^2.0.1", + "url-parse-lax": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/is-ci": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", + "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", + "dependencies": { + "ci-info": "^1.5.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/libnpx/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/is-installed-globally": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.1.0.tgz", + "integrity": "sha512-ERNhMg+i/XgDwPIPF3u24qpajVreaiSuvpb1Uu0jugw7KKcxGyCX8cgp8P5fwTmAuXku6beDHHECdKArjlg7tw==", + "dependencies": { + "global-dirs": "^0.1.0", + "is-path-inside": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/is-npm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "integrity": "sha512-9r39FIr3d+KD9SbX0sfMsHzb5PP3uimOiwr3YupUaUFG4W0l1U57Rx3utpttV7qz5U3jmrO5auUa04LU9pyHsg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha512-qhsCR/Esx4U4hg/9I19OVUAJkGWtjRYHMRgUMZE2TDdj+Ag+kttZanLupfddNyglzz50cUlmWzUaI37GDfNx/g==", + "dependencies": { + "path-is-inside": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/latest-version": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", + "integrity": "sha512-Be1YRHWWlZaSsrz2U+VInk+tO0EwLIyV+23RhWLINJYwg/UIikxjlj3MhH37/6/EDCAusjajvMkMMUXRaMWl/w==", + "dependencies": { + "package-json": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/package-json": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-4.0.1.tgz", + "integrity": "sha512-q/R5GrMek0vzgoomq6rm9OX+3PQve8sLwTirmK30YB3Cu0Bbt9OX9M/SIUnroN5BGJkzwGsFwDaRGD9EwBOlCA==", + "dependencies": { + "got": "^6.7.1", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/libnpx/node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/libnpx/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/libnpx/node_modules/semver-diff": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz", + "integrity": "sha512-gL8F8L4ORwsS0+iQ34yCYv///jsOq0ZL7WP55d1HnJ32o7tyFYEFQZQA22mrLIacZdU6xecaBBZ+uEiffGNyXw==", + "dependencies": { + "semver": "^5.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/term-size": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", + "integrity": "sha512-7dPUZQGy/+m3/wjVz3ZW5dobSoD/02NxJpoXUX0WIyjfVS3l0c+b/+9phIDFA7FHzkYtwtMFgeGZ/Y8jVTeqQQ==", + "dependencies": { + "execa": "^0.7.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/unique-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", + "integrity": "sha512-ODgiYu03y5g76A1I9Gt0/chLCzQjvzDy7DsZGsLOE/1MrF6wriEskSncj1+/C58Xk/kPZDppSctDybCwOSaGAg==", + "dependencies": { + "crypto-random-string": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/update-notifier": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-2.5.0.tgz", + "integrity": "sha512-gwMdhgJHGuj/+wHJJs9e6PcCszpxR1b236igrOkUofGhqJuG+amlIKwApH1IW1WWl7ovZxsX49lMBWLxSdm5Dw==", + "dependencies": { + "boxen": "^1.2.1", + "chalk": "^2.0.1", + "configstore": "^3.0.0", + "import-lazy": "^2.1.0", + "is-ci": "^1.0.10", + "is-installed-globally": "^0.1.0", + "is-npm": "^1.0.0", + "latest-version": "^3.0.0", + "semver-diff": "^2.0.0", + "xdg-basedir": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "dependencies": { + "prepend-http": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/libnpx/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/libnpx/node_modules/widest-line": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", + "integrity": "sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==", + "dependencies": { + "string-width": "^2.1.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/libnpx/node_modules/write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "node_modules/libnpx/node_modules/xdg-basedir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", + "integrity": "sha512-1Dly4xqlulvPD3fZUQJLY+FUIeqN3N2MM3uqe4rCJftAvOjFa3jFGfctOgluGx4ahPbUCsZkmJILiP0Vi4T6lQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/listify": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/listify/-/listify-1.0.3.tgz", + "integrity": "sha512-083swF7iH7bx8666zdzBColpgEuy46HjN3r1isD4zV6Ix7FuHfb/2/WVnl4CH8hjuoWeFF7P5KkKNXUnJCFEJg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "peer": true, + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, + "node_modules/loader-utils/node_modules/json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==" + }, + "node_modules/longest": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-2.0.1.tgz", + "integrity": "sha512-Ajzxb8CM6WAnFjgiloPsI3bF+WCxcvhdIG3KNA2KN962+tdBsHcuQ4k4qX/EcS/2CRkcc0iAkR956Nib6aXU/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/longest-streak": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.0.1.tgz", + "integrity": "sha512-cHlYSUpL2s7Fb3394mYxwTYj8niTaNHUCLr0qdiCXQfSjfuA7CKofpX2uSwEfFDQ0EB7JcnMnm+GjbqqoinYYg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/magic-string": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", + "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", + "dependencies": { + "sourcemap-codec": "^1.4.8" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/markdown-extensions": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-1.1.1.tgz", + "integrity": "sha512-WWC0ZuMzCyDHYCasEGs4IPvLyTGftYwh6wIEOULOF0HXcqZlhwRzrK0w2VUlxWA98xnvb/jszw4ZSkJ6ADpM6Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/markdown-table": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.2.tgz", + "integrity": "sha512-y8j3a5/DkJCmS5x4dMCQL+OR0+2EAq3DOtio1COSHsmW2BGXnNCK3v12hJt1LrUz5iZH5g0LmuYOjDdI+czghA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/md-attr-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/md-attr-parser/-/md-attr-parser-1.3.0.tgz", + "integrity": "sha512-KTVlfU5Oxo/6kd0YZ2mLP3eWJj+5vzh5mBCxLo3yGl1fzHIgxmtadbE9tHb7TbUBi3XZbl+P0xKeGmakat135w==" + }, + "node_modules/mdast-util-definitions": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-5.1.1.tgz", + "integrity": "sha512-rQ+Gv7mHttxHOBx2dkF4HWTg+EE+UR78ptQWDylzPKaQuVGdG4HIoY3SrS/pCp80nZ04greFvXbVFHT+uf0JVQ==", + "dependencies": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.1.tgz", + "integrity": "sha512-SobxkQXFAdd4b5WmEakmkVoh18icjQRxGy5OWTCzgsLRm1Fu/KCtwD1HIQSsmq5ZRjVH0Ehwg6/Fn3xIUk+nKw==", + "dependencies": { + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.2.0.tgz", + "integrity": "sha512-iZJyyvKD1+K7QX1b5jXdE7Sc5dtoTry1vzV28UZZe8Z1xVnB/czKntJ7ZAkG0tANqRnBF6p3p7GpU1y19DTf2Q==", + "dependencies": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "mdast-util-to-string": "^3.1.0", + "micromark": "^3.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-decode-string": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-stringify-position": "^3.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.1.tgz", + "integrity": "sha512-42yHBbfWIFisaAfV1eixlabbsa6q7vHeSPY+cg+BBjX51M8xhgMacqH9g6TftB/9+YkcI0ooV4ncfrJslzm/RQ==", + "dependencies": { + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-gfm-autolink-literal": "^1.0.0", + "mdast-util-gfm-footnote": "^1.0.0", + "mdast-util-gfm-strikethrough": "^1.0.0", + "mdast-util-gfm-table": "^1.0.0", + "mdast-util-gfm-task-list-item": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.2.tgz", + "integrity": "sha512-FzopkOd4xTTBeGXhXSBU0OCDDh5lUj2rd+HQqG92Ld+jL4lpUfgX2AT2OHAVP9aEeDKp7G92fuooSZcYJA3cRg==", + "dependencies": { + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-find-and-replace": "^2.0.0", + "micromark-util-character": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.1.tgz", + "integrity": "sha512-p+PrYlkw9DeCRkTVw1duWqPRHX6Ywh2BNKJQcZbCwAuP/59B0Lk9kakuAd7KbQprVO4GzdW8eS5++A9PUSqIyw==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0", + "micromark-util-normalize-identifier": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.2.tgz", + "integrity": "sha512-T/4DVHXcujH6jx1yqpcAYYwd+z5lAYMw4Ls6yhTfbMMtCt0PHY4gEfhW9+lKsLBtyhUGKRIzcUA2FATVqnvPDA==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.6.tgz", + "integrity": "sha512-uHR+fqFq3IvB3Rd4+kzXW8dmpxUhvgCQZep6KdjsLK4O6meK5dYZEayLtIxNus1XO3gfjfcIFe8a7L0HZRGgag==", + "dependencies": { + "@types/mdast": "^3.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.1.tgz", + "integrity": "sha512-KZ4KLmPdABXOsfnM6JHUIjxEvcx2ulk656Z/4Balw071/5qgnhz+H1uGtf2zIGnrnvDC8xR4Fj9uKbjAFGNIeA==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-2.0.0.tgz", + "integrity": "sha512-M09lW0CcBT1VrJUaF/PYxemxxHa7SLDHdSn94Q9FhxjCQfuW7nMAWKWimTmA3OyDMSTH981NN1csW1X+HPSluw==", + "dependencies": { + "mdast-util-mdx-expression": "^1.0.0", + "mdast-util-mdx-jsx": "^2.0.0", + "mdast-util-mdxjs-esm": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-1.3.1.tgz", + "integrity": "sha512-TTb6cKyTA1RD+1su1iStZ5PAv3rFfOUKcoU5EstUpv/IZo63uDX03R8+jXjMEhcobXnNOiG6/ccekvVl4eV1zQ==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-2.1.0.tgz", + "integrity": "sha512-KzgzfWMhdteDkrY4mQtyvTU5bc/W4ppxhe9SzelO6QUUiwLAM+Et2Dnjjprik74a336kHdo0zKm7Tp+n6FFeRg==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-to-markdown": "^1.3.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-remove-position": "^4.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-1.3.0.tgz", + "integrity": "sha512-7N5ihsOkAEGjFotIX9p/YPdl4TqUoMxL4ajNz7PbT89BqsdWJuBC9rvgt6wpbwTZqWWR0jKWqQbwsOWDBUZv4g==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "12.2.4", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.2.4.tgz", + "integrity": "sha512-a21xoxSef1l8VhHxS1Dnyioz6grrJkoaCUgGzMD/7dWHvboYX3VW53esRUfB5tgTyz4Yos1n25SPcj35dJqmAg==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-definitions": "^5.0.0", + "micromark-util-sanitize-uri": "^1.1.0", + "trim-lines": "^3.0.0", + "unist-builder": "^3.0.0", + "unist-util-generated": "^2.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.3.0.tgz", + "integrity": "sha512-6tUSs4r+KK4JGTTiQ7FfHmVOaDrLQJPmpjD6wPMlHGUVXoG9Vjc3jIeP+uyBWRf8clwB2blM+W7+KrlMYQnftA==", + "dependencies": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "longest-streak": "^3.0.0", + "mdast-util-to-string": "^3.0.0", + "micromark-util-decode-string": "^1.0.0", + "unist-util-visit": "^4.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.1.0.tgz", + "integrity": "sha512-n4Vypz/DZgwo0iMHLQL49dJzlp7YtAJP+N07MZHpjPf/5XJuHUWstviF4Mn2jEiR/GNmtnRRqnwsXExk3igfFA==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==" + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromark": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-3.1.0.tgz", + "integrity": "sha512-6Mj0yHLdUZjHnOPgr5xfWIMqMWS12zDN6iws9SLuSz76W8jTtAv24MN4/CL7gJrl5vtxGInkkqDv/JIoRsQOvA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "micromark-core-commonmark": "^1.0.1", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-1.0.6.tgz", + "integrity": "sha512-K+PkJTxqjFfSNkfAhp4GB+cZPfQd6dxtTXnf+RjZOV7T4EEXnvgzOcnp+eSTmpGk9d1S9sL6/lqrgSNn/s0HZA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-factory-destination": "^1.0.0", + "micromark-factory-label": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-factory-title": "^1.0.0", + "micromark-factory-whitespace": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-html-tag-name": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.1.tgz", + "integrity": "sha512-p2sGjajLa0iYiGQdT0oelahRYtMWvLjy8J9LOCxzIQsllMCGLbsLW+Nc+N4vi02jcRJvedVJ68cjelKIO6bpDA==", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^1.0.0", + "micromark-extension-gfm-footnote": "^1.0.0", + "micromark-extension-gfm-strikethrough": "^1.0.0", + "micromark-extension-gfm-table": "^1.0.0", + "micromark-extension-gfm-tagfilter": "^1.0.0", + "micromark-extension-gfm-task-list-item": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.3.tgz", + "integrity": "sha512-i3dmvU0htawfWED8aHMMAzAVp/F0Z+0bPh3YrbTPPL1v4YAlCZpy5rBO5p0LPYiZo0zFVkoYh7vDU7yQSiCMjg==", + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.0.4.tgz", + "integrity": "sha512-E/fmPmDqLiMUP8mLJ8NbJWJ4bTw6tS+FEQS8CcuDtZpILuOb2kjLqPEeAePF1djXROHXChM/wPJw0iS4kHCcIg==", + "dependencies": { + "micromark-core-commonmark": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.4.tgz", + "integrity": "sha512-/vjHU/lalmjZCT5xt7CcHVJGq8sYRm80z24qAKXzaHzem/xsDYb2yLL+NNVbYvmpLx3O7SYPuGL5pzusL9CLIQ==", + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.5.tgz", + "integrity": "sha512-xAZ8J1X9W9K3JTJTUL7G6wSKhp2ZYHrFk5qJgY/4B33scJzE2kpfRL6oiw/veJTbt7jiM/1rngLlOKPWr1G+vg==", + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.1.tgz", + "integrity": "sha512-Ty6psLAcAjboRa/UKUbbUcwjVAv5plxmpUTy2XC/3nJFL37eHej8jrHrRzkqcpipJliuBH30DTs7+3wqNcQUVA==", + "dependencies": { + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.3.tgz", + "integrity": "sha512-PpysK2S1Q/5VXi72IIapbi/jliaiOFzv7THH4amwXeYXLq3l1uo8/2Be0Ac1rEwK20MQEsGH2ltAZLNY2KI/0Q==", + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-1.0.3.tgz", + "integrity": "sha512-TjYtjEMszWze51NJCZmhv7MEBcgYRgb3tJeMAJ+HQCAaZHHRBaDCccqQzGizR/H4ODefP44wRTgOn2vE5I6nZA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-mdx-expression": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-1.0.3.tgz", + "integrity": "sha512-VfA369RdqUISF0qGgv2FfV7gGjHDfn9+Qfiv5hEwpyr1xscRj/CiVRkU7rywGFCO7JwJ5L0e7CJz60lY52+qOA==", + "dependencies": { + "@types/acorn": "^4.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "micromark-factory-mdx-expression": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-md": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-1.0.0.tgz", + "integrity": "sha512-xaRAMoSkKdqZXDAoSgp20Azm0aRQKGOl0RrS81yGu8Hr/JhMsBmfs4wR7m9kgVUIO36cMUQjNyiyDKPrsv8gOw==", + "dependencies": { + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-1.0.0.tgz", + "integrity": "sha512-TZZRZgeHvtgm+IhtgC2+uDMR7h8eTKF0QUX9YsgoL9+bADBpBY6SiLvWqnBlLbCEevITmTqmEuY3FoxMKVs1rQ==", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^1.0.0", + "micromark-extension-mdx-jsx": "^1.0.0", + "micromark-extension-mdx-md": "^1.0.0", + "micromark-extension-mdxjs-esm": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-1.0.3.tgz", + "integrity": "sha512-2N13ol4KMoxb85rdDwTAC6uzs8lMX0zeqpcyx7FhS7PxXomOnLactu8WI8iBNXW8AVyea3KIJd/1CKnUmwrK9A==", + "dependencies": { + "micromark-core-commonmark": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-position-from-estree": "^1.1.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.0.0.tgz", + "integrity": "sha512-eUBA7Rs1/xtTVun9TmV3gjfPz2wEwgK5R5xcbIM5ZYAtvGF6JkyaDsj0agx8urXnO31tEO6Ug83iVH3tdedLnw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-1.0.2.tgz", + "integrity": "sha512-CTIwxlOnU7dEshXDQ+dsr2n+yxpP0+fn271pu0bwDIS8uqfFcumXpj5mLn3hSC8iw2MUr6Gx8EcKng1dD7i6hg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-1.0.6.tgz", + "integrity": "sha512-WRQIc78FV7KrCfjsEf/sETopbYjElh3xAmNpLkd1ODPqxEngP42eVRGbiPEQWpRV27LzqW+XVTvQAMIIRLPnNA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-position-from-estree": "^1.0.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-1.0.0.tgz", + "integrity": "sha512-qUmqs4kj9a5yBnk3JMLyjtWYN6Mzfcx8uJfi5XAveBniDevmZasdGBba5b4QsvRcAkmvGo5ACmSUmyGiKTLZew==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-1.0.2.tgz", + "integrity": "sha512-zily+Nr4yFqgMGRKLpTVsNl5L4PMu485fGFDOQJQBl2NFpjGte1e86zC0da93wf97jrc4+2G2GQudFMHn3IX+A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-1.0.0.tgz", + "integrity": "sha512-Qx7uEyahU1lt1RnsECBiuEbfr9INjQTGa6Err+gF3g0Tx4YEviPbqqGKNv/NrBaE7dVHdn1bVZKM/n5I/Bak7A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-1.1.0.tgz", + "integrity": "sha512-agJ5B3unGNJ9rJvADMJ5ZiYjBRyDpzKAOk01Kpi1TKhlT1APx3XZk6eN7RtSz1erbWHC2L8T3xLZ81wdtGRZzg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-1.0.0.tgz", + "integrity": "sha512-5e8xTis5tEZKgesfbQMKRCyzvffRRUX+lK/y+DvsMFdabAicPkkZV6gO+FEWi9RfuKKoxxPwNL+dFF0SMImc1g==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-1.0.0.tgz", + "integrity": "sha512-F8oW2KKrQRb3vS5ud5HIqBVkCqQi224Nm55o5wYLzY/9PwHGXC01tr3d7+TqHHz6zrKQ72Okwtvm/xQm6OVNZA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-1.0.0.tgz", + "integrity": "sha512-J8H058vFBdo/6+AsjHp2NF7AJ02SZtWaVUjsayNFeAiydTxUwViQPxN0Hf8dp4FmCQi0UUFovFsEyRSUmFH3MA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-1.0.0.tgz", + "integrity": "sha512-OzO9AI5VUtrTD7KSdagf4MWgHMtET17Ua1fIpXTpuhclCqD8egFWo85GxSGvxgkGS74bEahvtM0WP0HjvV0e4w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-1.0.2.tgz", + "integrity": "sha512-DLT5Ho02qr6QWVNYbRZ3RYOSSWWFuH3tJexd3dgN1odEuPNxCngTCXJum7+ViRAd9BbdxCvMToPOD/IvVhzG6Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-1.0.1.tgz", + "integrity": "sha512-U2s5YdnAYexjKDel31SVMPbfi+eF8y1U4pfiRW/Y8EFVCy/vgxk/2wWTxzcqE71LHtCuCzlBDRU2a5CQ5j+mQA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-1.2.0.tgz", + "integrity": "sha512-WWp3bf7xT9MppNuw3yPjpnOxa8cj5ACivEzXJKu0WwnjBYfzaBvIAT9KfeyI0Qkll+bfQtfftSwdgTH6QhTOKw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "estree-util-visit": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0", + "vfile-location": "^4.0.0", + "vfile-message": "^3.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-1.1.0.tgz", + "integrity": "sha512-BKlClMmYROy9UiV03SwNmckkjn8QHVaWkqoAqzivabvdGcwNGMMMH/5szAnywmsTBUzDsU57/mFi0sp4BQO6dA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-1.0.0.tgz", + "integrity": "sha512-yg+zrL14bBTFrQ7n35CmByWUTFsgst5JhA4gJYoty4Dqzj4Z4Fr/DHekSS5aLfH9bdlfnSvKAWsAgJhIbogyBg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-1.0.0.tgz", + "integrity": "sha512-CB/AGk98u50k42kvgaMM94wzBqozSzDDaonKU7P7jwQIuH2RU0TeBqGYJz2WY1UdihhjweivStrJ2JdkdEmcfw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-types": "^1.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-1.1.0.tgz", + "integrity": "sha512-RoxtuSCX6sUNtxhbmsEFQfWzs8VN7cTctmBPvYivo98xb/kDEoTCtJQX5wyzIYEmk/lvNFTat4hL8oW0KndFpg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-1.0.2.tgz", + "integrity": "sha512-d90uqCnXp/cy4G881Ub4psE57Sf8YD0pim9QdjCRNjfas2M1u6Lbt+XZK9gnHL2XFhnozZiEdCa9CNfXSfQ6xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-1.0.1.tgz", + "integrity": "sha512-oKDEMK2u5qqAptasDAwWDXq0tG9AssVwAx3E9bBF3t/shRIGsWIRG+cGafs2p/SnDSOecnt6hZPCE2o6lHfFmQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-types": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.0.2.tgz", + "integrity": "sha512-DCfg/T8fcrhrRKTPjRrw/5LLvdGV7BHySf/1LOZx7TzWZdYRjogNtyNq885z3nNallwr3QUKARjqvHqX1/7t+w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/middleearth-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/middleearth-names/-/middleearth-names-1.1.0.tgz", + "integrity": "sha512-Oo1mbq9odpn6KHsDs8/UA5xFfX/gcrY+jWZpvd5MDaX0tjkxA7S7NTREQuqD7DWfluDgygjhKvETMWbwd3A9sA==", + "dependencies": { + "unique-random-array": "1.0.0" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "peer": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "peer": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/moment": { + "version": "2.21.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.21.0.tgz", + "integrity": "sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ==", + "engines": { + "node": "*" + } + }, + "node_modules/mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/mrm": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/mrm/-/mrm-3.0.10.tgz", + "integrity": "sha512-aRByZsPXMM8W0NHNH9afkKyk5OW4bB5pYNRIN+8iSVfpMAzqeMejmj/yIYcdFNJTksXmdPMfTaucm2NYdh4xIw==", + "dependencies": { + "git-username": "^1.0.0", + "glob": "^7.1.6", + "inquirer": "^7.0.4", + "is-directory": "^0.3.1", + "kleur": "^3.0.3", + "libnpx": "^10.2.4", + "listify": "^1.0.0", + "lodash": "^4.17.15", + "longest": "^2.0.1", + "middleearth-names": "^1.1.0", + "minimist": "^1.2.0", + "mrm-core": "^6.1.7", + "semver-utils": "^1.1.4", + "update-notifier": "^4.1.0", + "user-home": "^2.0.0", + "user-meta": "^1.0.0", + "which": "^2.0.2" + }, + "bin": { + "mrm": "bin/mrm.js" + }, + "engines": { + "node": ">=10.13" + } + }, + "node_modules/mrm-core": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/mrm-core/-/mrm-core-6.1.7.tgz", + "integrity": "sha512-jLGWrkupcgGIsLerrI/xmM/dFHbaoehRsuVbgYBrxYKXNMRBHN3Mgkd8cw+/ZCCoiZEXF8/SaZol0GCp6oBQ9g==", + "dependencies": { + "babel-code-frame": "^6.26.0", + "comment-json": "^2.2.0", + "detect-indent": "^6.0.0", + "editorconfig": "^0.15.3", + "find-up": "^4.1.0", + "fs-extra": "^8.1.0", + "kleur": "^3.0.3", + "listify": "^1.0.0", + "lodash": "^4.17.15", + "minimist": "^1.2.0", + "prop-ini": "^0.0.2", + "rc": "^1.2.8", + "readme-badger": "^0.3.0", + "semver": "^6.3.0", + "smpltmpl": "^1.0.2", + "split-lines": "^2.0.0", + "strip-bom": "^4.0.0", + "validate-npm-package-name": "^3.0.0", + "webpack-merge": "^4.2.2", + "yaml": "^2.0.0-1" + }, + "engines": { + "node": ">=10.13" + } + }, + "node_modules/mrm-core/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mrm-core/node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/mrm-core/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mrm-core/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mrm-core/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mrm-core/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/mrm-core/node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/mrm/node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" + }, + "node_modules/nanoid": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "peer": true + }, + "node_modules/next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/next/-/next-13.0.2.tgz", + "integrity": "sha512-uQ5z5e4D9mOe8+upy6bQdYYjo/kk1v3jMW87kTy2TgAyAsEO+CkwRnMgyZ4JoHEnhPZLHwh7dk0XymRNLe1gFw==", + "dependencies": { + "@next/env": "13.0.2", + "@swc/helpers": "0.4.11", + "caniuse-lite": "^1.0.30001406", + "postcss": "8.4.14", + "styled-jsx": "5.1.0", + "use-sync-external-store": "1.2.0" + }, + "bin": { + "next": "dist/bin/next" + }, + "engines": { + "node": ">=14.6.0" + }, + "optionalDependencies": { + "@next/swc-android-arm-eabi": "13.0.2", + "@next/swc-android-arm64": "13.0.2", + "@next/swc-darwin-arm64": "13.0.2", + "@next/swc-darwin-x64": "13.0.2", + "@next/swc-freebsd-x64": "13.0.2", + "@next/swc-linux-arm-gnueabihf": "13.0.2", + "@next/swc-linux-arm64-gnu": "13.0.2", + "@next/swc-linux-arm64-musl": "13.0.2", + "@next/swc-linux-x64-gnu": "13.0.2", + "@next/swc-linux-x64-musl": "13.0.2", + "@next/swc-win32-arm64-msvc": "13.0.2", + "@next/swc-win32-ia32-msvc": "13.0.2", + "@next/swc-win32-x64-msvc": "13.0.2" + }, + "peerDependencies": { + "fibers": ">= 3.1.0", + "node-sass": "^6.0.0 || ^7.0.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "sass": "^1.3.0" + }, + "peerDependenciesMeta": { + "fibers": { + "optional": true + }, + "node-sass": { + "optional": true + }, + "sass": { + "optional": true + } + } + }, + "node_modules/next-mdx-remote": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/next-mdx-remote/-/next-mdx-remote-4.2.0.tgz", + "integrity": "sha512-X5RhD7f7b78pH2abbuusObSGgII5l54OdusS/2iXljN7WN1cel6ToLlZeCZcyxx9cR4wmBGQYGongIttDYNmAA==", + "dependencies": { + "@mdx-js/mdx": "^2.0.0", + "@mdx-js/react": "^2.0.0", + "vfile": "^5.3.0", + "vfile-matter": "^3.0.1" + }, + "engines": { + "node": ">=14", + "npm": ">=7" + }, + "peerDependencies": { + "react": ">=16.x <=18.x", + "react-dom": ">=16.x <=18.x" + } + }, + "node_modules/next-plausible": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/next-plausible/-/next-plausible-3.6.5.tgz", + "integrity": "sha512-eGsyie68xIyX+If1PSIJsgv7pkiMUm66Yt9gELl3kUvljaxoJOCwUEALGk5JHvhByGsBiRswkrGioDMthL3gfQ==", + "funding": { + "url": "https://github.com/4lejandrito/next-plausible?sponsor=1" + }, + "peerDependencies": { + "next": "^11.1.0 || ^12.0.0 || ^13.0.0", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/next-pwa": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/next-pwa/-/next-pwa-5.6.0.tgz", + "integrity": "sha512-XV8g8C6B7UmViXU8askMEYhWwQ4qc/XqJGnexbLV68hzKaGHZDMtHsm2TNxFcbR7+ypVuth/wwpiIlMwpRJJ5A==", + "dependencies": { + "babel-loader": "^8.2.5", + "clean-webpack-plugin": "^4.0.0", + "globby": "^11.0.4", + "terser-webpack-plugin": "^5.3.3", + "workbox-webpack-plugin": "^6.5.4", + "workbox-window": "^6.5.4" + }, + "peerDependencies": { + "next": ">=9.0.0" + } + }, + "node_modules/next-sitemap": { + "version": "3.1.32", + "resolved": "https://registry.npmjs.org/next-sitemap/-/next-sitemap-3.1.32.tgz", + "integrity": "sha512-jkIKpwLXpWWTPfmDO46+6nu4+qpar4CjvUwCR9rYZHWtzE/wFfaCVFKpGtFMl6MFjpu8GjiE6kWFEa7uF3bzzg==", + "funding": [ + { + "url": "https://github.com/iamvishnusankar/next-sitemap.git" + } + ], + "dependencies": { + "@corex/deepmerge": "^4.0.29", + "minimist": "^1.2.6" + }, + "bin": { + "next-sitemap": "bin/next-sitemap.mjs", + "next-sitemap-cjs": "bin/next-sitemap.cjs" + }, + "engines": { + "node": ">=14.18" + }, + "peerDependencies": { + "@next/env": "*", + "next": "*" + } + }, + "node_modules/nlcst-to-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-3.1.0.tgz", + "integrity": "sha512-Y8HQWKw/zrHTCnu2zcFBN1dV6vN0NUG7s5fkEj380G8tF3R+vA2KG+tDl2QoHVQCTHGHVXwoni2RQkDSFQb1PA==", + "dependencies": { + "@types/nlcst": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/node-releases": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/nothing-mock": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/nothing-mock/-/nothing-mock-1.0.4.tgz", + "integrity": "sha512-wl5hpQruSQHt5M15oY/akWd7hFPyec0uTCUHYQa3WhBeTxonsFnMKmixaCDvb4U7MN9UjS2UvXw3s4KxjrNjKg==" + }, + "node_modules/npm-package-arg": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-6.1.1.tgz", + "integrity": "sha512-qBpssaL3IOZWi5vEKUKW0cO7kzLeT+EQO9W8RsLOZf76KF9E/K9+wH0C7t06HXPpaH8WH5xF1MExLuCwbTqRUg==", + "dependencies": { + "hosted-git-info": "^2.7.1", + "osenv": "^0.1.5", + "semver": "^5.6.0", + "validate-npm-package-name": "^3.0.0" + } + }, + "node_modules/npm-package-arg/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.entries": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.6.tgz", + "integrity": "sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.fromentries": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.6.tgz", + "integrity": "sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.hasown": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.2.tgz", + "integrity": "sha512-B5UIT3J1W+WuWIU55h0mjlwaqxiE5vYENJXIXZ4VFe05pNYrkKuK0U/6aFcb0pKywYJh7IhfoqUfKVmrJJHZHw==", + "dependencies": { + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.values": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.6.tgz", + "integrity": "sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dependencies": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "node_modules/p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dependencies": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/package-json/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-entities": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.0.tgz", + "integrity": "sha512-5nk9Fn03x3rEhGaX1FU6IDwG/k+GxLXlFAkgrbM1asuAFl3BhdQWvASaIsmwWypRNcZKHPYnIuOSfIWEyEQnPQ==", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-git-config": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/parse-git-config/-/parse-git-config-1.1.1.tgz", + "integrity": "sha512-S3LGXJZVSy/hswvbSkfdbKBRVsnqKrVu6j8fcvdtJ4TxosSELyQDsJPuGPXuZ+EyuYuJd3O4uAF8gcISR0OFrQ==", + "dependencies": { + "extend-shallow": "^2.0.1", + "fs-exists-sync": "^0.1.0", + "git-config-path": "^1.0.1", + "ini": "^1.3.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse-github-url": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-github-url/-/parse-github-url-1.0.2.tgz", + "integrity": "sha512-kgBf6avCbO3Cn6+RnzRGLkUsv4ZVqv/VfAYkRsyBcgkshNvVBkRn1FEZcW0Jb+npXQWm2vHPnnOqFteZxRRGNw==", + "bin": { + "parse-github-url": "cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse-latin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/parse-latin/-/parse-latin-5.0.1.tgz", + "integrity": "sha512-b/K8ExXaWC9t34kKeDV8kGXBkXZ1HCSAZRYE7HR14eA1GlXX5L8iWhs8USJNhQU9q5ci413jCKF0gOyovvyRBg==", + "dependencies": { + "nlcst-to-string": "^3.0.0", + "unist-util-modify-children": "^3.0.0", + "unist-util-visit-children": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-numeric-range": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz", + "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==" + }, + "node_modules/parse-passwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", + "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "node_modules/path-posix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-posix/-/path-posix-1.0.0.tgz", + "integrity": "sha512-1gJ0WpNIiYcQydgg3Ed8KzvIqTsDpNwq+cjBCssvBtuTWjEqY1AW+i+OepiEMqDCzyro9B2sLAe4RBPajMYFiA==" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/periscopic": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.0.4.tgz", + "integrity": "sha512-SFx68DxCv0Iyo6APZuw/AKewkkThGwssmU0QWtTlvov3VAtPX+QJ4CadwSaz8nrT5jPIuxdvJWB4PnD2KNDxQg==", + "dependencies": { + "estree-walker": "^3.0.0", + "is-reference": "^3.0.0" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "engines": { + "node": ">=4" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss": { + "version": "8.4.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.14.tgz", + "integrity": "sha512-E398TUmfAYFPBSdzgeieK2Y1+1cpdxJx8yXbK/m57nRhKSmk1GB2tO4lbLBtlkfPQTDKfe4Xqv1ASWPpayPEig==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + ], + "dependencies": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha512-ravE6m9Atw9Z/jjttRUZ+clIXogdghyZAuWJ3qEzjT+jI/dL1ifAqhZeC5VHzQp1MSt1+jxKkFNemj/iO7tVUA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/prettier": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-bytes": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", + "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/prismjs": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", + "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==", + "engines": { + "node": ">=6" + } + }, + "node_modules/prop-ini": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/prop-ini/-/prop-ini-0.0.2.tgz", + "integrity": "sha512-qyU57WvAvZDbzmRy9xDbJGVwrGJhmA+rYnVjy4xtX4Ny9c7gzvpmf/j7A3oq9ChbPh15MZQKjPep2mNdnAhtig==", + "dependencies": { + "extend": "^3.0.0" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/property-information": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.1.1.tgz", + "integrity": "sha512-hrzC564QIl0r0vy4l6MvRLhafmUowhO/O3KgVSoXIbbA2Sz4j8HGpJc6T2cubRVwMwpdiG/vKGfhT4IixmKN9w==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/pupa": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.1.1.tgz", + "integrity": "sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A==", + "dependencies": { + "escape-goat": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", + "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.0" + }, + "peerDependencies": { + "react": "^18.2.0" + } + }, + "node_modules/react-from-dom": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/react-from-dom/-/react-from-dom-0.6.2.tgz", + "integrity": "sha512-qvWWTL/4xw4k/Dywd41RBpLQUSq97csuv15qrxN+izNeLYlD9wn5W8LspbfYe5CWbaSdkZ72BsaYBPQf2x4VbQ==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-github-btn": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/react-github-btn/-/react-github-btn-1.4.0.tgz", + "integrity": "sha512-lV4FYClAfjWnBfv0iNlJUGhamDgIq6TayD0kPZED6VzHWdpcHmPfsYOZ/CFwLfPv4Zp+F4m8QKTj0oy2HjiGXg==", + "dependencies": { + "github-buttons": "^2.22.0" + }, + "peerDependencies": { + "react": ">=16.3.0" + } + }, + "node_modules/react-inlinesvg": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/react-inlinesvg/-/react-inlinesvg-3.0.1.tgz", + "integrity": "sha512-cBfoyfseNI2PkDA7ZKIlDoHq0eMfpoC3DhKBQNC+/X1M4ZQB+aXW+YiNPUDDDKXUsGDUIZWWiZWNFeauDIVdoA==", + "dependencies": { + "exenv": "^1.2.2", + "react-from-dom": "^0.6.2" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-intersection-observer": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-9.4.1.tgz", + "integrity": "sha512-IXpIsPe6BleFOEHKzKh5UjwRUaz/JYS0lT/HPsupWEQou2hDqjhLMStc5zyE3eQVT4Fk3FufM8Fw33qW1uyeiw==", + "peerDependencies": { + "react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/readme-badger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/readme-badger/-/readme-badger-0.3.0.tgz", + "integrity": "sha512-+sMOLSs1imZUISZ2Rhz7qqVd77QtpcAPbGeIraFdgJmijb04YtdlPjGNBvDChTNtLbeQ6JNGQy3pOgslWfaP3g==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/refractor": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", + "integrity": "sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==", + "dependencies": { + "hastscript": "^6.0.0", + "parse-entities": "^2.0.0", + "prismjs": "~1.27.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/character-entities": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", + "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/character-entities-legacy": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", + "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/character-reference-invalid": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz", + "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/is-alphabetical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", + "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/is-alphanumerical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", + "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==", + "dependencies": { + "is-alphabetical": "^1.0.0", + "is-decimal": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/is-decimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz", + "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/is-hexadecimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz", + "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "dependencies": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/refractor/node_modules/prismjs": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.27.0.tgz", + "integrity": "sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==" + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz", + "integrity": "sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ==", + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" + }, + "node_modules/regenerator-transform": { + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.1.tgz", + "integrity": "sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg==", + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regexp.prototype.flags": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/regexpu-core": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.2.2.tgz", + "integrity": "sha512-T0+1Zp2wjF/juXMrMxHxidqGYn8U4R+zleSJhX9tQ1PUsS8a9UtYfbsF9LdiVgNX3kiX8RNaKM42nfSgvFJjmw==", + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.1.0", + "regjsgen": "^0.7.1", + "regjsparser": "^0.9.1", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/registry-auth-token": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.2.tgz", + "integrity": "sha512-PC5ZysNb42zpFME6D/XlIgtNGdTl8bBOCw90xQLVMpzuuubJKYDWFAEuUNc+Cn8Z8724tg2SDhDRrkVEsqfDMg==", + "dependencies": { + "rc": "1.2.8" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dependencies": { + "rc": "^1.2.8" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/regjsgen": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.7.1.tgz", + "integrity": "sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA==" + }, + "node_modules/regjsparser": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.9.1.tgz", + "integrity": "sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==", + "dependencies": { + "jsesc": "~0.5.0" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/regjsparser/node_modules/jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "bin": { + "jsesc": "bin/jsesc" + } + }, + "node_modules/remark": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/remark/-/remark-14.0.2.tgz", + "integrity": "sha512-A3ARm2V4BgiRXaUo5K0dRvJ1lbogrbXnhkJRmD0yw092/Yl0kOCZt1k9ZeElEwkZsWGsMumz6qL5MfNJH9nOBA==", + "dependencies": { + "@types/mdast": "^3.0.0", + "remark-parse": "^10.0.0", + "remark-stringify": "^10.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-gfm": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", + "integrity": "sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-gfm": "^2.0.0", + "micromark-extension-gfm": "^2.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-mdx": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-2.1.5.tgz", + "integrity": "sha512-A8vw5s+BgOa968Irt8BO7DfWJTE0Fe7Ge3hX8zzDB1DnwMZTNdK6qF2IcFao+/7nzk1vSysKcFp+3ku4vhMpaQ==", + "dependencies": { + "mdast-util-mdx": "^2.0.0", + "micromark-extension-mdxjs": "^1.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-parse": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.1.tgz", + "integrity": "sha512-1fUyHr2jLsVOkhbvPRBJ5zTKZZyD6yZzYaWCS6BPBdQ8vEMBCH+9zNCDA6tET/zHCi/jLqjCWtlJZUPk+DbnFw==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-react": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/remark-react/-/remark-react-9.0.1.tgz", + "integrity": "sha512-NtpTMfUIreelaRdUVUtgSizTOSwV6JQvGAPn2gWNoWF+Etd0D8YUdDDFEV3vVOiMIoiNR2bQ0etMb/9uaWhjAw==", + "dependencies": { + "@mapbox/hast-util-table-cell-style": "^0.2.0", + "@types/mdast": "^3.0.0", + "@types/react": "^17.0.0", + "hast-to-hyperscript": "^10.0.0", + "hast-util-sanitize": "^4.0.0", + "mdast-util-to-hast": "^11.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-react/node_modules/@types/react": { + "version": "17.0.52", + "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.52.tgz", + "integrity": "sha512-vwk8QqVODi0VaZZpDXQCmEmiOuyjEFPY7Ttaw5vjM112LOq37yz1CDJGrRJwA1fYEq4Iitd5rnjd1yWAc/bT+A==", + "dependencies": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/remark-react/node_modules/mdast-util-to-hast": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-11.3.0.tgz", + "integrity": "sha512-4o3Cli3hXPmm1LhB+6rqhfsIUBjnKFlIUZvudaermXB+4/KONdd/W4saWWkC+LBLbPMqhFSSTSRgafHsT5fVJw==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "@types/mdurl": "^1.0.0", + "mdast-util-definitions": "^5.0.0", + "mdurl": "^1.0.0", + "unist-builder": "^3.0.0", + "unist-util-generated": "^2.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-10.1.0.tgz", + "integrity": "sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==", + "dependencies": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-to-hast": "^12.1.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-smartypants": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/remark-smartypants/-/remark-smartypants-2.0.0.tgz", + "integrity": "sha512-Rc0VDmr/yhnMQIz8n2ACYXlfw/P/XZev884QU1I5u+5DgJls32o97Vc1RbK3pfumLsJomS2yy8eT4Fxj/2MDVA==", + "dependencies": { + "retext": "^8.1.0", + "retext-smartypants": "^5.1.0", + "unist-util-visit": "^4.1.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/remark-stringify": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-10.0.2.tgz", + "integrity": "sha512-6wV3pvbPvHkbNnWB0wdDvVFHOe1hBRAx1Q/5g/EpH4RppAII6J8Gnwe7VbHuXaoKIF6LAg6ExTel/+kNqSQ7lw==", + "dependencies": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-unwrap-images": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-unwrap-images/-/remark-unwrap-images-3.0.1.tgz", + "integrity": "sha512-5VUY0n+J9lPTPfkct5S3/SbutryBjp8J/4mbgtlkDrOk3h8jde0hyqdYUJOoJKherZezS08tjd6i4+nnQ+wl5w==", + "dependencies": { + "@types/mdast": "^3.0.0", + "hast-util-whitespace": "^2.0.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remote-origin-url": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/remote-origin-url/-/remote-origin-url-1.0.0.tgz", + "integrity": "sha512-xHDM6IBqivpiQ1e4WOuFpM/T6rbzA/WBsu+3WLtgPOhHyjA0nYlijV3NprlTb4FcXlQ5+Q+z174sQ1NnUF5FwA==", + "dependencies": { + "parse-git-config": "^1.1.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "engines": { + "node": ">=4" + } + }, + "node_modules/responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ==", + "dependencies": { + "lowercase-keys": "^1.0.0" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retext": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/retext/-/retext-8.1.0.tgz", + "integrity": "sha512-N9/Kq7YTn6ZpzfiGW45WfEGJqFf1IM1q8OsRa1CGzIebCJBNCANDRmOrholiDRGKo/We7ofKR4SEvcGAWEMD3Q==", + "dependencies": { + "@types/nlcst": "^1.0.0", + "retext-latin": "^3.0.0", + "retext-stringify": "^3.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-latin": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/retext-latin/-/retext-latin-3.1.0.tgz", + "integrity": "sha512-5MrD1tuebzO8ppsja5eEu+ZbBeUNCjoEarn70tkXOS7Bdsdf6tNahsv2bY0Z8VooFF6cw7/6S+d3yI/TMlMVVQ==", + "dependencies": { + "@types/nlcst": "^1.0.0", + "parse-latin": "^5.0.0", + "unherit": "^3.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-smartypants": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/retext-smartypants/-/retext-smartypants-5.2.0.tgz", + "integrity": "sha512-Do8oM+SsjrbzT2UNIKgheP0hgUQTDDQYyZaIY3kfq0pdFzoPk+ZClYJ+OERNXveog4xf1pZL4PfRxNoVL7a/jw==", + "dependencies": { + "@types/nlcst": "^1.0.0", + "nlcst-to-string": "^3.0.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/retext-stringify": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/retext-stringify/-/retext-stringify-3.1.0.tgz", + "integrity": "sha512-767TLOaoXFXyOnjx/EggXlb37ZD2u4P1n0GJqVdpipqACsQP+20W+BNpMYrlJkq7hxffnFk+jc6mAK9qrbuB8w==", + "dependencies": { + "@types/nlcst": "^1.0.0", + "nlcst-to-string": "^3.0.0", + "unified": "^10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rollup": { + "version": "2.79.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.1.tgz", + "integrity": "sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw==", + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=10.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/rollup-plugin-terser": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz", + "integrity": "sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==", + "deprecated": "This package has been deprecated and is no longer maintained. Please use @rollup/plugin-terser", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "jest-worker": "^26.2.1", + "serialize-javascript": "^4.0.0", + "terser": "^5.0.0" + }, + "peerDependencies": { + "rollup": "^2.0.0" + } + }, + "node_modules/rollup-plugin-terser/node_modules/jest-worker": { + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", + "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/rollup-plugin-terser/node_modules/serialize-javascript": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", + "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rxjs": { + "version": "6.6.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", + "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", + "dependencies": { + "tslib": "^1.9.0" + }, + "engines": { + "npm": ">=2.0.0" + } + }, + "node_modules/rxjs/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, + "node_modules/sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "dependencies": { + "mri": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safe-regex-test": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", + "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==", + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.3", + "is-regex": "^1.1.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sass": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.56.1.tgz", + "integrity": "sha512-VpEyKpyBPCxE7qGDtOcdJ6fFbcpOM+Emu7uZLxVrkX8KVU/Dp5UF7WLvzqRuUhB6mqqQt1xffLoG+AndxTZrCQ==", + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/scheduler": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", + "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/schema-utils": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz", + "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==", + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dependencies": { + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/semver-diff/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/semver-utils": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/semver-utils/-/semver-utils-1.1.4.tgz", + "integrity": "sha512-EjnoLE5OGmDAVV/8YDoN5KiajNadjzIp9BAHOhYeQHt7j0UWxjmgsx4YD48wp4Ue1Qogq38F1GNUJNqF1kKKxA==" + }, + "node_modules/serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==" + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/smpltmpl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/smpltmpl/-/smpltmpl-1.0.2.tgz", + "integrity": "sha512-Hq23NNgeZigOzIiX1dkb6W3gFn2/XQj43KhPxu65IMieG/gIwf/lQb1IudjYv0c/5LwJeS/mPayYzyo+8WJMxQ==", + "dependencies": { + "babel-code-frame": "^6.26.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==" + }, + "node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", + "deprecated": "Please use @jridgewell/sourcemap-codec instead" + }, + "node_modules/space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/split-lines": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/split-lines/-/split-lines-2.1.0.tgz", + "integrity": "sha512-8dv+1zKgTpfTkOy8XZLFyWrfxO0NV/bj/3EaQ+hBrBxGv2DwiroljPjU8NlCr+59nLnsVm9WYT7lXKwe4TC6bw==", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/string.prototype.matchall": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.8.tgz", + "integrity": "sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "get-intrinsic": "^1.1.3", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "regexp.prototype.flags": "^1.4.3", + "side-channel": "^1.0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz", + "integrity": "sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz", + "integrity": "sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.3.tgz", + "integrity": "sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g==", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-object": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", + "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", + "dependencies": { + "get-own-enumerable-property-symbols": "^3.0.0", + "is-obj": "^1.0.1", + "is-regexp": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/stringify-object/node_modules/is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-comments/-/strip-comments-2.0.1.tgz", + "integrity": "sha512-ZprKx+bBLXv067WTCALv8SSz5l2+XhpYCsVtSqlMnkAXMWDq+/ekVbl1ghqP9rUHTzv6sm/DwCOiYutU/yp1fw==", + "engines": { + "node": ">=10" + } + }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/style-mod": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.0.0.tgz", + "integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw==" + }, + "node_modules/style-to-object": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.3.0.tgz", + "integrity": "sha512-CzFnRRXhzWIdItT3OmF8SQfWyahHhjq3HwcMNCNLn+N7klOOqPjMeG/4JSu77D7ypZdGvSzvkrbyeTMizz2VrA==", + "dependencies": { + "inline-style-parser": "0.1.1" + } + }, + "node_modules/styled-jsx": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.0.tgz", + "integrity": "sha512-/iHaRJt9U7T+5tp6TRelLnqBqiaIT0HsO0+vgyj8hK2KUk7aejFqRrumqPUlAqDwAj8IbS/1hk3IhBAAK/FCUQ==", + "dependencies": { + "client-only": "0.0.1" + }, + "engines": { + "node": ">= 12.0.0" + }, + "peerDependencies": { + "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "peer": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/temp-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", + "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/tempy": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-0.6.0.tgz", + "integrity": "sha512-G13vtMYPT/J8A4X2SjdtBTphZlrp1gKv6hZiOjw14RCWg6GbHuQBGtjlx75xLbYV/wEc0D7G5K4rxKP/cXk8Bw==", + "dependencies": { + "is-stream": "^2.0.0", + "temp-dir": "^2.0.0", + "type-fest": "^0.16.0", + "unique-string": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tempy/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tempy/node_modules/type-fest": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.16.0.tgz", + "integrity": "sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/term-size": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz", + "integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/terser": { + "version": "5.15.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.1.tgz", + "integrity": "sha512-K1faMUvpm/FBxjBXud0LWVAGxmvoPbZbfTCYbSgaaYQaIXI3/TdI7a7ZGA73Zrou6Q8Zmz3oeUTsp/dj+ag2Xw==", + "dependencies": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.6", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz", + "integrity": "sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ==", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.14", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.0", + "terser": "^5.14.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==" + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "node_modules/timed-out": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", + "integrity": "sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "engines": { + "node": ">=4" + } + }, + "node_modules/to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "engines": { + "node": ">=6" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.1.0.tgz", + "integrity": "sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tsconfig-paths": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", + "integrity": "sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==", + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, + "node_modules/tslib": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + }, + "node_modules/tsutils": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", + "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", + "dependencies": { + "tslib": "^1.8.1" + }, + "engines": { + "node": ">= 6" + }, + "peerDependencies": { + "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" + } + }, + "node_modules/tsutils/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/typescript": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/unbox-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "dependencies": { + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/unherit": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/unherit/-/unherit-3.0.1.tgz", + "integrity": "sha512-akOOQ/Yln8a2sgcLj4U0Jmx0R5jpIg2IUyRrWOzmEbjBtGzBdHtSeFKgoEcoH4KYIG/Pb8GQ/BwtYm0GCq1Sqg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz", + "integrity": "sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz", + "integrity": "sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/unified": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz", + "integrity": "sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==", + "dependencies": { + "@types/unist": "^2.0.0", + "bail": "^2.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unique-random": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-random/-/unique-random-1.0.0.tgz", + "integrity": "sha512-K1sUkPf9EXCZFNIlMCoX4icAqcvkR4FMPH4Z61HbyiWhQl1ZGo0zYeV2bJmocK8Cp6tnKYrCnpkeKGebXZoRTQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unique-random-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-random-array/-/unique-random-array-1.0.0.tgz", + "integrity": "sha512-vtj2yltjcHPa69nFjNJ3xnhsEwE8pMyjqUQDw2myz/iSezqf4YCAcygwFQEsOgMid5VNW/dCPbnb2BcmaDCCKg==", + "dependencies": { + "unique-random": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dependencies": { + "crypto-random-string": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/unist-builder": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-builder/-/unist-builder-3.0.0.tgz", + "integrity": "sha512-GFxmfEAa0vi9i5sd0R2kcrI9ks0r82NasRq5QHh2ysGngrc6GiqD5CDf1FjPenY4vApmFASBIIlk/jj5J5YbmQ==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-generated": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-generated/-/unist-util-generated-2.0.0.tgz", + "integrity": "sha512-TiWE6DVtVe7Ye2QxOVW9kqybs6cZexNwTwSMVgkfjEReqy/xwGpAXb99OxktoWwmL+Z+Epb0Dn8/GNDYP1wnUw==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz", + "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-modify-children": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/unist-util-modify-children/-/unist-util-modify-children-3.1.0.tgz", + "integrity": "sha512-L0UizdncPZ1NIwpmkwFdLo2NaK2Eb5LU/vaQ7lZGkAaOBZfsHp+8T/gVWPVmmMO1hj6gc+XeMoytut8jr7fdyA==", + "dependencies": { + "@types/unist": "^2.0.0", + "array-iterate": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-4.0.3.tgz", + "integrity": "sha512-p/5EMGIa1qwbXjA+QgcBXaPWjSnZfQ2Sc3yBEEfgPwsEmJd8Qh+DSk3LGnmOM4S1bY2C0AjmMnB8RuEYxpPwXQ==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-1.1.1.tgz", + "integrity": "sha512-xtoY50b5+7IH8tFbkw64gisG9tMSpxDjhX9TmaJJae/XuxQ9R/Kc8Nv1eOsf43Gt4KV/LkriMy9mptDr7XLcaw==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove-position": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-4.0.1.tgz", + "integrity": "sha512-0yDkppiIhDlPrfHELgB+NLQD5mfjup3a8UYclHruTJWmY74je8g+CIFr79x5f6AkmzSwlvKLbs63hC0meOMowQ==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-visit": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.2.tgz", + "integrity": "sha512-7A6eiDCs9UtjcwZOcCpM4aPII3bAAGv13E96IkawkOAW0OhH+yRxtY0lzo8KiHpzEMfH7Q+FizUmwp8Iqy5EWg==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-4.1.1.tgz", + "integrity": "sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.1.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-children": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-children/-/unist-util-visit-children-2.0.1.tgz", + "integrity": "sha512-2cEU3dhV1hMfO9ajwb8rJsDedMfsahsm6fCfR8LxDR/w7KcB5lzHQ9dBTQIXsWGNWBFH5MPmaFP3Xh0dWLqClQ==", + "dependencies": { + "@types/unist": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-5.1.1.tgz", + "integrity": "sha512-gks4baapT/kNRaWxuGkl5BIhoanZo7sC/cUT/JToSRNL1dYoXRFl75d++NkjYk4TAu2uv2Px+l8guMajogeuiw==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unzip-response": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-2.0.1.tgz", + "integrity": "sha512-N0XH6lqDtFH84JxptQoZYmloF4nzrQqqrAymNj+/gW60AO2AZgOcf4O/nUXJcYfyQkqvMo9lSupBZmmgvuVXlw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/upath": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/update-notifier": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", + "integrity": "sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A==", + "dependencies": { + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "pupa": "^2.0.1", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/yeoman/update-notifier?sponsor=1" + } + }, + "node_modules/update-notifier/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha512-NjFKA0DidqPa5ciFcSrXnAltTtzz84ogy+NebPvfEgAck0+TNg4UJ4IN+fB7zRZfbgUf0syOo9MDxFkDSMuFaQ==", + "dependencies": { + "prepend-http": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/user-home": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/user-home/-/user-home-2.0.0.tgz", + "integrity": "sha512-KMWqdlOcjCYdtIJpicDSFBQ8nFwS2i9sslAd6f4+CBGcU4gist2REnr2fxj2YocvJFxSF3ZOHLYLVZnUxv4BZQ==", + "dependencies": { + "os-homedir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/user-meta": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/user-meta/-/user-meta-1.0.0.tgz", + "integrity": "sha512-Q/opMgFhVbBkdlTs44UKzV7L5Uj2zrJ4MVPXTTzJmrU1bHb2cX6wJzBIqEf1gROTzZIH8u39WmHsa5EvfnMPrw==", + "dependencies": { + "rc": "^1.2.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/uvu": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/uvu/-/uvu-0.5.6.tgz", + "integrity": "sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==", + "dependencies": { + "dequal": "^2.0.0", + "diff": "^5.0.0", + "kleur": "^4.0.3", + "sade": "^1.7.3" + }, + "bin": { + "uvu": "bin.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/validate-npm-package-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz", + "integrity": "sha512-M6w37eVCMMouJ9V/sdPGnC5H4uDr73/+xdq0FBLO3TFFX1+7wiUY6Es328NN+y43tmY+doUdN9g9J21vqB7iLw==", + "dependencies": { + "builtins": "^1.0.3" + } + }, + "node_modules/vfile": { + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.5.tgz", + "integrity": "sha512-U1ho2ga33eZ8y8pkbQLH54uKqGhFJ6GYIHnnG5AhRpAh3OWjkrRHKa/KogbmQn8We+c0KVV3rTOgR9V/WowbXQ==", + "dependencies": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz", + "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==", + "dependencies": { + "@types/unist": "^2.0.0", + "vfile": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-matter": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/vfile-matter/-/vfile-matter-3.0.1.tgz", + "integrity": "sha512-CAAIDwnh6ZdtrqAuxdElUqQRQDQgbbIrYtDYI8gCjXS1qQ+1XdLoK8FIZWxJwn0/I+BkSSZpar3SOgjemQz4fg==", + "dependencies": { + "@types/js-yaml": "^4.0.0", + "is-buffer": "^2.0.0", + "js-yaml": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.1.2.tgz", + "integrity": "sha512-QjSNP6Yxzyycd4SVOtmKKyTsSvClqBPJcd00Z0zuPj3hOIjg0rUPG6DbFGPvUKRgYyaIWLPKpuEclcuvb3H8qA==", + "dependencies": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/w3c-keyname": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.6.tgz", + "integrity": "sha512-f+fciywl1SJEniZHD6H+kUO8gOnwIr7f4ijKA6+ZvJFjeGi1r4PDLl53Ayud9O/rk64RqgoQine0feoeOU0kXg==" + }, + "node_modules/watchpack": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", + "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", + "peer": true, + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "node_modules/webpack": { + "version": "5.75.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", + "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "peer": true, + "dependencies": { + "@types/eslint-scope": "^3.7.3", + "@types/estree": "^0.0.51", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.7.1", + "acorn-import-assertions": "^1.7.6", + "browserslist": "^4.14.5", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.10.0", + "es-module-lexer": "^0.9.0", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.9", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^3.1.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.1.3", + "watchpack": "^2.4.0", + "webpack-sources": "^3.2.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-merge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz", + "integrity": "sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g==", + "dependencies": { + "lodash": "^4.17.15" + } + }, + "node_modules/webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "peer": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webpack/node_modules/@types/estree": { + "version": "0.0.51", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz", + "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==", + "peer": true + }, + "node_modules/webpack/node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "peer": true, + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/webpack/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dependencies": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==" + }, + "node_modules/widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dependencies": { + "string-width": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/workbox-background-sync": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-background-sync/-/workbox-background-sync-6.5.4.tgz", + "integrity": "sha512-0r4INQZMyPky/lj4Ou98qxcThrETucOde+7mRGJl13MPJugQNKeZQOdIJe/1AchOP23cTqHcN/YVpD6r8E6I8g==", + "dependencies": { + "idb": "^7.0.1", + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-broadcast-update": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-broadcast-update/-/workbox-broadcast-update-6.5.4.tgz", + "integrity": "sha512-I/lBERoH1u3zyBosnpPEtcAVe5lwykx9Yg1k6f8/BGEPGaMMgZrwVrqL1uA9QZ1NGGFoyE6t9i7lBjOlDhFEEw==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-build": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-build/-/workbox-build-6.5.4.tgz", + "integrity": "sha512-kgRevLXEYvUW9WS4XoziYqZ8Q9j/2ziJYEtTrjdz5/L/cTUa2XfyMP2i7c3p34lgqJ03+mTiz13SdFef2POwbA==", + "dependencies": { + "@apideck/better-ajv-errors": "^0.3.1", + "@babel/core": "^7.11.1", + "@babel/preset-env": "^7.11.0", + "@babel/runtime": "^7.11.2", + "@rollup/plugin-babel": "^5.2.0", + "@rollup/plugin-node-resolve": "^11.2.1", + "@rollup/plugin-replace": "^2.4.1", + "@surma/rollup-plugin-off-main-thread": "^2.2.3", + "ajv": "^8.6.0", + "common-tags": "^1.8.0", + "fast-json-stable-stringify": "^2.1.0", + "fs-extra": "^9.0.1", + "glob": "^7.1.6", + "lodash": "^4.17.20", + "pretty-bytes": "^5.3.0", + "rollup": "^2.43.1", + "rollup-plugin-terser": "^7.0.0", + "source-map": "^0.8.0-beta.0", + "stringify-object": "^3.3.0", + "strip-comments": "^2.0.1", + "tempy": "^0.6.0", + "upath": "^1.2.0", + "workbox-background-sync": "6.5.4", + "workbox-broadcast-update": "6.5.4", + "workbox-cacheable-response": "6.5.4", + "workbox-core": "6.5.4", + "workbox-expiration": "6.5.4", + "workbox-google-analytics": "6.5.4", + "workbox-navigation-preload": "6.5.4", + "workbox-precaching": "6.5.4", + "workbox-range-requests": "6.5.4", + "workbox-recipes": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4", + "workbox-streams": "6.5.4", + "workbox-sw": "6.5.4", + "workbox-window": "6.5.4" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/workbox-build/node_modules/@apideck/better-ajv-errors": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz", + "integrity": "sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA==", + "dependencies": { + "json-schema": "^0.4.0", + "jsonpointer": "^5.0.0", + "leven": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "ajv": ">=8" + } + }, + "node_modules/workbox-build/node_modules/ajv": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.2.tgz", + "integrity": "sha512-E4bfmKAhGiSTvMfL1Myyycaub+cUEU2/IvpylXkUu7CHBkBj1f/ikdzbD7YQ6FKUbixDxeYvB/xY4fvyroDlQg==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/workbox-build/node_modules/fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/workbox-build/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "node_modules/workbox-build/node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/workbox-build/node_modules/source-map": { + "version": "0.8.0-beta.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", + "integrity": "sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==", + "dependencies": { + "whatwg-url": "^7.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/workbox-build/node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/workbox-cacheable-response": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-cacheable-response/-/workbox-cacheable-response-6.5.4.tgz", + "integrity": "sha512-DCR9uD0Fqj8oB2TSWQEm1hbFs/85hXXoayVwFKLVuIuxwJaihBsLsp4y7J9bvZbqtPJ1KlCkmYVGQKrBU4KAug==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-core": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-core/-/workbox-core-6.5.4.tgz", + "integrity": "sha512-OXYb+m9wZm8GrORlV2vBbE5EC1FKu71GGp0H4rjmxmF4/HLbMCoTFws87M3dFwgpmg0v00K++PImpNQ6J5NQ6Q==" + }, + "node_modules/workbox-expiration": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-expiration/-/workbox-expiration-6.5.4.tgz", + "integrity": "sha512-jUP5qPOpH1nXtjGGh1fRBa1wJL2QlIb5mGpct3NzepjGG2uFFBn4iiEBiI9GUmfAFR2ApuRhDydjcRmYXddiEQ==", + "dependencies": { + "idb": "^7.0.1", + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-google-analytics": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-google-analytics/-/workbox-google-analytics-6.5.4.tgz", + "integrity": "sha512-8AU1WuaXsD49249Wq0B2zn4a/vvFfHkpcFfqAFHNHwln3jK9QUYmzdkKXGIZl9wyKNP+RRX30vcgcyWMcZ9VAg==", + "dependencies": { + "workbox-background-sync": "6.5.4", + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "node_modules/workbox-navigation-preload": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-navigation-preload/-/workbox-navigation-preload-6.5.4.tgz", + "integrity": "sha512-IIwf80eO3cr8h6XSQJF+Hxj26rg2RPFVUmJLUlM0+A2GzB4HFbQyKkrgD5y2d84g2IbJzP4B4j5dPBRzamHrng==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-precaching": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-precaching/-/workbox-precaching-6.5.4.tgz", + "integrity": "sha512-hSMezMsW6btKnxHB4bFy2Qfwey/8SYdGWvVIKFaUm8vJ4E53JAY+U2JwLTRD8wbLWoP6OVUdFlXsTdKu9yoLTg==", + "dependencies": { + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "node_modules/workbox-range-requests": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-range-requests/-/workbox-range-requests-6.5.4.tgz", + "integrity": "sha512-Je2qR1NXCFC8xVJ/Lux6saH6IrQGhMpDrPXWZWWS8n/RD+WZfKa6dSZwU+/QksfEadJEr/NfY+aP/CXFFK5JFg==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-recipes": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-recipes/-/workbox-recipes-6.5.4.tgz", + "integrity": "sha512-QZNO8Ez708NNwzLNEXTG4QYSKQ1ochzEtRLGaq+mr2PyoEIC1xFW7MrWxrONUxBFOByksds9Z4//lKAX8tHyUA==", + "dependencies": { + "workbox-cacheable-response": "6.5.4", + "workbox-core": "6.5.4", + "workbox-expiration": "6.5.4", + "workbox-precaching": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "node_modules/workbox-routing": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-routing/-/workbox-routing-6.5.4.tgz", + "integrity": "sha512-apQswLsbrrOsBUWtr9Lf80F+P1sHnQdYodRo32SjiByYi36IDyL2r7BH1lJtFX8fwNHDa1QOVY74WKLLS6o5Pg==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-strategies": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-strategies/-/workbox-strategies-6.5.4.tgz", + "integrity": "sha512-DEtsxhx0LIYWkJBTQolRxG4EI0setTJkqR4m7r4YpBdxtWJH1Mbg01Cj8ZjNOO8etqfA3IZaOPHUxCs8cBsKLw==", + "dependencies": { + "workbox-core": "6.5.4" + } + }, + "node_modules/workbox-streams": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-streams/-/workbox-streams-6.5.4.tgz", + "integrity": "sha512-FXKVh87d2RFXkliAIheBojBELIPnWbQdyDvsH3t74Cwhg0fDheL1T8BqSM86hZvC0ZESLsznSYWw+Va+KVbUzg==", + "dependencies": { + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4" + } + }, + "node_modules/workbox-sw": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-sw/-/workbox-sw-6.5.4.tgz", + "integrity": "sha512-vo2RQo7DILVRoH5LjGqw3nphavEjK4Qk+FenXeUsknKn14eCNedHOXWbmnvP4ipKhlE35pvJ4yl4YYf6YsJArA==" + }, + "node_modules/workbox-webpack-plugin": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-webpack-plugin/-/workbox-webpack-plugin-6.5.4.tgz", + "integrity": "sha512-LmWm/zoaahe0EGmMTrSLUi+BjyR3cdGEfU3fS6PN1zKFYbqAKuQ+Oy/27e4VSXsyIwAw8+QDfk1XHNGtZu9nQg==", + "dependencies": { + "fast-json-stable-stringify": "^2.1.0", + "pretty-bytes": "^5.4.1", + "upath": "^1.2.0", + "webpack-sources": "^1.4.3", + "workbox-build": "6.5.4" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "webpack": "^4.4.0 || ^5.9.0" + } + }, + "node_modules/workbox-webpack-plugin/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/workbox-webpack-plugin/node_modules/webpack-sources": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", + "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dependencies": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + } + }, + "node_modules/workbox-window": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-window/-/workbox-window-6.5.4.tgz", + "integrity": "sha512-HnLZJDwYBE+hpG25AQBO8RUWBJRaCsI9ksQJEp3aCOFCaG5kqaToAYXFRAHxzRluM2cQbGzdQF5rjKPWPA1fug==", + "dependencies": { + "@types/trusted-types": "^2.0.2", + "workbox-core": "6.5.4" + } + }, + "node_modules/wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dependencies": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/ws": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yaml": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz", + "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/yargs": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz", + "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==", + "dependencies": { + "cliui": "^5.0.0", + "decamelize": "^1.2.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^15.0.1" + } + }, + "node_modules/yargs-parser": { + "version": "15.0.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.3.tgz", + "integrity": "sha512-/MVEVjTXy/cGAjdtQf8dW3V9b97bPN7rNn8ETj6BmAQL7ibC7O1Q9SPJbGjgh3SlwoBNXMzj/ZGIj8mBgl12YA==", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/yargs/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/yargs/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yargs/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zwitch": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.2.tgz", + "integrity": "sha512-JZxotl7SxAJH0j7dN4pxsTV6ZLXoLdGME+PsjkL/DaBrVryK9kTGq06GfKrwcSOqypP+fdXGoCHE36b99fWVoA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + }, + "dependencies": { + "@algolia/autocomplete-core": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.7.2.tgz", + "integrity": "sha512-eclwUDC6qfApNnEfu1uWcL/rudQsn59tjEoUYZYE2JSXZrHLRjBUGMxiCoknobU2Pva8ejb0eRxpIYDtVVqdsw==", + "requires": { + "@algolia/autocomplete-shared": "1.7.2" + } + }, + "@algolia/autocomplete-preset-algolia": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.7.2.tgz", + "integrity": "sha512-+RYEG6B0QiGGfRb2G3MtPfyrl0dALF3cQNTWBzBX6p5o01vCCGTTinAm2UKG3tfc2CnOMAtnPLkzNZyJUpnVJw==", + "requires": { + "@algolia/autocomplete-shared": "1.7.2" + } + }, + "@algolia/autocomplete-shared": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.7.2.tgz", + "integrity": "sha512-QCckjiC7xXHIUaIL3ektBtjJ0w7tTA3iqKcAE/Hjn1lZ5omp7i3Y4e09rAr9ZybqirL7AbxCLLq0Ra5DDPKeug==" + }, + "@algolia/cache-browser-local-storage": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.14.2.tgz", + "integrity": "sha512-FRweBkK/ywO+GKYfAWbrepewQsPTIEirhi1BdykX9mxvBPtGNKccYAxvGdDCumU1jL4r3cayio4psfzKMejBlA==", + "requires": { + "@algolia/cache-common": "4.14.2" + } + }, + "@algolia/cache-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.14.2.tgz", + "integrity": "sha512-SbvAlG9VqNanCErr44q6lEKD2qoK4XtFNx9Qn8FK26ePCI8I9yU7pYB+eM/cZdS9SzQCRJBbHUumVr4bsQ4uxg==" + }, + "@algolia/cache-in-memory": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.14.2.tgz", + "integrity": "sha512-HrOukWoop9XB/VFojPv1R5SVXowgI56T9pmezd/djh2JnVN/vXswhXV51RKy4nCpqxyHt/aGFSq2qkDvj6KiuQ==", + "requires": { + "@algolia/cache-common": "4.14.2" + } + }, + "@algolia/client-account": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.14.2.tgz", + "integrity": "sha512-WHtriQqGyibbb/Rx71YY43T0cXqyelEU0lB2QMBRXvD2X0iyeGl4qMxocgEIcbHyK7uqE7hKgjT8aBrHqhgc1w==", + "requires": { + "@algolia/client-common": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "@algolia/client-analytics": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.14.2.tgz", + "integrity": "sha512-yBvBv2mw+HX5a+aeR0dkvUbFZsiC4FKSnfqk9rrfX+QrlNOKEhCG0tJzjiOggRW4EcNqRmaTULIYvIzQVL2KYQ==", + "requires": { + "@algolia/client-common": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "@algolia/client-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.14.2.tgz", + "integrity": "sha512-43o4fslNLcktgtDMVaT5XwlzsDPzlqvqesRi4MjQz2x4/Sxm7zYg5LRYFol1BIhG6EwxKvSUq8HcC/KxJu3J0Q==", + "requires": { + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "@algolia/client-personalization": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.14.2.tgz", + "integrity": "sha512-ACCoLi0cL8CBZ1W/2juehSltrw2iqsQBnfiu/Rbl9W2yE6o2ZUb97+sqN/jBqYNQBS+o0ekTMKNkQjHHAcEXNw==", + "requires": { + "@algolia/client-common": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "@algolia/client-search": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.14.2.tgz", + "integrity": "sha512-L5zScdOmcZ6NGiVbLKTvP02UbxZ0njd5Vq9nJAmPFtjffUSOGEp11BmD2oMJ5QvARgx2XbX4KzTTNS5ECYIMWw==", + "requires": { + "@algolia/client-common": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "@algolia/logger-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.14.2.tgz", + "integrity": "sha512-/JGlYvdV++IcMHBnVFsqEisTiOeEr6cUJtpjz8zc0A9c31JrtLm318Njc72p14Pnkw3A/5lHHh+QxpJ6WFTmsA==" + }, + "@algolia/logger-console": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.14.2.tgz", + "integrity": "sha512-8S2PlpdshbkwlLCSAB5f8c91xyc84VM9Ar9EdfE9UmX+NrKNYnWR1maXXVDQQoto07G1Ol/tYFnFVhUZq0xV/g==", + "requires": { + "@algolia/logger-common": "4.14.2" + } + }, + "@algolia/requester-browser-xhr": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.14.2.tgz", + "integrity": "sha512-CEh//xYz/WfxHFh7pcMjQNWgpl4wFB85lUMRyVwaDPibNzQRVcV33YS+63fShFWc2+42YEipFGH2iPzlpszmDw==", + "requires": { + "@algolia/requester-common": "4.14.2" + } + }, + "@algolia/requester-common": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.14.2.tgz", + "integrity": "sha512-73YQsBOKa5fvVV3My7iZHu1sUqmjjfs9TteFWwPwDmnad7T0VTCopttcsM3OjLxZFtBnX61Xxl2T2gmG2O4ehg==" + }, + "@algolia/requester-node-http": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.14.2.tgz", + "integrity": "sha512-oDbb02kd1o5GTEld4pETlPZLY0e+gOSWjWMJHWTgDXbv9rm/o2cF7japO6Vj1ENnrqWvLBmW1OzV9g6FUFhFXg==", + "requires": { + "@algolia/requester-common": "4.14.2" + } + }, + "@algolia/transporter": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.14.2.tgz", + "integrity": "sha512-t89dfQb2T9MFQHidjHcfhh6iGMNwvuKUvojAj+JsrHAGbuSy7yE4BylhLX6R0Q1xYRoC4Vvv+O5qIw/LdnQfsQ==", + "requires": { + "@algolia/cache-common": "4.14.2", + "@algolia/logger-common": "4.14.2", + "@algolia/requester-common": "4.14.2" + } + }, + "@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "requires": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "dependencies": { + "@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "requires": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + } + } + }, + "@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "requires": { + "@babel/highlight": "^7.18.6" + } + }, + "@babel/compat-data": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.20.5.tgz", + "integrity": "sha512-KZXo2t10+/jxmkhNXc7pZTqRvSOIvVv/+lJwHS+B2rErwOyjuVRh60yVpb7liQ1U5t7lLJ1bz+t8tSypUZdm0g==" + }, + "@babel/core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.5.tgz", + "integrity": "sha512-UdOWmk4pNWTm/4DlPUl/Pt4Gz4rcEMb7CY0Y3eJl5Yz1vI8ZJGmHWaVE55LoxRjdpx0z259GE9U5STA9atUinQ==", + "requires": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.20.5", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-module-transforms": "^7.20.2", + "@babel/helpers": "^7.20.5", + "@babel/parser": "^7.20.5", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + }, + "dependencies": { + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==" + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "@babel/generator": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.20.5.tgz", + "integrity": "sha512-jl7JY2Ykn9S0yj4DQP82sYvPU+T3g0HFcWTqDLqiuA9tGRNIj9VfbtXGAYTTkyNEnQk1jkMGOdYka8aG/lulCA==", + "requires": { + "@babel/types": "^7.20.5", + "@jridgewell/gen-mapping": "^0.3.2", + "jsesc": "^2.5.1" + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.9.tgz", + "integrity": "sha512-yFQ0YCHoIqarl8BCRwBL8ulYUaZpz3bNsA7oFepAzee+8/+ImtADXNOmO5vJvsPff3qi+hvpkY/NYBTrBQgdNw==", + "requires": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.9" + } + }, + "@babel/helper-compilation-targets": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.20.0.tgz", + "integrity": "sha512-0jp//vDGp9e8hZzBc6N/KwA5ZK3Wsm/pfm4CrY7vzegkVxc65SgSn6wYOnwHe9Js9HRQ1YTCKLGPzDtaS3RoLQ==", + "requires": { + "@babel/compat-data": "^7.20.0", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.21.3", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "@babel/helper-create-class-features-plugin": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.20.5.tgz", + "integrity": "sha512-3RCdA/EmEaikrhayahwToF0fpweU/8o2p8vhc1c/1kftHOdTKuC65kik/TLc+qfbS8JKw4qqJbne4ovICDhmww==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-member-expression-to-functions": "^7.18.9", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.19.1", + "@babel/helper-split-export-declaration": "^7.18.6" + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.20.5.tgz", + "integrity": "sha512-m68B1lkg3XDGX5yCvGO0kPx3v9WIYLnzjKfPcQiwntEQa5ZeRkPmo2X/ISJc8qxWGfwUr+kvZAeEzAwLec2r2w==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.2.1" + } + }, + "@babel/helper-define-polyfill-provider": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.3.tgz", + "integrity": "sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==", + "requires": { + "@babel/helper-compilation-targets": "^7.17.7", + "@babel/helper-plugin-utils": "^7.16.7", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "@babel/helper-environment-visitor": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.9.tgz", + "integrity": "sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==" + }, + "@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-function-name": { + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.19.0.tgz", + "integrity": "sha512-WAwHBINyrpqywkUH0nTnNgI5ina5TFn85HKS0pbPDfxFfhyR/aNQEn4hGi1P1JyT//I0t4OgXUlofzWILRvS5w==", + "requires": { + "@babel/template": "^7.18.10", + "@babel/types": "^7.19.0" + } + }, + "@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.9.tgz", + "integrity": "sha512-RxifAh2ZoVU67PyKIO4AMi1wTenGfMR/O/ae0CCRqwgBAt5v7xjdtRw7UoSbsreKrQn5t7r89eruK/9JjYHuDg==", + "requires": { + "@babel/types": "^7.18.9" + } + }, + "@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-module-transforms": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.20.2.tgz", + "integrity": "sha512-zvBKyJXRbmK07XhMuujYoJ48B5yvvmM6+wcpv6Ivj4Yg6qO7NOZOSnvZN9CRl1zz1Z4cKf8YejmCMh8clOoOeA==", + "requires": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.20.2", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.19.1", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.1", + "@babel/types": "^7.20.2" + } + }, + "@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-plugin-utils": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.20.2.tgz", + "integrity": "sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ==" + }, + "@babel/helper-remap-async-to-generator": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.9.tgz", + "integrity": "sha512-dI7q50YKd8BAv3VEfgg7PS7yD3Rtbi2J1XMXaalXO0W0164hYLnh8zpjRS0mte9MfVp/tltvr/cfdXPvJr1opA==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-wrap-function": "^7.18.9", + "@babel/types": "^7.18.9" + } + }, + "@babel/helper-replace-supers": { + "version": "7.19.1", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.19.1.tgz", + "integrity": "sha512-T7ahH7wV0Hfs46SFh5Jz3s0B6+o8g3c+7TMxu7xKfmHikg7EAZ3I2Qk9LFhjxXq8sL7UkP5JflezNwoZa8WvWw==", + "requires": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-member-expression-to-functions": "^7.18.9", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.19.1", + "@babel/types": "^7.19.0" + } + }, + "@babel/helper-simple-access": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.20.2.tgz", + "integrity": "sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==", + "requires": { + "@babel/types": "^7.20.2" + } + }, + "@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.20.0.tgz", + "integrity": "sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==", + "requires": { + "@babel/types": "^7.20.0" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-string-parser": { + "version": "7.19.4", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz", + "integrity": "sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==" + }, + "@babel/helper-validator-identifier": { + "version": "7.19.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz", + "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==" + }, + "@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==" + }, + "@babel/helper-wrap-function": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.20.5.tgz", + "integrity": "sha512-bYMxIWK5mh+TgXGVqAtnu5Yn1un+v8DDZtqyzKRLUzrh70Eal2O3aZ7aPYiMADO4uKlkzOiRiZ6GX5q3qxvW9Q==", + "requires": { + "@babel/helper-function-name": "^7.19.0", + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5" + } + }, + "@babel/helpers": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.20.6.tgz", + "integrity": "sha512-Pf/OjgfgFRW5bApskEz5pvidpim7tEDPlFtKcNRXWmfHGn9IEI2W2flqRQXTFb7gIPTyK++N6rVHuwKut4XK6w==", + "requires": { + "@babel/template": "^7.18.10", + "@babel/traverse": "^7.20.5", + "@babel/types": "^7.20.5" + } + }, + "@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "requires": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "@babel/parser": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.20.5.tgz", + "integrity": "sha512-r27t/cy/m9uKLXQNWWebeCUHgnAZq0CpG1OwKRxzJMP1vpSU4bSIK2hq+/cp0bQxetkXx38n09rNu8jVkcK/zA==" + }, + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.9.tgz", + "integrity": "sha512-AHrP9jadvH7qlOj6PINbgSuphjQUAK7AOT7DPjBo9EHoLhQTnnK5u45e1Hd4DbSQEO9nqPWtQ89r+XEOWFScKg==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9", + "@babel/plugin-proposal-optional-chaining": "^7.18.9" + } + }, + "@babel/plugin-proposal-async-generator-functions": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.20.1.tgz", + "integrity": "sha512-Gh5rchzSwE4kC+o/6T8waD0WHEQIsDmjltY8WnWRXHUdH8axZhuH86Ov9M72YhJfDrZseQwuuWaaIT/TmePp3g==", + "requires": { + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-remap-async-to-generator": "^7.18.9", + "@babel/plugin-syntax-async-generators": "^7.8.4" + } + }, + "@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + } + }, + "@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + } + }, + "@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.9.tgz", + "integrity": "sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + } + }, + "@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + } + }, + "@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.9.tgz", + "integrity": "sha512-128YbMpjCrP35IOExw2Fq+x55LMP42DzhOhX2aNNIdI9avSWl2PI0yuBWarr3RYpZBSPtabfadkH2yeRiMD61Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + } + }, + "@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + } + }, + "@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + } + }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.20.2.tgz", + "integrity": "sha512-Ks6uej9WFK+fvIMesSqbAto5dD8Dz4VuuFvGJFKgIGSkJuRGcrwGECPA1fDgQK3/DbExBJpEkTeYeB8geIFCSQ==", + "requires": { + "@babel/compat-data": "^7.20.1", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.20.1" + } + }, + "@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + } + }, + "@babel/plugin-proposal-optional-chaining": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.9.tgz", + "integrity": "sha512-v5nwt4IqBXihxGsW2QmCWMDS3B3bzGIk/EQVZz2ei7f3NJl8NzAJVvUmpDW5q1CRNY+Beb/k58UAH1Km1N411w==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + } + }, + "@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-private-property-in-object": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.20.5.tgz", + "integrity": "sha512-Vq7b9dUA12ByzB4EjQTPo25sFhY+08pQDBSZRtUAkj7lb7jahaHR5igera16QZ+3my1nYR4dKsNdYj5IjPHilQ==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.20.5", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + } + }, + "@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "requires": { + "@babel/helper-plugin-utils": "^7.12.13" + } + }, + "@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.3" + } + }, + "@babel/plugin-syntax-import-assertions": { + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.20.0.tgz", + "integrity": "sha512-IUh1vakzNoWalR8ch/areW7qFopR2AEw03JlG7BbrDqmQ4X3q9uuipQwSGrUn7oGiemKjtSLDhNtQHzMHr1JdQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.19.0" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "requires": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoping": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.20.5.tgz", + "integrity": "sha512-WvpEIW9Cbj9ApF3yJCjIEEf1EiNJLtXagOrL5LNWEZOo3jv8pmPoYTSNJQvqej8OavVlgOoOPw6/htGZro6IkA==", + "requires": { + "@babel/helper-plugin-utils": "^7.20.2" + } + }, + "@babel/plugin-transform-classes": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.20.2.tgz", + "integrity": "sha512-9rbPp0lCVVoagvtEyQKSo5L8oo0nQS/iif+lwlAz29MccX2642vWDlSZK+2T2buxbopotId2ld7zZAzRfz9j1g==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-replace-supers": "^7.19.1", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + }, + "dependencies": { + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" + } + } + }, + "@babel/plugin-transform-computed-properties": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.9.tgz", + "integrity": "sha512-+i0ZU1bCDymKakLxn5srGHrsAPRELC2WIbzwjLhHW9SIE1cPYkLCL0NlnXMZaM1vhfgA2+M7hySk42VBvrkBRw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.20.2.tgz", + "integrity": "sha512-mENM+ZHrvEgxLTBXUiQ621rRXZes3KWUv6NdQlrnr1TkWVw+hUjQBZuP2X32qKlrlG2BzgR95gkuCRSkJl8vIw==", + "requires": { + "@babel/helper-plugin-utils": "^7.20.2" + } + }, + "@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-duplicate-keys": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.9.tgz", + "integrity": "sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "requires": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-for-of": { + "version": "7.18.8", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.8.tgz", + "integrity": "sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-function-name": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.9.tgz", + "integrity": "sha512-WvIBoRPaJQ5yVHzcnJFor7oS5Ls0PYixlTYE63lCj2RtdQEl15M68FXQlxnG6wdraJIXRdR7KI+hQ7q/9QjrCQ==", + "requires": { + "@babel/helper-compilation-targets": "^7.18.9", + "@babel/helper-function-name": "^7.18.9", + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-literals": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.9.tgz", + "integrity": "sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-modules-amd": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.19.6.tgz", + "integrity": "sha512-uG3od2mXvAtIFQIh0xrpLH6r5fpSQN04gIVovl+ODLdUMANokxQLZnPBHcjmv3GxRjnqwLuHvppjjcelqUFZvg==", + "requires": { + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0" + } + }, + "@babel/plugin-transform-modules-commonjs": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.19.6.tgz", + "integrity": "sha512-8PIa1ym4XRTKuSsOUXqDG0YaOlEuTVvHMe5JCfgBMOtHvJKw/4NGovEGN33viISshG/rZNVrACiBmPQLvWN8xQ==", + "requires": { + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-simple-access": "^7.19.4" + } + }, + "@babel/plugin-transform-modules-systemjs": { + "version": "7.19.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.19.6.tgz", + "integrity": "sha512-fqGLBepcc3kErfR9R3DnVpURmckXP7gj7bAlrTQyBxrigFqszZCkFkcoxzCp2v32XmwXLvbw+8Yq9/b+QqksjQ==", + "requires": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.19.6", + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-validator-identifier": "^7.19.1" + } + }, + "@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.20.5.tgz", + "integrity": "sha512-mOW4tTzi5iTLnw+78iEq3gr8Aoq4WNRGpmSlrogqaiCBoR1HFhpU4JkpQFOHfeYx3ReVIFWOQJS4aZBRvuZ6mA==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.20.5", + "@babel/helper-plugin-utils": "^7.20.2" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + } + }, + "@babel/plugin-transform-parameters": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.20.5.tgz", + "integrity": "sha512-h7plkOmcndIUWXZFLgpbrh2+fXAi47zcUX7IrOQuZdLD0I0KvjJ6cvo3BEcAOsDOcZhVKGJqv07mkSqK0y2isQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.20.2" + } + }, + "@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.20.5.tgz", + "integrity": "sha512-kW/oO7HPBtntbsahzQ0qSE3tFvkFwnbozz3NWFhLGqH75vLEg+sCGngLlhVkePlCs3Jv0dBBHDzCHxNiFAQKCQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.20.2", + "regenerator-transform": "^0.15.1" + } + }, + "@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-spread": { + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.19.0.tgz", + "integrity": "sha512-RsuMk7j6n+r752EtzyScnWkQyuJdli6LdO5Klv8Yx0OfPVTcQkIUfS8clx5e9yHXzlnhOZF3CbQ8C2uP5j074w==", + "requires": { + "@babel/helper-plugin-utils": "^7.19.0", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.9" + } + }, + "@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-template-literals": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.9.tgz", + "integrity": "sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-typeof-symbol": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.9.tgz", + "integrity": "sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-unicode-escapes": { + "version": "7.18.10", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.10.tgz", + "integrity": "sha512-kKAdAI+YzPgGY/ftStBFXTI1LZFju38rYThnfMykS+IXy8BVx+res7s2fxf1l8I35DV2T97ezo6+SGrXz6B3iQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.18.9" + } + }, + "@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/preset-env": { + "version": "7.20.2", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.20.2.tgz", + "integrity": "sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==", + "requires": { + "@babel/compat-data": "^7.20.1", + "@babel/helper-compilation-targets": "^7.20.0", + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.9", + "@babel/plugin-proposal-async-generator-functions": "^7.20.1", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.9", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.9", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.20.2", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.9", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.20.0", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.20.2", + "@babel/plugin-transform-classes": "^7.20.2", + "@babel/plugin-transform-computed-properties": "^7.18.9", + "@babel/plugin-transform-destructuring": "^7.20.2", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.9", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.8", + "@babel/plugin-transform-function-name": "^7.18.9", + "@babel/plugin-transform-literals": "^7.18.9", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.19.6", + "@babel/plugin-transform-modules-commonjs": "^7.19.6", + "@babel/plugin-transform-modules-systemjs": "^7.19.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.19.1", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.20.1", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.19.0", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.9", + "@babel/plugin-transform-typeof-symbol": "^7.18.9", + "@babel/plugin-transform-unicode-escapes": "^7.18.10", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.20.2", + "babel-plugin-polyfill-corejs2": "^0.3.3", + "babel-plugin-polyfill-corejs3": "^0.6.0", + "babel-plugin-polyfill-regenerator": "^0.4.1", + "core-js-compat": "^3.25.1", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "requires": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + } + }, + "@babel/runtime": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.1.tgz", + "integrity": "sha512-mrzLkl6U9YLF8qpqI7TB82PESyEGjm/0Ly91jG575eVxMMlb8fYfOXFZIJ8XfLrJZQbm7dlKry2bJmXBUEkdFg==", + "requires": { + "regenerator-runtime": "^0.13.10" + } + }, + "@babel/runtime-corejs3": { + "version": "7.20.1", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.20.1.tgz", + "integrity": "sha512-CGulbEDcg/ND1Im7fUNRZdGXmX2MTWVVZacQi/6DiKE5HNwZ3aVTm5PV4lO8HHz0B2h8WQyvKKjbX5XgTtydsg==", + "requires": { + "core-js-pure": "^3.25.1", + "regenerator-runtime": "^0.13.10" + } + }, + "@babel/template": { + "version": "7.18.10", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.10.tgz", + "integrity": "sha512-TI+rCtooWHr3QJ27kJxfjutghu44DLnasDMwpDqCXVTal9RLp3RSYNh4NdBrRP2cQAoG9A8juOQl6P6oZG4JxA==", + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.10", + "@babel/types": "^7.18.10" + } + }, + "@babel/traverse": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.20.5.tgz", + "integrity": "sha512-WM5ZNN3JITQIq9tFZaw1ojLU3WgWdtkxnhM1AegMS+PvHjkM5IXjmYEGY7yukz5XS4sJyEf2VzWjI8uAavhxBQ==", + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.20.5", + "@babel/helper-environment-visitor": "^7.18.9", + "@babel/helper-function-name": "^7.19.0", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.20.5", + "@babel/types": "^7.20.5", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "dependencies": { + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" + } + } + }, + "@babel/types": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.20.5.tgz", + "integrity": "sha512-c9fst/h2/dcF7H+MJKZ2T0KjEQ8hY/BNnDk/H3XY8C4Aw/eWQXWn/lWntHF9ooUBnGmEvbfGrTgLWc+um0YDUg==", + "requires": { + "@babel/helper-string-parser": "^7.19.4", + "@babel/helper-validator-identifier": "^7.19.1", + "to-fast-properties": "^2.0.0" + } + }, + "@codemirror/autocomplete": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.4.0.tgz", + "integrity": "sha512-HLF2PnZAm1s4kGs30EiqKMgD7XsYaQ0XJnMR0rofEWQ5t5D60SfqpDIkIh1ze5tiEbyUWm8+VJ6W1/erVvBMIA==", + "requires": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.6.0", + "@lezer/common": "^1.0.0" + } + }, + "@codemirror/commands": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.1.2.tgz", + "integrity": "sha512-sO3jdX1s0pam6lIdeSJLMN3DQ6mPEbM4yLvyKkdqtmd/UDwhXA5+AwFJ89rRXm6vTeOXBsE5cAmlos/t7MJdgg==", + "requires": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0" + } + }, + "@codemirror/lang-python": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/lang-python/-/lang-python-6.1.0.tgz", + "integrity": "sha512-a/JhyPYn5qz5T8WtAfZCuAZcfClgNVb7UZzdLr76bWUeG7Usd3Un5o8UQOkZ/5Xw+EM5YGHHG+T6ickMYkDcRQ==", + "requires": { + "@codemirror/autocomplete": "^6.3.2", + "@codemirror/language": "^6.0.0", + "@lezer/python": "^1.0.0" + } + }, + "@codemirror/language": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.3.1.tgz", + "integrity": "sha512-MK+G1QKaGfSEUg9YEFaBkMBI6j1ge4VMBPZv9fDYotw7w695c42x5Ba1mmwBkesYnzYFBfte6Hh9TDcKa6xORQ==", + "requires": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "@codemirror/lint": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.1.0.tgz", + "integrity": "sha512-mdvDQrjRmYPvQ3WrzF6Ewaao+NWERYtpthJvoQ3tK3t/44Ynhk8ZGjTSL9jMEv8CgSMogmt75X8ceOZRDSXHtQ==", + "requires": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "@codemirror/search": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.2.3.tgz", + "integrity": "sha512-V9n9233lopQhB1dyjsBK2Wc1i+8hcCqxl1wQ46c5HWWLePoe4FluV3TGHoZ04rBRlGjNyz9DTmpJErig8UE4jw==", + "requires": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "@codemirror/state": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.1.4.tgz", + "integrity": "sha512-g+3OJuRylV5qsXuuhrc6Cvs1NQluNioepYMM2fhnpYkNk7NgX+j0AFuevKSVKzTDmDyt9+Puju+zPdHNECzCNQ==" + }, + "@codemirror/theme-one-dark": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@codemirror/theme-one-dark/-/theme-one-dark-6.1.0.tgz", + "integrity": "sha512-AiTHtFRu8+vWT9wWUWDM+cog6ZwgivJogB1Tm/g40NIpLwph7AnmxrSzWfvJN5fBVufsuwBxecQCNmdcR5D7Aw==", + "requires": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/highlight": "^1.0.0" + } + }, + "@codemirror/view": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.7.1.tgz", + "integrity": "sha512-kYtS+uqYw/q/0ytYxpkqE1JVuK5NsbmBklWYhwLFTKO9gVuTdh/kDEeZPKorbqHcJ+P+ucrhcsS1czVweOpT2g==", + "requires": { + "@codemirror/state": "^6.1.4", + "style-mod": "^4.0.0", + "w3c-keyname": "^2.2.4" + } + }, + "@corex/deepmerge": { + "version": "4.0.29", + "resolved": "https://registry.npmjs.org/@corex/deepmerge/-/deepmerge-4.0.29.tgz", + "integrity": "sha512-q/yVUnqckA8Do+EvAfpy7RLdumnBy9ZsducMUtZTvpdbJC7azEf1hGtnYYxm0QfphYxjwggv6XtH64prvS1W+A==" + }, + "@docsearch/css": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.3.0.tgz", + "integrity": "sha512-rODCdDtGyudLj+Va8b6w6Y85KE85bXRsps/R4Yjwt5vueXKXZQKYw0aA9knxLBT6a/bI/GMrAcmCR75KYOM6hg==" + }, + "@docsearch/react": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.3.0.tgz", + "integrity": "sha512-fhS5adZkae2SSdMYEMVg6pxI5a/cE+tW16ki1V0/ur4Fdok3hBRkmN/H8VvlXnxzggkQIIRIVvYPn00JPjen3A==", + "requires": { + "@algolia/autocomplete-core": "1.7.2", + "@algolia/autocomplete-preset-algolia": "1.7.2", + "@docsearch/css": "3.3.0", + "algoliasearch": "^4.0.0" + } + }, + "@eslint/eslintrc": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.3.tgz", + "integrity": "sha512-uj3pT6Mg+3t39fvLrj8iuCIJ38zKO9FpGtJ4BBJebJhEwjoT+KLVNCcHT5QC9NGRIEi7fZ0ZR8YRb884auB4Lg==", + "requires": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.4.0", + "globals": "^13.15.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + } + }, + "@humanwhocodes/config-array": { + "version": "0.11.7", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.7.tgz", + "integrity": "sha512-kBbPWzN8oVMLb0hOUYXhmxggL/1cJE6ydvjDIGi9EnAGUyA7cLVKQg+d/Dsm+KZwx2czGHrCmMVLiyg8s5JPKw==", + "requires": { + "@humanwhocodes/object-schema": "^1.2.1", + "debug": "^4.1.1", + "minimatch": "^3.0.5" + } + }, + "@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==" + }, + "@humanwhocodes/object-schema": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz", + "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==" + }, + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==" + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==" + }, + "@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "requires": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "@jridgewell/trace-mapping": { + "version": "0.3.17", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz", + "integrity": "sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==", + "requires": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, + "@jupyterlab/coreutils": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@jupyterlab/coreutils/-/coreutils-2.2.1.tgz", + "integrity": "sha512-XkGMBXqEAnENC4fK/L3uEqqxyNUtf4TI/1XNDln7d5VOPHQJSBTbYlBAZ0AQotn2qbs4WqmV6icxje2ITVedqQ==", + "requires": { + "@phosphor/algorithm": "^1.1.2", + "@phosphor/coreutils": "^1.3.0", + "@phosphor/disposable": "^1.1.2", + "@phosphor/signaling": "^1.2.2", + "ajv": "~5.1.6", + "comment-json": "^1.1.3", + "minimist": "~1.2.0", + "moment": "~2.21.0", + "path-posix": "~1.0.0", + "url-parse": "~1.4.3" + }, + "dependencies": { + "ajv": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.1.6.tgz", + "integrity": "sha512-K/r7dMp3q7rKzhx6v6deMuxVuQCw0w/789F75BooHOOVBEXzejwUq3LwO4x41C/xzXNKSNzqoAAS48Sx2a2Qxg==", + "requires": { + "co": "^4.6.0", + "json-schema-traverse": "^0.3.0", + "json-stable-stringify": "^1.0.1" + } + }, + "comment-json": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-1.1.3.tgz", + "integrity": "sha512-gGSHSON1HdpFDb3WpU6C1tdksZV0lpfzoihXK+CD0l4xQ2TvHbVuiZy8Dg414ZqbeRxt2sU8d9kC0srpM2+RzQ==", + "requires": { + "json-parser": "^1.0.0" + } + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha512-4JD/Ivzg7PoW8NzdrBSr3UFwC9mHgvI7Z6z3QGBsSHgKaRTUDmyZAAKJo2UbG1kUVfS9WS8bi36N49U1xw43DA==" + } + } + }, + "@jupyterlab/observables": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@jupyterlab/observables/-/observables-2.4.0.tgz", + "integrity": "sha512-M/fhAnPqd6F4Zwt4IIsvHCkJmwbSw1Tko/hUXgdUQG86lPsJiTOh98sB3qwV1gtzb9oFF+kH21XsHnQZ6Yl6Pw==", + "requires": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/coreutils": "^1.3.1", + "@phosphor/disposable": "^1.3.0", + "@phosphor/messaging": "^1.3.0", + "@phosphor/signaling": "^1.3.0" + } + }, + "@jupyterlab/services": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@jupyterlab/services/-/services-3.2.1.tgz", + "integrity": "sha512-zCMruGGYxTe427ESK4YUO1V/liFOFYpebYPRsJ+j9CFdV+Hm760+nx4pFX6N6Z9TbS+5cs8BgZ+ebm8unRZrJg==", + "requires": { + "@jupyterlab/coreutils": "^2.2.1", + "@jupyterlab/observables": "^2.1.1", + "@phosphor/algorithm": "^1.1.2", + "@phosphor/coreutils": "^1.3.0", + "@phosphor/disposable": "^1.1.2", + "@phosphor/signaling": "^1.2.2" + } + }, + "@lezer/common": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz", + "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng==" + }, + "@lezer/highlight": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz", + "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==", + "requires": { + "@lezer/common": "^1.0.0" + } + }, + "@lezer/lr": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.2.5.tgz", + "integrity": "sha512-f9319YG1A/3ysgUE3bqCHEd7g+3ZZ71MWlwEc42mpnLVYXgfJJgtu1XAyBB4Kz8FmqmnFe9caopDqKeMMMAU6g==", + "requires": { + "@lezer/common": "^1.0.0" + } + }, + "@lezer/python": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@lezer/python/-/python-1.1.1.tgz", + "integrity": "sha512-ArUGh9kvdaOVu6IkSaYUS9WFQeMAFVWKRuZo6vexnxoeCLnxf0Y9DCFEAMMa7W9SQBGYE55OarSpPqSkdOXSCA==", + "requires": { + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "@mapbox/hast-util-table-cell-style": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@mapbox/hast-util-table-cell-style/-/hast-util-table-cell-style-0.2.0.tgz", + "integrity": "sha512-gqaTIGC8My3LVSnU38IwjHVKJC94HSonjvFHDk8/aSrApL8v4uWgm8zJkK7MJIIbHuNOr/+Mv2KkQKcxs6LEZA==", + "requires": { + "unist-util-visit": "^1.4.1" + }, + "dependencies": { + "unist-util-is": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-3.0.0.tgz", + "integrity": "sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==" + }, + "unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "requires": { + "unist-util-visit-parents": "^2.0.0" + } + }, + "unist-util-visit-parents": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", + "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", + "requires": { + "unist-util-is": "^3.0.0" + } + } + } + }, + "@mapbox/rehype-prism": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@mapbox/rehype-prism/-/rehype-prism-0.8.0.tgz", + "integrity": "sha512-bIz4a3oZ8g+pQBHSMMIxNpYOMX4yq4aZdpezoiCl9yJudh1Z9SRmHQqH6f+WtTBNOORzXMEGkvBze4PiH17wtA==", + "requires": { + "hast-util-to-string": "^1.0.4", + "mrm": "^3.0.9", + "refractor": "^3.4.0", + "unist-util-visit": "^2.0.3" + }, + "dependencies": { + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + } + } + }, + "@mdx-js/loader": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/loader/-/loader-2.1.5.tgz", + "integrity": "sha512-oXjfTa/iAcMmW8DdQ+hQFodPCLqw5VKT2yoZkLwrZfPVVpUgKrI+5/ZePYq328xxtAUStZmR3ed0PWJrwd5Pkg==", + "requires": { + "@mdx-js/mdx": "^2.0.0", + "source-map": "^0.7.0" + } + }, + "@mdx-js/mdx": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/mdx/-/mdx-2.1.5.tgz", + "integrity": "sha512-zEG0lt+Bl/r5U6e0TOS7qDbsXICtemfAPquxWFsMbdzrvlWaqMGemLl+sjVpqlyaaiCiGVQBSGdCk0t1qXjkQg==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "@types/mdx": "^2.0.0", + "estree-util-build-jsx": "^2.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "estree-util-to-js": "^1.1.0", + "estree-walker": "^3.0.0", + "hast-util-to-estree": "^2.0.0", + "markdown-extensions": "^1.0.0", + "periscopic": "^3.0.0", + "remark-mdx": "^2.0.0", + "remark-parse": "^10.0.0", + "remark-rehype": "^10.0.0", + "unified": "^10.0.0", + "unist-util-position-from-estree": "^1.0.0", + "unist-util-stringify-position": "^3.0.0", + "unist-util-visit": "^4.0.0", + "vfile": "^5.0.0" + } + }, + "@mdx-js/react": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-2.1.5.tgz", + "integrity": "sha512-3Az1I6SAWA9R38rYjz5rXBrGKeZhq96CSSyQtqY+maPj8stBsoUH5pNcmIixuGkufYsh8F5+ka2CVPo2fycWZw==", + "requires": { + "@types/mdx": "^2.0.0", + "@types/react": ">=16" + } + }, + "@next/env": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/env/-/env-13.0.2.tgz", + "integrity": "sha512-Qb6WPuRriGIQ19qd6NBxpcrFOfj8ziN7l9eZUfwff5gl4zLXluqtuZPddYZM/oWjN53ZYcuRXzL+oowKyJeYtA==" + }, + "@next/eslint-plugin-next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-13.0.2.tgz", + "integrity": "sha512-W+fIIIaFU7Kct7Okx91C7XDRGolv/w2RUenX2yZFeeNVcuVzDIKUcNmckrYbYcwrNQUSXmtwrs3g8xwast0YtA==", + "requires": { + "glob": "7.1.7" + } + }, + "@next/mdx": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/mdx/-/mdx-13.0.2.tgz", + "integrity": "sha512-udtSlWoOLoHdJvtuGyj3ctXW4fogsQDWU2YXp4Sx4JdsS7VKvJkhTU0Df47nAJsQKCJhQ92h+j7rA+1Dulxxqg==", + "requires": { + "source-map": "^0.7.0" + } + }, + "@next/swc-android-arm-eabi": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-13.0.2.tgz", + "integrity": "sha512-X54UQCTFyOGnJP//Z71dPPlp4BCYcQL2ncikKXQcPzVpqPs4C3m+tKC8ivBNH6edAXkppwsLRz1/yQwgSZ9Swg==", + "optional": true + }, + "@next/swc-android-arm64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-13.0.2.tgz", + "integrity": "sha512-1P00Kv8uKaLubqo7JzPrTqgFAzSOmfb8iwqJrOb9in5IvTRtNGlkR4hU0sXzqbQNM/+SaYxze6Z5ry1IDyb/cQ==", + "optional": true + }, + "@next/swc-darwin-arm64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.0.2.tgz", + "integrity": "sha512-1zGIOkInkOLRv0QQGZ+3wffYsyKI4vIy62LYTvDWUn7TAYqnmXwougp9NSLqDeagLwgsv2URrykyAFixA/YqxA==", + "optional": true + }, + "@next/swc-darwin-x64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.0.2.tgz", + "integrity": "sha512-ECDAjoMP1Y90cARaelS6X+k6BQx+MikAYJ8f/eaJrLur44NIOYc9HA/dgcTp5jenguY4yT8V+HCquLjAVle6fA==", + "optional": true + }, + "@next/swc-freebsd-x64": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-13.0.2.tgz", + "integrity": "sha512-2DcL/ofQdBnQX3IoI9sjlIAyLCD1oZoUBuhrhWbejvBQjutWrI0JTEv9uG69WcxWhVMm3BCsjv8GK2/68OKp7A==", + "optional": true + }, + "@next/swc-linux-arm-gnueabihf": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-13.0.2.tgz", + "integrity": "sha512-Y3OQF1CSBSWW2vGkmvOIuOUNqOq8qX7f1ZpcKUVWP3/Uq++DZmVi9d18lgnSe1I3QFqc+nXWyun9ljsN83j0sw==", + "optional": true + }, + "@next/swc-linux-arm64-gnu": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.0.2.tgz", + "integrity": "sha512-mNyzwsFF6kwZYEjnGicx9ksDZYEZvyzEc1BtCu8vdZi/v8UeixQwCiAT6FyYX9uxMPEkzk8qiU0t0u9gvltsKw==", + "optional": true + }, + "@next/swc-linux-arm64-musl": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.0.2.tgz", + "integrity": "sha512-M6SdYjWgRrY3tJBxz0663zCRPTu5BRONmxlftKWWHv9LjAJ59neTLaGj4rp0A08DkJglZIoCkLOzLrzST6TGag==", + "optional": true + }, + "@next/swc-linux-x64-gnu": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.0.2.tgz", + "integrity": "sha512-pi63RoxvG4ES1KS06Zpm0MATVIXTs/TIbLbdckeLoM40u1d3mQl/+hSSrLRSxzc2OtyL8fh92sM4gkJrQXAMAw==", + "optional": true + }, + "@next/swc-linux-x64-musl": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.0.2.tgz", + "integrity": "sha512-9Pv91gfYnDONgjtRm78n64b/c54+azeHtlnqBLTnIFWSMBDRl1/WDkhKWIj3fBGPLimtK7Tko3ULR3og9RRUPw==", + "optional": true + }, + "@next/swc-win32-arm64-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.0.2.tgz", + "integrity": "sha512-Nvewe6YZaizAkGHHprbMkYqQulBjZCHKBGKeFPwoPtOA+a2Qi4pZzc/qXFyC5/2A6Z0mr2U1zg9rd04WBYMwBw==", + "optional": true + }, + "@next/swc-win32-ia32-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.0.2.tgz", + "integrity": "sha512-ZUBYGZw5G3QrqDpRq1EWi3aHmvPZM8ijK5TFL6UbH16cYQ0JpANmuG2P66KB93Qe/lWWzbeAZk/tj1XqwoCuPA==", + "optional": true + }, + "@next/swc-win32-x64-msvc": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.0.2.tgz", + "integrity": "sha512-fA9uW1dm7C0mEYGcKlbmLcVm2sKcye+1kPxh2cM4jVR+kQQMtHWsjIzeSpe2grQLSDan06z4n6hbr8b1c3hA8w==", + "optional": true + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==" + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@phosphor/algorithm": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@phosphor/algorithm/-/algorithm-1.2.0.tgz", + "integrity": "sha512-C9+dnjXyU2QAkWCW6QVDGExk4hhwxzAKf5/FIuYlHAI9X5vFv99PYm0EREDxX1PbMuvfFBZhPNu0PvuSDQ7sFA==" + }, + "@phosphor/collections": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@phosphor/collections/-/collections-1.2.0.tgz", + "integrity": "sha512-T9/0EjSuY6+ga2LIFRZ0xupciOR3Qnyy8Q95lhGTC0FXZUFwC8fl9e8On6IcwasCszS+1n8dtZUWSIynfgdpzw==", + "requires": { + "@phosphor/algorithm": "^1.2.0" + } + }, + "@phosphor/coreutils": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/coreutils/-/coreutils-1.3.1.tgz", + "integrity": "sha512-9OHCn8LYRcPU/sbHm5v7viCA16Uev3gbdkwqoQqlV+EiauDHl70jmeL7XVDXdigl66Dz0LI11C99XOxp+s3zOA==" + }, + "@phosphor/disposable": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/disposable/-/disposable-1.3.1.tgz", + "integrity": "sha512-0NGzoTXTOizWizK/brKKd5EjJhuuEH4903tLika7q6wl/u0tgneJlTh7R+MBVeih0iNxtuJAfBa3IEY6Qmj+Sw==", + "requires": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/signaling": "^1.3.1" + } + }, + "@phosphor/messaging": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@phosphor/messaging/-/messaging-1.3.0.tgz", + "integrity": "sha512-k0JE+BTMKlkM335S2AmmJxoYYNRwOdW5jKBqLgjJdGRvUQkM0+2i60ahM45+J23atGJDv9esKUUBINiKHFhLew==", + "requires": { + "@phosphor/algorithm": "^1.2.0", + "@phosphor/collections": "^1.2.0" + } + }, + "@phosphor/signaling": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@phosphor/signaling/-/signaling-1.3.1.tgz", + "integrity": "sha512-Eq3wVCPQAhUd9+gUGaYygMr+ov7dhSGblSBXiDzpZlSIfa8OVD4P3cCvYXr/acDTNmZ/gHTcSFO8/n3rDkeXzg==", + "requires": { + "@phosphor/algorithm": "^1.2.0" + } + }, + "@rehooks/online-status": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@rehooks/online-status/-/online-status-1.1.2.tgz", + "integrity": "sha512-tuz6RtdgqkBMZr82UqTkzPCsHNx6tpEyhTZ5OD8WhDpU4/TLzljBCseuWLlbeQiygbM9F2A4Ea7P+wyjohJXBw==" + }, + "@rollup/plugin-babel": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz", + "integrity": "sha512-WFfdLWU/xVWKeRQnKmIAQULUI7Il0gZnBIH/ZFO069wYIfPu+8zrfp/KMW0atmELoRDq8FbiP3VCss9MhCut7Q==", + "requires": { + "@babel/helper-module-imports": "^7.10.4", + "@rollup/pluginutils": "^3.1.0" + } + }, + "@rollup/plugin-node-resolve": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz", + "integrity": "sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg==", + "requires": { + "@rollup/pluginutils": "^3.1.0", + "@types/resolve": "1.17.1", + "builtin-modules": "^3.1.0", + "deepmerge": "^4.2.2", + "is-module": "^1.0.0", + "resolve": "^1.19.0" + } + }, + "@rollup/plugin-replace": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-2.4.2.tgz", + "integrity": "sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==", + "requires": { + "@rollup/pluginutils": "^3.1.0", + "magic-string": "^0.25.7" + } + }, + "@rollup/pluginutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.1.0.tgz", + "integrity": "sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==", + "requires": { + "@types/estree": "0.0.39", + "estree-walker": "^1.0.1", + "picomatch": "^2.2.2" + }, + "dependencies": { + "@types/estree": { + "version": "0.0.39", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", + "integrity": "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==" + }, + "estree-walker": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-1.0.1.tgz", + "integrity": "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg==" + } + } + }, + "@rushstack/eslint-patch": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz", + "integrity": "sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==" + }, + "@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==" + }, + "@surma/rollup-plugin-off-main-thread": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@surma/rollup-plugin-off-main-thread/-/rollup-plugin-off-main-thread-2.2.3.tgz", + "integrity": "sha512-lR8q/9W7hZpMWweNiAKU7NQerBnzQQLvi8qnTDU/fxItPhtZVMbPV3lbCwjhIlNBe9Bbr5V+KHshvWmVSG9cxQ==", + "requires": { + "ejs": "^3.1.6", + "json5": "^2.2.0", + "magic-string": "^0.25.0", + "string.prototype.matchall": "^4.0.6" + }, + "dependencies": { + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==" + } + } + }, + "@swc/helpers": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.4.11.tgz", + "integrity": "sha512-rEUrBSGIoSFuYxwBYtlUFMlE2CwGhmW+w9355/5oduSw8e5h2+Tj4UrAGNNgP9915++wj5vkQo0UuOBqOAq4nw==", + "requires": { + "tslib": "^2.4.0" + } + }, + "@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "requires": { + "defer-to-connect": "^1.0.1" + } + }, + "@types/acorn": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz", + "integrity": "sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==", + "requires": { + "@types/estree": "*" + } + }, + "@types/debug": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", + "integrity": "sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==", + "requires": { + "@types/ms": "*" + } + }, + "@types/eslint": { + "version": "8.4.10", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.10.tgz", + "integrity": "sha512-Sl/HOqN8NKPmhWo2VBEPm0nvHnu2LL3v9vKo8MEq0EtbJ4eVzGPl41VNPvn5E1i5poMk4/XD8UriLHpJvEP/Nw==", + "peer": true, + "requires": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "@types/eslint-scope": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.4.tgz", + "integrity": "sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA==", + "peer": true, + "requires": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "@types/estree": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.0.tgz", + "integrity": "sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ==" + }, + "@types/estree-jsx": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.0.tgz", + "integrity": "sha512-3qvGd0z8F2ENTGr/GG1yViqfiKmRfrXVx5sJyHGFu3z7m5g5utCQtGp/g29JnjflhtQJBv1WDQukHiT58xPcYQ==", + "requires": { + "@types/estree": "*" + } + }, + "@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "@types/hast": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz", + "integrity": "sha512-wLEm0QvaoawEDoTRwzTXp4b4jpwiJDvR5KMnFnVodm3scufTlBOWRD6N1OBf9TZMhjlNsSfcO5V+7AF4+Vy+9g==", + "requires": { + "@types/unist": "*" + } + }, + "@types/js-yaml": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.5.tgz", + "integrity": "sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==" + }, + "@types/json-schema": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", + "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==" + }, + "@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==" + }, + "@types/mdast": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz", + "integrity": "sha512-W864tg/Osz1+9f4lrGTZpCSO5/z4608eUp19tbozkq2HJK6i3z1kT0H9tlADXuYIb1YYOBByU4Jsqkk75q48qA==", + "requires": { + "@types/unist": "*" + } + }, + "@types/mdurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", + "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==" + }, + "@types/mdx": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/mdx/-/mdx-2.0.3.tgz", + "integrity": "sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==" + }, + "@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" + }, + "@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==" + }, + "@types/nlcst": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/nlcst/-/nlcst-1.0.0.tgz", + "integrity": "sha512-3TGCfOcy8R8mMQ4CNSNOe3PG66HttvjcLzCoOpvXvDtfWOTi+uT/rxeOKm/qEwbM4SNe1O/PjdiBK2YcTjU4OQ==", + "requires": { + "@types/unist": "*" + } + }, + "@types/node": { + "version": "18.11.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.9.tgz", + "integrity": "sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg==" + }, + "@types/prop-types": { + "version": "15.7.5", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", + "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" + }, + "@types/react": { + "version": "18.0.25", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.25.tgz", + "integrity": "sha512-xD6c0KDT4m7n9uD4ZHi02lzskaiqcBxf4zi+tXZY98a04wvc0hi/TcCPC2FOESZi51Nd7tlUeOJY8RofL799/g==", + "requires": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "@types/react-dom": { + "version": "18.0.8", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.0.8.tgz", + "integrity": "sha512-C3GYO0HLaOkk9dDAz3Dl4sbe4AKUGTCfFIZsz3n/82dPNN8Du533HzKatDxeUYWu24wJgMP1xICqkWk1YOLOIw==", + "requires": { + "@types/react": "*" + } + }, + "@types/resolve": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", + "integrity": "sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==", + "requires": { + "@types/node": "*" + } + }, + "@types/scheduler": { + "version": "0.16.2", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", + "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" + }, + "@types/trusted-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.2.tgz", + "integrity": "sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==" + }, + "@types/unist": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz", + "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==" + }, + "@typescript-eslint/parser": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.42.1.tgz", + "integrity": "sha512-kAV+NiNBWVQDY9gDJDToTE/NO8BHi4f6b7zTsVAJoTkmB/zlfOpiEVBzHOKtlgTndCKe8vj9F/PuolemZSh50Q==", + "requires": { + "@typescript-eslint/scope-manager": "5.42.1", + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/typescript-estree": "5.42.1", + "debug": "^4.3.4" + } + }, + "@typescript-eslint/scope-manager": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.42.1.tgz", + "integrity": "sha512-QAZY/CBP1Emx4rzxurgqj3rUinfsh/6mvuKbLNMfJMMKYLRBfweus8brgXF8f64ABkIZ3zdj2/rYYtF8eiuksQ==", + "requires": { + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/visitor-keys": "5.42.1" + } + }, + "@typescript-eslint/types": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.42.1.tgz", + "integrity": "sha512-Qrco9dsFF5lhalz+lLFtxs3ui1/YfC6NdXu+RAGBa8uSfn01cjO7ssCsjIsUs484vny9Xm699FSKwpkCcqwWwA==" + }, + "@typescript-eslint/typescript-estree": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.42.1.tgz", + "integrity": "sha512-qElc0bDOuO0B8wDhhW4mYVgi/LZL+igPwXtV87n69/kYC/7NG3MES0jHxJNCr4EP7kY1XVsRy8C/u3DYeTKQmw==", + "requires": { + "@typescript-eslint/types": "5.42.1", + "@typescript-eslint/visitor-keys": "5.42.1", + "debug": "^4.3.4", + "globby": "^11.1.0", + "is-glob": "^4.0.3", + "semver": "^7.3.7", + "tsutils": "^3.21.0" + } + }, + "@typescript-eslint/visitor-keys": { + "version": "5.42.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.42.1.tgz", + "integrity": "sha512-LOQtSF4z+hejmpUvitPlc4hA7ERGoj2BVkesOcG91HCn8edLGUXbTrErmutmPbl8Bo9HjAvOO/zBKQHExXNA2A==", + "requires": { + "@typescript-eslint/types": "5.42.1", + "eslint-visitor-keys": "^3.3.0" + } + }, + "@uiw/codemirror-extensions-basic-setup": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/codemirror-extensions-basic-setup/-/codemirror-extensions-basic-setup-4.19.3.tgz", + "integrity": "sha512-Itt/cgaw2g3facr52DSdkcKDZPTyW7M2YGjfSbjYC5Xaap0R31cClR09UZTEokxZmE4yYq7QIs3DTeURExxXzw==", + "requires": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "@uiw/codemirror-themes": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/codemirror-themes/-/codemirror-themes-4.19.3.tgz", + "integrity": "sha512-2mqrrYZnUL/hseEeIYC0DBP4igst/w5g5JT1+3v+yMyvwsvSAWxxN1Zeh5FFMfVTZWjpYujOuoJ4V507taorBw==", + "requires": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "@uiw/react-codemirror": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@uiw/react-codemirror/-/react-codemirror-4.19.3.tgz", + "integrity": "sha512-/Rg7U0CjPpsKd1vxve9he86MMQLVDz4N6fSBGOIx7R+6UPBoDcUOPlcmLRFUQvmP+H5Xyk5IBygeGPUR7IEPog==", + "requires": { + "@babel/runtime": "^7.18.6", + "@codemirror/commands": "^6.1.0", + "@codemirror/state": "^6.1.1", + "@codemirror/theme-one-dark": "^6.0.0", + "@uiw/codemirror-extensions-basic-setup": "4.19.3", + "codemirror": "^6.0.0" + } + }, + "@webassemblyjs/ast": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", + "peer": true, + "requires": { + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" + } + }, + "@webassemblyjs/floating-point-hex-parser": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", + "peer": true + }, + "@webassemblyjs/helper-api-error": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", + "peer": true + }, + "@webassemblyjs/helper-buffer": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", + "peer": true + }, + "@webassemblyjs/helper-numbers": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", + "peer": true, + "requires": { + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/helper-wasm-bytecode": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", + "peer": true + }, + "@webassemblyjs/helper-wasm-section": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" + } + }, + "@webassemblyjs/ieee754": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", + "peer": true, + "requires": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "@webassemblyjs/leb128": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", + "peer": true, + "requires": { + "@xtuc/long": "4.2.2" + } + }, + "@webassemblyjs/utf8": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", + "peer": true + }, + "@webassemblyjs/wasm-edit": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" + } + }, + "@webassemblyjs/wasm-gen": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" + } + }, + "@webassemblyjs/wasm-opt": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" + } + }, + "@webassemblyjs/wasm-parser": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" + } + }, + "@webassemblyjs/wast-printer": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", + "peer": true, + "requires": { + "@webassemblyjs/ast": "1.11.1", + "@xtuc/long": "4.2.2" + } + }, + "@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "peer": true + }, + "@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "peer": true + }, + "acorn": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==" + }, + "acorn-import-assertions": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz", + "integrity": "sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw==", + "peer": true, + "requires": {} + }, + "acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "requires": {} + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "requires": {} + }, + "algoliasearch": { + "version": "4.14.2", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.14.2.tgz", + "integrity": "sha512-ngbEQonGEmf8dyEh5f+uOIihv4176dgbuOZspiuhmTTBRBuzWu3KCGHre6uHj5YyuC7pNvQGzB6ZNJyZi0z+Sg==", + "requires": { + "@algolia/cache-browser-local-storage": "4.14.2", + "@algolia/cache-common": "4.14.2", + "@algolia/cache-in-memory": "4.14.2", + "@algolia/client-account": "4.14.2", + "@algolia/client-analytics": "4.14.2", + "@algolia/client-common": "4.14.2", + "@algolia/client-personalization": "4.14.2", + "@algolia/client-search": "4.14.2", + "@algolia/logger-common": "4.14.2", + "@algolia/logger-console": "4.14.2", + "@algolia/requester-browser-xhr": "4.14.2", + "@algolia/requester-common": "4.14.2", + "@algolia/requester-node-http": "4.14.2", + "@algolia/transporter": "4.14.2" + } + }, + "ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "requires": { + "string-width": "^4.1.0" + } + }, + "ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "requires": { + "type-fest": "^0.21.3" + }, + "dependencies": { + "type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" + } + } + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "aria-query": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz", + "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==", + "requires": { + "@babel/runtime": "^7.10.2", + "@babel/runtime-corejs3": "^7.10.2" + } + }, + "array-includes": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.6.tgz", + "integrity": "sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "get-intrinsic": "^1.1.3", + "is-string": "^1.0.7" + } + }, + "array-iterate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/array-iterate/-/array-iterate-2.0.1.tgz", + "integrity": "sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg==" + }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==" + }, + "array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==" + }, + "array.prototype.flat": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", + "integrity": "sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "es-shim-unscopables": "^1.0.0" + } + }, + "array.prototype.flatmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz", + "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "es-shim-unscopables": "^1.0.0" + } + }, + "ast-types-flow": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", + "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==" + }, + "astring": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.8.3.tgz", + "integrity": "sha512-sRpyiNrx2dEYIMmUXprS8nlpRg2Drs8m9ElX9vVEXaCB4XEAJhKfs7IcX0IwShjuOAjLR6wzIrgoptz1n19i1A==" + }, + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" + }, + "axe-core": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.5.1.tgz", + "integrity": "sha512-1exVbW0X1O/HSr/WMwnaweyqcWOgZgLiVxdLG34pvSQk4NlYQr9OUy0JLwuhFfuVNQzzqgH57eYzkFBCb3bIsQ==" + }, + "axobject-query": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz", + "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==" + }, + "babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha512-XqYMR2dfdGMW+hd0IUZ2PwK+fGeFkOxZJ0wY+JaQAHzt1Zx8LcvpiZD2NiGkEG8qx0CfkAOr5xt76d1e8vG90g==", + "requires": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha512-RjTcuD4xjtthQkaWH7dFlH85L+QaVtSoOyGdZ3g6HFhS9dFNDfLyqgm2NFe2X6cQpeFmt0452FJjFG5UameExg==" + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==" + } + } + }, + "babel-loader": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.3.0.tgz", + "integrity": "sha512-H8SvsMF+m9t15HNLMipppzkC+Y2Yq+v3SonZyU70RBL/h1gxPkH08Ot8pEE9Z4Kd+czyWJClmFS8qzIP9OZ04Q==", + "requires": { + "find-cache-dir": "^3.3.1", + "loader-utils": "^2.0.0", + "make-dir": "^3.1.0", + "schema-utils": "^2.6.5" + }, + "dependencies": { + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + } + } + }, + "babel-plugin-polyfill-corejs2": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.3.tgz", + "integrity": "sha512-8hOdmFYFSZhqg2C/JgLUQ+t52o5nirNwaWM2B9LWteozwIvM14VSwdsCAUET10qT+kmySAlseadmfeeSWFCy+Q==", + "requires": { + "@babel/compat-data": "^7.17.7", + "@babel/helper-define-polyfill-provider": "^0.3.3", + "semver": "^6.1.1" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "babel-plugin-polyfill-corejs3": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.6.0.tgz", + "integrity": "sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==", + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.3", + "core-js-compat": "^3.25.1" + } + }, + "babel-plugin-polyfill-regenerator": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.4.1.tgz", + "integrity": "sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==", + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.3" + } + }, + "bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==" + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==" + }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" + }, + "boxen": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "requires": { + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" + }, + "dependencies": { + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==" + } + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "requires": { + "fill-range": "^7.0.1" + } + }, + "browser-monads": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-monads/-/browser-monads-1.0.0.tgz", + "integrity": "sha512-NgmpFaI9NtRKyzHYA+xVzPA5Bu+JgCBzgC3WJCXstlajCBB3fJTieBwImdG8JepK8GctHSnia866ceFLg1LLTQ==", + "requires": { + "nothing-mock": "^1.0.0" + } + }, + "browserslist": { + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", + "requires": { + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "builtin-modules": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", + "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==" + }, + "builtins": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz", + "integrity": "sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==" + }, + "cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "requires": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "dependencies": { + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "requires": { + "pump": "^3.0.0" + } + }, + "lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" + } + } + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==" + }, + "caniuse-lite": { + "version": "1.0.30001431", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001431.tgz", + "integrity": "sha512-zBUoFU0ZcxpvSt9IU66dXVT/3ctO1cy4y9cscs1szkPlcWb6pasYM144GqrUygUbT+k7cmUCW61cvskjcv0enQ==" + }, + "capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==" + }, + "ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==" + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==" + }, + "character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==" + }, + "character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==" + }, + "character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==" + }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" + }, + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, + "chrome-trace-event": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "peer": true + }, + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==" + }, + "classnames": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" + }, + "clean-webpack-plugin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/clean-webpack-plugin/-/clean-webpack-plugin-4.0.0.tgz", + "integrity": "sha512-WuWE1nyTNAyW5T7oNyys2EN0cfP2fdRxhxnIQWiAp0bMabPdHhoGxM8A6YL2GhqwgrPnnaemVE7nv5XJ2Fhh2w==", + "requires": { + "del": "^4.1.1" + } + }, + "cli-boxes": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", + "integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==" + }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==" + }, + "client-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", + "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==" + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "clone-response": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", + "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", + "requires": { + "mimic-response": "^1.0.0" + } + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==" + }, + "codemirror": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz", + "integrity": "sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==", + "requires": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "comma-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.2.tgz", + "integrity": "sha512-G5yTt3KQN4Yn7Yk4ed73hlZ1evrFKXeUW3086p3PRFNp7m2vIjI6Pg+Kgb+oyzhd9F2qdcoj67+y3SdxL5XWsg==" + }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "comment-json": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-2.4.2.tgz", + "integrity": "sha512-T+iXox779qsqneMYx/x5BZyz4xjCeQRmuNVzz8tko7qZUs3MlzpA3RAs+O1XsgcKToNBMIvfVzafGOeiU7RggA==", + "requires": { + "core-util-is": "^1.0.2", + "esprima": "^4.0.1", + "has-own-prop": "^2.0.0", + "repeat-string": "^1.6.1" + } + }, + "common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==" + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "requires": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + } + }, + "convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==" + }, + "core-js-compat": { + "version": "3.26.1", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.26.1.tgz", + "integrity": "sha512-622/KzTudvXCDLRw70iHW4KKs1aGpcRcowGWyYJr2DEBfRrd6hNJybxSWJFuZYD4ma86xhrwDDHxmDaIq4EA8A==", + "requires": { + "browserslist": "^4.21.4" + } + }, + "core-js-pure": { + "version": "3.26.0", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.26.0.tgz", + "integrity": "sha512-LiN6fylpVBVwT8twhhluD9TzXmZQQsr2I2eIKtWNbZI1XMfBT7CV18itaN6RA7EtQd/SDdRx/wzvAShX2HvhQA==" + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "requires": { + "capture-stack-trace": "^1.0.0" + } + }, + "crelt": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.5.tgz", + "integrity": "sha512-+BO9wPPi+DWTDcNYhr/W90myha8ptzftZT+LwcmUbbok0rcP/fequmFYCw8NMoH7pkAZQzU78b3kYrlua5a9eA==" + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==" + }, + "csstype": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", + "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==" + }, + "damerau-levenshtein": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", + "integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==" + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==" + }, + "decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "requires": { + "character-entities": "^2.0.0" + } + }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", + "requires": { + "mimic-response": "^1.0.0" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" + }, + "deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==" + }, + "defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" + }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "del": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", + "requires": { + "@types/glob": "^7.1.1", + "globby": "^6.1.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" + }, + "dependencies": { + "array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "requires": { + "array-uniq": "^1.0.1" + } + }, + "globby": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "integrity": "sha512-KVbFv2TQtbzCoxAnfD6JcHZTYCzyliEaaeM/gH8qQdkKr5s0OP9scEgvdcngyk7AVdY6YVW/TJHd+lQ/Df3Daw==", + "requires": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==" + } + } + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==" + }, + "detect-indent": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.1.0.tgz", + "integrity": "sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==" + }, + "diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==" + }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "requires": { + "path-type": "^4.0.0" + } + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "requires": { + "esutils": "^2.0.2" + } + }, + "dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "requires": { + "domelementtype": "^2.3.0" + } + }, + "domutils": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.0.1.tgz", + "integrity": "sha512-z08c1l761iKhDFtfXO04C7kTdPBLi41zwOZl00WS8b5eiaebNpY00HKbztwBq+e3vyqWNwWF3mP9YLUeqIrF+Q==", + "requires": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.1" + } + }, + "dot-prop": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", + "requires": { + "is-obj": "^2.0.0" + } + }, + "dotenv": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-5.0.1.tgz", + "integrity": "sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==" + }, + "duplexer3": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.5.tgz", + "integrity": "sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==" + }, + "editorconfig": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", + "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", + "requires": { + "commander": "^2.19.0", + "lru-cache": "^4.1.5", + "semver": "^5.6.0", + "sigmund": "^1.0.1" + }, + "dependencies": { + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + } + } + }, + "ejs": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz", + "integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==", + "requires": { + "jake": "^10.8.5" + } + }, + "electron-to-chromium": { + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==" + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==" + }, + "end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "requires": { + "once": "^1.4.0" + } + }, + "enhanced-resolve": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", + "peer": true, + "requires": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + } + }, + "entities": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz", + "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" + }, + "es-abstract": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.4.tgz", + "integrity": "sha512-0UtvRN79eMe2L+UNEF1BwRe364sj/DXhQ/k5FmivgoSdpM90b8Jc0mDzKMGo7QS0BVbOP/bTwBKNnDc9rNzaPA==", + "requires": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "function.prototype.name": "^1.1.5", + "get-intrinsic": "^1.1.3", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.2", + "object-keys": "^1.1.1", + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.4.3", + "safe-regex-test": "^1.0.0", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" + } + }, + "es-module-lexer": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz", + "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==", + "peer": true + }, + "es-shim-unscopables": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz", + "integrity": "sha512-Jm6GPcCdC30eMLbZ2x8z2WuRwAws3zTBBKuusffYVUrNj/GVSUAZ+xKMaUpfNDR5IbyNA5LJbaecoUVbmUcB1w==", + "requires": { + "has": "^1.0.3" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==" + }, + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" + }, + "eslint": { + "version": "8.27.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.27.0.tgz", + "integrity": "sha512-0y1bfG2ho7mty+SiILVf9PfuRA49ek4Nc60Wmmu62QlobNR+CeXa4xXIJgcuwSQgZiWaPH+5BDsctpIW0PR/wQ==", + "requires": { + "@eslint/eslintrc": "^1.3.3", + "@humanwhocodes/config-array": "^0.11.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.1.1", + "eslint-utils": "^3.0.0", + "eslint-visitor-keys": "^3.3.0", + "espree": "^9.4.0", + "esquery": "^1.4.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.15.0", + "grapheme-splitter": "^1.0.4", + "ignore": "^5.2.0", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-sdsl": "^4.1.4", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "regexpp": "^3.2.0", + "strip-ansi": "^6.0.1", + "strip-json-comments": "^3.1.0", + "text-table": "^0.2.0" + } + }, + "eslint-config-next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-13.0.2.tgz", + "integrity": "sha512-SrrHp+zBDYLjOFZdM5b9aW/pliK687Xxfa+qpDuL08Z04ReHhmz3L+maXaAqgrEVZHQximP7nh0El4yNDJW+CA==", + "requires": { + "@next/eslint-plugin-next": "13.0.2", + "@rushstack/eslint-patch": "^1.1.3", + "@typescript-eslint/parser": "^5.21.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-import-resolver-typescript": "^2.7.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-jsx-a11y": "^6.5.1", + "eslint-plugin-react": "^7.31.7", + "eslint-plugin-react-hooks": "^4.5.0" + } + }, + "eslint-import-resolver-node": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.6.tgz", + "integrity": "sha512-0En0w03NRVMn9Uiyn8YRPDKvWjxCWkslUEhGNTdGx15RvPJYQ+lbOlqrlNI2vEAs4pDYK4f/HN2TbDmk5TP0iw==", + "requires": { + "debug": "^3.2.7", + "resolve": "^1.20.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "eslint-import-resolver-typescript": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-2.7.1.tgz", + "integrity": "sha512-00UbgGwV8bSgUv34igBDbTOtKhqoRMy9bFjNehT40bXg6585PNIct8HhXZ0SybqB9rWtXj9crcku8ndDn/gIqQ==", + "requires": { + "debug": "^4.3.4", + "glob": "^7.2.0", + "is-glob": "^4.0.3", + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } + } + }, + "eslint-module-utils": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz", + "integrity": "sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA==", + "requires": { + "debug": "^3.2.7" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "eslint-plugin-import": { + "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", + "requires": { + "array-includes": "^3.1.4", + "array.prototype.flat": "^1.2.5", + "debug": "^2.6.9", + "doctrine": "^2.1.0", + "eslint-import-resolver-node": "^0.3.6", + "eslint-module-utils": "^2.7.3", + "has": "^1.0.3", + "is-core-module": "^2.8.1", + "is-glob": "^4.0.3", + "minimatch": "^3.1.2", + "object.values": "^1.1.5", + "resolve": "^1.22.0", + "tsconfig-paths": "^3.14.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "requires": { + "esutils": "^2.0.2" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, + "eslint-plugin-jsx-a11y": { + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", + "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", + "requires": { + "@babel/runtime": "^7.18.9", + "aria-query": "^4.2.2", + "array-includes": "^3.1.5", + "ast-types-flow": "^0.0.7", + "axe-core": "^4.4.3", + "axobject-query": "^2.2.0", + "damerau-levenshtein": "^1.0.8", + "emoji-regex": "^9.2.2", + "has": "^1.0.3", + "jsx-ast-utils": "^3.3.2", + "language-tags": "^1.0.5", + "minimatch": "^3.1.2", + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "eslint-plugin-react": { + "version": "7.31.10", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.31.10.tgz", + "integrity": "sha512-e4N/nc6AAlg4UKW/mXeYWd3R++qUano5/o+t+wnWxIf+bLsOaH3a4q74kX3nDjYym3VBN4HyO9nEn1GcAqgQOA==", + "requires": { + "array-includes": "^3.1.5", + "array.prototype.flatmap": "^1.3.0", + "doctrine": "^2.1.0", + "estraverse": "^5.3.0", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.1.2", + "object.entries": "^1.1.5", + "object.fromentries": "^2.0.5", + "object.hasown": "^1.1.1", + "object.values": "^1.1.5", + "prop-types": "^15.8.1", + "resolve": "^2.0.0-next.3", + "semver": "^6.3.0", + "string.prototype.matchall": "^4.0.7" + }, + "dependencies": { + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "requires": { + "esutils": "^2.0.2" + } + }, + "resolve": { + "version": "2.0.0-next.4", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.4.tgz", + "integrity": "sha512-iMDbmAWtfU+MHpxt/I5iWI7cY6YVEZUQ3MBgPQ++XD1PELuJHIl82xBmObyP2KyQmkNB2dsqF7seoQQiAn5yDQ==", + "requires": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "eslint-plugin-react-hooks": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz", + "integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==", + "requires": {} + }, + "eslint-scope": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", + "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + } + }, + "eslint-utils": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", + "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", + "requires": { + "eslint-visitor-keys": "^2.0.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==" + } + } + }, + "eslint-visitor-keys": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", + "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==" + }, + "espree": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", + "integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==", + "requires": { + "acorn": "^8.8.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "requires": { + "estraverse": "^5.1.0" + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "requires": { + "estraverse": "^5.2.0" + } + }, + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + }, + "estree-util-attach-comments": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/estree-util-attach-comments/-/estree-util-attach-comments-2.1.0.tgz", + "integrity": "sha512-rJz6I4L0GaXYtHpoMScgDIwM0/Vwbu5shbMeER596rB2D1EWF6+Gj0e0UKzJPZrpoOc87+Q2kgVFHfjAymIqmw==", + "requires": { + "@types/estree": "^1.0.0" + } + }, + "estree-util-build-jsx": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/estree-util-build-jsx/-/estree-util-build-jsx-2.2.0.tgz", + "integrity": "sha512-apsfRxF9uLrqosApvHVtYZjISPvTJ+lBiIydpC+9wE6cF6ssbhnjyQLqaIjgzGxvC2Hbmec1M7g91PoBayYoQQ==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "estree-walker": "^3.0.0" + } + }, + "estree-util-is-identifier-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-2.0.1.tgz", + "integrity": "sha512-rxZj1GkQhY4x1j/CSnybK9cGuMFQYFPLq0iNyopqf14aOVLFtMv7Esika+ObJWPWiOHuMOAHz3YkWoLYYRnzWQ==" + }, + "estree-util-to-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/estree-util-to-js/-/estree-util-to-js-1.1.0.tgz", + "integrity": "sha512-490lbfCcpLk+ofK6HCgqDfYs4KAfq6QVvDw3+Bm1YoKRgiOjKiKYGAVQE1uwh7zVxBgWhqp4FDtp5SqunpUk1A==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "astring": "^1.8.0", + "source-map": "^0.7.0" + } + }, + "estree-util-visit": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-1.2.0.tgz", + "integrity": "sha512-wdsoqhWueuJKsh5hqLw3j8lwFqNStm92VcwtAOAny8g/KS/l5Y8RISjR4k5W6skCj3Nirag/WUCMS0Nfy3sgsg==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^2.0.0" + } + }, + "estree-walker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.1.tgz", + "integrity": "sha512-woY0RUD87WzMBUiZLx8NsYr23N5BKsOMZHhu2hoNRVh6NXGfoiT1KOL8G3UHlJAnEDGmfa5ubNA/AacfG+Kb0g==" + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" + }, + "events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "peer": true + }, + "execa": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", + "integrity": "sha512-RztN09XglpYI7aBBrJCPW95jEH7YF1UEPOoX9yDhUTPdp7mK+CQvnLTuD10BNXZ3byLTu2uehZ8EcKT/4CGiFw==", + "requires": { + "cross-spawn": "^5.0.1", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==" + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==" + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "^2.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + } + } + }, + "exenv": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/exenv/-/exenv-1.2.2.tgz", + "integrity": "sha512-Z+ktTxTwv9ILfgKCk32OX3n/doe+OcLTRtqK9pcL+JsP3J1/VW8Uvl4ZjLlKqeW4rzK4oesDOGMEMRIZqtP4Iw==" + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" + }, + "fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "requires": { + "reusify": "^1.0.4" + } + }, + "figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "requires": { + "escape-string-regexp": "^1.0.5" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + } + } + }, + "file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "requires": { + "flat-cache": "^3.0.4" + } + }, + "filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "requires": { + "minimatch": "^5.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.1.tgz", + "integrity": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==", + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "find-cache-dir": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", + "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", + "requires": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + } + }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, + "flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "requires": { + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + } + }, + "flatted": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.7.tgz", + "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==" + }, + "fs-exists-sync": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/fs-exists-sync/-/fs-exists-sync-0.1.0.tgz", + "integrity": "sha512-cR/vflFyPZtrN6b38ZyWxpWdhlXrzZEBawlpBQMq7033xVY7/kg0GDMBK5jg8lDYQckdJ5x/YC88lM3C7VMsLg==" + }, + "fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "function.prototype.name": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.5.tgz", + "integrity": "sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.19.0", + "functions-have-names": "^1.2.2" + } + }, + "functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==" + }, + "gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "get-own-enumerable-property-symbols": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", + "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==" + }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } + }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, + "git-config-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/git-config-path/-/git-config-path-1.0.1.tgz", + "integrity": "sha512-KcJ2dlrrP5DbBnYIZ2nlikALfRhKzNSX0stvv3ImJ+fvC4hXKoV+U+74SV0upg+jlQZbrtQzc0bu6/Zh+7aQbg==", + "requires": { + "extend-shallow": "^2.0.1", + "fs-exists-sync": "^0.1.0", + "homedir-polyfill": "^1.0.0" + } + }, + "git-username": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/git-username/-/git-username-1.0.0.tgz", + "integrity": "sha512-xm45KwBR6Eu1jO4umx/o2M84v9TC7tdOBuzLx8ayhdR9H1FBiiG9azz31uC0esDvaWVBTDINpJ5USomk+ja8OQ==", + "requires": { + "parse-github-url": "^1.0.2", + "remote-origin-url": "^1.0.0" + } + }, + "github-buttons": { + "version": "2.22.1", + "resolved": "https://registry.npmjs.org/github-buttons/-/github-buttons-2.22.1.tgz", + "integrity": "sha512-937dpW009lbV8p1gg1SoUvUnJBnfJKYU6CEJcsdoLnBzI4YP7Y8oT/M0O7WTQwfErTYLCG9t0W1huaexSLx7yA==" + }, + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "requires": { + "is-glob": "^4.0.3" + } + }, + "glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "peer": true + }, + "global-dirs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.1.0.tgz", + "integrity": "sha512-MG6kdOUh/xBnyo9cJFeIKkLEc1AyFq42QTU4XiX51i2NEdxLxLWXIjEjmqKeSuKR7pAZjTqUVoT2b2huxVLgYQ==", + "requires": { + "ini": "1.3.7" + }, + "dependencies": { + "ini": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==" + } + } + }, + "globals": { + "version": "13.17.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", + "integrity": "sha512-1C+6nQRb1GwGMKm2dH/E7enFAMxGTmGI7/dEdhy/DNelv85w9B72t3uc5frtMNXIbzrarJJ/lTCjcaZwbLJmyw==", + "requires": { + "type-fest": "^0.20.2" + } + }, + "globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + } + }, + "got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "requires": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + } + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + }, + "grapheme-splitter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", + "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==" + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" + } + } + }, + "has-bigints": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==" + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "has-own-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-own-prop/-/has-own-prop-2.0.0.tgz", + "integrity": "sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ==" + }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "requires": { + "has-symbols": "^1.0.2" + } + }, + "has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==" + }, + "hast-to-hyperscript": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/hast-to-hyperscript/-/hast-to-hyperscript-10.0.1.tgz", + "integrity": "sha512-dhIVGoKCQVewFi+vz3Vt567E4ejMppS1haBRL6TEmeLeJVB1i/FJIIg/e6s1Bwn0g5qtYojHEKvyGA+OZuyifw==", + "requires": { + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.3.0", + "unist-util-is": "^5.0.0", + "web-namespaces": "^2.0.0" + } + }, + "hast-util-parse-selector": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", + "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==" + }, + "hast-util-sanitize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-sanitize/-/hast-util-sanitize-4.0.0.tgz", + "integrity": "sha512-pw56+69jq+QSr/coADNvWTmBPDy+XsmwaF5KnUys4/wM1jt/fZdl7GPxhXXXYdXnz3Gj3qMkbUCH2uKjvX0MgQ==", + "requires": { + "@types/hast": "^2.0.0" + } + }, + "hast-util-to-estree": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-2.1.0.tgz", + "integrity": "sha512-Vwch1etMRmm89xGgz+voWXvVHba2iiMdGMKmaMfYt35rbVtFDq8JNwwAIvi8zHMkO6Gvqo9oTMwJTmzVRfXh4g==", + "requires": { + "@types/estree": "^1.0.0", + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/unist": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "estree-util-attach-comments": "^2.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "hast-util-whitespace": "^2.0.0", + "mdast-util-mdx-expression": "^1.0.0", + "mdast-util-mdxjs-esm": "^1.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^0.3.0", + "unist-util-position": "^4.0.0", + "zwitch": "^2.0.0" + } + }, + "hast-util-to-string": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-1.0.4.tgz", + "integrity": "sha512-eK0MxRX47AV2eZ+Lyr18DCpQgodvaS3fAQO2+b9Two9F5HEoRPhiUMNzoXArMJfZi2yieFzUBMRl3HNJ3Jus3w==" + }, + "hast-util-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-2.0.0.tgz", + "integrity": "sha512-Pkw+xBHuV6xFeJprJe2BBEoDV+AvQySaz3pPDRUs5PNZEMQjpXJJueqrpcHIXxnWTcAGi/UOCgVShlkY6kLoqg==" + }, + "hastscript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", + "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==", + "requires": { + "@types/hast": "^2.0.0", + "comma-separated-tokens": "^1.0.0", + "hast-util-parse-selector": "^2.0.0", + "property-information": "^5.0.0", + "space-separated-tokens": "^1.0.0" + }, + "dependencies": { + "comma-separated-tokens": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", + "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==" + }, + "property-information": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", + "integrity": "sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==", + "requires": { + "xtend": "^4.0.0" + } + }, + "space-separated-tokens": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz", + "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==" + } + } + }, + "homedir-polyfill": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", + "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", + "requires": { + "parse-passwd": "^1.0.0" + } + }, + "hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "html-to-react": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/html-to-react/-/html-to-react-1.5.0.tgz", + "integrity": "sha512-tjihXBgaJZRRYzmkrJZ/Qf9jFayilFYcb+sJxXXE2BVLk2XsNrGeuNCVvhXmvREULZb9dz6NFTBC96DTR/lQCQ==", + "requires": { + "domhandler": "^5.0", + "htmlparser2": "^8.0", + "lodash.camelcase": "^4.3.0" + } + }, + "htmlparser2": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.1.tgz", + "integrity": "sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "entities": "^4.3.0" + } + }, + "http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "idb": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/idb/-/idb-7.1.1.tgz", + "integrity": "sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==" + }, + "ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==" + }, + "immutable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.1.0.tgz", + "integrity": "sha512-oNkuqVTA8jqG1Q6c+UglTOD1xhC1BtjKI7XkCXRkZHrN5m18/XsnUp8Q89GkQO/z+0WjonSvl0FLhDYftp46nQ==" + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha512-m7ZEHgtw69qOGw+jwxXkHlrlIPdTGkyh66zXZ1ajZbxkDBNjSY/LGbmjc7h0s2ELsUDTAhFr55TrPSSqJGPG0A==" + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "inline-style-parser": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", + "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==" + }, + "inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "requires": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + } + }, + "internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "requires": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + } + }, + "is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==" + }, + "is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "requires": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + } + }, + "is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "requires": { + "has-bigints": "^1.0.1" + } + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" + }, + "is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "requires": { + "ci-info": "^2.0.0" + } + }, + "is-core-module": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", + "requires": { + "has": "^1.0.3" + } + }, + "is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==" + }, + "is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==" + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==" + }, + "is-installed-globally": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "requires": { + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" + } + }, + "is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==" + }, + "is-negative-zero": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==" + }, + "is-npm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==" + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" + }, + "is-number-object": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" + }, + "is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==" + }, + "is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "requires": { + "is-path-inside": "^2.1.0" + }, + "dependencies": { + "is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "requires": { + "path-is-inside": "^1.0.2" + } + } + } + }, + "is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==" + }, + "is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==" + }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==" + }, + "is-reference": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.0.tgz", + "integrity": "sha512-Eo1W3wUoHWoCoVM4GVl/a+K0IgiqE5aIo4kJABFyMum1ZORlPkC+UC357sSQUL5w5QCE5kCC9upl75b7+7CY/Q==", + "requires": { + "@types/estree": "*" + } + }, + "is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", + "integrity": "sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==" + }, + "is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==" + }, + "is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + }, + "is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "requires": { + "has-symbols": "^1.0.2" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, + "is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jake": { + "version": "10.8.5", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.5.tgz", + "integrity": "sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==", + "requires": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.1", + "minimatch": "^3.0.4" + } + }, + "jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "requires": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "dependencies": { + "supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "jinja-to-js": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/jinja-to-js/-/jinja-to-js-3.2.3.tgz", + "integrity": "sha512-ktEBxQG17fYaFcHThB719+EbePBx+AkkORQMyuP0UuLPS2zx8uJXP5CsItXjUUwMHFPj3hCRkyqEYzLbeklYgQ==" + }, + "js-sdsl": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.1.5.tgz", + "integrity": "sha512-08bOAKweV2NUC1wqTtf3qZlnpOX/R2DU9ikpjOHs0H+ibQv3zpncVQg6um4uYtRtrwIX8M4Nh3ytK4HGlYAq7Q==" + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==" + }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha512-CuUqjv0FUZIdXkHPI8MezCnFCdaTAacej1TZYulLoAg1h/PhwkdXFN4V/gzY4g+fMBCOV2xF+rp7t2XD2ns/NQ==" + }, + "json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "peer": true + }, + "json-parser": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/json-parser/-/json-parser-1.1.5.tgz", + "integrity": "sha512-6wQrWqKdbvCQ4ccCM1+14Q2Z2XTXeUqwt3xEaGe/GpsObJRh1n1Z+5ZeHosmfSpN5bJKDfeNRy9jOE+wQRduOg==", + "requires": { + "esprima": "^2.7.0" + }, + "dependencies": { + "esprima": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz", + "integrity": "sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A==" + } + } + }, + "json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stable-stringify": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.2.tgz", + "integrity": "sha512-eunSSaEnxV12z+Z73y/j5N37/In40GK4GmsSy+tEHJMxknvqnA7/djeYtAgW0GsWHUfg+847WJjKaEylk2y09g==", + "requires": { + "jsonify": "^0.0.1" + } + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==" + }, + "json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "requires": { + "minimist": "^1.2.0" + } + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "jsonify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.1.tgz", + "integrity": "sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg==" + }, + "jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==" + }, + "jsx-ast-utils": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", + "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", + "requires": { + "array-includes": "^3.1.5", + "object.assign": "^4.1.3" + } + }, + "keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "requires": { + "json-buffer": "3.0.0" + } + }, + "kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==" + }, + "language-subtag-registry": { + "version": "0.3.22", + "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz", + "integrity": "sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w==" + }, + "language-tags": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.5.tgz", + "integrity": "sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ==", + "requires": { + "language-subtag-registry": "~0.3.2" + } + }, + "latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "requires": { + "package-json": "^6.3.0" + } + }, + "leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==" + }, + "levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "requires": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + } + }, + "libnpx": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/libnpx/-/libnpx-10.2.4.tgz", + "integrity": "sha512-BPc0D1cOjBeS8VIBKUu5F80s6njm0wbVt7CsGMrIcJ+SI7pi7V0uVPGpEMH9H5L8csOcclTxAXFE2VAsJXUhfA==", + "requires": { + "dotenv": "^5.0.1", + "npm-package-arg": "^6.0.0", + "rimraf": "^2.6.2", + "safe-buffer": "^5.1.0", + "update-notifier": "^2.3.0", + "which": "^1.3.0", + "y18n": "^4.0.0", + "yargs": "^14.2.3" + }, + "dependencies": { + "ansi-align": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", + "integrity": "sha512-TdlOggdA/zURfMYa7ABC66j+oqfMew58KpJMbUlH3bcZP1b+cBHIHDDn5uH9INsxrHBPjsqM0tDB4jPTF/vgJA==", + "requires": { + "string-width": "^2.0.0" + } + }, + "ansi-regex": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==" + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "boxen": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", + "integrity": "sha512-TNPjfTr432qx7yOjQyaXm3dSR0MH9vXp7eT1BFSl/C51g+EFnOR9hTg1IreahGBmDNCehscshe45f+C1TBZbLw==", + "requires": { + "ansi-align": "^2.0.0", + "camelcase": "^4.0.0", + "chalk": "^2.0.1", + "cli-boxes": "^1.0.0", + "string-width": "^2.0.0", + "term-size": "^1.2.0", + "widest-line": "^2.0.0" + } + }, + "camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==" + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "ci-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", + "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==" + }, + "cli-boxes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", + "integrity": "sha512-3Fo5wu8Ytle8q9iCzS4D2MWVL2X7JVWRiS1BnXbTFDhS9c/REkM9vd1AmabsoZoY5/dGi5TT9iKL8Kb6DeBRQg==" + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "configstore": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.5.tgz", + "integrity": "sha512-nlOhI4+fdzoK5xmJ+NY+1gZK56bwEaWZr8fYuXohZ9Vkc1o3a4T/R3M+yE/w7x/ZVJ1zF8c+oaOvF0dztdUgmA==", + "requires": { + "dot-prop": "^4.2.1", + "graceful-fs": "^4.1.2", + "make-dir": "^1.0.0", + "unique-string": "^1.0.0", + "write-file-atomic": "^2.0.0", + "xdg-basedir": "^3.0.0" + } + }, + "crypto-random-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", + "integrity": "sha512-GsVpkFPlycH7/fRR7Dhcmnoii54gV1nz7y4CWyeFS14N+JVBBhY+r8amRHE4BwSYal7BPTDp8isvAlCxyFt3Hg==" + }, + "dot-prop": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz", + "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==", + "requires": { + "is-obj": "^1.0.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==" + }, + "global-dirs": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz", + "integrity": "sha512-NknMLn7F2J7aflwFOlGdNIuCDpN3VGoSoB+aap3KABFWbHVn1TCgFC+np23J8W2BiZbjfEw3BFBycSMv1AFblg==", + "requires": { + "ini": "^1.3.4" + } + }, + "got": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", + "integrity": "sha512-Y/K3EDuiQN9rTZhBvPRWMLXIKdeD1Rj0nzunfoi0Yyn5WBEbzxXKU9Ub2X41oZBagVWOBU3MuDonFMgPWQFnwg==", + "requires": { + "create-error-class": "^3.0.0", + "duplexer3": "^0.1.4", + "get-stream": "^3.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "safe-buffer": "^5.0.1", + "timed-out": "^4.0.0", + "unzip-response": "^2.0.1", + "url-parse-lax": "^1.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + }, + "is-ci": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", + "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", + "requires": { + "ci-info": "^1.5.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==" + }, + "is-installed-globally": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.1.0.tgz", + "integrity": "sha512-ERNhMg+i/XgDwPIPF3u24qpajVreaiSuvpb1Uu0jugw7KKcxGyCX8cgp8P5fwTmAuXku6beDHHECdKArjlg7tw==", + "requires": { + "global-dirs": "^0.1.0", + "is-path-inside": "^1.0.0" + } + }, + "is-npm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "integrity": "sha512-9r39FIr3d+KD9SbX0sfMsHzb5PP3uimOiwr3YupUaUFG4W0l1U57Rx3utpttV7qz5U3jmrO5auUa04LU9pyHsg==" + }, + "is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==" + }, + "is-path-inside": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", + "integrity": "sha512-qhsCR/Esx4U4hg/9I19OVUAJkGWtjRYHMRgUMZE2TDdj+Ag+kttZanLupfddNyglzz50cUlmWzUaI37GDfNx/g==", + "requires": { + "path-is-inside": "^1.0.1" + } + }, + "latest-version": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", + "integrity": "sha512-Be1YRHWWlZaSsrz2U+VInk+tO0EwLIyV+23RhWLINJYwg/UIikxjlj3MhH37/6/EDCAusjajvMkMMUXRaMWl/w==", + "requires": { + "package-json": "^4.0.0" + } + }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "requires": { + "pify": "^3.0.0" + } + }, + "package-json": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-4.0.1.tgz", + "integrity": "sha512-q/R5GrMek0vzgoomq6rm9OX+3PQve8sLwTirmK30YB3Cu0Bbt9OX9M/SIUnroN5BGJkzwGsFwDaRGD9EwBOlCA==", + "requires": { + "got": "^6.7.1", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + } + }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==" + }, + "registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "requires": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "requires": { + "rc": "^1.0.1" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "semver-diff": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz", + "integrity": "sha512-gL8F8L4ORwsS0+iQ34yCYv///jsOq0ZL7WP55d1HnJ32o7tyFYEFQZQA22mrLIacZdU6xecaBBZ+uEiffGNyXw==", + "requires": { + "semver": "^5.0.3" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + }, + "term-size": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", + "integrity": "sha512-7dPUZQGy/+m3/wjVz3ZW5dobSoD/02NxJpoXUX0WIyjfVS3l0c+b/+9phIDFA7FHzkYtwtMFgeGZ/Y8jVTeqQQ==", + "requires": { + "execa": "^0.7.0" + } + }, + "unique-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", + "integrity": "sha512-ODgiYu03y5g76A1I9Gt0/chLCzQjvzDy7DsZGsLOE/1MrF6wriEskSncj1+/C58Xk/kPZDppSctDybCwOSaGAg==", + "requires": { + "crypto-random-string": "^1.0.0" + } + }, + "update-notifier": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-2.5.0.tgz", + "integrity": "sha512-gwMdhgJHGuj/+wHJJs9e6PcCszpxR1b236igrOkUofGhqJuG+amlIKwApH1IW1WWl7ovZxsX49lMBWLxSdm5Dw==", + "requires": { + "boxen": "^1.2.1", + "chalk": "^2.0.1", + "configstore": "^3.0.0", + "import-lazy": "^2.1.0", + "is-ci": "^1.0.10", + "is-installed-globally": "^0.1.0", + "is-npm": "^1.0.0", + "latest-version": "^3.0.0", + "semver-diff": "^2.0.0", + "xdg-basedir": "^3.0.0" + } + }, + "url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "requires": { + "prepend-http": "^1.0.1" + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "requires": { + "isexe": "^2.0.0" + } + }, + "widest-line": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", + "integrity": "sha512-Ba5m9/Fa4Xt9eb2ELXt77JxVDV8w7qQrH0zS/TWSJdLyAwQjWoOzpzj5lwVftDz6n/EOu3tNACS84v509qwnJA==", + "requires": { + "string-width": "^2.1.1" + } + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "xdg-basedir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", + "integrity": "sha512-1Dly4xqlulvPD3fZUQJLY+FUIeqN3N2MM3uqe4rCJftAvOjFa3jFGfctOgluGx4ahPbUCsZkmJILiP0Vi4T6lQ==" + } + } + }, + "listify": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/listify/-/listify-1.0.3.tgz", + "integrity": "sha512-083swF7iH7bx8666zdzBColpgEuy46HjN3r1isD4zV6Ix7FuHfb/2/WVnl4CH8hjuoWeFF7P5KkKNXUnJCFEJg==" + }, + "loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "peer": true + }, + "loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "dependencies": { + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==" + } + } + }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "requires": { + "p-locate": "^5.0.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==" + }, + "longest": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-2.0.1.tgz", + "integrity": "sha512-Ajzxb8CM6WAnFjgiloPsI3bF+WCxcvhdIG3KNA2KN962+tdBsHcuQ4k4qX/EcS/2CRkcc0iAkR956Nib6aXU/Q==" + }, + "longest-streak": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.0.1.tgz", + "integrity": "sha512-cHlYSUpL2s7Fb3394mYxwTYj8niTaNHUCLr0qdiCXQfSjfuA7CKofpX2uSwEfFDQ0EB7JcnMnm+GjbqqoinYYg==" + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "magic-string": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", + "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", + "requires": { + "sourcemap-codec": "^1.4.8" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "markdown-extensions": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-1.1.1.tgz", + "integrity": "sha512-WWC0ZuMzCyDHYCasEGs4IPvLyTGftYwh6wIEOULOF0HXcqZlhwRzrK0w2VUlxWA98xnvb/jszw4ZSkJ6ADpM6Q==" + }, + "markdown-table": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.2.tgz", + "integrity": "sha512-y8j3a5/DkJCmS5x4dMCQL+OR0+2EAq3DOtio1COSHsmW2BGXnNCK3v12hJt1LrUz5iZH5g0LmuYOjDdI+czghA==" + }, + "md-attr-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/md-attr-parser/-/md-attr-parser-1.3.0.tgz", + "integrity": "sha512-KTVlfU5Oxo/6kd0YZ2mLP3eWJj+5vzh5mBCxLo3yGl1fzHIgxmtadbE9tHb7TbUBi3XZbl+P0xKeGmakat135w==" + }, + "mdast-util-definitions": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-5.1.1.tgz", + "integrity": "sha512-rQ+Gv7mHttxHOBx2dkF4HWTg+EE+UR78ptQWDylzPKaQuVGdG4HIoY3SrS/pCp80nZ04greFvXbVFHT+uf0JVQ==", + "requires": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "unist-util-visit": "^4.0.0" + } + }, + "mdast-util-find-and-replace": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.1.tgz", + "integrity": "sha512-SobxkQXFAdd4b5WmEakmkVoh18icjQRxGy5OWTCzgsLRm1Fu/KCtwD1HIQSsmq5ZRjVH0Ehwg6/Fn3xIUk+nKw==", + "requires": { + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==" + } + } + }, + "mdast-util-from-markdown": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.2.0.tgz", + "integrity": "sha512-iZJyyvKD1+K7QX1b5jXdE7Sc5dtoTry1vzV28UZZe8Z1xVnB/czKntJ7ZAkG0tANqRnBF6p3p7GpU1y19DTf2Q==", + "requires": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "mdast-util-to-string": "^3.1.0", + "micromark": "^3.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-decode-string": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-stringify-position": "^3.0.0", + "uvu": "^0.5.0" + } + }, + "mdast-util-gfm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.1.tgz", + "integrity": "sha512-42yHBbfWIFisaAfV1eixlabbsa6q7vHeSPY+cg+BBjX51M8xhgMacqH9g6TftB/9+YkcI0ooV4ncfrJslzm/RQ==", + "requires": { + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-gfm-autolink-literal": "^1.0.0", + "mdast-util-gfm-footnote": "^1.0.0", + "mdast-util-gfm-strikethrough": "^1.0.0", + "mdast-util-gfm-table": "^1.0.0", + "mdast-util-gfm-task-list-item": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + } + }, + "mdast-util-gfm-autolink-literal": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.2.tgz", + "integrity": "sha512-FzopkOd4xTTBeGXhXSBU0OCDDh5lUj2rd+HQqG92Ld+jL4lpUfgX2AT2OHAVP9aEeDKp7G92fuooSZcYJA3cRg==", + "requires": { + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-find-and-replace": "^2.0.0", + "micromark-util-character": "^1.0.0" + } + }, + "mdast-util-gfm-footnote": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.1.tgz", + "integrity": "sha512-p+PrYlkw9DeCRkTVw1duWqPRHX6Ywh2BNKJQcZbCwAuP/59B0Lk9kakuAd7KbQprVO4GzdW8eS5++A9PUSqIyw==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0", + "micromark-util-normalize-identifier": "^1.0.0" + } + }, + "mdast-util-gfm-strikethrough": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.2.tgz", + "integrity": "sha512-T/4DVHXcujH6jx1yqpcAYYwd+z5lAYMw4Ls6yhTfbMMtCt0PHY4gEfhW9+lKsLBtyhUGKRIzcUA2FATVqnvPDA==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-gfm-table": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.6.tgz", + "integrity": "sha512-uHR+fqFq3IvB3Rd4+kzXW8dmpxUhvgCQZep6KdjsLK4O6meK5dYZEayLtIxNus1XO3gfjfcIFe8a7L0HZRGgag==", + "requires": { + "@types/mdast": "^3.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-gfm-task-list-item": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.1.tgz", + "integrity": "sha512-KZ4KLmPdABXOsfnM6JHUIjxEvcx2ulk656Z/4Balw071/5qgnhz+H1uGtf2zIGnrnvDC8xR4Fj9uKbjAFGNIeA==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.3.0" + } + }, + "mdast-util-mdx": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-2.0.0.tgz", + "integrity": "sha512-M09lW0CcBT1VrJUaF/PYxemxxHa7SLDHdSn94Q9FhxjCQfuW7nMAWKWimTmA3OyDMSTH981NN1csW1X+HPSluw==", + "requires": { + "mdast-util-mdx-expression": "^1.0.0", + "mdast-util-mdx-jsx": "^2.0.0", + "mdast-util-mdxjs-esm": "^1.0.0" + } + }, + "mdast-util-mdx-expression": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-1.3.1.tgz", + "integrity": "sha512-TTb6cKyTA1RD+1su1iStZ5PAv3rFfOUKcoU5EstUpv/IZo63uDX03R8+jXjMEhcobXnNOiG6/ccekvVl4eV1zQ==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + } + }, + "mdast-util-mdx-jsx": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-2.1.0.tgz", + "integrity": "sha512-KzgzfWMhdteDkrY4mQtyvTU5bc/W4ppxhe9SzelO6QUUiwLAM+Et2Dnjjprik74a336kHdo0zKm7Tp+n6FFeRg==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "ccount": "^2.0.0", + "mdast-util-to-markdown": "^1.3.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-remove-position": "^4.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + } + }, + "mdast-util-mdxjs-esm": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-1.3.0.tgz", + "integrity": "sha512-7N5ihsOkAEGjFotIX9p/YPdl4TqUoMxL4ajNz7PbT89BqsdWJuBC9rvgt6wpbwTZqWWR0jKWqQbwsOWDBUZv4g==", + "requires": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "mdast-util-to-markdown": "^1.0.0" + } + }, + "mdast-util-to-hast": { + "version": "12.2.4", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.2.4.tgz", + "integrity": "sha512-a21xoxSef1l8VhHxS1Dnyioz6grrJkoaCUgGzMD/7dWHvboYX3VW53esRUfB5tgTyz4Yos1n25SPcj35dJqmAg==", + "requires": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-definitions": "^5.0.0", + "micromark-util-sanitize-uri": "^1.1.0", + "trim-lines": "^3.0.0", + "unist-builder": "^3.0.0", + "unist-util-generated": "^2.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0" + } + }, + "mdast-util-to-markdown": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.3.0.tgz", + "integrity": "sha512-6tUSs4r+KK4JGTTiQ7FfHmVOaDrLQJPmpjD6wPMlHGUVXoG9Vjc3jIeP+uyBWRf8clwB2blM+W7+KrlMYQnftA==", + "requires": { + "@types/mdast": "^3.0.0", + "@types/unist": "^2.0.0", + "longest-streak": "^3.0.0", + "mdast-util-to-string": "^3.0.0", + "micromark-util-decode-string": "^1.0.0", + "unist-util-visit": "^4.0.0", + "zwitch": "^2.0.0" + } + }, + "mdast-util-to-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.1.0.tgz", + "integrity": "sha512-n4Vypz/DZgwo0iMHLQL49dJzlp7YtAJP+N07MZHpjPf/5XJuHUWstviF4Mn2jEiR/GNmtnRRqnwsXExk3igfFA==" + }, + "mdurl": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", + "integrity": "sha512-/sKlQJCBYVY9Ers9hqzKou4H6V5UWc/M59TH2dvkt+84itfnq7uFOMLpOiOS4ujvHP4etln18fmIxA5R5fll0g==" + }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" + }, + "micromark": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-3.1.0.tgz", + "integrity": "sha512-6Mj0yHLdUZjHnOPgr5xfWIMqMWS12zDN6iws9SLuSz76W8jTtAv24MN4/CL7gJrl5vtxGInkkqDv/JIoRsQOvA==", + "requires": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "micromark-core-commonmark": "^1.0.1", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "micromark-core-commonmark": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-1.0.6.tgz", + "integrity": "sha512-K+PkJTxqjFfSNkfAhp4GB+cZPfQd6dxtTXnf+RjZOV7T4EEXnvgzOcnp+eSTmpGk9d1S9sL6/lqrgSNn/s0HZA==", + "requires": { + "decode-named-character-reference": "^1.0.0", + "micromark-factory-destination": "^1.0.0", + "micromark-factory-label": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-factory-title": "^1.0.0", + "micromark-factory-whitespace": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-html-tag-name": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-subtokenize": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.1", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.1.tgz", + "integrity": "sha512-p2sGjajLa0iYiGQdT0oelahRYtMWvLjy8J9LOCxzIQsllMCGLbsLW+Nc+N4vi02jcRJvedVJ68cjelKIO6bpDA==", + "requires": { + "micromark-extension-gfm-autolink-literal": "^1.0.0", + "micromark-extension-gfm-footnote": "^1.0.0", + "micromark-extension-gfm-strikethrough": "^1.0.0", + "micromark-extension-gfm-table": "^1.0.0", + "micromark-extension-gfm-tagfilter": "^1.0.0", + "micromark-extension-gfm-task-list-item": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-gfm-autolink-literal": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.3.tgz", + "integrity": "sha512-i3dmvU0htawfWED8aHMMAzAVp/F0Z+0bPh3YrbTPPL1v4YAlCZpy5rBO5p0LPYiZo0zFVkoYh7vDU7yQSiCMjg==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-footnote": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.0.4.tgz", + "integrity": "sha512-E/fmPmDqLiMUP8mLJ8NbJWJ4bTw6tS+FEQS8CcuDtZpILuOb2kjLqPEeAePF1djXROHXChM/wPJw0iS4kHCcIg==", + "requires": { + "micromark-core-commonmark": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-normalize-identifier": "^1.0.0", + "micromark-util-sanitize-uri": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-strikethrough": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.4.tgz", + "integrity": "sha512-/vjHU/lalmjZCT5xt7CcHVJGq8sYRm80z24qAKXzaHzem/xsDYb2yLL+NNVbYvmpLx3O7SYPuGL5pzusL9CLIQ==", + "requires": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-classify-character": "^1.0.0", + "micromark-util-resolve-all": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-table": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.5.tgz", + "integrity": "sha512-xAZ8J1X9W9K3JTJTUL7G6wSKhp2ZYHrFk5qJgY/4B33scJzE2kpfRL6oiw/veJTbt7jiM/1rngLlOKPWr1G+vg==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-gfm-tagfilter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.1.tgz", + "integrity": "sha512-Ty6psLAcAjboRa/UKUbbUcwjVAv5plxmpUTy2XC/3nJFL37eHej8jrHrRzkqcpipJliuBH30DTs7+3wqNcQUVA==", + "requires": { + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-gfm-task-list-item": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.3.tgz", + "integrity": "sha512-PpysK2S1Q/5VXi72IIapbi/jliaiOFzv7THH4amwXeYXLq3l1uo8/2Be0Ac1rEwK20MQEsGH2ltAZLNY2KI/0Q==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-mdx-expression": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-1.0.3.tgz", + "integrity": "sha512-TjYtjEMszWze51NJCZmhv7MEBcgYRgb3tJeMAJ+HQCAaZHHRBaDCccqQzGizR/H4ODefP44wRTgOn2vE5I6nZA==", + "requires": { + "micromark-factory-mdx-expression": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-extension-mdx-jsx": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-1.0.3.tgz", + "integrity": "sha512-VfA369RdqUISF0qGgv2FfV7gGjHDfn9+Qfiv5hEwpyr1xscRj/CiVRkU7rywGFCO7JwJ5L0e7CJz60lY52+qOA==", + "requires": { + "@types/acorn": "^4.0.0", + "estree-util-is-identifier-name": "^2.0.0", + "micromark-factory-mdx-expression": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + } + }, + "micromark-extension-mdx-md": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-1.0.0.tgz", + "integrity": "sha512-xaRAMoSkKdqZXDAoSgp20Azm0aRQKGOl0RrS81yGu8Hr/JhMsBmfs4wR7m9kgVUIO36cMUQjNyiyDKPrsv8gOw==", + "requires": { + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-mdxjs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-1.0.0.tgz", + "integrity": "sha512-TZZRZgeHvtgm+IhtgC2+uDMR7h8eTKF0QUX9YsgoL9+bADBpBY6SiLvWqnBlLbCEevITmTqmEuY3FoxMKVs1rQ==", + "requires": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^1.0.0", + "micromark-extension-mdx-jsx": "^1.0.0", + "micromark-extension-mdx-md": "^1.0.0", + "micromark-extension-mdxjs-esm": "^1.0.0", + "micromark-util-combine-extensions": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-extension-mdxjs-esm": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-1.0.3.tgz", + "integrity": "sha512-2N13ol4KMoxb85rdDwTAC6uzs8lMX0zeqpcyx7FhS7PxXomOnLactu8WI8iBNXW8AVyea3KIJd/1CKnUmwrK9A==", + "requires": { + "micromark-core-commonmark": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-position-from-estree": "^1.1.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + } + }, + "micromark-factory-destination": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.0.0.tgz", + "integrity": "sha512-eUBA7Rs1/xtTVun9TmV3gjfPz2wEwgK5R5xcbIM5ZYAtvGF6JkyaDsj0agx8urXnO31tEO6Ug83iVH3tdedLnw==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-factory-label": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-1.0.2.tgz", + "integrity": "sha512-CTIwxlOnU7dEshXDQ+dsr2n+yxpP0+fn271pu0bwDIS8uqfFcumXpj5mLn3hSC8iw2MUr6Gx8EcKng1dD7i6hg==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-factory-mdx-expression": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-1.0.6.tgz", + "integrity": "sha512-WRQIc78FV7KrCfjsEf/sETopbYjElh3xAmNpLkd1ODPqxEngP42eVRGbiPEQWpRV27LzqW+XVTvQAMIIRLPnNA==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-events-to-acorn": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "unist-util-position-from-estree": "^1.0.0", + "uvu": "^0.5.0", + "vfile-message": "^3.0.0" + } + }, + "micromark-factory-space": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-1.0.0.tgz", + "integrity": "sha512-qUmqs4kj9a5yBnk3JMLyjtWYN6Mzfcx8uJfi5XAveBniDevmZasdGBba5b4QsvRcAkmvGo5ACmSUmyGiKTLZew==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-factory-title": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-1.0.2.tgz", + "integrity": "sha512-zily+Nr4yFqgMGRKLpTVsNl5L4PMu485fGFDOQJQBl2NFpjGte1e86zC0da93wf97jrc4+2G2GQudFMHn3IX+A==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-factory-whitespace": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-1.0.0.tgz", + "integrity": "sha512-Qx7uEyahU1lt1RnsECBiuEbfr9INjQTGa6Err+gF3g0Tx4YEviPbqqGKNv/NrBaE7dVHdn1bVZKM/n5I/Bak7A==", + "requires": { + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-util-character": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-1.1.0.tgz", + "integrity": "sha512-agJ5B3unGNJ9rJvADMJ5ZiYjBRyDpzKAOk01Kpi1TKhlT1APx3XZk6eN7RtSz1erbWHC2L8T3xLZ81wdtGRZzg==", + "requires": { + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-util-chunked": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-1.0.0.tgz", + "integrity": "sha512-5e8xTis5tEZKgesfbQMKRCyzvffRRUX+lK/y+DvsMFdabAicPkkZV6gO+FEWi9RfuKKoxxPwNL+dFF0SMImc1g==", + "requires": { + "micromark-util-symbol": "^1.0.0" + } + }, + "micromark-util-classify-character": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-1.0.0.tgz", + "integrity": "sha512-F8oW2KKrQRb3vS5ud5HIqBVkCqQi224Nm55o5wYLzY/9PwHGXC01tr3d7+TqHHz6zrKQ72Okwtvm/xQm6OVNZA==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-util-combine-extensions": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-1.0.0.tgz", + "integrity": "sha512-J8H058vFBdo/6+AsjHp2NF7AJ02SZtWaVUjsayNFeAiydTxUwViQPxN0Hf8dp4FmCQi0UUFovFsEyRSUmFH3MA==", + "requires": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-types": "^1.0.0" + } + }, + "micromark-util-decode-numeric-character-reference": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-1.0.0.tgz", + "integrity": "sha512-OzO9AI5VUtrTD7KSdagf4MWgHMtET17Ua1fIpXTpuhclCqD8egFWo85GxSGvxgkGS74bEahvtM0WP0HjvV0e4w==", + "requires": { + "micromark-util-symbol": "^1.0.0" + } + }, + "micromark-util-decode-string": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-1.0.2.tgz", + "integrity": "sha512-DLT5Ho02qr6QWVNYbRZ3RYOSSWWFuH3tJexd3dgN1odEuPNxCngTCXJum7+ViRAd9BbdxCvMToPOD/IvVhzG6Q==", + "requires": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^1.0.0", + "micromark-util-decode-numeric-character-reference": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "micromark-util-encode": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-1.0.1.tgz", + "integrity": "sha512-U2s5YdnAYexjKDel31SVMPbfi+eF8y1U4pfiRW/Y8EFVCy/vgxk/2wWTxzcqE71LHtCuCzlBDRU2a5CQ5j+mQA==" + }, + "micromark-util-events-to-acorn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-1.2.0.tgz", + "integrity": "sha512-WWp3bf7xT9MppNuw3yPjpnOxa8cj5ACivEzXJKu0WwnjBYfzaBvIAT9KfeyI0Qkll+bfQtfftSwdgTH6QhTOKw==", + "requires": { + "@types/acorn": "^4.0.0", + "@types/estree": "^1.0.0", + "estree-util-visit": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0", + "vfile-location": "^4.0.0", + "vfile-message": "^3.0.0" + } + }, + "micromark-util-html-tag-name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-1.1.0.tgz", + "integrity": "sha512-BKlClMmYROy9UiV03SwNmckkjn8QHVaWkqoAqzivabvdGcwNGMMMH/5szAnywmsTBUzDsU57/mFi0sp4BQO6dA==" + }, + "micromark-util-normalize-identifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-1.0.0.tgz", + "integrity": "sha512-yg+zrL14bBTFrQ7n35CmByWUTFsgst5JhA4gJYoty4Dqzj4Z4Fr/DHekSS5aLfH9bdlfnSvKAWsAgJhIbogyBg==", + "requires": { + "micromark-util-symbol": "^1.0.0" + } + }, + "micromark-util-resolve-all": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-1.0.0.tgz", + "integrity": "sha512-CB/AGk98u50k42kvgaMM94wzBqozSzDDaonKU7P7jwQIuH2RU0TeBqGYJz2WY1UdihhjweivStrJ2JdkdEmcfw==", + "requires": { + "micromark-util-types": "^1.0.0" + } + }, + "micromark-util-sanitize-uri": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-1.1.0.tgz", + "integrity": "sha512-RoxtuSCX6sUNtxhbmsEFQfWzs8VN7cTctmBPvYivo98xb/kDEoTCtJQX5wyzIYEmk/lvNFTat4hL8oW0KndFpg==", + "requires": { + "micromark-util-character": "^1.0.0", + "micromark-util-encode": "^1.0.0", + "micromark-util-symbol": "^1.0.0" + } + }, + "micromark-util-subtokenize": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-1.0.2.tgz", + "integrity": "sha512-d90uqCnXp/cy4G881Ub4psE57Sf8YD0pim9QdjCRNjfas2M1u6Lbt+XZK9gnHL2XFhnozZiEdCa9CNfXSfQ6xA==", + "requires": { + "micromark-util-chunked": "^1.0.0", + "micromark-util-symbol": "^1.0.0", + "micromark-util-types": "^1.0.0", + "uvu": "^0.5.0" + } + }, + "micromark-util-symbol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-1.0.1.tgz", + "integrity": "sha512-oKDEMK2u5qqAptasDAwWDXq0tG9AssVwAx3E9bBF3t/shRIGsWIRG+cGafs2p/SnDSOecnt6hZPCE2o6lHfFmQ==" + }, + "micromark-util-types": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-1.0.2.tgz", + "integrity": "sha512-DCfg/T8fcrhrRKTPjRrw/5LLvdGV7BHySf/1LOZx7TzWZdYRjogNtyNq885z3nNallwr3QUKARjqvHqX1/7t+w==" + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, + "middleearth-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/middleearth-names/-/middleearth-names-1.1.0.tgz", + "integrity": "sha512-Oo1mbq9odpn6KHsDs8/UA5xFfX/gcrY+jWZpvd5MDaX0tjkxA7S7NTREQuqD7DWfluDgygjhKvETMWbwd3A9sA==", + "requires": { + "unique-random-array": "1.0.0" + } + }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "peer": true + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "peer": true, + "requires": { + "mime-db": "1.52.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" + }, + "mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==" + }, + "moment": { + "version": "2.21.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.21.0.tgz", + "integrity": "sha512-TCZ36BjURTeFTM/CwRcViQlfkMvL1/vFISuNLO5GkcVm1+QHfbSiNqZuWeMFjj1/3+uAjXswgRk30j1kkLYJBQ==" + }, + "mri": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", + "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==" + }, + "mrm": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/mrm/-/mrm-3.0.10.tgz", + "integrity": "sha512-aRByZsPXMM8W0NHNH9afkKyk5OW4bB5pYNRIN+8iSVfpMAzqeMejmj/yIYcdFNJTksXmdPMfTaucm2NYdh4xIw==", + "requires": { + "git-username": "^1.0.0", + "glob": "^7.1.6", + "inquirer": "^7.0.4", + "is-directory": "^0.3.1", + "kleur": "^3.0.3", + "libnpx": "^10.2.4", + "listify": "^1.0.0", + "lodash": "^4.17.15", + "longest": "^2.0.1", + "middleearth-names": "^1.1.0", + "minimist": "^1.2.0", + "mrm-core": "^6.1.7", + "semver-utils": "^1.1.4", + "update-notifier": "^4.1.0", + "user-home": "^2.0.0", + "user-meta": "^1.0.0", + "which": "^2.0.2" + }, + "dependencies": { + "kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==" + } + } + }, + "mrm-core": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/mrm-core/-/mrm-core-6.1.7.tgz", + "integrity": "sha512-jLGWrkupcgGIsLerrI/xmM/dFHbaoehRsuVbgYBrxYKXNMRBHN3Mgkd8cw+/ZCCoiZEXF8/SaZol0GCp6oBQ9g==", + "requires": { + "babel-code-frame": "^6.26.0", + "comment-json": "^2.2.0", + "detect-indent": "^6.0.0", + "editorconfig": "^0.15.3", + "find-up": "^4.1.0", + "fs-extra": "^8.1.0", + "kleur": "^3.0.3", + "listify": "^1.0.0", + "lodash": "^4.17.15", + "minimist": "^1.2.0", + "prop-ini": "^0.0.2", + "rc": "^1.2.8", + "readme-badger": "^0.3.0", + "semver": "^6.3.0", + "smpltmpl": "^1.0.2", + "split-lines": "^2.0.0", + "strip-bom": "^4.0.0", + "validate-npm-package-name": "^3.0.0", + "webpack-merge": "^4.2.2", + "yaml": "^2.0.0-1" + }, + "dependencies": { + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==" + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, + "strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==" + } + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" + }, + "nanoid": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==" + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" + }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "peer": true + }, + "next": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/next/-/next-13.0.2.tgz", + "integrity": "sha512-uQ5z5e4D9mOe8+upy6bQdYYjo/kk1v3jMW87kTy2TgAyAsEO+CkwRnMgyZ4JoHEnhPZLHwh7dk0XymRNLe1gFw==", + "requires": { + "@next/env": "13.0.2", + "@next/swc-android-arm-eabi": "13.0.2", + "@next/swc-android-arm64": "13.0.2", + "@next/swc-darwin-arm64": "13.0.2", + "@next/swc-darwin-x64": "13.0.2", + "@next/swc-freebsd-x64": "13.0.2", + "@next/swc-linux-arm-gnueabihf": "13.0.2", + "@next/swc-linux-arm64-gnu": "13.0.2", + "@next/swc-linux-arm64-musl": "13.0.2", + "@next/swc-linux-x64-gnu": "13.0.2", + "@next/swc-linux-x64-musl": "13.0.2", + "@next/swc-win32-arm64-msvc": "13.0.2", + "@next/swc-win32-ia32-msvc": "13.0.2", + "@next/swc-win32-x64-msvc": "13.0.2", + "@swc/helpers": "0.4.11", + "caniuse-lite": "^1.0.30001406", + "postcss": "8.4.14", + "styled-jsx": "5.1.0", + "use-sync-external-store": "1.2.0" + } + }, + "next-mdx-remote": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/next-mdx-remote/-/next-mdx-remote-4.2.0.tgz", + "integrity": "sha512-X5RhD7f7b78pH2abbuusObSGgII5l54OdusS/2iXljN7WN1cel6ToLlZeCZcyxx9cR4wmBGQYGongIttDYNmAA==", + "requires": { + "@mdx-js/mdx": "^2.0.0", + "@mdx-js/react": "^2.0.0", + "vfile": "^5.3.0", + "vfile-matter": "^3.0.1" + } + }, + "next-plausible": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/next-plausible/-/next-plausible-3.6.5.tgz", + "integrity": "sha512-eGsyie68xIyX+If1PSIJsgv7pkiMUm66Yt9gELl3kUvljaxoJOCwUEALGk5JHvhByGsBiRswkrGioDMthL3gfQ==", + "requires": {} + }, + "next-pwa": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/next-pwa/-/next-pwa-5.6.0.tgz", + "integrity": "sha512-XV8g8C6B7UmViXU8askMEYhWwQ4qc/XqJGnexbLV68hzKaGHZDMtHsm2TNxFcbR7+ypVuth/wwpiIlMwpRJJ5A==", + "requires": { + "babel-loader": "^8.2.5", + "clean-webpack-plugin": "^4.0.0", + "globby": "^11.0.4", + "terser-webpack-plugin": "^5.3.3", + "workbox-webpack-plugin": "^6.5.4", + "workbox-window": "^6.5.4" + } + }, + "next-sitemap": { + "version": "3.1.32", + "resolved": "https://registry.npmjs.org/next-sitemap/-/next-sitemap-3.1.32.tgz", + "integrity": "sha512-jkIKpwLXpWWTPfmDO46+6nu4+qpar4CjvUwCR9rYZHWtzE/wFfaCVFKpGtFMl6MFjpu8GjiE6kWFEa7uF3bzzg==", + "requires": { + "@corex/deepmerge": "^4.0.29", + "minimist": "^1.2.6" + } + }, + "nlcst-to-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-3.1.0.tgz", + "integrity": "sha512-Y8HQWKw/zrHTCnu2zcFBN1dV6vN0NUG7s5fkEj380G8tF3R+vA2KG+tDl2QoHVQCTHGHVXwoni2RQkDSFQb1PA==", + "requires": { + "@types/nlcst": "^1.0.0" + } + }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + }, + "dependencies": { + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, + "node-releases": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==" + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "normalize-url": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz", + "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==" + }, + "nothing-mock": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/nothing-mock/-/nothing-mock-1.0.4.tgz", + "integrity": "sha512-wl5hpQruSQHt5M15oY/akWd7hFPyec0uTCUHYQa3WhBeTxonsFnMKmixaCDvb4U7MN9UjS2UvXw3s4KxjrNjKg==" + }, + "npm-package-arg": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-6.1.1.tgz", + "integrity": "sha512-qBpssaL3IOZWi5vEKUKW0cO7kzLeT+EQO9W8RsLOZf76KF9E/K9+wH0C7t06HXPpaH8WH5xF1MExLuCwbTqRUg==", + "requires": { + "hosted-git-info": "^2.7.1", + "osenv": "^0.1.5", + "semver": "^5.6.0", + "validate-npm-package-name": "^3.0.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, + "npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "requires": { + "path-key": "^2.0.0" + }, + "dependencies": { + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==" + } + } + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==" + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object.assign": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + } + }, + "object.entries": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.6.tgz", + "integrity": "sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "object.fromentries": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.6.tgz", + "integrity": "sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "object.hasown": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.2.tgz", + "integrity": "sha512-B5UIT3J1W+WuWIU55h0mjlwaqxiE5vYENJXIXZ4VFe05pNYrkKuK0U/6aFcb0pKywYJh7IhfoqUfKVmrJJHZHw==", + "requires": { + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "object.values": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.6.tgz", + "integrity": "sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "requires": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + } + }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==" + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" + }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==" + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==" + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "requires": { + "p-limit": "^3.0.2" + } + }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "requires": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "requires": { + "callsites": "^3.0.0" + } + }, + "parse-entities": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.0.tgz", + "integrity": "sha512-5nk9Fn03x3rEhGaX1FU6IDwG/k+GxLXlFAkgrbM1asuAFl3BhdQWvASaIsmwWypRNcZKHPYnIuOSfIWEyEQnPQ==", + "requires": { + "@types/unist": "^2.0.0", + "character-entities": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + } + }, + "parse-git-config": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/parse-git-config/-/parse-git-config-1.1.1.tgz", + "integrity": "sha512-S3LGXJZVSy/hswvbSkfdbKBRVsnqKrVu6j8fcvdtJ4TxosSELyQDsJPuGPXuZ+EyuYuJd3O4uAF8gcISR0OFrQ==", + "requires": { + "extend-shallow": "^2.0.1", + "fs-exists-sync": "^0.1.0", + "git-config-path": "^1.0.1", + "ini": "^1.3.4" + } + }, + "parse-github-url": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/parse-github-url/-/parse-github-url-1.0.2.tgz", + "integrity": "sha512-kgBf6avCbO3Cn6+RnzRGLkUsv4ZVqv/VfAYkRsyBcgkshNvVBkRn1FEZcW0Jb+npXQWm2vHPnnOqFteZxRRGNw==" + }, + "parse-latin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/parse-latin/-/parse-latin-5.0.1.tgz", + "integrity": "sha512-b/K8ExXaWC9t34kKeDV8kGXBkXZ1HCSAZRYE7HR14eA1GlXX5L8iWhs8USJNhQU9q5ci413jCKF0gOyovvyRBg==", + "requires": { + "nlcst-to-string": "^3.0.0", + "unist-util-modify-children": "^3.0.0", + "unist-util-visit-children": "^2.0.0" + } + }, + "parse-numeric-range": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz", + "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==" + }, + "parse-passwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", + "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==" + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "path-posix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-posix/-/path-posix-1.0.0.tgz", + "integrity": "sha512-1gJ0WpNIiYcQydgg3Ed8KzvIqTsDpNwq+cjBCssvBtuTWjEqY1AW+i+OepiEMqDCzyro9B2sLAe4RBPajMYFiA==" + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" + }, + "periscopic": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.0.4.tgz", + "integrity": "sha512-SFx68DxCv0Iyo6APZuw/AKewkkThGwssmU0QWtTlvov3VAtPX+QJ4CadwSaz8nrT5jPIuxdvJWB4PnD2KNDxQg==", + "requires": { + "estree-walker": "^3.0.0", + "is-reference": "^3.0.0" + } + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==" + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "requires": { + "pinkie": "^2.0.0" + } + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "requires": { + "find-up": "^4.0.0" + }, + "dependencies": { + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + } + } + }, + "postcss": { + "version": "8.4.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.14.tgz", + "integrity": "sha512-E398TUmfAYFPBSdzgeieK2Y1+1cpdxJx8yXbK/m57nRhKSmk1GB2tO4lbLBtlkfPQTDKfe4Xqv1ASWPpayPEig==", + "requires": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + } + }, + "prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==" + }, + "prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha512-ravE6m9Atw9Z/jjttRUZ+clIXogdghyZAuWJ3qEzjT+jI/dL1ifAqhZeC5VHzQp1MSt1+jxKkFNemj/iO7tVUA==" + }, + "prettier": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", + "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==" + }, + "pretty-bytes": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", + "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==" + }, + "prismjs": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", + "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==" + }, + "prop-ini": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/prop-ini/-/prop-ini-0.0.2.tgz", + "integrity": "sha512-qyU57WvAvZDbzmRy9xDbJGVwrGJhmA+rYnVjy4xtX4Ny9c7gzvpmf/j7A3oq9ChbPh15MZQKjPep2mNdnAhtig==", + "requires": { + "extend": "^3.0.0" + } + }, + "prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "property-information": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.1.1.tgz", + "integrity": "sha512-hrzC564QIl0r0vy4l6MvRLhafmUowhO/O3KgVSoXIbbA2Sz4j8HGpJc6T2cubRVwMwpdiG/vKGfhT4IixmKN9w==" + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "pupa": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.1.1.tgz", + "integrity": "sha512-l1jNAspIBSFqbT+y+5FosojNpVpF94nlI+wDUpqP9enwOTfHx9f0gh5nB96vl+6yTpsJsypeNrwfzPrKuHB41A==", + "requires": { + "escape-goat": "^2.0.0" + } + }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + } + } + }, + "react": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "requires": { + "loose-envify": "^1.1.0" + } + }, + "react-dom": { + "version": "18.2.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", + "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "requires": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.0" + } + }, + "react-from-dom": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/react-from-dom/-/react-from-dom-0.6.2.tgz", + "integrity": "sha512-qvWWTL/4xw4k/Dywd41RBpLQUSq97csuv15qrxN+izNeLYlD9wn5W8LspbfYe5CWbaSdkZ72BsaYBPQf2x4VbQ==", + "requires": {} + }, + "react-github-btn": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/react-github-btn/-/react-github-btn-1.4.0.tgz", + "integrity": "sha512-lV4FYClAfjWnBfv0iNlJUGhamDgIq6TayD0kPZED6VzHWdpcHmPfsYOZ/CFwLfPv4Zp+F4m8QKTj0oy2HjiGXg==", + "requires": { + "github-buttons": "^2.22.0" + } + }, + "react-inlinesvg": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/react-inlinesvg/-/react-inlinesvg-3.0.1.tgz", + "integrity": "sha512-cBfoyfseNI2PkDA7ZKIlDoHq0eMfpoC3DhKBQNC+/X1M4ZQB+aXW+YiNPUDDDKXUsGDUIZWWiZWNFeauDIVdoA==", + "requires": { + "exenv": "^1.2.2", + "react-from-dom": "^0.6.2" + } + }, + "react-intersection-observer": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-9.4.1.tgz", + "integrity": "sha512-IXpIsPe6BleFOEHKzKh5UjwRUaz/JYS0lT/HPsupWEQou2hDqjhLMStc5zyE3eQVT4Fk3FufM8Fw33qW1uyeiw==", + "requires": {} + }, + "react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "requires": { + "picomatch": "^2.2.1" + } + }, + "readme-badger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/readme-badger/-/readme-badger-0.3.0.tgz", + "integrity": "sha512-+sMOLSs1imZUISZ2Rhz7qqVd77QtpcAPbGeIraFdgJmijb04YtdlPjGNBvDChTNtLbeQ6JNGQy3pOgslWfaP3g==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "refractor": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", + "integrity": "sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==", + "requires": { + "hastscript": "^6.0.0", + "parse-entities": "^2.0.0", + "prismjs": "~1.27.0" + }, + "dependencies": { + "character-entities": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", + "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==" + }, + "character-entities-legacy": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", + "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==" + }, + "character-reference-invalid": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz", + "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==" + }, + "is-alphabetical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", + "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==" + }, + "is-alphanumerical": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", + "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==", + "requires": { + "is-alphabetical": "^1.0.0", + "is-decimal": "^1.0.0" + } + }, + "is-decimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz", + "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==" + }, + "is-hexadecimal": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz", + "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==" + }, + "parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "requires": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + } + }, + "prismjs": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.27.0.tgz", + "integrity": "sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==" + } + } + }, + "regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==" + }, + "regenerate-unicode-properties": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.0.tgz", + "integrity": "sha512-d1VudCLoIGitcU/hEg2QqvyGZQmdC0Lf8BqdOMXGFSvJP4bNV1+XqbPQeHHLD51Jh4QJJ225dlIFvY4Ly6MXmQ==", + "requires": { + "regenerate": "^1.4.2" + } + }, + "regenerator-runtime": { + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" + }, + "regenerator-transform": { + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.1.tgz", + "integrity": "sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg==", + "requires": { + "@babel/runtime": "^7.8.4" + } + }, + "regexp.prototype.flags": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" + } + }, + "regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==" + }, + "regexpu-core": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.2.2.tgz", + "integrity": "sha512-T0+1Zp2wjF/juXMrMxHxidqGYn8U4R+zleSJhX9tQ1PUsS8a9UtYfbsF9LdiVgNX3kiX8RNaKM42nfSgvFJjmw==", + "requires": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.1.0", + "regjsgen": "^0.7.1", + "regjsparser": "^0.9.1", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.1.0" + } + }, + "registry-auth-token": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.2.tgz", + "integrity": "sha512-PC5ZysNb42zpFME6D/XlIgtNGdTl8bBOCw90xQLVMpzuuubJKYDWFAEuUNc+Cn8Z8724tg2SDhDRrkVEsqfDMg==", + "requires": { + "rc": "1.2.8" + } + }, + "registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "requires": { + "rc": "^1.2.8" + } + }, + "regjsgen": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.7.1.tgz", + "integrity": "sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA==" + }, + "regjsparser": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.9.1.tgz", + "integrity": "sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==", + "requires": { + "jsesc": "~0.5.0" + }, + "dependencies": { + "jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==" + } + } + }, + "remark": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/remark/-/remark-14.0.2.tgz", + "integrity": "sha512-A3ARm2V4BgiRXaUo5K0dRvJ1lbogrbXnhkJRmD0yw092/Yl0kOCZt1k9ZeElEwkZsWGsMumz6qL5MfNJH9nOBA==", + "requires": { + "@types/mdast": "^3.0.0", + "remark-parse": "^10.0.0", + "remark-stringify": "^10.0.0", + "unified": "^10.0.0" + } + }, + "remark-gfm": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz", + "integrity": "sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-gfm": "^2.0.0", + "micromark-extension-gfm": "^2.0.0", + "unified": "^10.0.0" + } + }, + "remark-mdx": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/remark-mdx/-/remark-mdx-2.1.5.tgz", + "integrity": "sha512-A8vw5s+BgOa968Irt8BO7DfWJTE0Fe7Ge3hX8zzDB1DnwMZTNdK6qF2IcFao+/7nzk1vSysKcFp+3ku4vhMpaQ==", + "requires": { + "mdast-util-mdx": "^2.0.0", + "micromark-extension-mdxjs": "^1.0.0" + } + }, + "remark-parse": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.1.tgz", + "integrity": "sha512-1fUyHr2jLsVOkhbvPRBJ5zTKZZyD6yZzYaWCS6BPBdQ8vEMBCH+9zNCDA6tET/zHCi/jLqjCWtlJZUPk+DbnFw==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-from-markdown": "^1.0.0", + "unified": "^10.0.0" + } + }, + "remark-react": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/remark-react/-/remark-react-9.0.1.tgz", + "integrity": "sha512-NtpTMfUIreelaRdUVUtgSizTOSwV6JQvGAPn2gWNoWF+Etd0D8YUdDDFEV3vVOiMIoiNR2bQ0etMb/9uaWhjAw==", + "requires": { + "@mapbox/hast-util-table-cell-style": "^0.2.0", + "@types/mdast": "^3.0.0", + "@types/react": "^17.0.0", + "hast-to-hyperscript": "^10.0.0", + "hast-util-sanitize": "^4.0.0", + "mdast-util-to-hast": "^11.0.0", + "unified": "^10.0.0" + }, + "dependencies": { + "@types/react": { + "version": "17.0.52", + "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.52.tgz", + "integrity": "sha512-vwk8QqVODi0VaZZpDXQCmEmiOuyjEFPY7Ttaw5vjM112LOq37yz1CDJGrRJwA1fYEq4Iitd5rnjd1yWAc/bT+A==", + "requires": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "mdast-util-to-hast": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-11.3.0.tgz", + "integrity": "sha512-4o3Cli3hXPmm1LhB+6rqhfsIUBjnKFlIUZvudaermXB+4/KONdd/W4saWWkC+LBLbPMqhFSSTSRgafHsT5fVJw==", + "requires": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "@types/mdurl": "^1.0.0", + "mdast-util-definitions": "^5.0.0", + "mdurl": "^1.0.0", + "unist-builder": "^3.0.0", + "unist-util-generated": "^2.0.0", + "unist-util-position": "^4.0.0", + "unist-util-visit": "^4.0.0" + } + } + } + }, + "remark-rehype": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-10.1.0.tgz", + "integrity": "sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==", + "requires": { + "@types/hast": "^2.0.0", + "@types/mdast": "^3.0.0", + "mdast-util-to-hast": "^12.1.0", + "unified": "^10.0.0" + } + }, + "remark-smartypants": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/remark-smartypants/-/remark-smartypants-2.0.0.tgz", + "integrity": "sha512-Rc0VDmr/yhnMQIz8n2ACYXlfw/P/XZev884QU1I5u+5DgJls32o97Vc1RbK3pfumLsJomS2yy8eT4Fxj/2MDVA==", + "requires": { + "retext": "^8.1.0", + "retext-smartypants": "^5.1.0", + "unist-util-visit": "^4.1.0" + } + }, + "remark-stringify": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-10.0.2.tgz", + "integrity": "sha512-6wV3pvbPvHkbNnWB0wdDvVFHOe1hBRAx1Q/5g/EpH4RppAII6J8Gnwe7VbHuXaoKIF6LAg6ExTel/+kNqSQ7lw==", + "requires": { + "@types/mdast": "^3.0.0", + "mdast-util-to-markdown": "^1.0.0", + "unified": "^10.0.0" + } + }, + "remark-unwrap-images": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/remark-unwrap-images/-/remark-unwrap-images-3.0.1.tgz", + "integrity": "sha512-5VUY0n+J9lPTPfkct5S3/SbutryBjp8J/4mbgtlkDrOk3h8jde0hyqdYUJOoJKherZezS08tjd6i4+nnQ+wl5w==", + "requires": { + "@types/mdast": "^3.0.0", + "hast-util-whitespace": "^2.0.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0" + } + }, + "remote-origin-url": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/remote-origin-url/-/remote-origin-url-1.0.0.tgz", + "integrity": "sha512-xHDM6IBqivpiQ1e4WOuFpM/T6rbzA/WBsu+3WLtgPOhHyjA0nYlijV3NprlTb4FcXlQ5+Q+z174sQ1NnUF5FwA==", + "requires": { + "parse-git-config": "^1.1.1" + } + }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==" + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "requires": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==" + }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha512-/Fpe5guzJk1gPqdJLJR5u7eG/gNY4nImjbRDaVWVMRhne55TCmj2i9Q+54PBRfatRC8v/rIiv9BN0pMd9OV5EQ==", + "requires": { + "lowercase-keys": "^1.0.0" + } + }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, + "retext": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/retext/-/retext-8.1.0.tgz", + "integrity": "sha512-N9/Kq7YTn6ZpzfiGW45WfEGJqFf1IM1q8OsRa1CGzIebCJBNCANDRmOrholiDRGKo/We7ofKR4SEvcGAWEMD3Q==", + "requires": { + "@types/nlcst": "^1.0.0", + "retext-latin": "^3.0.0", + "retext-stringify": "^3.0.0", + "unified": "^10.0.0" + } + }, + "retext-latin": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/retext-latin/-/retext-latin-3.1.0.tgz", + "integrity": "sha512-5MrD1tuebzO8ppsja5eEu+ZbBeUNCjoEarn70tkXOS7Bdsdf6tNahsv2bY0Z8VooFF6cw7/6S+d3yI/TMlMVVQ==", + "requires": { + "@types/nlcst": "^1.0.0", + "parse-latin": "^5.0.0", + "unherit": "^3.0.0", + "unified": "^10.0.0" + } + }, + "retext-smartypants": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/retext-smartypants/-/retext-smartypants-5.2.0.tgz", + "integrity": "sha512-Do8oM+SsjrbzT2UNIKgheP0hgUQTDDQYyZaIY3kfq0pdFzoPk+ZClYJ+OERNXveog4xf1pZL4PfRxNoVL7a/jw==", + "requires": { + "@types/nlcst": "^1.0.0", + "nlcst-to-string": "^3.0.0", + "unified": "^10.0.0", + "unist-util-visit": "^4.0.0" + } + }, + "retext-stringify": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/retext-stringify/-/retext-stringify-3.1.0.tgz", + "integrity": "sha512-767TLOaoXFXyOnjx/EggXlb37ZD2u4P1n0GJqVdpipqACsQP+20W+BNpMYrlJkq7hxffnFk+jc6mAK9qrbuB8w==", + "requires": { + "@types/nlcst": "^1.0.0", + "nlcst-to-string": "^3.0.0", + "unified": "^10.0.0" + } + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, + "rollup": { + "version": "2.79.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.1.tgz", + "integrity": "sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw==", + "requires": { + "fsevents": "~2.3.2" + } + }, + "rollup-plugin-terser": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz", + "integrity": "sha512-w3iIaU4OxcF52UUXiZNsNeuXIMDvFrr+ZXK6bFZ0Q60qyVfq4uLptoS4bbq3paG3x216eQllFZX7zt6TIImguQ==", + "requires": { + "@babel/code-frame": "^7.10.4", + "jest-worker": "^26.2.1", + "serialize-javascript": "^4.0.0", + "terser": "^5.0.0" + }, + "dependencies": { + "jest-worker": { + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz", + "integrity": "sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ==", + "requires": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^7.0.0" + } + }, + "serialize-javascript": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", + "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "requires": { + "randombytes": "^2.1.0" + } + } + } + }, + "run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==" + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "rxjs": { + "version": "6.6.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", + "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", + "requires": { + "tslib": "^1.9.0" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, + "sade": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", + "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", + "requires": { + "mri": "^1.1.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "safe-regex-test": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", + "integrity": "sha512-JBUUzyOgEwXQY1NuPtvcj/qcBDbDmEvWufhlnXZIm75DEHp+afM1r1ujJpJsV/gSM4t59tpDyPi1sd6ZaPFfsA==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.3", + "is-regex": "^1.1.4" + } + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sass": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.56.1.tgz", + "integrity": "sha512-VpEyKpyBPCxE7qGDtOcdJ6fFbcpOM+Emu7uZLxVrkX8KVU/Dp5UF7WLvzqRuUhB6mqqQt1xffLoG+AndxTZrCQ==", + "requires": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + } + }, + "scheduler": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", + "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "requires": { + "loose-envify": "^1.1.0" + } + }, + "schema-utils": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz", + "integrity": "sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw==", + "requires": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "requires": { + "semver": "^6.3.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "semver-utils": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/semver-utils/-/semver-utils-1.1.4.tgz", + "integrity": "sha512-EjnoLE5OGmDAVV/8YDoN5KiajNadjzIp9BAHOhYeQHt7j0UWxjmgsx4YD48wp4Ue1Qogq38F1GNUJNqF1kKKxA==" + }, + "serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "requires": { + "randombytes": "^2.1.0" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, + "sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==" + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==" + }, + "smpltmpl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/smpltmpl/-/smpltmpl-1.0.2.tgz", + "integrity": "sha512-Hq23NNgeZigOzIiX1dkb6W3gFn2/XQj43KhPxu65IMieG/gIwf/lQb1IudjYv0c/5LwJeS/mPayYzyo+8WJMxQ==", + "requires": { + "babel-code-frame": "^6.26.0" + } + }, + "source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==" + }, + "source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==" + }, + "source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==" + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + } + } + }, + "sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==" + }, + "space-separated-tokens": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.1.tgz", + "integrity": "sha512-ekwEbFp5aqSPKaqeY1PGrlGQxPNaq+Cnx4+bE2D8sciBQrHpbwoBbawqTN2+6jPs9IdWxxiUcN0K2pkczD3zmw==" + }, + "split-lines": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/split-lines/-/split-lines-2.1.0.tgz", + "integrity": "sha512-8dv+1zKgTpfTkOy8XZLFyWrfxO0NV/bj/3EaQ+hBrBxGv2DwiroljPjU8NlCr+59nLnsVm9WYT7lXKwe4TC6bw==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "dependencies": { + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + } + } + }, + "string.prototype.matchall": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.8.tgz", + "integrity": "sha512-6zOCOcJ+RJAQshcTvXPHoxoQGONa3e/Lqx90wUA+wEzX78sg5Bo+1tQo4N0pohS0erG9qtCqJDjNCQBjeWVxyg==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "get-intrinsic": "^1.1.3", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.3", + "regexp.prototype.flags": "^1.4.3", + "side-channel": "^1.0.4" + } + }, + "string.prototype.trimend": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz", + "integrity": "sha512-JySq+4mrPf9EsDBEDYMOb/lM7XQLulwg5R/m1r0PXEFqrV0qHvl58sdTilSXtKOflCsK2E8jxf+GKC0T07RWwQ==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "string.prototype.trimstart": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.6.tgz", + "integrity": "sha512-omqjMDaY92pbn5HOX7f9IccLA+U1tA9GvtU4JrodiXFfYB7jPzzHpRzpglLAjtUV6bB557zwClJezTqnAiYnQA==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4" + } + }, + "stringify-entities": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.3.tgz", + "integrity": "sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g==", + "requires": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + } + }, + "stringify-object": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", + "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", + "requires": { + "get-own-enumerable-property-symbols": "^3.0.0", + "is-obj": "^1.0.1", + "is-regexp": "^1.0.0" + }, + "dependencies": { + "is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==" + } + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==" + }, + "strip-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-comments/-/strip-comments-2.0.1.tgz", + "integrity": "sha512-ZprKx+bBLXv067WTCALv8SSz5l2+XhpYCsVtSqlMnkAXMWDq+/ekVbl1ghqP9rUHTzv6sm/DwCOiYutU/yp1fw==" + }, + "strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==" + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" + }, + "style-mod": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.0.0.tgz", + "integrity": "sha512-OPhtyEjyyN9x3nhPsu76f52yUGXiZcgvsrFVtvTkyGRQJ0XK+GPc6ov1z+lRpbeabka+MYEQxOYRnt5nF30aMw==" + }, + "style-to-object": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.3.0.tgz", + "integrity": "sha512-CzFnRRXhzWIdItT3OmF8SQfWyahHhjq3HwcMNCNLn+N7klOOqPjMeG/4JSu77D7ypZdGvSzvkrbyeTMizz2VrA==", + "requires": { + "inline-style-parser": "0.1.1" + } + }, + "styled-jsx": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.0.tgz", + "integrity": "sha512-/iHaRJt9U7T+5tp6TRelLnqBqiaIT0HsO0+vgyj8hK2KUk7aejFqRrumqPUlAqDwAj8IbS/1hk3IhBAAK/FCUQ==", + "requires": { + "client-only": "0.0.1" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + }, + "tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "peer": true + }, + "temp-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", + "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==" + }, + "tempy": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-0.6.0.tgz", + "integrity": "sha512-G13vtMYPT/J8A4X2SjdtBTphZlrp1gKv6hZiOjw14RCWg6GbHuQBGtjlx75xLbYV/wEc0D7G5K4rxKP/cXk8Bw==", + "requires": { + "is-stream": "^2.0.0", + "temp-dir": "^2.0.0", + "type-fest": "^0.16.0", + "unique-string": "^2.0.0" + }, + "dependencies": { + "is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" + }, + "type-fest": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.16.0.tgz", + "integrity": "sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==" + } + } + }, + "term-size": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz", + "integrity": "sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==" + }, + "terser": { + "version": "5.15.1", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.1.tgz", + "integrity": "sha512-K1faMUvpm/FBxjBXud0LWVAGxmvoPbZbfTCYbSgaaYQaIXI3/TdI7a7ZGA73Zrou6Q8Zmz3oeUTsp/dj+ag2Xw==", + "requires": { + "@jridgewell/source-map": "^0.3.2", + "acorn": "^8.5.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + } + }, + "terser-webpack-plugin": { + "version": "5.3.6", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz", + "integrity": "sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ==", + "requires": { + "@jridgewell/trace-mapping": "^0.3.14", + "jest-worker": "^27.4.5", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.0", + "terser": "^5.14.1" + } + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==" + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "timed-out": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", + "integrity": "sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==" + }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "requires": { + "os-tmpdir": "~1.0.2" + } + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==" + }, + "to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==" + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "requires": { + "is-number": "^7.0.0" + } + }, + "tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==", + "requires": { + "punycode": "^2.1.0" + } + }, + "trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==" + }, + "trough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.1.0.tgz", + "integrity": "sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==" + }, + "tsconfig-paths": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", + "integrity": "sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==", + "requires": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.6", + "strip-bom": "^3.0.0" + } + }, + "tslib": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + }, + "tsutils": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", + "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", + "requires": { + "tslib": "^1.8.1" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "requires": { + "prelude-ls": "^1.2.1" + } + }, + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==" + }, + "typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "requires": { + "is-typedarray": "^1.0.0" + } + }, + "typescript": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", + "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==" + }, + "unbox-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "requires": { + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + } + }, + "unherit": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/unherit/-/unherit-3.0.1.tgz", + "integrity": "sha512-akOOQ/Yln8a2sgcLj4U0Jmx0R5jpIg2IUyRrWOzmEbjBtGzBdHtSeFKgoEcoH4KYIG/Pb8GQ/BwtYm0GCq1Sqg==" + }, + "unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==" + }, + "unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "requires": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + } + }, + "unicode-match-property-value-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz", + "integrity": "sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==" + }, + "unicode-property-aliases-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz", + "integrity": "sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==" + }, + "unified": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz", + "integrity": "sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==", + "requires": { + "@types/unist": "^2.0.0", + "bail": "^2.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^5.0.0" + } + }, + "unique-random": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-random/-/unique-random-1.0.0.tgz", + "integrity": "sha512-K1sUkPf9EXCZFNIlMCoX4icAqcvkR4FMPH4Z61HbyiWhQl1ZGo0zYeV2bJmocK8Cp6tnKYrCnpkeKGebXZoRTQ==" + }, + "unique-random-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unique-random-array/-/unique-random-array-1.0.0.tgz", + "integrity": "sha512-vtj2yltjcHPa69nFjNJ3xnhsEwE8pMyjqUQDw2myz/iSezqf4YCAcygwFQEsOgMid5VNW/dCPbnb2BcmaDCCKg==", + "requires": { + "unique-random": "^1.0.0" + } + }, + "unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "requires": { + "crypto-random-string": "^2.0.0" + } + }, + "unist-builder": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unist-builder/-/unist-builder-3.0.0.tgz", + "integrity": "sha512-GFxmfEAa0vi9i5sd0R2kcrI9ks0r82NasRq5QHh2ysGngrc6GiqD5CDf1FjPenY4vApmFASBIIlk/jj5J5YbmQ==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-generated": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-generated/-/unist-util-generated-2.0.0.tgz", + "integrity": "sha512-TiWE6DVtVe7Ye2QxOVW9kqybs6cZexNwTwSMVgkfjEReqy/xwGpAXb99OxktoWwmL+Z+Epb0Dn8/GNDYP1wnUw==" + }, + "unist-util-is": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.1.1.tgz", + "integrity": "sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==" + }, + "unist-util-modify-children": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/unist-util-modify-children/-/unist-util-modify-children-3.1.0.tgz", + "integrity": "sha512-L0UizdncPZ1NIwpmkwFdLo2NaK2Eb5LU/vaQ7lZGkAaOBZfsHp+8T/gVWPVmmMO1hj6gc+XeMoytut8jr7fdyA==", + "requires": { + "@types/unist": "^2.0.0", + "array-iterate": "^2.0.0" + } + }, + "unist-util-position": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-4.0.3.tgz", + "integrity": "sha512-p/5EMGIa1qwbXjA+QgcBXaPWjSnZfQ2Sc3yBEEfgPwsEmJd8Qh+DSk3LGnmOM4S1bY2C0AjmMnB8RuEYxpPwXQ==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-position-from-estree": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-1.1.1.tgz", + "integrity": "sha512-xtoY50b5+7IH8tFbkw64gisG9tMSpxDjhX9TmaJJae/XuxQ9R/Kc8Nv1eOsf43Gt4KV/LkriMy9mptDr7XLcaw==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-remove-position": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-4.0.1.tgz", + "integrity": "sha512-0yDkppiIhDlPrfHELgB+NLQD5mfjup3a8UYclHruTJWmY74je8g+CIFr79x5f6AkmzSwlvKLbs63hC0meOMowQ==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-visit": "^4.0.0" + } + }, + "unist-util-stringify-position": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-3.0.2.tgz", + "integrity": "sha512-7A6eiDCs9UtjcwZOcCpM4aPII3bAAGv13E96IkawkOAW0OhH+yRxtY0lzo8KiHpzEMfH7Q+FizUmwp8Iqy5EWg==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-visit": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-4.1.1.tgz", + "integrity": "sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^5.1.1" + } + }, + "unist-util-visit-children": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-children/-/unist-util-visit-children-2.0.1.tgz", + "integrity": "sha512-2cEU3dhV1hMfO9ajwb8rJsDedMfsahsm6fCfR8LxDR/w7KcB5lzHQ9dBTQIXsWGNWBFH5MPmaFP3Xh0dWLqClQ==", + "requires": { + "@types/unist": "^2.0.0" + } + }, + "unist-util-visit-parents": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-5.1.1.tgz", + "integrity": "sha512-gks4baapT/kNRaWxuGkl5BIhoanZo7sC/cUT/JToSRNL1dYoXRFl75d++NkjYk4TAu2uv2Px+l8guMajogeuiw==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0" + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, + "unzip-response": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-2.0.1.tgz", + "integrity": "sha512-N0XH6lqDtFH84JxptQoZYmloF4nzrQqqrAymNj+/gW60AO2AZgOcf4O/nUXJcYfyQkqvMo9lSupBZmmgvuVXlw==" + }, + "upath": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==" + }, + "update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + } + }, + "update-notifier": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", + "integrity": "sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A==", + "requires": { + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "pupa": "^2.0.1", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + } + } + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "requires": { + "punycode": "^2.1.0" + } + }, + "url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha512-NjFKA0DidqPa5ciFcSrXnAltTtzz84ogy+NebPvfEgAck0+TNg4UJ4IN+fB7zRZfbgUf0syOo9MDxFkDSMuFaQ==", + "requires": { + "prepend-http": "^2.0.0" + } + }, + "use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "requires": {} + }, + "user-home": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/user-home/-/user-home-2.0.0.tgz", + "integrity": "sha512-KMWqdlOcjCYdtIJpicDSFBQ8nFwS2i9sslAd6f4+CBGcU4gist2REnr2fxj2YocvJFxSF3ZOHLYLVZnUxv4BZQ==", + "requires": { + "os-homedir": "^1.0.0" + } + }, + "user-meta": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/user-meta/-/user-meta-1.0.0.tgz", + "integrity": "sha512-Q/opMgFhVbBkdlTs44UKzV7L5Uj2zrJ4MVPXTTzJmrU1bHb2cX6wJzBIqEf1gROTzZIH8u39WmHsa5EvfnMPrw==", + "requires": { + "rc": "^1.2.1" + } + }, + "uvu": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/uvu/-/uvu-0.5.6.tgz", + "integrity": "sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==", + "requires": { + "dequal": "^2.0.0", + "diff": "^5.0.0", + "kleur": "^4.0.3", + "sade": "^1.7.3" + } + }, + "validate-npm-package-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz", + "integrity": "sha512-M6w37eVCMMouJ9V/sdPGnC5H4uDr73/+xdq0FBLO3TFFX1+7wiUY6Es328NN+y43tmY+doUdN9g9J21vqB7iLw==", + "requires": { + "builtins": "^1.0.3" + } + }, + "vfile": { + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-5.3.5.tgz", + "integrity": "sha512-U1ho2ga33eZ8y8pkbQLH54uKqGhFJ6GYIHnnG5AhRpAh3OWjkrRHKa/KogbmQn8We+c0KVV3rTOgR9V/WowbXQ==", + "requires": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^3.0.0", + "vfile-message": "^3.0.0" + } + }, + "vfile-location": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-4.0.1.tgz", + "integrity": "sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==", + "requires": { + "@types/unist": "^2.0.0", + "vfile": "^5.0.0" + } + }, + "vfile-matter": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/vfile-matter/-/vfile-matter-3.0.1.tgz", + "integrity": "sha512-CAAIDwnh6ZdtrqAuxdElUqQRQDQgbbIrYtDYI8gCjXS1qQ+1XdLoK8FIZWxJwn0/I+BkSSZpar3SOgjemQz4fg==", + "requires": { + "@types/js-yaml": "^4.0.0", + "is-buffer": "^2.0.0", + "js-yaml": "^4.0.0" + } + }, + "vfile-message": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-3.1.2.tgz", + "integrity": "sha512-QjSNP6Yxzyycd4SVOtmKKyTsSvClqBPJcd00Z0zuPj3hOIjg0rUPG6DbFGPvUKRgYyaIWLPKpuEclcuvb3H8qA==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^3.0.0" + } + }, + "w3c-keyname": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.6.tgz", + "integrity": "sha512-f+fciywl1SJEniZHD6H+kUO8gOnwIr7f4ijKA6+ZvJFjeGi1r4PDLl53Ayud9O/rk64RqgoQine0feoeOU0kXg==" + }, + "watchpack": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", + "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", + "peer": true, + "requires": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + } + }, + "web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==" + }, + "webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "webpack": { + "version": "5.75.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", + "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "peer": true, + "requires": { + "@types/eslint-scope": "^3.7.3", + "@types/estree": "^0.0.51", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.7.1", + "acorn-import-assertions": "^1.7.6", + "browserslist": "^4.14.5", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.10.0", + "es-module-lexer": "^0.9.0", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.9", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^3.1.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.1.3", + "watchpack": "^2.4.0", + "webpack-sources": "^3.2.3" + }, + "dependencies": { + "@types/estree": { + "version": "0.0.51", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.51.tgz", + "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==", + "peer": true + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "peer": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "peer": true + } + } + }, + "webpack-merge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.2.2.tgz", + "integrity": "sha512-TUE1UGoTX2Cd42j3krGYqObZbOD+xF7u28WB7tfUordytSjbWTIjK/8V0amkBfTYN4/pB/GIDlJZZ657BGG19g==", + "requires": { + "lodash": "^4.17.15" + } + }, + "webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "peer": true + }, + "whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==" + }, + "widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "requires": { + "string-width": "^4.0.0" + } + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" + }, + "workbox-background-sync": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-background-sync/-/workbox-background-sync-6.5.4.tgz", + "integrity": "sha512-0r4INQZMyPky/lj4Ou98qxcThrETucOde+7mRGJl13MPJugQNKeZQOdIJe/1AchOP23cTqHcN/YVpD6r8E6I8g==", + "requires": { + "idb": "^7.0.1", + "workbox-core": "6.5.4" + } + }, + "workbox-broadcast-update": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-broadcast-update/-/workbox-broadcast-update-6.5.4.tgz", + "integrity": "sha512-I/lBERoH1u3zyBosnpPEtcAVe5lwykx9Yg1k6f8/BGEPGaMMgZrwVrqL1uA9QZ1NGGFoyE6t9i7lBjOlDhFEEw==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-build": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-build/-/workbox-build-6.5.4.tgz", + "integrity": "sha512-kgRevLXEYvUW9WS4XoziYqZ8Q9j/2ziJYEtTrjdz5/L/cTUa2XfyMP2i7c3p34lgqJ03+mTiz13SdFef2POwbA==", + "requires": { + "@apideck/better-ajv-errors": "^0.3.1", + "@babel/core": "^7.11.1", + "@babel/preset-env": "^7.11.0", + "@babel/runtime": "^7.11.2", + "@rollup/plugin-babel": "^5.2.0", + "@rollup/plugin-node-resolve": "^11.2.1", + "@rollup/plugin-replace": "^2.4.1", + "@surma/rollup-plugin-off-main-thread": "^2.2.3", + "ajv": "^8.6.0", + "common-tags": "^1.8.0", + "fast-json-stable-stringify": "^2.1.0", + "fs-extra": "^9.0.1", + "glob": "^7.1.6", + "lodash": "^4.17.20", + "pretty-bytes": "^5.3.0", + "rollup": "^2.43.1", + "rollup-plugin-terser": "^7.0.0", + "source-map": "^0.8.0-beta.0", + "stringify-object": "^3.3.0", + "strip-comments": "^2.0.1", + "tempy": "^0.6.0", + "upath": "^1.2.0", + "workbox-background-sync": "6.5.4", + "workbox-broadcast-update": "6.5.4", + "workbox-cacheable-response": "6.5.4", + "workbox-core": "6.5.4", + "workbox-expiration": "6.5.4", + "workbox-google-analytics": "6.5.4", + "workbox-navigation-preload": "6.5.4", + "workbox-precaching": "6.5.4", + "workbox-range-requests": "6.5.4", + "workbox-recipes": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4", + "workbox-streams": "6.5.4", + "workbox-sw": "6.5.4", + "workbox-window": "6.5.4" + }, + "dependencies": { + "@apideck/better-ajv-errors": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz", + "integrity": "sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA==", + "requires": { + "json-schema": "^0.4.0", + "jsonpointer": "^5.0.0", + "leven": "^3.1.0" + } + }, + "ajv": { + "version": "8.11.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.2.tgz", + "integrity": "sha512-E4bfmKAhGiSTvMfL1Myyycaub+cUEU2/IvpylXkUu7CHBkBj1f/ikdzbD7YQ6FKUbixDxeYvB/xY4fvyroDlQg==", + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "source-map": { + "version": "0.8.0-beta.0", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.8.0-beta.0.tgz", + "integrity": "sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==", + "requires": { + "whatwg-url": "^7.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" + } + } + }, + "workbox-cacheable-response": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-cacheable-response/-/workbox-cacheable-response-6.5.4.tgz", + "integrity": "sha512-DCR9uD0Fqj8oB2TSWQEm1hbFs/85hXXoayVwFKLVuIuxwJaihBsLsp4y7J9bvZbqtPJ1KlCkmYVGQKrBU4KAug==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-core": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-core/-/workbox-core-6.5.4.tgz", + "integrity": "sha512-OXYb+m9wZm8GrORlV2vBbE5EC1FKu71GGp0H4rjmxmF4/HLbMCoTFws87M3dFwgpmg0v00K++PImpNQ6J5NQ6Q==" + }, + "workbox-expiration": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-expiration/-/workbox-expiration-6.5.4.tgz", + "integrity": "sha512-jUP5qPOpH1nXtjGGh1fRBa1wJL2QlIb5mGpct3NzepjGG2uFFBn4iiEBiI9GUmfAFR2ApuRhDydjcRmYXddiEQ==", + "requires": { + "idb": "^7.0.1", + "workbox-core": "6.5.4" + } + }, + "workbox-google-analytics": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-google-analytics/-/workbox-google-analytics-6.5.4.tgz", + "integrity": "sha512-8AU1WuaXsD49249Wq0B2zn4a/vvFfHkpcFfqAFHNHwln3jK9QUYmzdkKXGIZl9wyKNP+RRX30vcgcyWMcZ9VAg==", + "requires": { + "workbox-background-sync": "6.5.4", + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "workbox-navigation-preload": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-navigation-preload/-/workbox-navigation-preload-6.5.4.tgz", + "integrity": "sha512-IIwf80eO3cr8h6XSQJF+Hxj26rg2RPFVUmJLUlM0+A2GzB4HFbQyKkrgD5y2d84g2IbJzP4B4j5dPBRzamHrng==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-precaching": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-precaching/-/workbox-precaching-6.5.4.tgz", + "integrity": "sha512-hSMezMsW6btKnxHB4bFy2Qfwey/8SYdGWvVIKFaUm8vJ4E53JAY+U2JwLTRD8wbLWoP6OVUdFlXsTdKu9yoLTg==", + "requires": { + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "workbox-range-requests": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-range-requests/-/workbox-range-requests-6.5.4.tgz", + "integrity": "sha512-Je2qR1NXCFC8xVJ/Lux6saH6IrQGhMpDrPXWZWWS8n/RD+WZfKa6dSZwU+/QksfEadJEr/NfY+aP/CXFFK5JFg==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-recipes": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-recipes/-/workbox-recipes-6.5.4.tgz", + "integrity": "sha512-QZNO8Ez708NNwzLNEXTG4QYSKQ1ochzEtRLGaq+mr2PyoEIC1xFW7MrWxrONUxBFOByksds9Z4//lKAX8tHyUA==", + "requires": { + "workbox-cacheable-response": "6.5.4", + "workbox-core": "6.5.4", + "workbox-expiration": "6.5.4", + "workbox-precaching": "6.5.4", + "workbox-routing": "6.5.4", + "workbox-strategies": "6.5.4" + } + }, + "workbox-routing": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-routing/-/workbox-routing-6.5.4.tgz", + "integrity": "sha512-apQswLsbrrOsBUWtr9Lf80F+P1sHnQdYodRo32SjiByYi36IDyL2r7BH1lJtFX8fwNHDa1QOVY74WKLLS6o5Pg==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-strategies": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-strategies/-/workbox-strategies-6.5.4.tgz", + "integrity": "sha512-DEtsxhx0LIYWkJBTQolRxG4EI0setTJkqR4m7r4YpBdxtWJH1Mbg01Cj8ZjNOO8etqfA3IZaOPHUxCs8cBsKLw==", + "requires": { + "workbox-core": "6.5.4" + } + }, + "workbox-streams": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-streams/-/workbox-streams-6.5.4.tgz", + "integrity": "sha512-FXKVh87d2RFXkliAIheBojBELIPnWbQdyDvsH3t74Cwhg0fDheL1T8BqSM86hZvC0ZESLsznSYWw+Va+KVbUzg==", + "requires": { + "workbox-core": "6.5.4", + "workbox-routing": "6.5.4" + } + }, + "workbox-sw": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-sw/-/workbox-sw-6.5.4.tgz", + "integrity": "sha512-vo2RQo7DILVRoH5LjGqw3nphavEjK4Qk+FenXeUsknKn14eCNedHOXWbmnvP4ipKhlE35pvJ4yl4YYf6YsJArA==" + }, + "workbox-webpack-plugin": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-webpack-plugin/-/workbox-webpack-plugin-6.5.4.tgz", + "integrity": "sha512-LmWm/zoaahe0EGmMTrSLUi+BjyR3cdGEfU3fS6PN1zKFYbqAKuQ+Oy/27e4VSXsyIwAw8+QDfk1XHNGtZu9nQg==", + "requires": { + "fast-json-stable-stringify": "^2.1.0", + "pretty-bytes": "^5.4.1", + "upath": "^1.2.0", + "webpack-sources": "^1.4.3", + "workbox-build": "6.5.4" + }, + "dependencies": { + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "webpack-sources": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", + "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "requires": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + } + } + } + }, + "workbox-window": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/workbox-window/-/workbox-window-6.5.4.tgz", + "integrity": "sha512-HnLZJDwYBE+hpG25AQBO8RUWBJRaCsI9ksQJEp3aCOFCaG5kqaToAYXFRAHxzRluM2cQbGzdQF5rjKPWPA1fug==", + "requires": { + "@types/trusted-types": "^2.0.2", + "workbox-core": "6.5.4" + } + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==" + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "requires": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "ws": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", + "requires": {} + }, + "xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "y18n": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "yaml": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.3.tgz", + "integrity": "sha512-AacA8nRULjKMX2DvWvOAdBZMOfQlypSFkjcOcu9FalllIDJ1kvlREzcdIZmidQUqqeMv7jorHjq2HlLv/+c2lg==" + }, + "yargs": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz", + "integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==", + "requires": { + "cliui": "^5.0.0", + "decamelize": "^1.2.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^15.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==" + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "requires": { + "locate-path": "^3.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==" + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "requires": { + "p-limit": "^2.0.0" + } + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==" + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "yargs-parser": { + "version": "15.0.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.3.tgz", + "integrity": "sha512-/MVEVjTXy/cGAjdtQf8dW3V9b97bPN7rNn8ETj6BmAQL7ibC7O1Q9SPJbGjgh3SlwoBNXMzj/ZGIj8mBgl12YA==", + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" + }, + "zwitch": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.2.tgz", + "integrity": "sha512-JZxotl7SxAJH0j7dN4pxsTV6ZLXoLdGME+PsjkL/DaBrVryK9kTGq06GfKrwcSOqypP+fdXGoCHE36b99fWVoA==" + } + } +} diff --git a/website/package.json b/website/package.json new file mode 100644 index 0000000..5f8bae4 --- /dev/null +++ b/website/package.json @@ -0,0 +1,64 @@ +{ + "name": "next-blog", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build && npm run sitemap && next export", + "prebuild": "pip install -r setup/requirements.txt && sh setup/setup.sh", + "predev": "npm run prebuild", + "sitemap": "next-sitemap --config next-sitemap.config.mjs", + "start": "next start", + "lint": "next lint", + "prettier": "prettier . --write" + }, + "dependencies": { + "@codemirror/lang-python": "^6.1.0", + "@docsearch/react": "^3.3.0", + "@jupyterlab/services": "^3.2.1", + "@lezer/highlight": "^1.1.3", + "@mapbox/rehype-prism": "^0.8.0", + "@mdx-js/loader": "^2.1.5", + "@mdx-js/react": "^2.1.5", + "@next/mdx": "^13.0.2", + "@rehooks/online-status": "^1.1.2", + "@types/node": "18.11.9", + "@types/react": "18.0.25", + "@types/react-dom": "18.0.8", + "@uiw/codemirror-themes": "^4.19.3", + "@uiw/react-codemirror": "^4.19.3", + "acorn": "^8.8.1", + "browser-monads": "^1.0.0", + "classnames": "^2.3.2", + "eslint": "8.27.0", + "eslint-config-next": "13.0.2", + "html-to-react": "^1.5.0", + "jinja-to-js": "^3.2.3", + "md-attr-parser": "^1.3.0", + "next": "13.0.2", + "next-mdx-remote": "^4.2.0", + "next-plausible": "^3.6.5", + "next-pwa": "^5.6.0", + "next-sitemap": "^3.1.32", + "node-fetch": "^2.6.7", + "parse-numeric-range": "^1.3.0", + "prettier": "^2.7.1", + "prismjs": "^1.29.0", + "prop-types": "^15.8.1", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-github-btn": "^1.4.0", + "react-inlinesvg": "^3.0.1", + "react-intersection-observer": "^9.4.0", + "remark": "^14.0.2", + "remark-gfm": "^3.0.1", + "remark-react": "^9.0.1", + "remark-smartypants": "^2.0.0", + "remark-unwrap-images": "^3.0.1", + "sass": "^1.56.1", + "typescript": "4.8.4", + "unist-util-visit": "^4.1.1", + "ws": "^8.11.0" + }, + "engine": 18 +} diff --git a/website/pages/404.js b/website/pages/404.js new file mode 100644 index 0000000..3eb621e --- /dev/null +++ b/website/pages/404.js @@ -0,0 +1,32 @@ +import React from 'react' +import { window } from 'browser-monads' +import classNames from 'classnames' + +import Template from '../src/templates/index' +import { LandingHeader, LandingTitle } from '../src/components/landing' +import { nightly, legacy } from '../meta/dynamicMeta' +import classes from '../src/styles/button.module.sass' + +const page404 = () => { + const pageContext = { title: '404 Error', searchExclude: true, isIndex: false } + return ( + + ) +} + +export default page404 diff --git a/website/pages/[...listPathPage].tsx b/website/pages/[...listPathPage].tsx new file mode 100644 index 0000000..b65073f --- /dev/null +++ b/website/pages/[...listPathPage].tsx @@ -0,0 +1,150 @@ +import type { GetStaticPaths, GetStaticProps } from 'next' +import { serialize } from 'next-mdx-remote/serialize' +import fs from 'fs' +import { MDXRemote, MDXRemoteSerializeResult } from 'next-mdx-remote' +import path from 'path' +import Layout from '../src/templates' +import remarkPlugins from '../plugins/index.mjs' + +import recordSection from '../meta/recordSections' +import { sidebarUsageFlat } from '../meta/sidebarFlat' + +type ApiDetails = { + stringName: string | null + baseClass: { + title: string + slug: string + } | null + trainable: string | null +} + +export type PropsPageBase = { + /** + * TODO: This is only here for legacy support of the old code base + * It should be refactort to pass the file path and page path instead. + */ + slug: string + sectionTitle: string | null + theme: string | null + section: string + isIndex: boolean +} + +export type PropsPage = PropsPageBase & { + mdx: MDXRemoteSerializeResult + apiDetails: ApiDetails +} + +const PostPage = ({ mdx: mdx, ...props }: PropsPage) => { + return ( + + + + ) +} + +export default PostPage + +type ParsedUrlQuery = { + listPathPage: Array +} + +export const getStaticPaths: GetStaticPaths = async () => { + // This function needs to be defined inside `getStaticPath` to be executed in executed in the correct context + const loadFolder = (pathBase: Array = []): Array<{ params: ParsedUrlQuery }> => + fs + .readdirSync(path.join('docs', ...pathBase), { withFileTypes: true }) + .flatMap((dirent: fs.Dirent) => { + if (dirent.isDirectory()) { + return loadFolder([...pathBase, dirent.name]) + } + if (!dirent.name.includes('.mdx') || dirent.name[0] === '_') { + return [] + } + + return { + params: { + listPathPage: + dirent.name === 'index.mdx' + ? pathBase + : [...pathBase, dirent.name.replace('.mdx', '')], + }, + } + }) + + return { + paths: loadFolder(), + fallback: false, + } +} + +const getPathFileWithExtension = (listPathFile: ReadonlyArray) => + `${path.join(...listPathFile)}.mdx` + +export const getStaticProps: GetStaticProps = async (args) => { + if (!args.params) { + return { notFound: true } + } + + const listPathFile = ['docs', ...args.params.listPathPage] + const isIndex = fs.existsSync(getPathFileWithExtension(listPathFile)) !== true + const listPathFileWithIndex = isIndex ? [...listPathFile, 'index'] : listPathFile + const pathFileWithIndexAndExtension = getPathFileWithExtension(listPathFileWithIndex) + + const mdx = await serialize(fs.readFileSync(pathFileWithIndexAndExtension, 'utf-8'), { + parseFrontmatter: true, + mdxOptions: { remarkPlugins }, + }) + + if (!mdx.frontmatter) { + throw new Error(`Frontmatter missing for ${pathFileWithIndexAndExtension}`) + } + + const parentFolder = + listPathFileWithIndex.length > 1 + ? listPathFileWithIndex[listPathFileWithIndex.length - 2] + : null + const section = mdx.frontmatter.section ?? parentFolder + const sectionMeta = section ? recordSection[section] ?? null : null + const baseClass = null + const apiDetails: ApiDetails = { + stringName: mdx.frontmatter.api_string_name ?? null, + baseClass: baseClass + ? { + title: mdx.frontmatter.title, + slug: mdx.frontmatter.api_base_class, + } + : null, + trainable: mdx.frontmatter.api_trainable ?? null, + } + + const slug = `/${args.params.listPathPage.join('/')}` + + const next = + section === 'usage' + ? sidebarUsageFlat.find((item, index) => { + return ( + index > 0 && sidebarUsageFlat[index - 1].url === slug && item.url[0] === '/' + ) + }) + : undefined + + return { + props: { + ...mdx.frontmatter, + slug, + mdx, + sectionTitle: sectionMeta?.title ?? null, + theme: sectionMeta?.theme ?? null, + section: section, + apiDetails: apiDetails, + isIndex, + next: next + ? { + slug: next.url, + title: next.text, + } + : null, + }, + } +} diff --git a/website/pages/_app.tsx b/website/pages/_app.tsx new file mode 100644 index 0000000..a837d9c --- /dev/null +++ b/website/pages/_app.tsx @@ -0,0 +1,33 @@ +import '../src/styles/layout.sass' +import '../src/styles/search.sass' + +import type { AppProps } from 'next/app' +import Head from 'next/head' +import PlausibleProvider from 'next-plausible' +import { MDXProvider } from '@mdx-js/react' +import { remarkComponents } from '../src/remark' +import { domain } from '../meta/dynamicMeta.mjs' + +export default function App({ Component, pageProps }: AppProps) { + return ( + + + + + + + + + + + + + + + + + ) +} diff --git a/website/pages/_document.tsx b/website/pages/_document.tsx new file mode 100644 index 0000000..718fb43 --- /dev/null +++ b/website/pages/_document.tsx @@ -0,0 +1,13 @@ +import { Html, Head, Main, NextScript } from 'next/document' + +export default function Document() { + return ( + + + +
+ + + + ) +} diff --git a/website/pages/index.tsx b/website/pages/index.tsx new file mode 100644 index 0000000..089d75b --- /dev/null +++ b/website/pages/index.tsx @@ -0,0 +1,306 @@ +import React from 'react' +import PropTypes from 'prop-types' + +import { + LandingHeader, + LandingTitle, + LandingSubtitle, + LandingGrid, + LandingCard, + LandingCol, + LandingDemo, + LandingBannerGrid, + LandingBanner, +} from '../src/components/landing' +import { H2 } from '../src/components/typography' +import { InlineCode } from '../src/components/inlineCode' +import { Ul, Li } from '../src/components/list' +import Button from '../src/components/button' +import Link from '../src/components/link' + +import QuickstartTraining from '../src/widgets/quickstart-training' +import Project from '../src/widgets/project' +import Features from '../src/widgets/features' +import Layout from '../src/templates' +import courseImage from '../public/images/course.jpg' +import prodigyImage from '../public/images/prodigy_overview.jpg' +import projectsImage from '../public/images/projects.png' +import tailoredPipelinesImage from '../public/images/spacy-tailored-pipelines_wide.png' +import { nightly, legacy } from '../meta/dynamicMeta.mjs' + +import Benchmarks from '../docs/usage/_benchmarks-models.mdx' +import { ImageFill } from '../src/components/embed' + +function getCodeExample(nightly) { + return `# pip install -U ${nightly ? 'spacy-nightly --pre' : 'spacy'} +# python -m spacy download en_core_web_sm +import spacy + +# Load English tokenizer, tagger, parser and NER +nlp = spacy.load("en_core_web_sm") + +# Process whole documents +text = ("When Sebastian Thrun started working on self-driving cars at " + "Google in 2007, few people outside of the company took him " + "seriously. “I can tell you very senior CEOs of major American " + "car companies would shake my hand and turn away because I wasn’t " + "worth talking to,” said Thrun, in an interview with Recode earlier " + "this week.") +doc = nlp(text) + +# Analyze syntax +print("Noun phrases:", [chunk.text for chunk in doc.noun_chunks]) +print("Verbs:", [token.lemma_ for token in doc if token.pos_ == "VERB"]) + +# Find named entities, phrases and concepts +for entity in doc.ents: + print(entity.text, entity.label_) +` +} + +const Landing = () => { + const codeExample = getCodeExample(nightly) + return ( + + + + Industrial-Strength +
+ Natural Language +
+ Processing + + in Python + + + + spaCy is designed to help you do real work — to build real products, or gather + real insights. The library respects your time, and tries to avoid wasting it. + It's easy to install, and its API is simple and productive. + + + spaCy excels at large-scale information extraction tasks. It's written from + the ground up in carefully memory-managed Cython. If your application needs to + process entire web dumps, spaCy is the library you want to be using. + + + + Since its release in 2015, spaCy has become an industry standard with a huge + ecosystem. Choose from a variety of plugins, integrate with your machine + learning stack and build custom components and workflows. + + + + + {codeExample} + + +

Features

+ + + + + + +

+ + The spacy-llm package + {' '} + integrates Large Language Models (LLMs) into spaCy, featuring a modular + system for fast prototyping and prompting, + and turning unstructured responses into robust outputs for + various NLP tasks, no training data required. +

+ + + +

+ + + +

+

+ Prodigy is an annotation tool so efficient that data + scientists can do the annotation themselves, enabling a new level of rapid + iteration. Whether you're working on entity recognition, intent + detection or image classification, Prodigy can help you{' '} + train and evaluate your models faster. +

+ + + + + +

Reproducible training for custom pipelines

+

+ spaCy v3.0 introduces a comprehensive and extensible system for{' '} + configuring your training runs. Your configuration file + will describe every detail of your training run, with no hidden defaults, + making it easy to rerun your experiments and track changes. + You can use the quickstart widget or the{' '} + + init config + {' '} + command to get started, or clone a project template for an end-to-end + workflow. +

+

+ +

+ + + + + + + + + + + +
+
+
+ + The easiest way to get started is to clone a project template and run it + – for example, this template for training a{' '} + part-of-speech tagger and{' '} + dependency parser on a Universal Dependencies treebank. + + + +

End-to-end workflows from prototype to production

+

+ spaCy's new project system gives you a smooth path from prototype to + production. It lets you keep track of all those{' '} + data transformation, preprocessing and{' '} + training steps, so you can make sure your project is always + ready to hand over for automation. It features source asset download, + command execution, checksum verification, and caching with a variety of + backends and integrations. +

+

+ +

+ + + + + +

+ + + +

+

+ + Get a custom spaCy pipeline, tailor-made for your NLP problem by + spaCy's core developers. + +

+
    +
  • + Streamlined. Nobody knows spaCy better than we do. Send + us your pipeline requirements and we'll be ready to start producing + your solution in no time at all. +
    • +
  • + Production ready. spaCy pipelines are robust and easy + to deploy. You'll get a complete spaCy project folder which is + ready to spacy project run. +
    • +
  • + Predictable. You'll know exactly what you're + going to get and what it's going to cost. We quote fees up-front, + let you try before you buy, and don't charge for over-runs at our + end — all the risk is on us. +
    • +
  • + Maintainable. spaCy is an industry standard, and + we'll deliver your pipeline with full code, data, tests and + documentation, so your team can retrain, update and extend the solution + as your requirements change. +
    • +
+ + +

+ + + +

+

+ In this free and interactive online course you’ll learn how + to use spaCy to build advanced natural language understanding systems, using + both rule-based and machine learning approaches. It includes{' '} + 55 exercises featuring videos, slide decks, multiple-choice + questions and interactive coding practice in the browser. +

+ + + + + +

Benchmarks

+

+ spaCy v3.0 introduces transformer-based pipelines that bring spaCy's + accuracy right up to the current state-of-the-art. You can + also use a CPU-optimized pipeline, which is less accurate but much cheaper + to run. +

+

+ +

+ + + + + + + + ) +} + +export default Landing diff --git a/website/pages/models/[slug].tsx b/website/pages/models/[slug].tsx new file mode 100644 index 0000000..f4908be --- /dev/null +++ b/website/pages/models/[slug].tsx @@ -0,0 +1,66 @@ +import type { GetStaticPaths, GetStaticProps } from 'next' +import models from '../../meta/languages.json' +import recordSection from '../../meta/recordSections' +import recordLanguages from '../../meta/recordLanguages' +import Layout from '../../src/templates' +import { PropsPageBase } from '../[...listPathPage]' +import { languagesSorted } from '../../meta/languageSorted' + +type PropsPageModel = PropsPageBase & { + next: { title: string; slug: string } | null + meta: { models?: ReadonlyArray; example?: string; hasExamples?: boolean } +} + +const PostPageModel = (props: PropsPageModel) => { + return +} + +export default PostPageModel + +export const getStaticPaths: GetStaticPaths<{ slug: string }> = async () => { + return { + paths: models.languages + .filter(({ models }) => models && models.length) + .map((language) => `/models/${language.code}`), + fallback: false, + } +} + +export const getStaticProps: GetStaticProps< + PropsPageModel, + { + slug: string + } +> = async (args) => { + const getSlug = (languageCode: string) => `/${['models', languageCode].join('/')}` + + if (args.params === undefined) { + return { notFound: true } + } + + const language = recordLanguages[args.params.slug] + + const nextLanguage = languagesSorted.find( + (item, index) => index > 0 && languagesSorted[index - 1].code === language.code + ) + + return { + props: { + id: language.code, + slug: getSlug(language.code), + isIndex: false, + title: language.name, + section: 'models', + sectionTitle: recordSection.models.title, + theme: recordSection.models.theme, + next: nextLanguage + ? { title: nextLanguage.name, slug: getSlug(nextLanguage.code) } + : null, + meta: { + models: language.models || null, + example: language.example || null, + hasExamples: language.has_examples || null, + }, + }, + } +} diff --git a/website/pages/universe/category/[slug].tsx b/website/pages/universe/category/[slug].tsx new file mode 100644 index 0000000..b110213 --- /dev/null +++ b/website/pages/universe/category/[slug].tsx @@ -0,0 +1,43 @@ +import { GetStaticPaths, GetStaticProps } from 'next' +import recordSections from '../../../meta/recordSections' +import { recordUniverseCategories } from '../../../meta/recordUniverse' +import universe from '../../../meta/universe.json' +import Layout from '../../../src/templates' +import { PropsPageBase } from '../../[...listPathPage]' + +type ParsedUrlQuery = { + slug: string +} + +export default Layout + +export const getStaticPaths: GetStaticPaths = async () => { + return { + paths: universe.categories.flatMap((category) => + category.items.map((item) => `/universe/category/${item.id}`) + ), + fallback: false, + } +} + +export const getStaticProps: GetStaticProps = async (args) => { + if (!args.params) { + return { notFound: true } + } + + const item = recordUniverseCategories[args.params.slug] + + return { + props: { + id: item.id, + title: item.title, + teaser: item.description, + slug: `/universe/category/${args.params.slug}`, + isIndex: false, + data: { ...item, isCategory: true }, + section: 'universe', + sectionTitle: recordSections.universe.title, + theme: recordSections.universe.theme, + }, + } +} diff --git a/website/pages/universe/index.tsx b/website/pages/universe/index.tsx new file mode 100644 index 0000000..69e5ad4 --- /dev/null +++ b/website/pages/universe/index.tsx @@ -0,0 +1,17 @@ +import recordSections from '../../meta/recordSections' +import Layout from '../../src/templates' + +const Universe = () => { + return ( + + ) +} + +export default Universe diff --git a/website/pages/universe/project/[slug].tsx b/website/pages/universe/project/[slug].tsx new file mode 100644 index 0000000..7452470 --- /dev/null +++ b/website/pages/universe/project/[slug].tsx @@ -0,0 +1,41 @@ +import { GetStaticPaths, GetStaticProps } from 'next' +import recordSections from '../../../meta/recordSections' +import { recordUniverseResources } from '../../../meta/recordUniverse' +import universe from '../../../meta/universe.json' +import Layout from '../../../src/templates' +import { PropsPageBase } from '../../[...listPathPage]' + +type ParsedUrlQuery = { + slug: string +} + +export default Layout + +export const getStaticPaths: GetStaticPaths = async () => { + return { + paths: universe.resources.flatMap((resource) => `/universe/project/${resource.id}`), + fallback: false, + } +} + +export const getStaticProps: GetStaticProps = async (args) => { + if (!args.params) { + return { notFound: true } + } + + const resource = recordUniverseResources[args.params.slug] + + return { + props: { + id: resource.id, + title: resource.title || resource.id, + teaser: resource.slogan || null, + slug: `/universe/project/${args.params.slug}`, + isIndex: false, + data: { ...resource, isProject: true }, + section: 'universe', + sectionTitle: recordSections.universe.title, + theme: recordSections.universe.theme, + }, + } +} diff --git a/website/plugins/getProps.mjs b/website/plugins/getProps.mjs new file mode 100644 index 0000000..6eb2fdc --- /dev/null +++ b/website/plugins/getProps.mjs @@ -0,0 +1,39 @@ +const parseAttribute = (expression) => { + if (expression.type !== 'AssignmentExpression' || !expression.left || !expression.right) { + return + } + + const { left, right } = expression + + if (left.type !== 'Identifier' || right.type !== 'Literal' || !left.name || !right.value) { + return + } + + return [left.name, right.value] +} + +const getProps = (estree) => { + if (estree.type !== 'Program' || !estree.body || estree.body.length <= 0 || !estree.body[0]) { + return {} + } + + const estreeBodyFirstNode = + estree.body[0].type === 'BlockStatement' ? estree.body[0].body[0] : estree.body[0] + + if (estreeBodyFirstNode.type !== 'ExpressionStatement' || !estreeBodyFirstNode.expression) { + return {} + } + + const statement = estreeBodyFirstNode.expression + + const attributeExpressions = [ + ...(statement.type === 'SequenceExpression' && statement.expressions + ? statement.expressions + : []), + ...(statement.type === 'AssignmentExpression' ? [statement] : []), + ] + + return Object.fromEntries(attributeExpressions.map(parseAttribute)) +} + +export default getProps diff --git a/website/plugins/index.mjs b/website/plugins/index.mjs new file mode 100644 index 0000000..d896ca4 --- /dev/null +++ b/website/plugins/index.mjs @@ -0,0 +1,20 @@ +import remarkGfm from 'remark-gfm' +import remarkUnwrapImages from 'remark-unwrap-images' +import remarkSmartypants from 'remark-smartypants' + +import remarkCustomAttrs from './remarkCustomAttrs.mjs' +import remarkWrapSections from './remarkWrapSections.mjs' +import remarkCodeBlocks from './remarkCodeBlocks.mjs' +import remarkFindAndReplace from './remarkFindAndReplace.mjs' + +const remarkPlugins = [ + remarkGfm, + remarkSmartypants, + remarkFindAndReplace, + remarkUnwrapImages, + remarkCustomAttrs, + remarkCodeBlocks, + remarkWrapSections, +] + +export default remarkPlugins diff --git a/website/plugins/remarkCodeBlocks.mjs b/website/plugins/remarkCodeBlocks.mjs new file mode 100644 index 0000000..f553a55 --- /dev/null +++ b/website/plugins/remarkCodeBlocks.mjs @@ -0,0 +1,77 @@ +/** + * Support titles, line highlights and more for code blocks + */ +import { Parser } from 'acorn' +import { visit } from 'unist-util-visit' +import parseAttr from 'md-attr-parser' + +import getProps from './getProps.mjs' + +const defaultOptions = { + defaultPrefix: '###', + prefixMap: { + javascript: '///', + jsx: '///', + }, + languageAliases: {}, + prompts: ['$'], +} + +function remarkCodeBlocks(userOptions = {}) { + const options = Object.assign({}, defaultOptions, userOptions) + + function transformer(tree) { + visit(tree, 'code', (node) => { + if (node.value) { + const langName = node.lang || 'none' + const lang = options.languageAliases[langName] || langName + const prefix = options.prefixMap[lang] || options.defaultPrefix + const lines = node.value.split('\n') + let attrs = { lang } + const firstLine = lines[0].trim() + if (firstLine && firstLine.startsWith(prefix)) { + const title = firstLine.slice(prefix.length).trim() + attrs.title = title + // Check for attributes in title, e.g. {executable="true"} + const parsed = title.split(/\{(.*?)\}/) + if (parsed.length >= 2 && parsed[1]) { + const { prop } = parseAttr(parsed[1]) + attrs = { ...attrs, ...prop, title: parsed[0].trim(), lang } + } + // Overwrite the code text with the rest of the lines + node.value = lines.slice(1).join('\n') + } else if ( + (firstLine && /^https:\/\/github.com/.test(firstLine)) || + firstLine.startsWith('%%GITHUB_') + ) { + // GitHub URL + attrs.github = node.value + } + + const data = node.data || (node.data = {}) + const hProps = data.hProperties || (data.hProperties = {}) + + const meta = getProps(Parser.parse(node.meta, { ecmaVersion: 'latest' })) + + node.data.hProperties = Object.assign({}, hProps, attrs, meta) + } + }) + + visit(tree, 'inlineCode', (node) => { + node.type = 'mdxJsxTextElement' + node.name = 'InlineCode' + node.children = [ + { + type: 'text', + value: node.value, + }, + ] + node.data = { _mdxExplicitJsx: true } + }) + + return tree + } + return transformer +} + +export default remarkCodeBlocks diff --git a/website/plugins/remarkCustomAttrs.mjs b/website/plugins/remarkCustomAttrs.mjs new file mode 100644 index 0000000..18274a0 --- /dev/null +++ b/website/plugins/remarkCustomAttrs.mjs @@ -0,0 +1,38 @@ +import getProps from './getProps.mjs' + +const handleNode = (node) => { + if (node.type === 'section' && node.children) { + return { + ...node, + children: node.children.map(handleNode), + } + } + + if (node.type !== 'heading' || !node.children || node.children < 2) { + return node + } + + const indexLast = node.children.length - 1 + const lastNode = node.children[indexLast] + + if (lastNode.type !== 'mdxTextExpression' || !lastNode.data || !lastNode.data.estree) { + return node + } + + const data = node.data || (node.data = {}) + data.hProperties = getProps(lastNode.data.estree) + + // Only keep the text, drop the rest + node.children = [node.children[0]] + + return node +} + +const parseAstTree = (markdownAST) => ({ + ...markdownAST, + children: markdownAST.children.map(handleNode), +}) + +const remarkCustomAttrs = () => parseAstTree + +export default remarkCustomAttrs diff --git a/website/plugins/remarkFindAndReplace.mjs b/website/plugins/remarkFindAndReplace.mjs new file mode 100644 index 0000000..ddd5573 --- /dev/null +++ b/website/plugins/remarkFindAndReplace.mjs @@ -0,0 +1,42 @@ +import { visit } from 'unist-util-visit' +import { replacements } from '../meta/dynamicMeta.mjs' + +const prefix = '%%' + +// Attaches prefix to the start of the string. +const attachPrefix = (str) => (prefix || '') + str + +// RegExp to find any replacement keys. +const regexp = RegExp( + '(' + + Object.keys(replacements) + .map((key) => attachPrefix(key)) + .join('|') + + ')', + 'g' +) + +// Removes prefix from the start of the string. +const stripPrefix = (str) => (prefix ? str.replace(RegExp(`^${prefix}`), '') : str) + +const replacer = (_match, name) => replacements[stripPrefix(name)] + +const remarkFindAndReplace = () => (markdownAST) => { + // Go through all text, html, code, inline code, and links. + visit(markdownAST, ['text', 'html', 'code', 'link'], (node) => { + if (node.type === 'link') { + // For links, the text value is replaced by text node, so we change the + // URL value. + const processedText = node.url.replace(regexp, replacer) + node.url = processedText + } else { + // For all other nodes, replace the node value. + const processedText = node.value.replace(regexp, replacer) + node.value = processedText + } + }) + + return markdownAST +} + +export default remarkFindAndReplace diff --git a/website/plugins/remarkWrapSections.mjs b/website/plugins/remarkWrapSections.mjs new file mode 100644 index 0000000..c93fc97 --- /dev/null +++ b/website/plugins/remarkWrapSections.mjs @@ -0,0 +1,66 @@ +/** + * Check the tree for headlines of a certain depth and wrap the headline and + * all content up to the next headline in a section. + * Based on: https://github.com/luhmann/tufte-markdown + */ + +import { visit } from 'unist-util-visit' + +const defaultOptions = { + element: 'section', + prefix: 'section-', + depth: 2, + slugify: true, +} + +function remarkWrapSection(userOptions = {}) { + const options = Object.assign({}, defaultOptions, userOptions) + + function transformer(tree) { + const headingsMap = [] + const newTree = [] + + visit(tree, 'import', (node) => { + // For compatibility with MDX / gatsby-mdx, make sure import nodes + // are not moved further down into children (which means they're not + // recognized and interpreted anymore). Add them to the very start + // of the tree. + newTree.push(node) + }) + + visit(tree, 'heading', (node, index) => { + if (node.depth === options.depth) { + const data = node.data || (node.data = {}) + const hProps = data.hProperties || (data.hProperties = {}) + headingsMap.push({ index, id: hProps.id }) + } + }) + + if (headingsMap.length) { + for (let index = 0; index <= headingsMap.length; index++) { + const sectionStartIndex = index === 0 ? 0 : headingsMap[index - 1].index + const sectionEndIndex = + index === headingsMap.length ? tree.children.length : headingsMap[index].index + const children = tree.children + .slice(sectionStartIndex, sectionEndIndex) + .filter((node) => node.type !== 'import') + + if (children.length) { + const headingId = index === 0 ? 0 : headingsMap[index - 1].id + const sectionId = headingId ? options.prefix + headingId : undefined + const wrapperNode = { + type: 'section', + children, + data: { hName: options.element, hProperties: { id: sectionId } }, + } + newTree.push(wrapperNode) + } + } + tree.children = newTree + } + return tree + } + return transformer +} + +export default remarkWrapSection diff --git a/website/public/favicon.ico b/website/public/favicon.ico new file mode 100644 index 0000000..718d6fe Binary files /dev/null and b/website/public/favicon.ico differ diff --git a/website/public/icons/icon-192x192.png b/website/public/icons/icon-192x192.png new file mode 100644 index 0000000..875e82b Binary files /dev/null and b/website/public/icons/icon-192x192.png differ diff --git a/website/public/icons/icon-256x256.png b/website/public/icons/icon-256x256.png new file mode 100644 index 0000000..2dedf8e Binary files /dev/null and b/website/public/icons/icon-256x256.png differ diff --git a/website/public/icons/icon-384x384.png b/website/public/icons/icon-384x384.png new file mode 100644 index 0000000..3296bb6 Binary files /dev/null and b/website/public/icons/icon-384x384.png differ diff --git a/website/public/icons/icon-512x512.png b/website/public/icons/icon-512x512.png new file mode 100644 index 0000000..f698700 Binary files /dev/null and b/website/public/icons/icon-512x512.png differ diff --git a/website/public/images/architecture.svg b/website/public/images/architecture.svg new file mode 100644 index 0000000..2e271e9 --- /dev/null +++ b/website/public/images/architecture.svg @@ -0,0 +1,226 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/cli_init_fill-config_diff.jpg b/website/public/images/cli_init_fill-config_diff.jpg new file mode 100644 index 0000000..3e37517 Binary files /dev/null and b/website/public/images/cli_init_fill-config_diff.jpg differ diff --git a/website/public/images/course.jpg b/website/public/images/course.jpg new file mode 100644 index 0000000..b4f271c Binary files /dev/null and b/website/public/images/course.jpg differ diff --git a/website/public/images/dep-match-diagram.svg b/website/public/images/dep-match-diagram.svg new file mode 100644 index 0000000..676be41 --- /dev/null +++ b/website/public/images/dep-match-diagram.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/displacy-compact.svg b/website/public/images/displacy-compact.svg new file mode 100644 index 0000000..b2c7dda --- /dev/null +++ b/website/public/images/displacy-compact.svg @@ -0,0 +1,123 @@ + + + This + DET + + + + is + VERB + + + + a + DET + + + + sentence + NOUN + + + + in + ADP + + + + compact + ADJ + + + + mode + NOUN + + + + with + ADP + + + + custom + NOUN + + + + styles. + NOUN + + + + + + nsubj + + + + + + + + det + + + + + + + + attr + + + + + + + + prep + + + + + + + + amod + + + + + + + + pobj + + + + + + + + prep + + + + + + + + compound + + + + + + + + pobj + + + + diff --git a/website/public/images/displacy-custom-parser.svg b/website/public/images/displacy-custom-parser.svg new file mode 100644 index 0000000..1ed2294 --- /dev/null +++ b/website/public/images/displacy-custom-parser.svg @@ -0,0 +1,55 @@ + + + find + ROOT + + + + a + + + + + hotel + + + + + with + + + + + good + + + + + wifi + + + + + + + PLACE + + + + + + + + QUALITY + + + + + + + + ATTRIBUTE + + + + diff --git a/website/public/images/displacy-dep-founded.svg b/website/public/images/displacy-dep-founded.svg new file mode 100644 index 0000000..8e3c475 --- /dev/null +++ b/website/public/images/displacy-dep-founded.svg @@ -0,0 +1,155 @@ + + + Smith + + + + + founded + + + + + a + + + + + healthcare + + + + + company + + + + + + + + nsubj + + + + + + + + + + det + + + + + + + + + + compound + + + + + + + + + + dobj + + + + + diff --git a/website/public/images/displacy-long.svg b/website/public/images/displacy-long.svg new file mode 100644 index 0000000..e298610 --- /dev/null +++ b/website/public/images/displacy-long.svg @@ -0,0 +1,328 @@ + + + Apple + PROPN + + + + is + VERB + + + + looking + VERB + + + + at + ADP + + + + buying + VERB + + + + U.K. + PROPN + + + + startup + NOUN + + + + for + ADP + + + + $ + SYM + + + + 1 + NUM + + + + billion + NUM + + + + + + + nsubj + + + + + + + + + + aux + + + + + + + + + + prep + + + + + + + + + + pcomp + + + + + + + + + + compound + + + + + + + + + + dobj + + + + + + + + + + prep + + + + + + + + + + quantmod + + + + + + + + + + compound + + + + + + + + + + pobj + + + + + diff --git a/website/public/images/displacy-long2.svg b/website/public/images/displacy-long2.svg new file mode 100644 index 0000000..c428bd2 --- /dev/null +++ b/website/public/images/displacy-long2.svg @@ -0,0 +1,212 @@ + + + Autonomous + ADJ + + + + cars + NOUN + + + + shift + VERB + + + + insurance + NOUN + + + + liability + NOUN + + + + toward + ADP + + + + manufacturers + NOUN + + + + + + + amod + + + + + + + + + + nsubj + + + + + + + + + + compound + + + + + + + + + + dobj + + + + + + + + + + prep + + + + + + + + + + pobj + + + + + diff --git a/website/public/images/displacy-model-rules.svg b/website/public/images/displacy-model-rules.svg new file mode 100644 index 0000000..71f7ffc --- /dev/null +++ b/website/public/images/displacy-model-rules.svg @@ -0,0 +1,45 @@ + + + Alex Smith + NNP + + + + worked + VBD + + + + at + IN + + + + Acme Corp Inc. + NNP + + + + + + nsubj + + + + + + + + prep + + + + + + + + pobj + + + + diff --git a/website/public/images/displacy-model-rules2.svg b/website/public/images/displacy-model-rules2.svg new file mode 100644 index 0000000..85e86c4 --- /dev/null +++ b/website/public/images/displacy-model-rules2.svg @@ -0,0 +1,58 @@ + + + Alex Smith + NNP + + + + was + VBD + + + + working + VBG + + + + at + IN + + + + Acme Corp Inc. + NNP + + + + + + nsubj + + + + + + + + aux + + + + + + + + prep + + + + + + + + pobj + + + + diff --git a/website/public/images/displacy-small.svg b/website/public/images/displacy-small.svg new file mode 100644 index 0000000..7f9d6ba --- /dev/null +++ b/website/public/images/displacy-small.svg @@ -0,0 +1,33 @@ + + + This + DET + + + is + VERB + + + a + DET + + + sentence. + NOUN + + + + nsubj + + + + + det + + + + + attr + + + diff --git a/website/public/images/displacy.svg b/website/public/images/displacy.svg new file mode 100644 index 0000000..738e0ca --- /dev/null +++ b/website/public/images/displacy.svg @@ -0,0 +1,45 @@ + + + This + DET + + + + is + VERB + + + + a + DET + + + + sentence. + NOUN + + + + + + nsubj + + + + + + + + det + + + + + + + + attr + + + + diff --git a/website/public/images/displacy_jupyter.jpg b/website/public/images/displacy_jupyter.jpg new file mode 100644 index 0000000..4f81530 Binary files /dev/null and b/website/public/images/displacy_jupyter.jpg differ diff --git a/website/public/images/huggingface_hub.jpg b/website/public/images/huggingface_hub.jpg new file mode 100644 index 0000000..5618df0 Binary files /dev/null and b/website/public/images/huggingface_hub.jpg differ diff --git a/website/public/images/lifecycle.svg b/website/public/images/lifecycle.svg new file mode 100644 index 0000000..2f4b304 --- /dev/null +++ b/website/public/images/lifecycle.svg @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/matcher-demo.jpg b/website/public/images/matcher-demo.jpg new file mode 100644 index 0000000..601131e Binary files /dev/null and b/website/public/images/matcher-demo.jpg differ diff --git a/website/public/images/pipeline-design.svg b/website/public/images/pipeline-design.svg new file mode 100644 index 0000000..3b528ea --- /dev/null +++ b/website/public/images/pipeline-design.svg @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/pipeline.svg b/website/public/images/pipeline.svg new file mode 100644 index 0000000..9ece70e --- /dev/null +++ b/website/public/images/pipeline.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/pipeline_transformer.svg b/website/public/images/pipeline_transformer.svg new file mode 100644 index 0000000..cfbf470 --- /dev/null +++ b/website/public/images/pipeline_transformer.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/prodigy.jpg b/website/public/images/prodigy.jpg new file mode 100644 index 0000000..9291f0d Binary files /dev/null and b/website/public/images/prodigy.jpg differ diff --git a/website/public/images/prodigy_overview.jpg b/website/public/images/prodigy_overview.jpg new file mode 100644 index 0000000..84326cc Binary files /dev/null and b/website/public/images/prodigy_overview.jpg differ diff --git a/website/public/images/prodigy_spans-manual.jpg b/website/public/images/prodigy_spans-manual.jpg new file mode 100644 index 0000000..d67f347 Binary files /dev/null and b/website/public/images/prodigy_spans-manual.jpg differ diff --git a/website/public/images/prodigy_train_curve.jpg b/website/public/images/prodigy_train_curve.jpg new file mode 100644 index 0000000..af22cd0 Binary files /dev/null and b/website/public/images/prodigy_train_curve.jpg differ diff --git a/website/public/images/project_document.jpg b/website/public/images/project_document.jpg new file mode 100644 index 0000000..7942619 Binary files /dev/null and b/website/public/images/project_document.jpg differ diff --git a/website/public/images/projects.png b/website/public/images/projects.png new file mode 100644 index 0000000..934e98e Binary files /dev/null and b/website/public/images/projects.png differ diff --git a/website/public/images/projects.svg b/website/public/images/projects.svg new file mode 100644 index 0000000..c7518d4 --- /dev/null +++ b/website/public/images/projects.svg @@ -0,0 +1,92 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/sense2vec.jpg b/website/public/images/sense2vec.jpg new file mode 100644 index 0000000..3a17725 Binary files /dev/null and b/website/public/images/sense2vec.jpg differ diff --git a/website/public/images/spacy-extension-demo.gif b/website/public/images/spacy-extension-demo.gif new file mode 100644 index 0000000..a857bbe Binary files /dev/null and b/website/public/images/spacy-extension-demo.gif differ diff --git a/website/public/images/spacy-ray.svg b/website/public/images/spacy-ray.svg new file mode 100644 index 0000000..4c2fd81 --- /dev/null +++ b/website/public/images/spacy-ray.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/spacy-streamlit.png b/website/public/images/spacy-streamlit.png new file mode 100644 index 0000000..8f617d4 Binary files /dev/null and b/website/public/images/spacy-streamlit.png differ diff --git a/website/public/images/spacy-tailored-pipelines_wide.png b/website/public/images/spacy-tailored-pipelines_wide.png new file mode 100644 index 0000000..d1a762e Binary files /dev/null and b/website/public/images/spacy-tailored-pipelines_wide.png differ diff --git a/website/public/images/thinc_mypy.jpg b/website/public/images/thinc_mypy.jpg new file mode 100644 index 0000000..c0f7ee6 Binary files /dev/null and b/website/public/images/thinc_mypy.jpg differ diff --git a/website/public/images/tok2vec-listener.svg b/website/public/images/tok2vec-listener.svg new file mode 100644 index 0000000..bb67d21 --- /dev/null +++ b/website/public/images/tok2vec-listener.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/tok2vec.svg b/website/public/images/tok2vec.svg new file mode 100644 index 0000000..5338b62 --- /dev/null +++ b/website/public/images/tok2vec.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/website/public/images/tokenization.svg b/website/public/images/tokenization.svg new file mode 100644 index 0000000..d676fda --- /dev/null +++ b/website/public/images/tokenization.svg @@ -0,0 +1,305 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/trainable_component.svg b/website/public/images/trainable_component.svg new file mode 100644 index 0000000..621ff90 --- /dev/null +++ b/website/public/images/trainable_component.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/training.svg b/website/public/images/training.svg new file mode 100644 index 0000000..4d6bcc0 --- /dev/null +++ b/website/public/images/training.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/vocab_stringstore.svg b/website/public/images/vocab_stringstore.svg new file mode 100644 index 0000000..e10ff3c --- /dev/null +++ b/website/public/images/vocab_stringstore.svg @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/website/public/images/wandb1.jpg b/website/public/images/wandb1.jpg new file mode 100644 index 0000000..3baf4ab Binary files /dev/null and b/website/public/images/wandb1.jpg differ diff --git a/website/public/images/wandb2.jpg b/website/public/images/wandb2.jpg new file mode 100644 index 0000000..cd67c9a Binary files /dev/null and b/website/public/images/wandb2.jpg differ diff --git a/website/public/manifest.webmanifest b/website/public/manifest.webmanifest new file mode 100644 index 0000000..248d603 --- /dev/null +++ b/website/public/manifest.webmanifest @@ -0,0 +1,31 @@ +{ + "theme_color": "#09a3d5", + "background_color": "#09a3d5", + "display": "minimal-ui", + "scope": "/", + "start_url": "/", + "name": "spaCy", + "short_name": "spaCy", + "icons": [ + { + "src": "/icons/icon-192x192.png", + "sizes": "192x192", + "type": "image/png" + }, + { + "src": "/icons/icon-256x256.png", + "sizes": "256x256", + "type": "image/png" + }, + { + "src": "/icons/icon-384x384.png", + "sizes": "384x384", + "type": "image/png" + }, + { + "src": "/icons/icon-512x512.png", + "sizes": "512x512", + "type": "image/png" + } + ] +} diff --git a/website/public/vercel.svg b/website/public/vercel.svg new file mode 100644 index 0000000..fbf0e25 --- /dev/null +++ b/website/public/vercel.svg @@ -0,0 +1,4 @@ + + + \ No newline at end of file diff --git a/website/runtime.txt b/website/runtime.txt new file mode 100644 index 0000000..cc1923a --- /dev/null +++ b/website/runtime.txt @@ -0,0 +1 @@ +3.8 diff --git a/website/setup/jinja_to_js.py b/website/setup/jinja_to_js.py new file mode 100644 index 0000000..3e1963f --- /dev/null +++ b/website/setup/jinja_to_js.py @@ -0,0 +1,1272 @@ +# Forked from: https://github.com/jonbretman/jinja-to-js +# With additional functionality: in/not in, replace, pprint, round, + for lists, +# rendering empty dicts +# This script is mostly used to generate the JavaScript function for the +# training quickstart widget. +import contextlib +import json +import re +import os +from os import path +from io import StringIO +from jinja2 import Environment, FileSystemLoader, nodes +from pathlib import Path +import srsly +import sys + + +OPERANDS = { + "eq": "===", + "ne": "!==", + "lt": " < ", + "gt": " > ", + "lteq": " <= ", + "gteq": " >= ", +} + +DICT_ITER_METHODS = ("iteritems", "items", "values", "keys") + +STATE_DEFAULT = 0 +STATE_EXECUTING = 1 +STATE_INTERPOLATING = 2 + +LOOP_HELPER_INDEX = "index" +LOOP_HELPER_INDEX_0 = "index0" +LOOP_HELPER_FIRST = "first" +LOOP_HELPER_LAST = "last" +LOOP_HELPER_LENGTH = "length" +LOOP_HELPERS = ( + LOOP_HELPER_INDEX, + LOOP_HELPER_INDEX_0, + LOOP_HELPER_FIRST, + LOOP_HELPER_LAST, + LOOP_HELPER_LENGTH, +) + + +def amd_format(dependencies, template_function): + result = "define([" + result += ",".join('"{0}"'.format(x[0]) for x in dependencies) + result += "], function (" + result += ",".join(x[1] for x in dependencies) + result += ") { return " + result += template_function + result += "; });" + return result + + +def commonjs_format(dependencies, template_function): + result = "".join('var {0} = require("{1}");'.format(y, x) for x, y in dependencies) + result += "module.exports = {0};".format(template_function) + return result + + +def es6_format(dependencies, template_function): + result = "".join('import {0} from "{1}";'.format(y, x) for x, y in dependencies) + result += "export default {0}".format(template_function) + return result + + +JS_MODULE_FORMATS = { + None: lambda dependencies, template_function: template_function, + "amd": amd_format, + "commonjs": commonjs_format, + "es6": es6_format, +} + + +# This string has to double all the '{' and '}' due to Python's string formatting. +# See - https://docs.python.org/2/library/string.html#formatstrings +TEMPLATE_WRAPPER = """ +function {function_name}(ctx) {{ + var __result = ""; + var __tmp; + var __runtime = jinjaToJS.runtime; + var __filters = jinjaToJS.filters; + var __globals = jinjaToJS.globals; + var context = jinjaToJS.createContext(ctx); + {template_code} + return __result; +}} +""" + + +class ExtendsException(Exception): + """ + Raised when an {% extends %} is encountered. At this point the parent template is + loaded and all blocks defined in the current template passed to it. + """ + + pass + + +@contextlib.contextmanager +def option(current_kwargs, **kwargs): + """ + Context manager for temporarily setting a keyword argument and + then restoring it to whatever it was before. + """ + + tmp_kwargs = dict((key, current_kwargs.get(key)) for key, value in kwargs.items()) + current_kwargs.update(kwargs) + yield + current_kwargs.update(tmp_kwargs) + + +def is_method_call(node, method_name): + """ + Returns True if `node` is a method call for `method_name`. `method_name` + can be either a string or an iterable of strings. + """ + + if not isinstance(node, nodes.Call): + return False + + if isinstance(node.node, nodes.Getattr): + # e.g. foo.bar() + method = node.node.attr + + elif isinstance(node.node, nodes.Name): + # e.g. bar() + method = node.node.name + + elif isinstance(node.node, nodes.Getitem): + # e.g. foo["bar"]() + method = node.node.arg.value + + else: + return False + + if isinstance(method_name, (list, tuple)): + return method in method_name + + return method == method_name + + +def is_loop_helper(node): + """ + Returns True is node is a loop helper e.g. {{ loop.index }} or {{ loop.first }} + """ + return ( + hasattr(node, "node") + and isinstance(node.node, nodes.Name) + and node.node.name == "loop" + ) + + +def temp_var_names_generator(): + x = 0 + while True: + yield "__$%s" % x + x += 1 + + +class JinjaToJS(object): + def __init__( + self, + template_root, + template_name, + js_module_format=None, + runtime_path="jinja-to-js", + include_prefix="", + include_ext="", + child_blocks=None, + dependencies=None, + custom_filters=None, + ): + """ + Args: + template_root (str): The path to where templates should be loaded from. + template_name (str): The name of the template to compile (relative to `template_root`). + js_module_format (str, optional): The JavaScript module format to use. + One of ('amd', 'commonjs', 'es6') + runtime_path (str, optional): If `js_module_format` is specified then the JavaScript + runtime will be imported using the appropriate method. + It defaults to assuming it will be imported from + `node_modules` but you can change it using this option. + include_prefix (str, optional): If using the `amd` module format you can use this option + to add a prefix to every include path as AMD imports are + generally relative to the main file, not the module + importing. + include_ext (str, optional): By default any includes will be references without an + extension, as neither AMD, commonJS or ES6 require the + '.js' extension. If you want to use an extension, say + '.template' then set this option to a string including + the leading '.' + child_blocks (dict, optional): Used internally when handling templates that extend + other templates. + dependencies (list of tuple, optional): Used internally when handling templates that + extend other templates. + custom_filters (list of str, optional): List of custom filters which should be allowed. + These may be filters supported by Jinja but not + supported by jinja-to-js. These filters MUST be + registered with the jinja-to-js JS runtime. + """ + + self.environment = Environment( + loader=FileSystemLoader(template_root), + autoescape=True, + ) + self.output = StringIO() + self.stored_names = set() + self.temp_var_names = temp_var_names_generator() + self.state = STATE_DEFAULT + self.child_blocks = child_blocks or {} + self.dependencies = dependencies or [] + self._runtime_function_cache = [] + self.js_module_format = js_module_format + self.runtime_path = runtime_path + self.include_prefix = include_prefix + self.include_ext = include_ext + self.template_root = template_root + self.template_name = template_name + self.custom_filters = custom_filters or [] + + # The name of the JavaScript function that will output this template. By using a named + # function the template can call itself which is required to support recursive includes. + self.js_function_name = "template" + "".join( + x.title() + for x in re.split(r"[^\w]|_", path.splitext(self.template_name)[0]) + ) + + self.context_name = "context" + + self._add_dependency(self.runtime_path, "jinjaToJS") + + # Jinja2 doesn't accept Windows filepaths + if os.name == "nt": + self.template_name = self.template_name.replace(os.pathsep, "/") + + template_string, template_path, _ = self.environment.loader.get_source( + self.environment, self.template_name + ) + + # It is assumed that this will be the absolute path to the template. It is used to work out + # related paths for inclues. + self.template_path = template_path + + if self.js_module_format not in JS_MODULE_FORMATS.keys(): + raise ValueError( + "The js_module_format option must be one of: %s" + % JS_MODULE_FORMATS.keys() + ) + + self.ast = self.environment.parse(template_string) + + try: + for node in self.ast.body: + self._process_node(node) + except ExtendsException: + pass + + def get_output(self): + """ + Returns the generated JavaScript code. + + Returns: + str + """ + # generate the JS function string + template_function = TEMPLATE_WRAPPER.format( + function_name=self.js_function_name, template_code=self.output.getvalue() + ).strip() + + # get the correct module format template + module_format = JS_MODULE_FORMATS[self.js_module_format] + + # generate the module code + return module_format(self.dependencies, template_function) + + def _get_depencency_var_name(self, dependency): + """ + Returns the variable name assigned to the given dependency or None if the dependency has + not yet been registered. + + Args: + dependency (str): Thet dependency that needs to be imported. + + Returns: + str or None + """ + for dep_path, var_name in self.dependencies: + if dep_path == dependency: + return var_name + + def _add_dependency(self, dependency, var_name=None): + """ + Adds the given dependency and returns the variable name to use to access it. If `var_name` + is not given then a random one will be created. + + Args: + dependency (str): + var_name (str, optional): + + Returns: + str + """ + if var_name is None: + var_name = next(self.temp_var_names) + # Don't add duplicate dependencies + if (dependency, var_name) not in self.dependencies: + self.dependencies.append((dependency, var_name)) + return var_name + + def _process_node(self, node, **kwargs): + node_name = node.__class__.__name__.lower() + handler = getattr(self, "_process_" + node_name, None) + if callable(handler): + handler(node, **kwargs) + else: + raise Exception(f"Unknown node {node} ({node_name})") + + def _process_extends(self, node, **kwargs): + """ + Processes an extends block e.g. `{% extends "some/template.jinja" %}` + """ + + # find all the blocks in this template + for b in self.ast.find_all(nodes.Block): + + # if not already in `child_blocks` then this is the first time a + # block with this name has been encountered. + if b.name not in self.child_blocks: + self.child_blocks[b.name] = b + else: + + # otherwise we have seen this block before, so we need to find the last + # super_block and add the block from this template to the end. + block = self.child_blocks.get(b.name) + while hasattr(block, "super_block"): + block = block.super_block + block.super_block = b + + # load the parent template + parent_template = JinjaToJS( + template_root=self.template_root, + template_name=node.template.value, + js_module_format=self.js_module_format, + runtime_path=self.runtime_path, + include_prefix=self.include_prefix, + include_ext=self.include_ext, + child_blocks=self.child_blocks, + dependencies=self.dependencies, + ) + + # add the parent templates output to the current output + self.output.write(parent_template.output.getvalue()) + + # Raise an exception so we stop parsing this template + raise ExtendsException + + def _process_block(self, node, **kwargs): + """ + Processes a block e.g. `{% block my_block %}{% endblock %}` + """ + + # check if this node already has a 'super_block' attribute + if not hasattr(node, "super_block"): + + # since it doesn't it must be the last block in the inheritance chain + node.super_block = None + + # see if there has been a child block defined - if there is this + # will be the first block in the inheritance chain + child_block = self.child_blocks.get(node.name) + + if child_block: + + # we have child nodes so we need to set `node` as the + # super of the last one in the chain + last_block = child_block + while hasattr(last_block, "super_block"): + last_block = child_block.super_block + + # once we have found it, set this node as it's super block + last_block.super_block = node + + # this is the node we want to process as it's the first in the inheritance chain + node = child_block + + # process the block passing the it's super along, if this block + # calls super() it will be handled by `_process_call` + for n in node.body: + self._process_node(n, super_block=node.super_block, **kwargs) + + def _process_output(self, node, **kwargs): + """ + Processes an output node, which will contain things like `Name` and `TemplateData` nodes. + """ + for n in node.nodes: + self._process_node(n, **kwargs) + + def _process_templatedata(self, node, **_): + """ + Processes a `TemplateData` node, this is just a bit of as-is text + to be written to the output. + """ + + # escape double quotes + value = re.sub('"', r'\\"', node.data) + + # escape new lines + value = re.sub("\n", r"\\n", value) + + # append value to the result + self.output.write('__result += "' + value + '";') + + def _process_name(self, node, **kwargs): + """ + Processes a `Name` node. Some examples of `Name` nodes: + {{ foo }} -> 'foo' is a Name + {% if foo }} -> 'foo' is a Name + """ + + with self._interpolation(): + with self._python_bool_wrapper(**kwargs): + + if node.name not in self.stored_names and node.ctx != "store": + self.output.write(self.context_name) + self.output.write(".") + + if node.ctx == "store": + self.stored_names.add(node.name) + + self.output.write(node.name) + + def _process_dict(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs): + if node.items: + err = f"Can't process non-empty dict in expression: {node}" + raise ValueError(err) + self.output.write("{}") + + def _process_getattr(self, node, **kwargs): + """ + Processes a `GetAttr` node. e.g. {{ foo.bar }} + """ + + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + if is_loop_helper(node): + self._process_loop_helper(node, **new_kwargs) + else: + self._process_node(node.node, **new_kwargs) + self.output.write(".") + self.output.write(node.attr) + + def _process_getitem(self, node, **kwargs): + """ + Processes a `GetItem` node e.g. {{ foo["bar"] }} + """ + + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + + if isinstance(node.arg, nodes.Slice): + self.output.write(".slice(") + + if node.arg.step is not None: + raise Exception( + "The step argument is not supported when slicing." + ) + + if node.arg.start is None: + self.output.write("0") + else: + self._process_node(node.arg.start, **new_kwargs) + + if node.arg.stop is None: + self.output.write(")") + else: + self.output.write(",") + self._process_node(node.arg.stop, **new_kwargs) + self.output.write(")") + else: + self.output.write("[") + self._process_node(node.arg, **new_kwargs) + self.output.write("]") + + def _process_for(self, node, **kwargs): + """ + Processes a for loop. e.g. + {% for number in numbers %} + {{ number }} + {% endfor %} + {% for key, value in somemap.items() %} + {{ key }} -> {{ value }} + {% %} + """ + + # since a for loop can introduce new names into the context + # we need to remember the ones that existed outside the loop + previous_stored_names = self.stored_names.copy() + + with self._execution(): + self.output.write("__runtime.each(") + + if is_method_call(node.iter, dict.keys.__name__): + self.output.write("Object.keys(") + + self._process_node(node.iter, **kwargs) + + if is_method_call(node.iter, dict.keys.__name__): + self.output.write(")") + + self.output.write(",") + self.output.write("function") + self.output.write("(") + + # javascript iterations put the value first, then the key + if isinstance(node.target, nodes.Tuple): + if len(node.target.items) > 2: + raise Exception( + "De-structuring more than 2 items is not supported." + ) + + for i, item in enumerate(reversed(node.target.items)): + self._process_node(item, **kwargs) + if i < len(node.target.items) - 1: + self.output.write(",") + else: + self._process_node(node.target, **kwargs) + + self.output.write(")") + self.output.write("{") + + if node.test: + self.output.write("if (!(") + self._process_node(node.test, **kwargs) + self.output.write(")) { return; }") + + assigns = ( + node.target.items if isinstance(node.target, nodes.Tuple) else [node.target] + ) + + with self._scoped_variables(assigns, **kwargs): + for n in node.body: + self._process_node(n, **kwargs) + + with self._execution(): + self.output.write("}") + self.output.write(")") + self.output.write(";") + + # restore the stored names + self.stored_names = previous_stored_names + + def _process_if(self, node, execute_end=None, **kwargs): + """ + Processes an if block e.g. `{% if foo %} do something {% endif %}` + """ + + with self._execution(): + self.output.write("if") + self.output.write("(") + + with option(kwargs, use_python_bool_wrapper=True): + self._process_node(node.test, **kwargs) + + self.output.write(")") + self.output.write("{") + + # We accept an `execute_end` function as a keyword argument as this function is + # recursive in the case of something like if-elif-elif-else. In these cases this + # invocation of this function may have to close execution opened by a previous + # invocation of this function. + if execute_end: + execute_end() + + # body + for n in node.body: + self._process_node(n, **kwargs) + + if not node.else_ and not node.elif_: + # no else - just close the if + with self._execution(): + self.output.write("}") + + else: + # either an else or an elif + with self._execution() as execute_end: + self.output.write("}") + self.output.write(" else ") + + # check for elif + for n in node.elif_: + self._process_node(n, execute_end=execute_end, **kwargs) + + if node.elif_ and node.else_: + self.output.write(" else ") + + # open up the body + self.output.write("{") + + # process the body of the else + for n in node.else_: + self._process_node(n, **kwargs) + + # close the body + with self._execution(): + self.output.write("}") + + def _process_condexpr(self, node, **kwargs): + with self._interpolation(): + self.output.write("(") + + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.test, **new_kwargs) + + self.output.write(" ? ") + self._process_node(node.expr1, **kwargs) + self.output.write(" : ") + self._process_node(node.expr2, **kwargs) + self.output.write(")") + + def _process_not(self, node, **kwargs): + self.output.write("!") + + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + + def _process_or(self, node, **kwargs): + self._process_node(node.left, **kwargs) + self.output.write(" || ") + self._process_node(node.right, **kwargs) + + def _process_and(self, node, **kwargs): + self._process_node(node.left, **kwargs) + self.output.write(" && ") + self._process_node(node.right, **kwargs) + + def _process_tuple(self, node, **kwargs): + self.output.write("[") + for i, item in enumerate(node.items): + self._process_node(item, **kwargs) + if i < len(node.items) - 1: + self.output.write(",") + self.output.write("]") + + def _process_call(self, node, super_block=None, **kwargs): + if is_method_call(node, DICT_ITER_METHODS): + # special case for dict methods + self._process_node(node.node.node, **kwargs) + + elif is_method_call(node, "super"): + # special case for the super() method which is available inside blocks + if not super_block: + raise Exception("super() called outside of a block with a parent.") + self._process_node(super_block, **kwargs) + + else: + # just a normal function call on a context variable + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + self.output.write("(") + self._process_args(node, **new_kwargs) + self.output.write(")") + + # only output the semi-colon if we are not interpolating + if self.state != STATE_INTERPOLATING: + self.output.write("") + + def _process_filter(self, node, **kwargs): + method_name = getattr(self, "_process_filter_%s" % node.name, None) + if callable(method_name): + method_name(node, **kwargs) + elif node.name in self.custom_filters: + with self._interpolation(safe=True): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.%s(" % node.name) + self._process_node(node.node, **new_kwargs) + if getattr(node, "args", None): + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + else: + raise Exception("Unsupported filter: %s" % node.name) + + def _process_filter_safe(self, node, **kwargs): + with self._interpolation(safe=True): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + + def _process_filter_capitalize(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.capitalize(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_abs(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("Math.abs(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_replace(self, node, **kwargs): + # We're getting a quoted string from Python/Jinja as the pattern to + # replace, but to replace all occurrences in JS, we typically need a + # regex, which would be annoying to convert. So we're using split/join + # instead here. + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + self.output.write(".split(") + self._process_node(node.args[0], **new_kwargs) + self.output.write(").join(") + self._process_node(node.args[1], **new_kwargs) + self.output.write(")") + + def _process_filter_pprint(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("JSON.stringify(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_attr(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.node, **new_kwargs) + self.output.write("[") + self._process_node(node.args[0], **new_kwargs) + self.output.write("]") + + def _process_filter_batch(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.batch(") + self._process_node(node.node, **new_kwargs) + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + + def _process_filter_default(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.default(") + self._process_node(node.node, **new_kwargs) + if node.args: + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + + def _process_filter_first(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.first(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_int(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.int(") + self._process_node(node.node, **new_kwargs) + if node.args: + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + + def _process_filter_round(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("Math.round((") + self._process_node(node.node, **new_kwargs) + self.output.write("+ Number.EPSILON) * 10**") + self._process_node(node.args[0], **new_kwargs) + self.output.write(") / 10**") + self._process_node(node.args[0], **new_kwargs) + + def _process_filter_last(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.last(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_length(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.size(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_lower(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("(") + self._process_node(node.node, **new_kwargs) + self.output.write(' + "").toLowerCase()') + + def _process_filter_slice(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.slice(") + self._process_node(node.node, **new_kwargs) + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + + def _process_filter_title(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.title(") + self._process_node(node.node, **new_kwargs) + self.output.write(")") + + def _process_filter_trim(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("(") + self._process_node(node.node, **new_kwargs) + self.output.write(' + "").trim()') + + def _process_filter_upper(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("(") + self._process_node(node.node, **new_kwargs) + self.output.write(' + "").toUpperCase()') + + def _process_filter_truncate(self, node, **kwargs): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self.output.write("__filters.truncate(") + self._process_node(node.node, **new_kwargs) + self.output.write(",") + self._process_args(node, **new_kwargs) + self.output.write(")") + + def _process_assign(self, node, **kwargs): + with self._execution(): + self.output.write("var ") + self._process_node(node.target, **kwargs) + self.output.write(" = ") + self._process_node(node.node, **kwargs) + self.output.write(";") + + def _process_with(self, node, **kwargs): + + # keep a copy of the stored names before the scope + previous_stored_names = self.stored_names.copy() + + # assigns in the with tag + # e.g. {% with var = "something %} + assigns_in_tag = [nodes.Assign(t, v) for t, v in zip(node.targets, node.values)] + + # assigns in the with body + # e.g. {% set name = 'John' %} + assigns_in_body = [x for x in node.body if isinstance(x, nodes.Assign)] + + # remove assigns from the body + node.body = [x for x in node.body if not isinstance(x, nodes.Assign)] + + # get a list of all the assigns in this with block + # both on the tag, and within the body of the block + all_assigns = assigns_in_tag + assigns_in_body + + with self._execution(): + self.output.write("(function () {") + + with self._scoped_variables(all_assigns, **kwargs): + for node in node.body: + self._process_node(node, **kwargs) + + with self._execution(): + self.output.write("})();") + + # restore previous stored names + self.stored_names = previous_stored_names + + def _process_compare(self, node, **kwargs): + + if len(node.ops) > 1: + raise Exception("Multiple operands are not supported.") + + operand = node.ops[0] + is_equality = operand.op in ("eq", "ne") + left_hand_is_const = isinstance(node.expr, nodes.Const) + right_hand_is_const = isinstance(operand.expr, nodes.Const) + + # If the operand is equality and neither the left or right hand side are constants then we + # will need to use the JavaScript deep equals function. Ideally we want to avoid using this + # as it is quite a big function. + use_is_equal_function = is_equality and not ( + left_hand_is_const or right_hand_is_const + ) + + with option(kwargs, use_python_bool_wrapper=False): + if operand.op == "in" or operand.op == "notin": + # Special case for "in" operator + if operand.op == "notin": + self.output.write("!") + self._process_node(operand.expr, **kwargs) + self.output.write(".includes(") + self._process_node(node.expr, **kwargs) + self.output.write(")") + else: + if use_is_equal_function: + if operand.op == "ne": + self.output.write("!") + self.output.write("__runtime.isEqual(") + + self._process_node(node.expr, **kwargs) + + if use_is_equal_function: + self.output.write(",") + else: + self.output.write(OPERANDS.get(operand.op)) + + self._process_node(operand.expr, **kwargs) + + if use_is_equal_function: + self.output.write(")") + + def _process_operand(self, node, **kwargs): + self.output.write(OPERANDS.get(node.op)) + self._process_node(node.expr, **kwargs) + + def _process_const(self, node, **_): + with self._interpolation(): + self.output.write(json.dumps(node.value)) + + def _process_nonetype(self, node, **_): + with self._interpolation(): + self.output.write("null") + + def _process_neg(self, node, **kwargs): + with self._interpolation(): + self.output.write("-") + self._process_node(node.node, **kwargs) + + def _process_list(self, node, **kwargs): + self.output.write("[") + for i, item in enumerate(node.items): + self._process_node(item, **kwargs) + if i < len(node.items) - 1: + self.output.write(",") + self.output.write("]") + + def _process_test(self, node, **kwargs): + with option(kwargs, use_python_bool_wrapper=False): + method_name = getattr(self, "_process_test_%s" % node.name, None) + if callable(method_name): + method_name(node, **kwargs) + else: + raise Exception("Unsupported test: %s" % node.name) + + def _process_test_defined(self, node, **kwargs): + self.output.write("(typeof ") + self._process_node(node.node, **kwargs) + self.output.write(' !== "undefined")') + + def _process_test_undefined(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(" === undefined") + + def _process_test_callable(self, node, **kwargs): + self.output.write("__runtime.type(") + self._process_node(node.node, **kwargs) + self.output.write(') === "Function"') + + def _process_test_divisibleby(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(" % ") + self._process_node(node.args[0], **kwargs) + self.output.write(" === 0") + + def _process_test_even(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(" % 2 === 0") + + def _process_test_odd(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(" % 2 === 1") + + def _process_test_none(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(" === null") + + def _process_test_upper(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(".toUpperCase() === ") + self._process_node(node.node, **kwargs) + + def _process_test_lower(self, node, **kwargs): + self._process_node(node.node, **kwargs) + self.output.write(".toLowerCase() === ") + self._process_node(node.node, **kwargs) + + def _process_test_string(self, node, **kwargs): + self.output.write("__runtime.type(") + self._process_node(node.node, **kwargs) + self.output.write(') === "String"') + + def _process_test_mapping(self, node, **kwargs): + self.output.write("__runtime.type(") + self._process_node(node.node, **kwargs) + self.output.write(') === "Object"') + + def _process_test_number(self, node, **kwargs): + self.output.write("(__runtime.type(") + self._process_node(node.node, **kwargs) + self.output.write(') === "Number" && !isNaN(') + self._process_node(node.node, **kwargs) + self.output.write("))") + + def _process_include(self, node, **kwargs): + with self._interpolation(safe=True): + include_path = node.template.value + + if include_path == self.template_name: + # template is including itself + include_var_name = self.js_function_name + else: + if self.include_prefix: + include_path = self.include_prefix + node.template.value + elif ( + self.js_module_format in ("es6", "commonjs",) and self.template_name + ): + _, absolute_include_path, _ = self.environment.loader.get_source( + self.environment, node.template.value + ) + include_path = os.path.relpath( + absolute_include_path, os.path.dirname(self.template_path) + ) + if not include_path.startswith("."): + include_path = "./" + include_path + + # Jinja2 doesn't accept Windows filepaths (but does output them!) + if os.name == "nt": + include_path = include_path.replace(os.pathsep, "/") + + include_path = path.splitext(include_path)[0] + self.include_ext + include_var_name = self._get_depencency_var_name(include_path) + + if not include_var_name: + include_var_name = self._add_dependency(include_path) + + if self.js_module_format is None: + self.output.write('jinjaToJS.include("') + self.output.write(include_path) + self.output.write('");') + else: + self.output.write(include_var_name) + + self.output.write("(") + self.output.write(self.context_name) + self.output.write(")") + + def _process_add(self, node, **kwargs): + # Handle + operator for lists, which behaves differently in JS. Currently + # only works if we have an explicit list node on either side (in which + # case we assume both are lists). + if isinstance(node.left, nodes.List) or isinstance(node.right, nodes.List): + with self._interpolation(): + with self._python_bool_wrapper(**kwargs) as new_kwargs: + self._process_node(node.left, **new_kwargs) + self.output.write(".concat(") + self._process_node(node.right, **new_kwargs) + self.output.write(")") + else: + self._process_math(node, math_operator=" + ", **kwargs) + + def _process_sub(self, node, **kwargs): + self._process_math(node, math_operator=" - ", **kwargs) + + def _process_div(self, node, **kwargs): + self._process_math(node, math_operator=" / ", **kwargs) + + def _process_floordiv(self, node, **kwargs): + self._process_math(node, math_operator=" / ", function="Math.floor", **kwargs) + + def _process_mul(self, node, **kwargs): + self._process_math(node, math_operator=" * ", **kwargs) + + def _process_mod(self, node, **kwargs): + self._process_math(node, math_operator=" % ", **kwargs) + + def _process_math(self, node, math_operator=None, function=None, **kwargs): + """ + Processes a math node e.g. `Div`, `Sub`, `Add`, `Mul` etc... + If `function` is provided the expression is wrapped in a call to that function. + """ + + with self._interpolation(): + if function: + self.output.write(function) + self.output.write("(") + + self._process_node(node.left, **kwargs) + self.output.write(math_operator) + self._process_node(node.right, **kwargs) + + if function: + self.output.write(")") + + def _process_loop_helper(self, node, **kwargs): + """ + Processes a loop helper e.g. {{ loop.first }} or {{ loop.index }} + """ + + if node.attr == LOOP_HELPER_INDEX: + self.output.write("(arguments[1] + 1)") + elif node.attr == LOOP_HELPER_INDEX_0: + self.output.write("arguments[1]") + elif node.attr == LOOP_HELPER_FIRST: + self.output.write("(arguments[1] == 0)") + elif node.attr == LOOP_HELPER_LAST: + self.output.write("(arguments[1] == arguments[2].length - 1)") + elif node.attr == LOOP_HELPER_LENGTH: + self.output.write("arguments[2].length") + + def _process_args(self, node, **kwargs): + args = getattr(node, "args", None) + if not args: + return + for i, item in enumerate(args): + self._process_node(item, **kwargs) + if i < len(node.args) - 1: + self.output.write(",") + + @contextlib.contextmanager + def _execution(self): + """ + Context manager for executing some JavaScript inside a template. + """ + + did_start_executing = False + + if self.state == STATE_DEFAULT: + did_start_executing = True + self.state = STATE_EXECUTING + + def close(): + if did_start_executing and self.state == STATE_EXECUTING: + self.state = STATE_DEFAULT + + yield close + close() + + @contextlib.contextmanager + def _interpolation(self, safe=False): + + did_start_interpolating = False + + if self.state == STATE_DEFAULT: + did_start_interpolating = True + self.output.write('__result += "" + ') + if safe is not True: + self.output.write("__runtime.escape") + self.output.write("((__tmp = (") + self.state = STATE_INTERPOLATING + + def close(): + if did_start_interpolating and self.state == STATE_INTERPOLATING: + self.output.write(')) == null ? "" : __tmp);') + self.state = STATE_DEFAULT + + yield close + close() + + @contextlib.contextmanager + def _scoped_variables(self, nodes_list, **kwargs): + """ + Context manager for creating scoped variables defined by the nodes in `nodes_list`. + These variables will be added to the context, and when the context manager exits the + context object will be restored to it's previous state. + """ + + tmp_vars = [] + for node in nodes_list: + + is_assign_node = isinstance(node, nodes.Assign) + name = node.target.name if is_assign_node else node.name + + # create a temp variable name + tmp_var = next(self.temp_var_names) + + # save previous context value + with self._execution(): + + # save the current value of this name + self.output.write( + "var %s = %s.%s;" % (tmp_var, self.context_name, name) + ) + + # add new value to context + self.output.write("%s.%s = " % (self.context_name, name)) + + if is_assign_node: + self._process_node(node.node, **kwargs) + else: + self.output.write(node.name) + + self.output.write(";") + + tmp_vars.append((tmp_var, name)) + + yield + + # restore context + for tmp_var, name in tmp_vars: + with self._execution(): + self.output.write("%s.%s = %s;" % (self.context_name, name, tmp_var)) + + @contextlib.contextmanager + def _python_bool_wrapper(self, **kwargs): + + use_python_bool_wrapper = kwargs.get("use_python_bool_wrapper") + + if use_python_bool_wrapper: + self.output.write("__runtime.boolean(") + + with option(kwargs, use_python_bool_wrapper=False): + yield kwargs + + if use_python_bool_wrapper: + self.output.write(")") + + +def main(template_path, output=None, data_path=None): + """Convert a jinja2 template to a JavaScript module. + + template_path (Path): Path to .jijna file. + output (Optional[Path]): Path to output .js module (stdout if unset). + data_path (Optional[Path]): Optional JSON or YAML file with additional data + to be included in the JS module as the exported variable DATA. + """ + data = "{}" + if data_path is not None: + if data_path.suffix in (".yml", ".yaml"): + data = srsly.read_yaml(data_path) + else: + data = srsly.read_json(data_path) + data = srsly.json_dumps(data) # dump and load for compactness + template_path = Path(template_path) + tpl_file = template_path.parts[-1] + compiler = JinjaToJS(template_path.parent, tpl_file, js_module_format="es6") + header = f"// This file was auto-generated by {__file__} based on {tpl_file}" + data_str = f"export const DATA = {data}" + result = compiler.get_output() + if output is not None: + with output.open("w", encoding="utf8") as f: + f.write(f"{header}\n{result}\n{data_str}") + print(f"Updated {output.parts[-1]}") + else: + print(result) + + +if __name__ == "__main__": + args = sys.argv[1:] + if not len(args): + raise ValueError("Need at least one argument: path to .jinja template") + template_path = Path(args[0]) + output = Path(args[1]) if len(args) > 1 else None + data_path = Path(args[2]) if len(args) > 2 else None + main(template_path, output, data_path) diff --git a/website/setup/requirements.txt b/website/setup/requirements.txt new file mode 100644 index 0000000..cbd306c --- /dev/null +++ b/website/setup/requirements.txt @@ -0,0 +1,3 @@ +# These are used to compile the training quickstart config +jinja2>=3.1.0 +srsly diff --git a/website/setup/setup.sh b/website/setup/setup.sh new file mode 100755 index 0000000..8207c6a --- /dev/null +++ b/website/setup/setup.sh @@ -0,0 +1 @@ +python setup/jinja_to_js.py ../spacy/cli/templates/quickstart_training.jinja src/widgets/quickstart-training-generator.js ../spacy/cli/templates/quickstart_training_recommendations.yml diff --git a/website/src/components/accordion.js b/website/src/components/accordion.js new file mode 100644 index 0000000..9ff145b --- /dev/null +++ b/website/src/components/accordion.js @@ -0,0 +1,66 @@ +import React, { useState, useEffect } from 'react' +import PropTypes from 'prop-types' +import classNames from 'classnames' + +import Link from './link' +import classes from '../styles/accordion.module.sass' + +export default function Accordion({ title, id, expanded = false, spaced = false, children }) { + const [isExpanded, setIsExpanded] = useState(true) + const rootClassNames = classNames(classes.root, { + [classes.spaced]: !!spaced, + }) + const contentClassNames = classNames(classes.content, { + [classes.hidden]: !isExpanded, + }) + const iconClassNames = classNames({ + [classes.hidden]: isExpanded, + }) + // Make sure accordion is expanded if JS is disabled + useEffect(() => setIsExpanded(expanded), [expanded]) + return ( +
+
+

+ +

+
{children}
+
+
+ ) +} + +Accordion.propTypes = { + title: PropTypes.string, + id: PropTypes.string, + children: PropTypes.node.isRequired, +} diff --git a/website/src/components/alert.js b/website/src/components/alert.js new file mode 100644 index 0000000..bc928a0 --- /dev/null +++ b/website/src/components/alert.js @@ -0,0 +1,29 @@ +import React, { useState } from 'react' +import PropTypes from 'prop-types' +import classNames from 'classnames' + +import Icon from './icon' +import classes from '../styles/alert.module.sass' + +export default function Alert({ title, icon, variant, closeOnClick = true, children }) { + const [visible, setVisible] = useState(true) + const alertClassNames = classNames(classes.root, { + [classes.warning]: variant === 'warning', + [classes.clickable]: !!closeOnClick, + }) + const handleClick = () => !!closeOnClick && setVisible(false) + return !visible ? null : ( + + ) +} + +Alert.propTypes = { + title: PropTypes.string, + icon: PropTypes.string, + variant: PropTypes.oneOf(['warning']), + closeOnClick: PropTypes.bool, + children: PropTypes.node, +} diff --git a/website/src/components/aside.js b/website/src/components/aside.js new file mode 100644 index 0000000..e7b94ee --- /dev/null +++ b/website/src/components/aside.js @@ -0,0 +1,22 @@ +import React from 'react' +import PropTypes from 'prop-types' + +import classes from '../styles/aside.module.sass' + +export default function Aside({ title, children }) { + return ( + + ) +} + +Aside.propTypes = { + title: PropTypes.string, + children: PropTypes.node.isRequired, +} diff --git a/website/src/components/button.js b/website/src/components/button.js new file mode 100644 index 0000000..990599e --- /dev/null +++ b/website/src/components/button.js @@ -0,0 +1,38 @@ +import React from 'react' +import PropTypes from 'prop-types' +import classNames from 'classnames' + +import Link from './link' +import Icon from './icon' +import classes from '../styles/button.module.sass' + +export default function Button({ + to, + variant = 'secondary', + large = false, + icon, + className, + children, + ...props +}) { + const buttonClassNames = classNames(classes.root, className, { + [classes.large]: large, + [classes.primary]: variant === 'primary', + [classes.secondary]: variant === 'secondary', + [classes.tertiary]: variant === 'tertiary', + }) + return ( + + {icon && } + {children} + + ) +} + +Button.propTypes = { + to: PropTypes.string, + variant: PropTypes.oneOf(['primary', 'secondary', 'tertiary']), + large: PropTypes.bool, + icon: PropTypes.string, + className: PropTypes.string, +} diff --git a/website/src/components/card.js b/website/src/components/card.js new file mode 100644 index 0000000..ef43eb8 --- /dev/null +++ b/website/src/components/card.js @@ -0,0 +1,46 @@ +import React from 'react' +import PropTypes from 'prop-types' +import classNames from 'classnames' +import ImageNext from 'next/image' + +import Link from './link' +import { H5 } from './typography' +import classes from '../styles/card.module.sass' + +export default function Card({ title, to, image, header, small, onClick, children }) { + return ( +
+ {header && ( + + {header} + + )} + {(title || image) && ( +
+ {image && ( +
+ +
+ )} + {title && ( + + {title} + + )} +
+ )} + + {children} + +
+ ) +} + +Card.propTypes = { + title: PropTypes.node, + header: PropTypes.node, + to: PropTypes.string, + image: PropTypes.string, + onClick: PropTypes.func, + children: PropTypes.node, +} diff --git a/website/src/components/code.js b/website/src/components/code.js new file mode 100644 index 0000000..e733dba --- /dev/null +++ b/website/src/components/code.js @@ -0,0 +1,311 @@ +import React, { Fragment, useEffect, useState } from 'react' +import PropTypes from 'prop-types' +import classNames from 'classnames' +import rangeParser from 'parse-numeric-range' +import Prism from 'prismjs' + +// We manually load all the languages that are needed, which are currently only those: +import 'prismjs/components/prism-diff.min.js' +import 'prismjs/components/prism-bash.min.js' +import 'prismjs/components/prism-ini.min.js' +import 'prismjs/components/prism-jsx.min.js' +import 'prismjs/components/prism-json.min.js' +import 'prismjs/components/prism-markdown.min.js' +import 'prismjs/components/prism-python.min.js' +import 'prismjs/components/prism-yaml.min.js' +import 'prismjs/components/prism-docker.min.js' +import 'prismjs/components/prism-r.min.js' + +import { isString } from './util' +import Link, { OptionalLink } from './link' +import GitHubCode from './github' +import classes from '../styles/code.module.sass' +import siteMetadata from '../../meta/site.json' +import { binderBranch } from '../../meta/dynamicMeta.mjs' +import dynamic from 'next/dynamic' + +const CLI_GROUPS = ['init', 'debug', 'project', 'ray', 'huggingface-hub'] + +const splitLines = (children) => { + const listChildrenPerLine = [] + + if (typeof children === 'string') { + listChildrenPerLine.push(...children.split('\n')) + } else { + listChildrenPerLine.push([]) + let indexLine = 0 + if (Array.isArray(children)) { + children.forEach((child) => { + if (typeof child === 'string' && child.includes('\n')) { + const listString = child.split('\n') + listString.forEach((string, index) => { + listChildrenPerLine[indexLine].push(string) + + if (index !== listString.length - 1) { + indexLine += 1 + listChildrenPerLine[indexLine] = [] + } + }) + } else { + listChildrenPerLine[indexLine].push(child) + } + }) + } else { + listChildrenPerLine[indexLine].push(children) + indexLine += 1 + listChildrenPerLine[indexLine] = [] + } + } + + const listLine = listChildrenPerLine[listChildrenPerLine.length - 1] + if (listLine === '' || (listLine.length === 1 && listLine[0] === '')) { + listChildrenPerLine.pop() + } + + return listChildrenPerLine.map((childrenPerLine, index) => ( + <> + {childrenPerLine} + {index !== listChildrenPerLine.length - 1 && '\n'} + + )) +} + +function parseArgs(raw) { + let args = raw.split(' ').filter((arg) => arg) + const result = {} + while (args.length) { + let opt = args.shift() + if (opt.length > 1 && opt.startsWith('-')) { + const isFlag = !args.length || (args[0].length > 1 && args[0].startsWith('-')) + result[opt] = isFlag ? true : args.shift() + } else { + let key = opt + if (CLI_GROUPS.includes(opt)) { + if (args.length && !args[0].startsWith('-')) { + key = `${opt} ${args.shift()}` + } + } + result[key] = null + } + } + return result +} + +const flattenReact = (children) => { + if (children === null || children === undefined || children === false) { + return [] + } + + if (typeof children === 'string') { + return [children] + } + + if (children.props) { + return flattenReact(children.props.children) + } + + return children.flatMap(flattenReact) +} + +const checkoutForComment = (line) => { + const lineParts = line.split(' # ') + + if (lineParts.length !== 2) { + return line + } + + return ( + <> + {lineParts[0]} + {` `} + + {`# `} + {lineParts[1]} + + + ) +} + +const handlePromot = ({ lineFlat, prompt }) => { + const lineWithoutPrompt = lineFlat.slice(prompt.length + 1) + + const cliRegex = /^python -m spacy/ + + if (!cliRegex.test(lineWithoutPrompt)) { + return {checkoutForComment(lineWithoutPrompt)} + } + + const text = lineWithoutPrompt.replace(cliRegex, '') + const args = parseArgs(text) + const cmd = Object.keys(args).map((key, i) => { + const value = args[key] + return value === null || value === true || i === 0 ? key : `${key} ${value}` + }) + return ( + + python -m spacy{' '} + {cmd.map((item, j) => { + const isCmd = j === 0 + const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null + const isAbstract = isString(item) && /^\[(.+)\]$/.test(item) + const itemClassNames = classNames(classes['cli-arg'], { + [classes['cli-arg-highlight']]: isCmd, + [classes['cli-arg-emphasis']]: isAbstract, + }) + const text = isAbstract ? item.slice(1, -1) : item + return ( + + {j !== 0 && ' '} + + + {text} + + + + ) + })} + + ) +} + +const convertLine = ({ line, prompt, lang }) => { + const lineFlat = flattenReact(line).join('') + if (lineFlat.startsWith(`${prompt} `)) { + return handlePromot({ lineFlat, prompt }) + } + + return lang === 'none' || !lineFlat || !(lang in Prism.languages) ? ( + lineFlat + ) : ( + + ) +} + +const addLineHighlight = (children, highlight) => { + if (!highlight) { + return children + } + const listHighlight = rangeParser(highlight) + + if (listHighlight.length === 0) { + return children + } + + return children.map((child, index) => { + const isHighlight = listHighlight.includes(index + 1) + return ( + + {child} + + ) + }) +} + +const CodeHighlighted = ({ children, highlight, lang }) => { + const [html, setHtml] = useState() + + useEffect( + () => + setHtml( + addLineHighlight( + splitLines(children).map((line) => convertLine({ line, prompt: '$', lang })), + highlight + ) + ), + [children, highlight, lang] + ) + + return <>{html} +} + +export default class Code extends React.Component { + static defaultProps = { + lang: 'none', + executable: null, + } + + static propTypes = { + lang: PropTypes.string, + title: PropTypes.string, + executable: PropTypes.oneOf(['true', 'false', true, false, null]), + github: PropTypes.string, + prompt: PropTypes.string, + highlight: PropTypes.string, + className: PropTypes.string, + children: PropTypes.node, + } + + render() { + const { lang, title, executable, github, wrap, highlight, className, children } = this.props + const codeClassNames = classNames(classes['code'], className, `language-${lang}`, { + [classes['wrap']]: !!highlight || !!wrap || lang === 'cli', + [classes['cli']]: lang === 'cli', + }) + const ghClassNames = classNames(codeClassNames, classes['max-height']) + + if (github) { + return + } + if (!!executable) { + return ( + + {children} + + ) + } + + return ( + <> + {title &&

{title}

} + + + {children} + + + + ) + } +} + +const JuniperDynamic = dynamic(() => import('./juniper')) + +const JuniperWrapper = ({ title, lang, children }) => { + const { binderUrl, binderVersion } = siteMetadata + const juniperTitle = title || 'Editable Code' + return ( +
+

+ {juniperTitle} + + spaCy v{binderVersion} · Python 3 · via{' '} + + Binder + + +

+ + + {children} + +
+ ) +} diff --git a/website/src/components/codeBlock.js b/website/src/components/codeBlock.js new file mode 100644 index 0000000..d990b93 --- /dev/null +++ b/website/src/components/codeBlock.js @@ -0,0 +1,14 @@ +import React from 'react' +import Code from './codeDynamic' +import classes from '../styles/code.module.sass' + +export const Pre = (props) => { + return 
{props.children}
+} + +const CodeBlock = (props) => ( + 
+        
+
+) +export default CodeBlock diff --git a/website/src/components/codeDynamic.js b/website/src/components/codeDynamic.js new file mode 100644 index 0000000..8c94835 --- /dev/null +++ b/website/src/components/codeDynamic.js @@ -0,0 +1,5 @@ +import dynamic from 'next/dynamic' + +export default dynamic(() => import('./code'), { + loading: () =>
Loading...
, +}) diff --git a/website/src/components/copy.js b/website/src/components/copy.js new file mode 100644 index 0000000..bc73271 --- /dev/null +++ b/website/src/components/copy.js @@ -0,0 +1,53 @@ +import React, { useState, useRef, useEffect } from 'react' + +import Icon from './icon' +import classes from '../styles/copy.module.sass' + +export function copyToClipboard(ref, callback) { + const isClient = typeof window !== 'undefined' + if (ref.current && isClient) { + ref.current.select() + document.execCommand('copy') + callback(true) + ref.current.blur() + setTimeout(() => callback(false), 1000) + } +} + +export default function CopyInput({ text, description, prefix }) { + const isClient = typeof window !== 'undefined' + const [supportsCopy, setSupportsCopy] = useState(false) + + useEffect(() => { + setSupportsCopy(isClient && document.queryCommandSupported('copy')) + }, [isClient]) + const textareaRef = useRef() + const [copySuccess, setCopySuccess] = useState(false) + const onClick = () => copyToClipboard(textareaRef, setCopySuccess) + + function selectText() { + if (textareaRef.current && isClient) { + textareaRef.current.select() + } + } + + return ( +
+ {prefix && {prefix}} +